@fabrika/app 0.9.0-dev.3 → 0.9.0-dev.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (115) hide show
  1. package/package.json +14 -9
  2. package/skill/SKILL.md +41 -14
  3. package/skill/references/app-setup.md +43 -12
  4. package/skill/references/dev.md +2 -1
  5. package/skill/references/endpoint.md +30 -24
  6. package/skill/references/errors.md +40 -35
  7. package/skill/references/files.md +4 -2
  8. package/skill/references/jobs.md +21 -4
  9. package/skill/references/table-and-migration.md +19 -2
  10. package/skill/references/tests.md +40 -1
  11. package/skill/references/webhooks-links-connectors.md +1 -1
  12. package/src/build-recipe.ts +198 -0
  13. package/src/build.ts +60 -4
  14. package/src/bun.ts +181 -13
  15. package/src/cloudflare/application.ts +822 -0
  16. package/src/cloudflare/bindings.ts +159 -0
  17. package/src/cloudflare/export.ts +641 -0
  18. package/src/cloudflare/import.ts +1021 -0
  19. package/src/cloudflare/maintenance.ts +518 -0
  20. package/src/cloudflare/names.ts +16 -0
  21. package/src/cloudflare/ops.ts +375 -0
  22. package/src/cloudflare/r2-files.ts +172 -0
  23. package/src/cloudflare/restore.ts +167 -0
  24. package/src/cloudflare/tracker.ts +192 -0
  25. package/src/cloudflare.ts +100 -146
  26. package/src/contract/app-id.ts +5 -0
  27. package/src/contract/artifact-schema.ts +9 -1
  28. package/src/contract/build.ts +40 -2
  29. package/src/contract/compile.ts +1 -0
  30. package/src/contract/context.ts +35 -9
  31. package/src/contract/declaration.ts +2 -0
  32. package/src/contract/defaults.ts +22 -2
  33. package/src/contract/define.ts +29 -3
  34. package/src/contract/kit.ts +4 -1
  35. package/src/contract/platform.ts +3 -2
  36. package/src/contract/vocabulary.ts +26 -8
  37. package/src/dev/launcher.ts +84 -13
  38. package/src/dev/output.ts +13 -3
  39. package/src/disk-assets.ts +8 -3
  40. package/src/files/archive.ts +2 -2
  41. package/src/files/job-names.ts +12 -0
  42. package/src/files/kinds.ts +73 -31
  43. package/src/files/scan.ts +3 -4
  44. package/src/files/streams.ts +3 -1
  45. package/src/implement/anonymous.ts +12 -11
  46. package/src/implement/boot.ts +75 -6
  47. package/src/implement/bounded.ts +21 -2
  48. package/src/implement/capability.ts +15 -3
  49. package/src/implement/dispatch.ts +419 -177
  50. package/src/implement/failures.ts +17 -1
  51. package/src/implement/faults.ts +1 -0
  52. package/src/implement/index.ts +1 -0
  53. package/src/implement/invocation.ts +58 -10
  54. package/src/implement/principal.ts +25 -2
  55. package/src/implement/terminals.ts +93 -0
  56. package/src/implement/workload.ts +42 -16
  57. package/src/jobs/mail.ts +8 -0
  58. package/src/jobs/report.ts +5 -1
  59. package/src/jobs/runner.ts +672 -113
  60. package/src/jobs/schedule.ts +7 -1
  61. package/src/jobs/scrub.ts +14 -1
  62. package/src/jobs/types.ts +6 -0
  63. package/src/spa-fallback.ts +12 -0
  64. package/src/testing/endpoint-access.ts +307 -152
  65. package/src/testing/isolation.ts +8 -1
  66. package/src/testing/kit.ts +80 -2
  67. package/src/testing/proxy.ts +15 -10
  68. package/types/src/build-recipe.d.ts +70 -0
  69. package/types/src/build.d.ts +19 -1
  70. package/types/src/bun.d.ts +34 -1
  71. package/types/src/cloudflare/application.d.ts +73 -0
  72. package/types/src/cloudflare/bindings.d.ts +33 -0
  73. package/types/src/cloudflare/export.d.ts +16 -0
  74. package/types/src/cloudflare/import.d.ts +51 -0
  75. package/types/src/cloudflare/maintenance.d.ts +117 -0
  76. package/types/src/cloudflare/names.d.ts +11 -0
  77. package/types/src/cloudflare/ops.d.ts +81 -0
  78. package/types/src/cloudflare/r2-files.d.ts +9 -0
  79. package/types/src/cloudflare/restore.d.ts +9 -0
  80. package/types/src/cloudflare/tracker.d.ts +29 -0
  81. package/types/src/cloudflare.d.ts +12 -18
  82. package/types/src/contract/app-id.d.ts +2 -0
  83. package/types/src/contract/artifact-schema.d.ts +2 -0
  84. package/types/src/contract/context.d.ts +35 -11
  85. package/types/src/contract/declaration.d.ts +4 -0
  86. package/types/src/contract/defaults.d.ts +19 -1
  87. package/types/src/contract/define.d.ts +20 -3
  88. package/types/src/contract/kit.d.ts +2 -1
  89. package/types/src/contract/platform.d.ts +2 -2
  90. package/types/src/contract/vocabulary.d.ts +9 -7
  91. package/types/src/dev/output.d.ts +8 -2
  92. package/types/src/disk-assets.d.ts +4 -1
  93. package/types/src/files/job-names.d.ts +9 -0
  94. package/types/src/files/kinds.d.ts +21 -14
  95. package/types/src/files/scan.d.ts +1 -3
  96. package/types/src/implement/anonymous.d.ts +2 -4
  97. package/types/src/implement/boot.d.ts +17 -1
  98. package/types/src/implement/bounded.d.ts +11 -1
  99. package/types/src/implement/capability.d.ts +4 -2
  100. package/types/src/implement/dispatch.d.ts +34 -1
  101. package/types/src/implement/failures.d.ts +5 -0
  102. package/types/src/implement/faults.d.ts +1 -0
  103. package/types/src/implement/index.d.ts +1 -0
  104. package/types/src/implement/invocation.d.ts +22 -7
  105. package/types/src/implement/principal.d.ts +6 -1
  106. package/types/src/implement/terminals.d.ts +33 -0
  107. package/types/src/implement/workload.d.ts +5 -0
  108. package/types/src/jobs/runner.d.ts +66 -2
  109. package/types/src/jobs/schedule.d.ts +2 -0
  110. package/types/src/jobs/types.d.ts +6 -0
  111. package/types/src/spa-fallback.d.ts +2 -0
  112. package/types/src/testing/endpoint-access.d.ts +37 -5
  113. package/types/src/testing/isolation.d.ts +1 -1
  114. package/types/src/testing/kit.d.ts +22 -1
  115. package/types/src/testing/proxy.d.ts +4 -4
@@ -0,0 +1,1021 @@
1
+ // One step of loading a consistent backup into a fresh object (ADR-0070 §6): the ops operation `import`, served before
2
+ // the application boots. Control drives the steps and holds their state; the object keeps only its import marker and
3
+ // the steps' receipts in KV storage, which every step that writes updates in the same `transactionSync` as its rows.
4
+ // Every step runs in the maintenance barrier's turn, so no transition interleaves with it.
5
+
6
+ import {
7
+ decodeOpsDumpLine,
8
+ encodeOpsDumpRow,
9
+ foldOpsTableHash,
10
+ OPS_DATA_REFUSED_STATUS,
11
+ OPS_DUMP_KEY,
12
+ OPS_DUMP_MANIFEST_MAX_BYTES,
13
+ OPS_DUMP_PAGE_ROWS,
14
+ OPS_DUMP_PART_MAX_RAW_BYTES,
15
+ OPS_DUMP_PART_MAX_ROWS,
16
+ OPS_DUMP_PART_SIZE,
17
+ OPS_EMPTY_TABLE_HASH,
18
+ OPS_STALE_STATUS,
19
+ type OpsDataRefusal,
20
+ type OpsDataRefusalReason,
21
+ type OpsDumpBinding,
22
+ opsDumpColumnProjection,
23
+ type OpsDumpCursor,
24
+ type OpsDumpLine,
25
+ type OpsDumpManifest,
26
+ type OpsDumpSchema,
27
+ type OpsDumpSchemaColumn,
28
+ type OpsDumpSchemaEntry,
29
+ type OpsDumpSchemaForeignKey,
30
+ type OpsDumpSchemaIndex,
31
+ type OpsDumpSchemaTable,
32
+ type OpsDumpSource,
33
+ type OpsDumpTable,
34
+ type OpsDumpTablePlan,
35
+ type OpsDumpValue,
36
+ opsDumpValueBinding,
37
+ type OpsImportBeginAck,
38
+ type OpsImportCompleteAck,
39
+ type OpsImportDiscardAck,
40
+ type OpsImportLinkAck,
41
+ type OpsImportPartAck,
42
+ type OpsImportRequest,
43
+ type OpsImportVerifyAck,
44
+ type OpsStaleRefusal,
45
+ parseOpsImportCompleteAck,
46
+ parseOpsImportLinkAck,
47
+ parseOpsImportPartAck,
48
+ parseOpsImportVerifyAck,
49
+ planOpsDump,
50
+ readOpsDumpManifest,
51
+ readOpsDumpValue,
52
+ sha256Hex,
53
+ } from '@fabrika/auth-core'
54
+ import type { SqliteStorage, SqliteValue } from '@fabrika/db/sqlite'
55
+ import { migrateSqlite, type SqliteMigrationBundle } from '@fabrika/db/sqlite-migrate'
56
+ import { jsonResponse } from '../errors.js'
57
+ import { method } from './bindings.js'
58
+ import type { MaintenanceBarrier, TurnStep } from './maintenance.js'
59
+ import type { OpsTarget } from './ops.js'
60
+
61
+ /**
62
+ * One step of loading a dump into a fresh object (ADR-0070); `serving` is the object of ordinary traffic. A runtime
63
+ * whose storage has no synchronous KV or `deleteAll` does not serve the operation.
64
+ */
65
+ export async function serveImport(target: OpsTarget, barrier: MaintenanceBarrier, step: OpsImportRequest, serving: string): Promise<Response> {
66
+ const { storage } = target
67
+ const { kv, deleteAll } = storage
68
+ if (kv === undefined || deleteAll === undefined) {
69
+ return jsonResponse({ refused: 'not_implemented' }, 501)
70
+ }
71
+ const importing: ImportStorage = {
72
+ sql: storage.sql,
73
+ transaction: (closure) => storage.transaction(closure),
74
+ transactionSync: (closure) => storage.transactionSync(closure),
75
+ kv,
76
+ deleteAll: () => deleteAll.call(storage),
77
+ }
78
+ return await importStep({ storage: importing, bucket: target.bucket, migrations: target.migrations, serving }, barrier, step)
79
+ }
80
+
81
+ /** workerd's synchronous KV storage of a SQLite-backed object: it lives in the same database, so it commits with a transaction. */
82
+ export interface ImportKv {
83
+ get(key: string): unknown
84
+ put(key: string, value: unknown): void
85
+ delete(key: string): boolean
86
+ }
87
+
88
+ /** What the importer needs of the object's storage; a SQLite-backed `DurableObjectStorage` fits it. */
89
+ export interface ImportStorage extends SqliteStorage {
90
+ readonly kv: ImportKv
91
+ /** Wipes the object's database, its KV storage and its alarm. */
92
+ deleteAll(): Promise<void>
93
+ }
94
+
95
+ /** The object an import step loads into. */
96
+ export interface ImportTarget {
97
+ readonly storage: ImportStorage
98
+ /** The environment's bucket (the `FABRIKA_FILES` R2 binding), where the backup lies. */
99
+ readonly bucket: unknown
100
+ /** The release's migration bundles in the order the object applies them: the framework's, then the application's. */
101
+ readonly migrations: readonly SqliteMigrationBundle[]
102
+ /** The object of ordinary traffic (`FABRIKA_APPLICATION_OBJECT`), which `discard` never wipes. */
103
+ readonly serving: string
104
+ }
105
+
106
+ /** The KV key of the import marker. Invariant: an object that holds it never boots and never leaves `inspect`. */
107
+ export const IMPORT_MARKER_KEY = 'fabrika:import'
108
+
109
+ /** Whether the object holds an import marker, so it must neither boot nor leave `inspect`. */
110
+ export function importInProgress(storage: { readonly kv: ImportKv }): boolean {
111
+ return storage.kv.get(IMPORT_MARKER_KEY) !== undefined
112
+ }
113
+
114
+ /** The KV key under which part `part` records its acknowledgement once loaded, so a retry loads nothing twice. */
115
+ function loadedPartKey(part: number): string {
116
+ return `${IMPORT_MARKER_KEY}:part:${part}`
117
+ }
118
+
119
+ /** The KV key under which part `part`'s link pass records its acknowledgement. */
120
+ function linkedPartKey(part: number): string {
121
+ return `${IMPORT_MARKER_KEY}:link:${part}`
122
+ }
123
+
124
+ /**
125
+ * The KV key of the receipt `complete` leaves for `source`, so a retried `complete` answers the same. It blocks nothing
126
+ * and stays for the object's life: `begin` takes only a fresh object, so an object completes one import.
127
+ */
128
+ function completedKey(source: OpsDumpSource): string {
129
+ return `${IMPORT_MARKER_KEY}:complete:${source.object}/${source.dump}`
130
+ }
131
+
132
+ /**
133
+ * The marker: the backup the object is loading, whether `begin` cleared the target, how many parts are loaded, and the
134
+ * verification of what is loaded now: `null` for none, else the cursor its next step must carry (`null` once every
135
+ * table matched the manifest) and the last step's cursor with its answer, which a retry of that step gets again. A
136
+ * part or a link pass that writes resets it.
137
+ */
138
+ interface ImportMarker {
139
+ readonly source: OpsDumpSource
140
+ readonly begun: boolean
141
+ readonly loaded: number
142
+ readonly verified: Verification | null
143
+ }
144
+
145
+ interface Verification {
146
+ readonly next: string | null
147
+ /** `null` after a step that started the verification, which a retry starts again. */
148
+ readonly replay: { readonly cursor: string; readonly ack: OpsImportVerifyAck } | null
149
+ }
150
+
151
+ /**
152
+ * One step of loading the consistent backup `step.source` into the fresh object `target` (ADR-0070 §6), in the
153
+ * barrier's turn: ADR-0069's stale refusal unless `step.generation` is the published one, then `serving_object` on the
154
+ * object of ordinary traffic, then `wrong_mode` unless `inspect`. The turn is held across the step's R2 reads, so the
155
+ * generation and mode it was admitted under still hold when it commits.
156
+ */
157
+ export async function importStep(target: ImportTarget, barrier: MaintenanceBarrier, step: OpsImportRequest): Promise<Response> {
158
+ const result = await barrier.step(step.generation, async (mode): Promise<TurnStep<Response>> => {
159
+ if (step.object === target.serving) {
160
+ return answered(refused('serving_object'))
161
+ }
162
+ if (mode !== 'inspect') {
163
+ return answered(refused('wrong_mode'))
164
+ }
165
+ if (step.step !== 'verify') {
166
+ await barrier.restrictedWrite()
167
+ }
168
+ return answered(await admittedStep(target, step))
169
+ })
170
+ if (result.kind === 'ran') {
171
+ return result.value
172
+ }
173
+ const refusal: OpsStaleRefusal = { refused: 'stale_generation', published: result.published }
174
+ return jsonResponse(refusal, OPS_STALE_STATUS)
175
+ }
176
+
177
+ async function admittedStep(target: ImportTarget, step: OpsImportRequest): Promise<Response> {
178
+ const { kv } = target.storage
179
+ if (step.step === 'discard') {
180
+ return await discard(target)
181
+ }
182
+ if (step.step === 'begin') {
183
+ return await begin(target, step.source)
184
+ }
185
+ if (importing(kv, step.source) === null) {
186
+ const completed = step.step === 'complete' ? readCompleted(kv, step.source) : null
187
+ return completed === null ? refused('no_import') : jsonResponse(completed, 200)
188
+ }
189
+ const manifest = await readManifest(target.bucket, step.source)
190
+ if (manifest === null) {
191
+ return refused('invalid_manifest')
192
+ }
193
+ if (step.step === 'complete') {
194
+ return complete(target.storage, step.source, manifest)
195
+ }
196
+ const plan = currentPlan(target.storage, manifest)
197
+ if ('refused' in plan) {
198
+ return jsonResponse(plan, OPS_DATA_REFUSED_STATUS)
199
+ }
200
+ switch (step.step) {
201
+ case 'part':
202
+ return await loadPart(target, step.source, manifest, plan, step.part)
203
+ case 'link':
204
+ return await linkPart(target, step.source, manifest, plan, step.part)
205
+ case 'verify':
206
+ return verify(target.storage, step.source, manifest, plan, step.cursor)
207
+ }
208
+ }
209
+
210
+ function answered(response: Response): TurnStep<Response> {
211
+ return { value: response, holdUntil: null }
212
+ }
213
+
214
+ /** The marker, when it names `source` and records `begin` done; `null` otherwise. */
215
+ function importing(kv: ImportKv, source: OpsDumpSource): ImportMarker | null {
216
+ const marker = readMarker(kv)
217
+ return marker?.begun === true && sameSource(marker.source, source) ? marker : null
218
+ }
219
+
220
+ /**
221
+ * Verifies the manifest, checks the target is fresh, writes the marker, migrates the target to exactly the manifest's
222
+ * ledger, checks the schema and its plan, then clears the target in one transaction that records `begin` done. A retry
223
+ * under the same backup resumes after what the marker records.
224
+ */
225
+ async function begin(target: ImportTarget, source: OpsDumpSource): Promise<Response> {
226
+ const { storage } = target
227
+ const manifest = await readManifest(target.bucket, source)
228
+ if (manifest === null) {
229
+ return refused('invalid_manifest')
230
+ }
231
+ if (!manifest.consistent) {
232
+ return refused('fuzzy_dump')
233
+ }
234
+ const marker = readMarker(storage.kv)
235
+ if (marker !== null && !sameSource(marker.source, source)) {
236
+ return refused('not_fresh')
237
+ }
238
+ if (marker?.begun === true) {
239
+ return beginAck(manifest)
240
+ }
241
+ if (marker === null && holdsTables(storage)) {
242
+ return refused('not_fresh')
243
+ }
244
+ const bundles = ledgerBundles(target.migrations, manifest)
245
+ if (bundles === null) {
246
+ return refused('ledger_mismatch')
247
+ }
248
+ if (marker === null) {
249
+ writeMarker(storage.kv, { source, begun: false, loaded: 0, verified: null })
250
+ }
251
+ // A database the framework never migrated has no ledger, and gets none.
252
+ if (manifest.migrations.length > 0) {
253
+ await migrateSqlite(
254
+ { exec: (query, ...bindings) => storage.sql.exec(query, ...bindings), transactionSync: (fn) => storage.transactionSync(fn) },
255
+ bundles,
256
+ )
257
+ }
258
+ if (!sameDdl(readDdl(storage), manifest.ddl)) {
259
+ return refused('schema_mismatch')
260
+ }
261
+ const plan = currentPlan(storage, manifest)
262
+ if ('refused' in plan) {
263
+ return jsonResponse(plan, OPS_DATA_REFUSED_STATUS)
264
+ }
265
+ storage.transactionSync(() => {
266
+ for (const entry of readDdl(storage)) {
267
+ if (entry.type === 'trigger') {
268
+ storage.sql.exec(`DROP TRIGGER ${quote(entry.name)}`)
269
+ }
270
+ }
271
+ // Every deferred column takes a temporary NULL (the schema rule), so no row of its cycle references another one after this.
272
+ for (const table of plan) {
273
+ if (table.deferred.length > 0) {
274
+ storage.sql.exec(`UPDATE ${quote(table.name)} SET ${table.deferred.map((column) => `${quote(column)} = NULL`).join(', ')}`)
275
+ }
276
+ }
277
+ for (const table of [...plan].reverse()) {
278
+ storage.sql.exec(`DELETE FROM ${quote(table.name)}`)
279
+ }
280
+ writeMarker(storage.kv, { source, begun: true, loaded: 0, verified: null })
281
+ })
282
+ return beginAck(manifest)
283
+ }
284
+
285
+ function beginAck(manifest: OpsDumpManifest): Response {
286
+ const ack: OpsImportBeginAck = { step: 'begin', parts: manifest.parts.length }
287
+ return jsonResponse(ack, 200)
288
+ }
289
+
290
+ /**
291
+ * Inserts part `part`'s rows with their rowids, every value through the wire's binding and every deferred column as
292
+ * NULL, in one transaction that reads the marker and the part's receipt again, records the part and resets the
293
+ * verification.
294
+ */
295
+ async function loadPart(target: ImportTarget, source: OpsDumpSource, manifest: OpsDumpManifest, plan: readonly OpsDumpTablePlan[], part: number) {
296
+ const { storage } = target
297
+ const recorded = readLoadedPart(storage.kv, part)
298
+ if (recorded !== null) {
299
+ return jsonResponse(recorded, 200)
300
+ }
301
+ const lines = await readPart(target.bucket, source, manifest, plan, part)
302
+ if (!Array.isArray(lines)) {
303
+ return jsonResponse(lines, OPS_DATA_REFUSED_STATUS)
304
+ }
305
+ const tables = new Map(plan.map((table) => [table.name, table]))
306
+ const ack = storage.transactionSync((): OpsImportPartAck | OpsDataRefusal => {
307
+ const marker = importing(storage.kv, source)
308
+ if (marker === null) {
309
+ return { refused: 'no_import', table: null }
310
+ }
311
+ const again = readLoadedPart(storage.kv, part)
312
+ if (again !== null) {
313
+ return again
314
+ }
315
+ let deferred = 0
316
+ for (const { table: name, row } of lines) {
317
+ const table = planned(tables, name)
318
+ if (insertRow(storage, table, row)) {
319
+ deferred++
320
+ }
321
+ }
322
+ const loaded: OpsImportPartAck = { step: 'part', part, rows: lines.length, deferred }
323
+ storage.kv.put(loadedPartKey(part), JSON.stringify(loaded))
324
+ writeMarker(storage.kv, { ...marker, loaded: marker.loaded + 1, verified: null })
325
+ return loaded
326
+ })
327
+ return jsonResponse(ack, 'refused' in ack ? OPS_DATA_REFUSED_STATUS : 200)
328
+ }
329
+
330
+ /** Writes one row; answers whether it has a deferred key for the link pass to set. */
331
+ function insertRow(storage: SqliteStorage, table: OpsDumpTablePlan, row: readonly OpsDumpValue[]): boolean {
332
+ const [rowid, ...values] = row
333
+ const key = opsDumpValueBinding(rowid ?? null)
334
+ const columns: string[] = [table.keyset]
335
+ const bindings = [key]
336
+ let waits = false
337
+ for (const [index, column] of table.columns.entries()) {
338
+ const value = values[index] ?? null
339
+ // An INTEGER PRIMARY KEY column is the rowid itself, written once.
340
+ if (column === table.keyset) {
341
+ continue
342
+ }
343
+ const deferred = table.deferred.includes(column)
344
+ waits ||= deferred && value !== null
345
+ columns.push(column)
346
+ bindings.push(opsDumpValueBinding(deferred ? null : value))
347
+ }
348
+ if (table.name === SQLITE_SEQUENCE) {
349
+ // Loading an AUTOINCREMENT table writes its own counter row; the source's row replaces it, whichever table loads first.
350
+ const name = opsDumpValueBinding(values[0] ?? null)
351
+ storage.sql.exec(`DELETE FROM ${SQLITE_SEQUENCE} WHERE rowid = ${key.sql} OR name = ${name.sql}`, ...params([key, name]))
352
+ }
353
+ storage.sql.exec(
354
+ `INSERT INTO ${quote(table.name)} (${columns.map(quote).join(', ')}) VALUES (${bindings.map((binding) => binding.sql).join(', ')})`,
355
+ ...params(bindings),
356
+ )
357
+ return waits
358
+ }
359
+
360
+ /**
361
+ * Sets the deferred keys of part `part`'s rows by their rowids, once every part is loaded, in one transaction that
362
+ * reads the marker and the pass's receipt again, records the pass and resets the verification.
363
+ */
364
+ async function linkPart(target: ImportTarget, source: OpsDumpSource, manifest: OpsDumpManifest, plan: readonly OpsDumpTablePlan[], part: number) {
365
+ const { storage } = target
366
+ const recorded = readLinkedPart(storage.kv, part)
367
+ if (recorded !== null) {
368
+ return jsonResponse(recorded, 200)
369
+ }
370
+ if (importing(storage.kv, source)?.loaded !== manifest.parts.length) {
371
+ return refused('part_mismatch')
372
+ }
373
+ const lines = await readPart(target.bucket, source, manifest, plan, part)
374
+ if (!Array.isArray(lines)) {
375
+ return jsonResponse(lines, OPS_DATA_REFUSED_STATUS)
376
+ }
377
+ const tables = new Map(plan.map((table) => [table.name, table]))
378
+ const ack = storage.transactionSync((): OpsImportLinkAck | OpsDataRefusal => {
379
+ const marker = importing(storage.kv, source)
380
+ if (marker === null) {
381
+ return { refused: 'no_import', table: null }
382
+ }
383
+ if (marker.loaded !== manifest.parts.length) {
384
+ return { refused: 'part_mismatch', table: null }
385
+ }
386
+ const again = readLinkedPart(storage.kv, part)
387
+ if (again !== null) {
388
+ return again
389
+ }
390
+ let linked = 0
391
+ for (const { table: name, row } of lines) {
392
+ const table = planned(tables, name)
393
+ const [rowid, ...values] = row
394
+ const deferred = table.columns.flatMap((column, index) => table.deferred.includes(column) ? [{ column, value: values[index] ?? null }] : [])
395
+ if (deferred.every(({ value }) => value === null)) {
396
+ continue
397
+ }
398
+ const sets = deferred.map(({ column, value }) => ({ column, binding: opsDumpValueBinding(value) }))
399
+ const key = opsDumpValueBinding(rowid ?? null)
400
+ storage.sql.exec(
401
+ `UPDATE ${quote(table.name)} SET ${sets.map(({ column, binding }) => `${quote(column)} = ${binding.sql}`).join(', ')} WHERE ${
402
+ quote(table.keyset)
403
+ } = ${key.sql}`,
404
+ ...params([...sets.map(({ binding }) => binding), key]),
405
+ )
406
+ linked++
407
+ }
408
+ const done: OpsImportLinkAck = { step: 'link', part, rows: linked }
409
+ storage.kv.put(linkedPartKey(part), JSON.stringify(done))
410
+ writeMarker(storage.kv, { ...marker, verified: null })
411
+ return done
412
+ })
413
+ return jsonResponse(ack, 'refused' in ack ? OPS_DATA_REFUSED_STATUS : 200)
414
+ }
415
+
416
+ /**
417
+ * Reads the loaded tables back from `cursor` with the producer's projection and fold, at most a part's row bound per
418
+ * step, and compares each finished table with the manifest. A verification starts at `null`; any other cursor must be
419
+ * the one the marker records this verification handed out last, or the one its last step was sent, whose answer it
420
+ * repeats. It writes no row: only its progress, into the marker, which a mismatch clears.
421
+ */
422
+ function verify(
423
+ storage: ImportStorage,
424
+ source: OpsDumpSource,
425
+ manifest: OpsDumpManifest,
426
+ plan: readonly OpsDumpTablePlan[],
427
+ cursor: OpsDumpCursor | null,
428
+ ): Response {
429
+ const marker = importing(storage.kv, source)
430
+ if (marker === null) {
431
+ return refused('no_import')
432
+ }
433
+ const mismatch = (table: string | null): Response => {
434
+ writeMarker(storage.kv, { ...marker, verified: null })
435
+ return refused('verify_mismatch', table)
436
+ }
437
+ if (cursor !== null) {
438
+ const key = cursorKey(cursor)
439
+ const replay = marker.verified?.replay ?? null
440
+ // Its answer was lost: what is loaded has not changed since, or the verification would be reset.
441
+ if (replay?.cursor === key) {
442
+ return jsonResponse(replay.ack, 200)
443
+ }
444
+ if ((marker.verified?.next ?? null) !== key) {
445
+ return mismatch(cursor.table)
446
+ }
447
+ }
448
+ const tables = new Map(plan.map((table) => [table.name, table]))
449
+ const order = manifest.tables
450
+ let index = cursor === null ? 0 : order.findIndex((table) => table.name === cursor.table)
451
+ if (index < 0) {
452
+ return mismatch(cursor?.table ?? null)
453
+ }
454
+ const startOf = (table: OpsDumpTable | undefined): OpsDumpCursor | null =>
455
+ table === undefined ? null : { table: table.name, after: null, rows: 0, hash: OPS_EMPTY_TABLE_HASH }
456
+ let current = cursor ?? startOf(order[0])
457
+ const finished: OpsDumpTable[] = []
458
+ let budget = OPS_DUMP_PART_MAX_ROWS
459
+ while (current !== null && budget > 0) {
460
+ const expected = order[index]
461
+ if (expected === undefined) {
462
+ throw new Error('a verify cursor past the last table of the manifest')
463
+ }
464
+ const page = readPage(storage, planned(tables, expected.name), current.after)
465
+ const last = page.at(-1)?.[0]
466
+ current = {
467
+ table: expected.name,
468
+ after: last?.type === 'integer' ? last.value : current.after,
469
+ rows: current.rows + page.length,
470
+ hash: foldOpsTableHash(current.hash, page.map((row) => encodeOpsDumpRow(row))),
471
+ }
472
+ budget -= page.length
473
+ if (current.rows > expected.rows) {
474
+ return mismatch(expected.name)
475
+ }
476
+ if (page.length < OPS_DUMP_PAGE_ROWS) {
477
+ if (current.rows !== expected.rows || current.hash !== expected.hash) {
478
+ return mismatch(expected.name)
479
+ }
480
+ finished.push(expected)
481
+ index++
482
+ current = startOf(order[index])
483
+ }
484
+ }
485
+ const ack: OpsImportVerifyAck = { step: 'verify', tables: finished, next: current }
486
+ const next = current === null ? null : cursorKey(current)
487
+ writeMarker(storage.kv, { ...marker, verified: { next, replay: cursor === null ? null : { cursor: cursorKey(cursor), ack } } })
488
+ return jsonResponse(ack, 200)
489
+ }
490
+
491
+ /** A cursor as the marker records it: its fields in a fixed order, whatever order a request named them in. */
492
+ function cursorKey(cursor: OpsDumpCursor): string {
493
+ return JSON.stringify([cursor.table, cursor.after, cursor.rows, cursor.hash])
494
+ }
495
+
496
+ /** One keyset page of `table` after the rowid `after`: each row its rowid, then its columns, as the producer reads them. */
497
+ function readPage(storage: SqliteStorage, table: OpsDumpTablePlan, after: string | null): (readonly OpsDumpValue[])[] {
498
+ const keyset = quote(table.keyset)
499
+ // The CTE names each projected column, so no two result columns share a name whatever the table's columns are called.
500
+ const names = table.columns.flatMap((_, index) => [`t${index}`, `v${index}`])
501
+ const select = [keyset, `CAST(${keyset} AS TEXT)`, ...table.columns.map((column) => opsDumpColumnProjection(column))].join(', ')
502
+ const where = after === null ? '' : ` WHERE ${keyset} > CAST(? AS INTEGER)`
503
+ // Qualified, so a table named `page` is not the CTE itself.
504
+ const query =
505
+ `WITH page(r, k, ${names.join(', ')}) AS (SELECT ${select} FROM main.${quote(table.name)}${where} ORDER BY ${keyset} LIMIT ${OPS_DUMP_PAGE_ROWS}) `
506
+ + `SELECT k, ${names.join(', ')} FROM page ORDER BY r`
507
+ const rows = after === null ? storage.sql.exec(query).toArray() : storage.sql.exec(query, after).toArray()
508
+ return rows.map((row) => [
509
+ readOpsDumpValue('integer', row['k']),
510
+ ...table.columns.map((_, index) => readOpsDumpValue(row[`t${index}`], row[`v${index}`])),
511
+ ])
512
+ }
513
+
514
+ /**
515
+ * Once every part is loaded, every link pass a part needs ran, and a verification of what is loaded now matched every
516
+ * table: recreates the triggers from the manifest's DDL, compares the whole schema again, clears the marker and the
517
+ * steps' receipts, and leaves the completion's receipt.
518
+ */
519
+ function complete(storage: ImportStorage, source: OpsDumpSource, manifest: OpsDumpManifest): Response {
520
+ const answer = storage.transactionSync((): OpsImportCompleteAck | OpsDataRefusal => {
521
+ const marker = importing(storage.kv, source)
522
+ if (marker === null) {
523
+ return { refused: 'no_import', table: null }
524
+ }
525
+ const loaded = manifest.parts.map((part) => readLoadedPart(storage.kv, part.part))
526
+ const unlinked = loaded.some((ack) => ack === null || (ack.deferred > 0 && readLinkedPart(storage.kv, ack.part) === null))
527
+ if (marker.loaded !== manifest.parts.length || unlinked) {
528
+ return { refused: 'part_mismatch', table: null }
529
+ }
530
+ if (marker.verified === null || marker.verified.next !== null) {
531
+ return { refused: 'verify_mismatch', table: null }
532
+ }
533
+ const present = new Set(readDdl(storage).filter((entry) => entry.type === 'trigger').map((entry) => entry.name))
534
+ for (const entry of manifest.ddl) {
535
+ if (entry.type !== 'trigger' || present.has(entry.name)) {
536
+ continue
537
+ }
538
+ if (!TRIGGER_DDL.test(entry.sql)) {
539
+ return { refused: 'schema_mismatch', table: null }
540
+ }
541
+ storage.sql.exec(entry.sql)
542
+ }
543
+ if (!sameDdl(readDdl(storage), manifest.ddl)) {
544
+ return { refused: 'schema_mismatch', table: null }
545
+ }
546
+ for (const part of manifest.parts) {
547
+ storage.kv.delete(loadedPartKey(part.part))
548
+ storage.kv.delete(linkedPartKey(part.part))
549
+ }
550
+ storage.kv.delete(IMPORT_MARKER_KEY)
551
+ const ack: OpsImportCompleteAck = { step: 'complete', source: { object: source.object, dump: source.dump } }
552
+ storage.kv.put(completedKey(source), JSON.stringify(ack))
553
+ return ack
554
+ })
555
+ return jsonResponse(answer, 'refused' in answer ? OPS_DATA_REFUSED_STATUS : 200)
556
+ }
557
+
558
+ /** Wipes an `inspect` object that holds an import marker; the caller refused the object of ordinary traffic. */
559
+ async function discard(target: ImportTarget): Promise<Response> {
560
+ if (readMarker(target.storage.kv) === null) {
561
+ return refused('no_import')
562
+ }
563
+ await target.storage.deleteAll()
564
+ const ack: OpsImportDiscardAck = { step: 'discard' }
565
+ return jsonResponse(ack, 200)
566
+ }
567
+
568
+ /** The plan of the target's schema, which must list the manifest's tables in the manifest's order. */
569
+ function currentPlan(storage: SqliteStorage, manifest: OpsDumpManifest): readonly OpsDumpTablePlan[] | OpsDataRefusal {
570
+ const plan = planOpsDump(describeDumpSchema(storage))
571
+ if ('refused' in plan) {
572
+ return plan
573
+ }
574
+ const names = plan.tables.map((table) => table.name)
575
+ const expected = manifest.tables.map((table) => table.name)
576
+ if (names.length !== expected.length || names.some((name, index) => name !== expected[index])) {
577
+ return { refused: 'schema_mismatch', table: null }
578
+ }
579
+ return plan.tables
580
+ }
581
+
582
+ function planned(tables: ReadonlyMap<string, OpsDumpTablePlan>, name: string): OpsDumpTablePlan {
583
+ const table = tables.get(name)
584
+ if (table === undefined) {
585
+ throw new Error('a dump line names a table the plan does not hold')
586
+ }
587
+ return table
588
+ }
589
+
590
+ /**
591
+ * Part `part`'s lines: its byte range read and checked against the manifest's size and SHA-256, decompressed up to the
592
+ * raw bound, at most the row bound, each line in the wire's encoding and fitting its table's plan.
593
+ */
594
+ async function readPart(
595
+ bucket: unknown,
596
+ source: OpsDumpSource,
597
+ manifest: OpsDumpManifest,
598
+ plan: readonly OpsDumpTablePlan[],
599
+ part: number,
600
+ ): Promise<OpsDumpLine[] | OpsDataRefusal> {
601
+ const mismatch: OpsDataRefusal = { refused: 'part_mismatch', table: null }
602
+ const described = manifest.parts[part - 1]
603
+ if (described === undefined) {
604
+ return mismatch
605
+ }
606
+ // Every part but the last is exactly the fixed size, so part n starts at (n - 1) parts.
607
+ const body = await r2Get(bucket, OPS_DUMP_KEY({ ...source, consistent: true }, 'data'), {
608
+ range: { offset: (part - 1) * OPS_DUMP_PART_SIZE, length: described.size },
609
+ })
610
+ const bytes = body === null ? null : await bodyBytes(body)
611
+ if (bytes === null || bytes.length !== described.size || (await sha256OfBytes(bytes)) !== described.sha256) {
612
+ return mismatch
613
+ }
614
+ const text = await gunzipWithin(bytes, OPS_DUMP_PART_MAX_RAW_BYTES)
615
+ const raw = text?.split('\n')
616
+ // Each line ends with its newline, so the text ends with one.
617
+ if (raw === undefined || raw.pop() !== '' || raw.length > OPS_DUMP_PART_MAX_ROWS) {
618
+ return mismatch
619
+ }
620
+ const tables = new Map(plan.map((table) => [table.name, table]))
621
+ const lines: OpsDumpLine[] = []
622
+ for (const encoded of raw) {
623
+ const line = decodeOpsDumpLine(encoded)
624
+ const table = line === null ? undefined : tables.get(line.table)
625
+ if (line === null || table === undefined) {
626
+ return mismatch
627
+ }
628
+ if (line.row.length !== table.columns.length + 1 || line.row[0]?.type !== 'integer') {
629
+ return { refused: 'part_mismatch', table: table.name }
630
+ }
631
+ lines.push(line)
632
+ }
633
+ return lines
634
+ }
635
+
636
+ /** `bytes` as one gzip member, or `null` when it does not decompress, decompresses past `bound`, or is not UTF-8. */
637
+ async function gunzipWithin(bytes: Uint8Array, bound: number): Promise<string | null> {
638
+ const reader = new Blob([bytes.slice()]).stream().pipeThrough(new DecompressionStream('gzip')).getReader()
639
+ const chunks: Uint8Array[] = []
640
+ let total = 0
641
+ try {
642
+ for (let read = await reader.read(); !read.done; read = await reader.read()) {
643
+ total += read.value.byteLength
644
+ if (total > bound) {
645
+ await reader.cancel()
646
+ return null
647
+ }
648
+ chunks.push(read.value)
649
+ }
650
+ } catch {
651
+ return null
652
+ }
653
+ const raw = new Uint8Array(total)
654
+ let offset = 0
655
+ for (const chunk of chunks) {
656
+ raw.set(chunk, offset)
657
+ offset += chunk.byteLength
658
+ }
659
+ try {
660
+ return new TextDecoder('utf-8', { fatal: true }).decode(raw)
661
+ } catch {
662
+ return null
663
+ }
664
+ }
665
+
666
+ async function sha256OfBytes(bytes: Uint8Array): Promise<string> {
667
+ const digest = await crypto.subtle.digest('SHA-256', bytes.slice())
668
+ return [...new Uint8Array(digest)].map((byte) => byte.toString(16).padStart(2, '0')).join('')
669
+ }
670
+
671
+ /** The manifest at the backup's key, or `null` when it is missing, too large, does not parse or names another dump. */
672
+ async function readManifest(bucket: unknown, source: OpsDumpSource): Promise<OpsDumpManifest | null> {
673
+ const body = await r2Get(bucket, OPS_DUMP_KEY({ ...source, consistent: true }, 'manifest'))
674
+ if (body === null) {
675
+ return null
676
+ }
677
+ const size: unknown = typeof body === 'object' ? Reflect.get(body, 'size') : undefined
678
+ if (typeof size === 'number' && size > OPS_DUMP_MANIFEST_MAX_BYTES) {
679
+ return null
680
+ }
681
+ const text = method(body, 'text')
682
+ if (text === null) {
683
+ throw new Error('R2 answered the manifest without its body')
684
+ }
685
+ const content: unknown = await text()
686
+ const manifest = typeof content === 'string' ? readOpsDumpManifest(content) : null
687
+ return manifest !== null && manifest.object === source.object && manifest.dump === source.dump ? manifest : null
688
+ }
689
+
690
+ /** R2's `get` on the bucket, by method lookup at the binding's edge; `null` for a missing object. */
691
+ async function r2Get(bucket: unknown, key: string, options?: { readonly range: { readonly offset: number; readonly length: number } }) {
692
+ const get = method(bucket, 'get')
693
+ if (get === null) {
694
+ throw new Error("the FABRIKA_FILES binding has no get; a backup lies in the environment's bucket")
695
+ }
696
+ const body: unknown = await (options === undefined ? get(key) : get(key, options))
697
+ return body ?? null
698
+ }
699
+
700
+ async function bodyBytes(body: unknown): Promise<Uint8Array> {
701
+ const arrayBuffer = method(body, 'arrayBuffer')
702
+ if (arrayBuffer === null) {
703
+ throw new Error('R2 answered a dump part without its body')
704
+ }
705
+ const bytes: unknown = await arrayBuffer()
706
+ if (!(bytes instanceof ArrayBuffer)) {
707
+ throw new Error('R2 answered a dump part body that is not bytes')
708
+ }
709
+ return new Uint8Array(bytes)
710
+ }
711
+
712
+ /**
713
+ * Each bundle of the release cut to the manifest's ledger, which each must begin with exactly (by name and checksum);
714
+ * `null` for any difference, a bundle or a file the release lacks included.
715
+ */
716
+ function ledgerBundles(release: readonly SqliteMigrationBundle[], manifest: OpsDumpManifest): readonly SqliteMigrationBundle[] | null {
717
+ const names = new Set(release.map((bundle) => bundle.bundle))
718
+ if (manifest.migrations.some((migration) => !names.has(migration.bundle))) {
719
+ return null
720
+ }
721
+ const bundles: SqliteMigrationBundle[] = []
722
+ for (const bundle of release) {
723
+ const recorded = manifest.migrations.filter((migration) => migration.bundle === bundle.bundle).sort((left, right) =>
724
+ compareNames(left.name, right.name)
725
+ )
726
+ const files = [...bundle.migrations].sort((left, right) => compareNames(left.name, right.name))
727
+ const prefix = files.slice(0, recorded.length)
728
+ const same = prefix.length === recorded.length
729
+ && prefix.every((file, index) => file.name === recorded[index]?.name && sha256Hex(file.sql) === recorded[index]?.checksum)
730
+ if (!same) {
731
+ return null
732
+ }
733
+ bundles.push({ bundle: bundle.bundle, migrations: prefix })
734
+ }
735
+ return bundles
736
+ }
737
+
738
+ const SQLITE_SEQUENCE = 'sqlite_sequence'
739
+
740
+ /** `CREATE TRIGGER` as `sqlite_schema` stores it, the only DDL `complete` runs from the manifest. */
741
+ const TRIGGER_DDL = /^CREATE\s+(?:TEMP\s+|TEMPORARY\s+)?TRIGGER\s/i
742
+
743
+ /** Whether a dump holds `name`: no SQLite internal table but `sqlite_sequence`, and none of Cloudflare's `_cf_` ones. */
744
+ function dumped(name: string): boolean {
745
+ return !name.startsWith('_cf_') && (!name.startsWith('sqlite_') || name === SQLITE_SEQUENCE)
746
+ }
747
+
748
+ /** Whether the object holds any table but Cloudflare's own. */
749
+ function holdsTables(storage: SqliteStorage): boolean {
750
+ return storage.sql.exec(`SELECT name FROM sqlite_schema WHERE type = 'table'`).toArray().some((row) => !text(row['name']).startsWith('_cf_'))
751
+ }
752
+
753
+ /** Every `sqlite_schema` entry with SQL a dump's DDL lists, in no particular order. */
754
+ function readDdl(storage: SqliteStorage): OpsDumpSchemaEntry[] {
755
+ return storage.sql.exec('SELECT type, name, tbl_name, sql FROM sqlite_schema WHERE sql IS NOT NULL').toArray().flatMap((row) => {
756
+ const name = text(row['name'])
757
+ const table = text(row['tbl_name'])
758
+ const type = row['type']
759
+ if (!dumped(name) || !dumped(table) || (type !== 'table' && type !== 'index' && type !== 'trigger' && type !== 'view')) {
760
+ return []
761
+ }
762
+ return [{ type, name, table, sql: text(row['sql']) }]
763
+ })
764
+ }
765
+
766
+ function sameDdl(left: readonly OpsDumpSchemaEntry[], right: readonly OpsDumpSchemaEntry[]): boolean {
767
+ const canonical = (entries: readonly OpsDumpSchemaEntry[]) =>
768
+ entries.map((entry) => JSON.stringify([entry.type, entry.name, entry.table, entry.sql])).sort(compareNames).join('\n')
769
+ return left.length === right.length && canonical(left) === canonical(right)
770
+ }
771
+
772
+ /**
773
+ * The schema `planOpsDump` reads, from the PRAGMAs workerd admits, of every table a dump holds. A `CHECK` or an
774
+ * `INITIALLY DEFERRED` clause is read from the table's SQL outside its quotes and comments; one such clause marks every
775
+ * foreign key of its table.
776
+ */
777
+ export function describeDumpSchema(storage: SqliteStorage): OpsDumpSchema {
778
+ const tables: OpsDumpSchemaTable[] = []
779
+ for (const row of storage.sql.exec('PRAGMA table_list').toArray()) {
780
+ const name = text(row['name'])
781
+ const type = text(row['type'])
782
+ if (text(row['schema']) !== 'main' || type === 'view' || !dumped(name)) {
783
+ continue
784
+ }
785
+ const virtual = type === 'virtual'
786
+ const sql = storage.sql.exec(`SELECT sql FROM sqlite_schema WHERE type = 'table' AND name = ?`, name).toArray()[0]?.['sql']
787
+ const clauses = typeof sql === 'string' ? unquotedSql(sql) : null
788
+ tables.push({
789
+ name,
790
+ virtual,
791
+ withoutRowid: flag(row['wr']),
792
+ checked: clauses === null || /\bcheck\b/i.test(clauses),
793
+ // A virtual table is refused whatever it holds, and its module may not be loaded to describe it.
794
+ columns: virtual ? [] : describeColumns(storage, name),
795
+ indexes: virtual ? [] : describeIndexes(storage, name),
796
+ foreignKeys: virtual ? [] : describeForeignKeys(storage, name, clauses === null || /\binitially\s+deferred\b/i.test(clauses)),
797
+ })
798
+ }
799
+ return { tables }
800
+ }
801
+
802
+ function describeColumns(storage: SqliteStorage, table: string): OpsDumpSchemaColumn[] {
803
+ return storage.sql.exec(`PRAGMA table_xinfo(${literal(table)})`).toArray().map((row) => {
804
+ const hidden = integer(row['hidden'])
805
+ return {
806
+ name: text(row['name']),
807
+ type: text(row['type']),
808
+ notNull: flag(row['notnull']),
809
+ primaryKey: integer(row['pk']),
810
+ generated: hidden === 2 || hidden === 3,
811
+ }
812
+ })
813
+ }
814
+
815
+ function describeIndexes(storage: SqliteStorage, table: string): OpsDumpSchemaIndex[] {
816
+ return storage.sql.exec(`PRAGMA index_list(${literal(table)})`).toArray().map((row) => {
817
+ const origin = row['origin']
818
+ if (origin !== 'c' && origin !== 'u' && origin !== 'pk') {
819
+ throw new Error('PRAGMA index_list answered an origin SQLite does not write')
820
+ }
821
+ const columns = storage.sql.exec(`PRAGMA index_xinfo(${literal(text(row['name']))})`).toArray()
822
+ .filter((column) => flag(column['key']))
823
+ .sort((left, right) => integer(left['seqno']) - integer(right['seqno']))
824
+ .map((column) => typeof column['name'] === 'string' ? column['name'] : null)
825
+ return { unique: flag(row['unique']), origin, partial: flag(row['partial']), columns }
826
+ })
827
+ }
828
+
829
+ function describeForeignKeys(storage: SqliteStorage, table: string, initiallyDeferred: boolean): OpsDumpSchemaForeignKey[] {
830
+ const keys = new Map<number, { parent: string; rows: { seq: number; from: string; to: string | null }[] }>()
831
+ for (const row of storage.sql.exec(`PRAGMA foreign_key_list(${literal(table)})`).toArray()) {
832
+ const id = integer(row['id'])
833
+ const to = row['to']
834
+ const entry = { seq: integer(row['seq']), from: text(row['from']), to: typeof to === 'string' ? to : null }
835
+ const key = keys.get(id)
836
+ if (key === undefined) {
837
+ keys.set(id, { parent: text(row['table']), rows: [entry] })
838
+ } else {
839
+ key.rows.push(entry)
840
+ }
841
+ }
842
+ return [...keys.entries()].sort(([left], [right]) => left - right).map(([, key]) => {
843
+ const rows = key.rows.sort((left, right) => left.seq - right.seq)
844
+ const references = rows.flatMap((row) => row.to === null ? [] : [row.to])
845
+ if (references.length !== 0 && references.length !== rows.length) {
846
+ throw new Error('PRAGMA foreign_key_list named the parent columns of a key only in part')
847
+ }
848
+ return { parent: key.parent, columns: rows.map((row) => row.from), references: references.length === 0 ? null : references, initiallyDeferred }
849
+ })
850
+ }
851
+
852
+ /** `sql` with its string literals, quoted identifiers and comments blanked, so a keyword search finds only clauses. */
853
+ function unquotedSql(sql: string): string {
854
+ let out = ''
855
+ let index = 0
856
+ while (index < sql.length) {
857
+ const char = sql.charAt(index)
858
+ const close = char === "'" ? "'" : char === '"' ? '"' : char === '`' ? '`' : char === '[' ? ']' : null
859
+ if (close !== null) {
860
+ let end = index + 1
861
+ for (;;) {
862
+ end = sql.indexOf(close, end)
863
+ if (end < 0 || close === ']' || sql.charAt(end + 1) !== close) break
864
+ end += 2
865
+ }
866
+ index = end < 0 ? sql.length : end + 1
867
+ out += ' '
868
+ } else if (sql.startsWith('--', index)) {
869
+ const end = sql.indexOf('\n', index)
870
+ index = end < 0 ? sql.length : end + 1
871
+ out += ' '
872
+ } else if (sql.startsWith('/*', index)) {
873
+ const end = sql.indexOf('*/', index + 2)
874
+ index = end < 0 ? sql.length : end + 2
875
+ out += ' '
876
+ } else {
877
+ out += char
878
+ index++
879
+ }
880
+ }
881
+ return out
882
+ }
883
+
884
+ /** The import marker, parsed strictly; `null` when the object holds none. A marker that does not parse is a corrupt object. */
885
+ function readMarker(kv: ImportKv): ImportMarker | null {
886
+ const raw = kv.get(IMPORT_MARKER_KEY)
887
+ if (raw === undefined) {
888
+ return null
889
+ }
890
+ const value: unknown = typeof raw === 'string' ? parseJson(raw) : undefined
891
+ const source: unknown = isRecord(value) ? value['source'] : undefined
892
+ const verified = isRecord(value) ? readVerification(value['verified']) : undefined
893
+ if (
894
+ !isRecord(value) || !isRecord(source) || Object.keys(value).length !== 4 || Object.keys(source).length !== 2
895
+ || typeof source['object'] !== 'string' || typeof source['dump'] !== 'string' || typeof value['begun'] !== 'boolean'
896
+ || typeof value['loaded'] !== 'number' || !Number.isSafeInteger(value['loaded']) || value['loaded'] < 0 || verified === undefined
897
+ ) {
898
+ throw new Error('the import marker in KV storage does not parse')
899
+ }
900
+ return { source: { object: source['object'], dump: source['dump'] }, begun: value['begun'], loaded: value['loaded'], verified }
901
+ }
902
+
903
+ /** The marker's verification; `undefined` when it does not parse. */
904
+ function readVerification(value: unknown): Verification | null | undefined {
905
+ if (value === null) {
906
+ return null
907
+ }
908
+ if (!isRecord(value) || Object.keys(value).length !== 2) {
909
+ return undefined
910
+ }
911
+ const next = value['next']
912
+ const replay = value['replay']
913
+ if (next !== null && typeof next !== 'string') {
914
+ return undefined
915
+ }
916
+ if (replay === null) {
917
+ return { next, replay: null }
918
+ }
919
+ const cursor = isRecord(replay) ? replay['cursor'] : undefined
920
+ const ack = isRecord(replay) && Object.keys(replay).length === 2 ? parseOpsImportVerifyAck(replay['ack']) : null
921
+ return typeof cursor === 'string' && ack !== null ? { next, replay: { cursor, ack } } : undefined
922
+ }
923
+
924
+ function writeMarker(kv: ImportKv, marker: ImportMarker): void {
925
+ kv.put(IMPORT_MARKER_KEY, JSON.stringify(marker))
926
+ }
927
+
928
+ function readLoadedPart(kv: ImportKv, part: number): OpsImportPartAck | null {
929
+ const raw = kv.get(loadedPartKey(part))
930
+ if (raw === undefined) {
931
+ return null
932
+ }
933
+ const ack = typeof raw === 'string' ? parseOpsImportPartAck(parseJson(raw)) : null
934
+ if (ack === null || ack.part !== part) {
935
+ throw new Error('a loaded part recorded in KV storage does not parse')
936
+ }
937
+ return ack
938
+ }
939
+
940
+ function readLinkedPart(kv: ImportKv, part: number): OpsImportLinkAck | null {
941
+ const raw = kv.get(linkedPartKey(part))
942
+ if (raw === undefined) {
943
+ return null
944
+ }
945
+ const ack = typeof raw === 'string' ? parseOpsImportLinkAck(parseJson(raw)) : null
946
+ if (ack === null || ack.part !== part) {
947
+ throw new Error('a link pass recorded in KV storage does not parse')
948
+ }
949
+ return ack
950
+ }
951
+
952
+ /** The acknowledgement `complete` answered for `source`, `null` when it never completed. */
953
+ function readCompleted(kv: ImportKv, source: OpsDumpSource): OpsImportCompleteAck | null {
954
+ const raw = kv.get(completedKey(source))
955
+ if (raw === undefined) {
956
+ return null
957
+ }
958
+ const ack = typeof raw === 'string' ? parseOpsImportCompleteAck(parseJson(raw)) : null
959
+ if (ack === null || !sameSource(ack.source, source)) {
960
+ throw new Error('a completed import recorded in KV storage does not parse')
961
+ }
962
+ return ack
963
+ }
964
+
965
+ function sameSource(left: OpsDumpSource, right: OpsDumpSource): boolean {
966
+ return left.object === right.object && left.dump === right.dump
967
+ }
968
+
969
+ function refused(reason: OpsDataRefusalReason, table: string | null = null): Response {
970
+ const refusal: OpsDataRefusal = { refused: reason, table }
971
+ return jsonResponse(refusal, OPS_DATA_REFUSED_STATUS)
972
+ }
973
+
974
+ /** The parameters of `bindings`; `sql.exec` binds bytes as an `ArrayBuffer`. */
975
+ function params(bindings: readonly OpsDumpBinding[]): SqliteValue[] {
976
+ return bindings.map(({ param }) => param instanceof Uint8Array ? param.slice().buffer : param)
977
+ }
978
+
979
+ function quote(identifier: string): string {
980
+ return `"${identifier.replaceAll('"', '""')}"`
981
+ }
982
+
983
+ function literal(value: string): string {
984
+ return `'${value.replaceAll("'", "''")}'`
985
+ }
986
+
987
+ /** Code-unit order, which no locale changes; the migration runner orders files the same way. */
988
+ function compareNames(left: string, right: string): number {
989
+ return left < right ? -1 : left > right ? 1 : 0
990
+ }
991
+
992
+ function parseJson(text: string): unknown {
993
+ try {
994
+ return JSON.parse(text)
995
+ } catch {
996
+ return undefined
997
+ }
998
+ }
999
+
1000
+ function isRecord(value: unknown): value is Record<string, unknown> {
1001
+ return typeof value === 'object' && value !== null && !Array.isArray(value)
1002
+ }
1003
+
1004
+ // SQLite's own answers: a value of another type is a broken runtime, never input to refuse.
1005
+ function text(value: SqliteValue | undefined): string {
1006
+ if (typeof value !== 'string') {
1007
+ throw new Error('SQLite answered a schema name or SQL that is not text')
1008
+ }
1009
+ return value
1010
+ }
1011
+
1012
+ function integer(value: SqliteValue | undefined): number {
1013
+ if (typeof value !== 'number' || !Number.isSafeInteger(value)) {
1014
+ throw new Error('SQLite answered a schema number that is not an integer')
1015
+ }
1016
+ return value
1017
+ }
1018
+
1019
+ function flag(value: SqliteValue | undefined): boolean {
1020
+ return integer(value) !== 0
1021
+ }