@fabrika/app 0.9.0-cloudflare.1 → 0.9.0-cloudflare.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. package/package.json +9 -9
  2. package/skill/SKILL.md +21 -17
  3. package/skill/references/app-setup.md +42 -11
  4. package/skill/references/endpoint.md +29 -22
  5. package/skill/references/errors.md +32 -31
  6. package/skill/references/jobs.md +22 -3
  7. package/skill/references/tests.md +29 -1
  8. package/src/build.ts +6 -1
  9. package/src/bun.ts +168 -5
  10. package/src/cloudflare/application.ts +546 -76
  11. package/src/cloudflare/bindings.ts +26 -5
  12. package/src/cloudflare/export.ts +621 -0
  13. package/src/cloudflare/import.ts +1097 -0
  14. package/src/cloudflare/maintenance.ts +518 -0
  15. package/src/cloudflare/names.ts +6 -0
  16. package/src/cloudflare/ops.ts +147 -18
  17. package/src/cloudflare/restore.ts +167 -0
  18. package/src/cloudflare/tracker.ts +192 -0
  19. package/src/cloudflare.ts +3 -2
  20. package/src/contract/artifact-schema.ts +13 -2
  21. package/src/contract/build.ts +38 -11
  22. package/src/contract/context.ts +26 -9
  23. package/src/contract/defaults.ts +22 -2
  24. package/src/contract/define.ts +1 -1
  25. package/src/contract/group.ts +10 -1
  26. package/src/contract/kit.ts +2 -1
  27. package/src/disk-assets.ts +8 -3
  28. package/src/files/kinds.ts +54 -24
  29. package/src/implement/anonymous.ts +1 -10
  30. package/src/implement/boot.ts +30 -12
  31. package/src/implement/capability.ts +3 -0
  32. package/src/implement/dispatch.ts +175 -37
  33. package/src/implement/egress.ts +367 -8
  34. package/src/implement/invocation.ts +60 -19
  35. package/src/implement/workload.ts +18 -3
  36. package/src/jobs/runner.ts +646 -98
  37. package/src/spa-fallback.ts +12 -0
  38. package/src/testing/endpoint-access.ts +135 -23
  39. package/src/testing/isolation.ts +8 -1
  40. package/src/testing/kit.ts +60 -0
  41. package/types/src/build.d.ts +1 -1
  42. package/types/src/bun.d.ts +34 -1
  43. package/types/src/cloudflare/application.d.ts +36 -5
  44. package/types/src/cloudflare/bindings.d.ts +6 -0
  45. package/types/src/cloudflare/export.d.ts +16 -0
  46. package/types/src/cloudflare/import.d.ts +58 -0
  47. package/types/src/cloudflare/maintenance.d.ts +117 -0
  48. package/types/src/cloudflare/names.d.ts +5 -0
  49. package/types/src/cloudflare/ops.d.ts +39 -4
  50. package/types/src/cloudflare/restore.d.ts +9 -0
  51. package/types/src/cloudflare/tracker.d.ts +29 -0
  52. package/types/src/cloudflare.d.ts +2 -1
  53. package/types/src/contract/artifact-schema.d.ts +2 -0
  54. package/types/src/contract/context.d.ts +24 -9
  55. package/types/src/contract/defaults.d.ts +19 -1
  56. package/types/src/contract/define.d.ts +1 -1
  57. package/types/src/contract/group.d.ts +9 -1
  58. package/types/src/contract/kit.d.ts +2 -2
  59. package/types/src/disk-assets.d.ts +4 -1
  60. package/types/src/files/kinds.d.ts +17 -6
  61. package/types/src/implement/anonymous.d.ts +0 -4
  62. package/types/src/implement/boot.d.ts +17 -0
  63. package/types/src/implement/capability.d.ts +2 -0
  64. package/types/src/implement/dispatch.d.ts +33 -1
  65. package/types/src/implement/egress.d.ts +36 -1
  66. package/types/src/implement/invocation.d.ts +20 -10
  67. package/types/src/implement/workload.d.ts +5 -0
  68. package/types/src/jobs/runner.d.ts +63 -2
  69. package/types/src/spa-fallback.d.ts +2 -0
  70. package/types/src/testing/endpoint-access.d.ts +25 -1
  71. package/types/src/testing/isolation.d.ts +1 -1
  72. package/types/src/testing/kit.d.ts +16 -0
@@ -0,0 +1,1097 @@
1
+ // One step of loading a consistent backup into a fresh object (ADR-0070 §6): the ops operation `import`, served before
2
+ // the application boots. Control drives the steps and holds their state; the object keeps only its import marker and
3
+ // the steps' receipts in KV storage, which every step that writes updates in the same `transactionSync` as its rows.
4
+ // Every step runs in the maintenance barrier's turn, so no transition interleaves with it.
5
+
6
+ import {
7
+ decodeOpsDumpLine,
8
+ encodeOpsDumpRow,
9
+ foldOpsTableHash,
10
+ OPS_DATA_REFUSED_STATUS,
11
+ OPS_DUMP_KEY,
12
+ OPS_DUMP_MANIFEST_MAX_BYTES,
13
+ OPS_DUMP_PAGE_ROWS,
14
+ OPS_DUMP_PART_MAX_RAW_BYTES,
15
+ OPS_DUMP_PART_MAX_ROWS,
16
+ OPS_DUMP_PART_SIZE,
17
+ OPS_EMPTY_TABLE_HASH,
18
+ OPS_STALE_STATUS,
19
+ type OpsDataRefusal,
20
+ type OpsDataRefusalReason,
21
+ type OpsDumpBinding,
22
+ type OpsDumpCursor,
23
+ type OpsDumpLine,
24
+ type OpsDumpManifest,
25
+ opsDumpRowidSql,
26
+ type OpsDumpSchema,
27
+ type OpsDumpSchemaColumn,
28
+ type OpsDumpSchemaEntry,
29
+ type OpsDumpSchemaForeignKey,
30
+ type OpsDumpSchemaIndex,
31
+ type OpsDumpSchemaTable,
32
+ type OpsDumpSource,
33
+ type OpsDumpTable,
34
+ type OpsDumpTablePlan,
35
+ type OpsDumpValue,
36
+ opsDumpValueBinding,
37
+ type OpsImportBeginAck,
38
+ type OpsImportCompleteAck,
39
+ type OpsImportDiscardAck,
40
+ type OpsImportLinkAck,
41
+ type OpsImportPartAck,
42
+ type OpsImportRequest,
43
+ type OpsImportVerifyAck,
44
+ type OpsStaleRefusal,
45
+ parseOpsImportCompleteAck,
46
+ parseOpsImportLinkAck,
47
+ parseOpsImportPartAck,
48
+ parseOpsImportVerifyAck,
49
+ planOpsDump,
50
+ readOpsDumpManifest,
51
+ readOpsDumpPage,
52
+ sha256Hex,
53
+ } from '@fabrika/auth-core'
54
+ import type { SqliteStorage, SqliteValue } from '@fabrika/db/sqlite'
55
+ import { migrateSqlite, type SqliteMigrationBundle } from '@fabrika/db/sqlite-migrate'
56
+ import { jsonResponse } from '../errors.js'
57
+ import { method } from './bindings.js'
58
+ import type { MaintenanceBarrier, TurnStep } from './maintenance.js'
59
+ import type { OpsTarget } from './ops.js'
60
+
61
+ /**
62
+ * One step of loading a dump into a fresh object (ADR-0070); `serving` is the object of ordinary traffic. A runtime
63
+ * whose storage has no synchronous KV or `deleteAll` does not serve the operation.
64
+ */
65
+ export async function serveImport(target: OpsTarget, barrier: MaintenanceBarrier, step: OpsImportRequest, serving: string): Promise<Response> {
66
+ const { storage } = target
67
+ const { kv, deleteAll } = storage
68
+ if (kv === undefined || deleteAll === undefined) {
69
+ return jsonResponse({ refused: 'not_implemented' }, 501)
70
+ }
71
+ const importing: ImportStorage = {
72
+ sql: storage.sql,
73
+ transaction: (closure) => storage.transaction(closure),
74
+ transactionSync: (closure) => storage.transactionSync(closure),
75
+ kv,
76
+ deleteAll: () => deleteAll.call(storage),
77
+ }
78
+ return await importStep({ storage: importing, bucket: target.bucket, migrations: target.migrations, serving }, barrier, step)
79
+ }
80
+
81
+ /** workerd's synchronous KV storage of a SQLite-backed object: it lives in the same database, so it commits with a transaction. */
82
+ export interface ImportKv {
83
+ get(key: string): unknown
84
+ put(key: string, value: unknown): void
85
+ delete(key: string): boolean
86
+ }
87
+
88
+ /** What the importer needs of the object's storage; a SQLite-backed `DurableObjectStorage` fits it. */
89
+ export interface ImportStorage extends SqliteStorage {
90
+ readonly kv: ImportKv
91
+ /** Wipes the object's database, its KV storage and its alarm. */
92
+ deleteAll(): Promise<void>
93
+ }
94
+
95
+ /** The object an import step loads into. */
96
+ export interface ImportTarget {
97
+ readonly storage: ImportStorage
98
+ /** The environment's bucket (the `FABRIKA_FILES` R2 binding), where the backup lies. */
99
+ readonly bucket: unknown
100
+ /** The release's migration bundles in the order the object applies them: the framework's, then the application's. */
101
+ readonly migrations: readonly SqliteMigrationBundle[]
102
+ /** The object of ordinary traffic (`FABRIKA_APPLICATION_OBJECT`), which `discard` never wipes. */
103
+ readonly serving: string
104
+ }
105
+
106
+ /** The KV key of the import marker. Invariant: an object that holds it never boots and never leaves `inspect`. */
107
+ export const IMPORT_MARKER_KEY = 'fabrika:import'
108
+
109
+ /** Whether the object holds an import marker, so it must neither boot nor leave `inspect`. */
110
+ export function importInProgress(storage: { readonly kv: ImportKv }): boolean {
111
+ return storage.kv.get(IMPORT_MARKER_KEY) !== undefined
112
+ }
113
+
114
+ /** The KV key under which part `part` records its acknowledgement once loaded, so a retry loads nothing twice. */
115
+ function loadedPartKey(part: number): string {
116
+ return `${IMPORT_MARKER_KEY}:part:${part}`
117
+ }
118
+
119
+ /** The KV key under which part `part`'s link pass records its acknowledgement. */
120
+ function linkedPartKey(part: number): string {
121
+ return `${IMPORT_MARKER_KEY}:link:${part}`
122
+ }
123
+
124
+ /**
125
+ * The KV key of the receipt `complete` leaves for `source`, so a retried `complete` answers the same. It blocks nothing
126
+ * and stays for the object's life: `begin` takes only a fresh object, so an object completes one import.
127
+ */
128
+ function completedKey(source: OpsDumpSource): string {
129
+ return `${IMPORT_MARKER_KEY}:complete:${source.object}/${source.dump}`
130
+ }
131
+
132
+ /**
133
+ * The marker: the backup the object is loading, whether `begin` cleared the target, how many parts are loaded, and the
134
+ * verification of what is loaded now: `null` for none, else the cursor its next step must carry (`null` once every
135
+ * table matched the manifest) and the last step's cursor with its answer, which a retry of that step gets again. A
136
+ * part or a link pass that writes resets it.
137
+ */
138
+ interface ImportMarker {
139
+ readonly source: OpsDumpSource
140
+ readonly begun: boolean
141
+ readonly loaded: number
142
+ readonly verified: Verification | null
143
+ }
144
+
145
+ interface Verification {
146
+ readonly next: string | null
147
+ /** `null` after a step that started the verification, which a retry starts again. */
148
+ readonly replay: { readonly cursor: string; readonly ack: OpsImportVerifyAck } | null
149
+ }
150
+
151
+ /**
152
+ * One step of loading the consistent backup `step.source` into the fresh object `target` (ADR-0070 §6), in the
153
+ * barrier's turn: ADR-0069's stale refusal unless `step.generation` is the published one, then `serving_object` on the
154
+ * object of ordinary traffic, then `wrong_mode` unless `inspect`. The turn is held across the step's R2 reads, so the
155
+ * generation and mode it was admitted under still hold when it commits.
156
+ */
157
+ export async function importStep(target: ImportTarget, barrier: MaintenanceBarrier, step: OpsImportRequest): Promise<Response> {
158
+ const result = await barrier.step(step.generation, async (mode): Promise<TurnStep<Response>> => {
159
+ if (step.object === target.serving) {
160
+ return answered(refused('serving_object'))
161
+ }
162
+ if (mode !== 'inspect') {
163
+ return answered(refused('wrong_mode'))
164
+ }
165
+ if (step.step !== 'verify') {
166
+ await barrier.restrictedWrite()
167
+ }
168
+ return answered(await admittedStep(target, step))
169
+ })
170
+ if (result.kind === 'ran') {
171
+ return result.value
172
+ }
173
+ const refusal: OpsStaleRefusal = { refused: 'stale_generation', published: result.published }
174
+ return jsonResponse(refusal, OPS_STALE_STATUS)
175
+ }
176
+
177
+ async function admittedStep(target: ImportTarget, step: OpsImportRequest): Promise<Response> {
178
+ const { kv } = target.storage
179
+ if (step.step === 'discard') {
180
+ return await discard(target)
181
+ }
182
+ if (step.step === 'begin') {
183
+ return await begin(target, step.source)
184
+ }
185
+ if (importing(kv, step.source) === null) {
186
+ const completed = step.step === 'complete' ? readCompleted(kv, step.source) : null
187
+ return completed === null ? refused('no_import') : jsonResponse(completed, 200)
188
+ }
189
+ const manifest = await readManifest(target.bucket, step.source)
190
+ if (manifest === null) {
191
+ return refused('invalid_manifest')
192
+ }
193
+ if (step.step === 'complete') {
194
+ return complete(target.storage, step.source, manifest)
195
+ }
196
+ const plan = currentPlan(target.storage, manifest)
197
+ if ('refused' in plan) {
198
+ return jsonResponse(plan, OPS_DATA_REFUSED_STATUS)
199
+ }
200
+ switch (step.step) {
201
+ case 'part':
202
+ return await loadPart(target, step.source, manifest, plan, step.part)
203
+ case 'link':
204
+ return await linkPart(target, step.source, manifest, plan, step.part)
205
+ case 'verify':
206
+ return verify(target.storage, step.source, manifest, plan, step.cursor)
207
+ }
208
+ }
209
+
210
+ function answered(response: Response): TurnStep<Response> {
211
+ return { value: response, holdUntil: null }
212
+ }
213
+
214
+ /** The marker, when it names `source` and records `begin` done; `null` otherwise. */
215
+ function importing(kv: ImportKv, source: OpsDumpSource): ImportMarker | null {
216
+ const marker = readMarker(kv)
217
+ return marker?.begun === true && sameSource(marker.source, source) ? marker : null
218
+ }
219
+
220
+ /**
221
+ * Verifies the manifest, checks the target is fresh, writes the marker, migrates the target to exactly the manifest's
222
+ * ledger, checks the schema and its plan, then clears the target in one transaction that records `begin` done. A retry
223
+ * under the same backup resumes after what the marker records.
224
+ */
225
+ async function begin(target: ImportTarget, source: OpsDumpSource): Promise<Response> {
226
+ const { storage } = target
227
+ const manifest = await readManifest(target.bucket, source)
228
+ if (manifest === null) {
229
+ return refused('invalid_manifest')
230
+ }
231
+ if (!manifest.consistent) {
232
+ return refused('fuzzy_dump')
233
+ }
234
+ const marker = readMarker(storage.kv)
235
+ if (marker !== null && !sameSource(marker.source, source)) {
236
+ return refused('not_fresh')
237
+ }
238
+ if (marker?.begun === true) {
239
+ return beginAck(manifest)
240
+ }
241
+ if (marker === null && holdsTables(storage)) {
242
+ return refused('not_fresh')
243
+ }
244
+ const bundles = ledgerBundles(target.migrations, manifest)
245
+ if (bundles === null) {
246
+ return refused('ledger_mismatch')
247
+ }
248
+ if (marker === null) {
249
+ writeMarker(storage.kv, { source, begun: false, loaded: 0, verified: null })
250
+ }
251
+ // A database the framework never migrated has no ledger, and gets none.
252
+ if (manifest.migrations.length > 0) {
253
+ await migrateSqlite(
254
+ { exec: (query, ...bindings) => storage.sql.exec(query, ...bindings), transactionSync: (fn) => storage.transactionSync(fn) },
255
+ bundles,
256
+ )
257
+ }
258
+ if (!sameDdl(readDdl(storage), manifest.ddl)) {
259
+ return refused('schema_mismatch')
260
+ }
261
+ const plan = currentPlan(storage, manifest)
262
+ if ('refused' in plan) {
263
+ return jsonResponse(plan, OPS_DATA_REFUSED_STATUS)
264
+ }
265
+ storage.transactionSync(() => {
266
+ for (const entry of readDdl(storage)) {
267
+ if (entry.type === 'trigger') {
268
+ storage.sql.exec(`DROP TRIGGER ${quote(entry.name)}`)
269
+ }
270
+ }
271
+ // Every deferred column takes a temporary NULL (the schema rule), so no row of its cycle references another one after this.
272
+ for (const table of plan) {
273
+ if (table.deferred.length > 0) {
274
+ storage.sql.exec(`UPDATE ${quote(table.name)} SET ${table.deferred.map((column) => `${quote(column)} = NULL`).join(', ')}`)
275
+ }
276
+ }
277
+ for (const table of [...plan].reverse()) {
278
+ storage.sql.exec(`DELETE FROM ${quote(table.name)}`)
279
+ }
280
+ writeMarker(storage.kv, { source, begun: true, loaded: 0, verified: null })
281
+ })
282
+ return beginAck(manifest)
283
+ }
284
+
285
+ function beginAck(manifest: OpsDumpManifest): Response {
286
+ const ack: OpsImportBeginAck = { step: 'begin', parts: manifest.parts.length }
287
+ return jsonResponse(ack, 200)
288
+ }
289
+
290
+ /**
291
+ * Inserts part `part`'s rows with their rowids, every value through the wire's binding and every deferred column as
292
+ * NULL, in one transaction that reads the marker and the part's receipt again, records the part and resets the
293
+ * verification.
294
+ */
295
+ async function loadPart(target: ImportTarget, source: OpsDumpSource, manifest: OpsDumpManifest, plan: readonly OpsDumpTablePlan[], part: number) {
296
+ const { storage } = target
297
+ const recorded = readLoadedPart(storage.kv, part)
298
+ if (recorded !== null) {
299
+ return jsonResponse(recorded, 200)
300
+ }
301
+ const lines = await readPart(target.bucket, source, manifest, plan, part)
302
+ if (!Array.isArray(lines)) {
303
+ return jsonResponse(lines, OPS_DATA_REFUSED_STATUS)
304
+ }
305
+ const tables = new Map(plan.map((table) => [table.name, table]))
306
+ const ack = storage.transactionSync((): OpsImportPartAck | OpsDataRefusal => {
307
+ const marker = importing(storage.kv, source)
308
+ if (marker === null) {
309
+ return { refused: 'no_import', table: null }
310
+ }
311
+ const again = readLoadedPart(storage.kv, part)
312
+ if (again !== null) {
313
+ return again
314
+ }
315
+ let deferred = 0
316
+ for (const { table: name, row } of lines) {
317
+ const table = planned(tables, name)
318
+ if (insertRow(storage, table, row)) {
319
+ deferred++
320
+ }
321
+ }
322
+ const loaded: OpsImportPartAck = { step: 'part', part, rows: lines.length, deferred }
323
+ storage.kv.put(loadedPartKey(part), JSON.stringify(loaded))
324
+ writeMarker(storage.kv, { ...marker, loaded: marker.loaded + 1, verified: null })
325
+ return loaded
326
+ })
327
+ return jsonResponse(ack, 'refused' in ack ? OPS_DATA_REFUSED_STATUS : 200)
328
+ }
329
+
330
+ /** Writes one row; answers whether it has a deferred key for the link pass to set. */
331
+ function insertRow(storage: SqliteStorage, table: OpsDumpTablePlan, row: readonly OpsDumpValue[]): boolean {
332
+ const [rowid, ...values] = row
333
+ // Inline, so a row of a table at workerd's column limit binds no more parameters than its columns.
334
+ const key = opsDumpRowidSql(rowid ?? null)
335
+ const columns: string[] = [table.keyset]
336
+ const keys: string[] = [key]
337
+ const bindings: OpsDumpBinding[] = []
338
+ let waits = false
339
+ for (const [index, column] of table.columns.entries()) {
340
+ const value = values[index] ?? null
341
+ // An INTEGER PRIMARY KEY column is the rowid itself, written once.
342
+ if (column === table.keyset) {
343
+ continue
344
+ }
345
+ const deferred = table.deferred.includes(column)
346
+ waits ||= deferred && value !== null
347
+ const binding = opsDumpValueBinding(deferred ? null : value)
348
+ columns.push(column)
349
+ keys.push(binding.sql)
350
+ bindings.push(binding)
351
+ }
352
+ if (table.name === SQLITE_SEQUENCE) {
353
+ // Loading an AUTOINCREMENT table writes its own counter row; the source's row replaces it, whichever table loads first.
354
+ const name = opsDumpValueBinding(values[0] ?? null)
355
+ storage.sql.exec(`DELETE FROM ${SQLITE_SEQUENCE} WHERE rowid = ${key} OR name = ${name.sql}`, ...params([name]))
356
+ }
357
+ storage.sql.exec(
358
+ `INSERT INTO ${quote(table.name)} (${columns.map(quote).join(', ')}) VALUES (${keys.join(', ')})`,
359
+ ...params(bindings),
360
+ )
361
+ return waits
362
+ }
363
+
364
+ /**
365
+ * Sets the deferred keys of part `part`'s rows by their rowids, once every part is loaded, in one transaction that
366
+ * reads the marker and the pass's receipt again, records the pass and resets the verification.
367
+ */
368
+ async function linkPart(target: ImportTarget, source: OpsDumpSource, manifest: OpsDumpManifest, plan: readonly OpsDumpTablePlan[], part: number) {
369
+ const { storage } = target
370
+ const recorded = readLinkedPart(storage.kv, part)
371
+ if (recorded !== null) {
372
+ return jsonResponse(recorded, 200)
373
+ }
374
+ if (importing(storage.kv, source)?.loaded !== manifest.parts.length) {
375
+ return refused('part_mismatch')
376
+ }
377
+ const lines = await readPart(target.bucket, source, manifest, plan, part)
378
+ if (!Array.isArray(lines)) {
379
+ return jsonResponse(lines, OPS_DATA_REFUSED_STATUS)
380
+ }
381
+ const tables = new Map(plan.map((table) => [table.name, table]))
382
+ const ack = storage.transactionSync((): OpsImportLinkAck | OpsDataRefusal => {
383
+ const marker = importing(storage.kv, source)
384
+ if (marker === null) {
385
+ return { refused: 'no_import', table: null }
386
+ }
387
+ if (marker.loaded !== manifest.parts.length) {
388
+ return { refused: 'part_mismatch', table: null }
389
+ }
390
+ const again = readLinkedPart(storage.kv, part)
391
+ if (again !== null) {
392
+ return again
393
+ }
394
+ let linked = 0
395
+ for (const { table: name, row } of lines) {
396
+ const table = planned(tables, name)
397
+ const [rowid, ...values] = row
398
+ const deferred = table.columns.flatMap((column, index) => table.deferred.includes(column) ? [{ column, value: values[index] ?? null }] : [])
399
+ if (deferred.every(({ value }) => value === null)) {
400
+ continue
401
+ }
402
+ const sets = deferred.map(({ column, value }) => ({ column, binding: opsDumpValueBinding(value) }))
403
+ storage.sql.exec(
404
+ `UPDATE ${quote(table.name)} SET ${sets.map(({ column, binding }) => `${quote(column)} = ${binding.sql}`).join(', ')} WHERE ${
405
+ quote(table.keyset)
406
+ } = ${opsDumpRowidSql(rowid ?? null)}`,
407
+ ...params(sets.map(({ binding }) => binding)),
408
+ )
409
+ linked++
410
+ }
411
+ const done: OpsImportLinkAck = { step: 'link', part, rows: linked }
412
+ storage.kv.put(linkedPartKey(part), JSON.stringify(done))
413
+ writeMarker(storage.kv, { ...marker, verified: null })
414
+ return done
415
+ })
416
+ return jsonResponse(ack, 'refused' in ack ? OPS_DATA_REFUSED_STATUS : 200)
417
+ }
418
+
419
+ /**
420
+ * Reads the loaded tables back from `cursor` with the producer's projection and fold, at most a part's row bound per
421
+ * step, and compares each finished table with the manifest. A verification starts at `null`; any other cursor must be
422
+ * the one the marker records this verification handed out last, or the one its last step was sent, whose answer it
423
+ * repeats. It writes no row: only its progress, into the marker, which a mismatch clears.
424
+ */
425
+ function verify(
426
+ storage: ImportStorage,
427
+ source: OpsDumpSource,
428
+ manifest: OpsDumpManifest,
429
+ plan: readonly OpsDumpTablePlan[],
430
+ cursor: OpsDumpCursor | null,
431
+ ): Response {
432
+ const marker = importing(storage.kv, source)
433
+ if (marker === null) {
434
+ return refused('no_import')
435
+ }
436
+ const mismatch = (table: string | null): Response => {
437
+ writeMarker(storage.kv, { ...marker, verified: null })
438
+ return refused('verify_mismatch', table)
439
+ }
440
+ if (cursor !== null) {
441
+ const key = cursorKey(cursor)
442
+ const replay = marker.verified?.replay ?? null
443
+ // Its answer was lost: what is loaded has not changed since, or the verification would be reset.
444
+ if (replay?.cursor === key) {
445
+ return jsonResponse(replay.ack, 200)
446
+ }
447
+ if ((marker.verified?.next ?? null) !== key) {
448
+ return mismatch(cursor.table)
449
+ }
450
+ }
451
+ const tables = new Map(plan.map((table) => [table.name, table]))
452
+ const order = manifest.tables
453
+ let index = cursor === null ? 0 : order.findIndex((table) => table.name === cursor.table)
454
+ if (index < 0) {
455
+ return mismatch(cursor?.table ?? null)
456
+ }
457
+ const startOf = (table: OpsDumpTable | undefined): OpsDumpCursor | null =>
458
+ table === undefined ? null : { table: table.name, after: null, rows: 0, hash: OPS_EMPTY_TABLE_HASH }
459
+ let current = cursor ?? startOf(order[0])
460
+ const finished: OpsDumpTable[] = []
461
+ let budget = OPS_DUMP_PART_MAX_ROWS
462
+ while (current !== null && budget > 0) {
463
+ const expected = order[index]
464
+ if (expected === undefined) {
465
+ throw new Error('a verify cursor past the last table of the manifest')
466
+ }
467
+ const page = readPage(storage, planned(tables, expected.name), current.after)
468
+ const last = page.at(-1)?.[0]
469
+ current = {
470
+ table: expected.name,
471
+ after: last?.type === 'integer' ? last.value : current.after,
472
+ rows: current.rows + page.length,
473
+ hash: foldOpsTableHash(current.hash, page.map((row) => encodeOpsDumpRow(row))),
474
+ }
475
+ budget -= page.length
476
+ if (current.rows > expected.rows) {
477
+ return mismatch(expected.name)
478
+ }
479
+ if (page.length < OPS_DUMP_PAGE_ROWS) {
480
+ if (current.rows !== expected.rows || current.hash !== expected.hash) {
481
+ return mismatch(expected.name)
482
+ }
483
+ finished.push(expected)
484
+ index++
485
+ current = startOf(order[index])
486
+ }
487
+ }
488
+ const ack: OpsImportVerifyAck = { step: 'verify', tables: finished, next: current }
489
+ const next = current === null ? null : cursorKey(current)
490
+ writeMarker(storage.kv, { ...marker, verified: { next, replay: cursor === null ? null : { cursor: cursorKey(cursor), ack } } })
491
+ return jsonResponse(ack, 200)
492
+ }
493
+
494
+ /** A cursor as the marker records it: its fields in a fixed order, whatever order a request named them in. */
495
+ function cursorKey(cursor: OpsDumpCursor): string {
496
+ return JSON.stringify([cursor.table, cursor.after, cursor.rows, cursor.hash])
497
+ }
498
+
499
+ /** One keyset page of `table` after the rowid `after`: each row its rowid, then its columns, as the producer reads them. */
500
+ function readPage(storage: SqliteStorage, table: OpsDumpTablePlan, after: string | null): (readonly OpsDumpValue[])[] {
501
+ const condition = after === null ? '' : `WHERE ${quote(table.keyset)} > CAST(? AS INTEGER)`
502
+ const query = (sql: string) => after === null ? storage.sql.exec(sql).toArray() : storage.sql.exec(sql, after).toArray()
503
+ return readOpsDumpPage(query, table, condition, OPS_DUMP_PAGE_ROWS).map((row) => row.values)
504
+ }
505
+
506
+ /**
507
+ * Once every part is loaded, every link pass a part needs ran, and a verification of what is loaded now matched every
508
+ * table: recreates the triggers from the manifest's DDL, compares the whole schema again, clears the marker and the
509
+ * steps' receipts, and leaves the completion's receipt.
510
+ */
511
+ function complete(storage: ImportStorage, source: OpsDumpSource, manifest: OpsDumpManifest): Response {
512
+ const answer = storage.transactionSync((): OpsImportCompleteAck | OpsDataRefusal => {
513
+ const marker = importing(storage.kv, source)
514
+ if (marker === null) {
515
+ return { refused: 'no_import', table: null }
516
+ }
517
+ const loaded = manifest.parts.map((part) => readLoadedPart(storage.kv, part.part))
518
+ const unlinked = loaded.some((ack) => ack === null || (ack.deferred > 0 && readLinkedPart(storage.kv, ack.part) === null))
519
+ if (marker.loaded !== manifest.parts.length || unlinked) {
520
+ return { refused: 'part_mismatch', table: null }
521
+ }
522
+ if (marker.verified === null || marker.verified.next !== null) {
523
+ return { refused: 'verify_mismatch', table: null }
524
+ }
525
+ const present = new Set(readDdl(storage).filter((entry) => entry.type === 'trigger').map((entry) => entry.name))
526
+ for (const entry of manifest.ddl) {
527
+ if (entry.type !== 'trigger' || present.has(entry.name)) {
528
+ continue
529
+ }
530
+ if (!TRIGGER_DDL.test(entry.sql)) {
531
+ return { refused: 'schema_mismatch', table: null }
532
+ }
533
+ storage.sql.exec(entry.sql)
534
+ }
535
+ if (!sameDdl(readDdl(storage), manifest.ddl)) {
536
+ return { refused: 'schema_mismatch', table: null }
537
+ }
538
+ for (const part of manifest.parts) {
539
+ storage.kv.delete(loadedPartKey(part.part))
540
+ storage.kv.delete(linkedPartKey(part.part))
541
+ }
542
+ storage.kv.delete(IMPORT_MARKER_KEY)
543
+ const ack: OpsImportCompleteAck = { step: 'complete', source: { object: source.object, dump: source.dump } }
544
+ storage.kv.put(completedKey(source), JSON.stringify(ack))
545
+ return ack
546
+ })
547
+ return jsonResponse(answer, 'refused' in answer ? OPS_DATA_REFUSED_STATUS : 200)
548
+ }
549
+
550
+ /** Wipes an `inspect` object that holds an import marker; the caller refused the object of ordinary traffic. */
551
+ async function discard(target: ImportTarget): Promise<Response> {
552
+ if (readMarker(target.storage.kv) === null) {
553
+ return refused('no_import')
554
+ }
555
+ await target.storage.deleteAll()
556
+ const ack: OpsImportDiscardAck = { step: 'discard' }
557
+ return jsonResponse(ack, 200)
558
+ }
559
+
560
+ /** The plan of the target's schema, which must list the manifest's tables in the manifest's order. */
561
+ function currentPlan(storage: SqliteStorage, manifest: OpsDumpManifest): readonly OpsDumpTablePlan[] | OpsDataRefusal {
562
+ const plan = planOpsDump(describeDumpSchema(storage))
563
+ if ('refused' in plan) {
564
+ return plan
565
+ }
566
+ const names = plan.tables.map((table) => table.name)
567
+ const expected = manifest.tables.map((table) => table.name)
568
+ if (names.length !== expected.length || names.some((name, index) => name !== expected[index])) {
569
+ return { refused: 'schema_mismatch', table: null }
570
+ }
571
+ return plan.tables
572
+ }
573
+
574
+ function planned(tables: ReadonlyMap<string, OpsDumpTablePlan>, name: string): OpsDumpTablePlan {
575
+ const table = tables.get(name)
576
+ if (table === undefined) {
577
+ throw new Error('a dump line names a table the plan does not hold')
578
+ }
579
+ return table
580
+ }
581
+
582
+ /**
583
+ * Part `part`'s lines: its byte range read and checked against the manifest's size and SHA-256, decompressed up to the
584
+ * raw bound, at most the row bound, each line in the wire's encoding and fitting its table's plan.
585
+ */
586
+ async function readPart(
587
+ bucket: unknown,
588
+ source: OpsDumpSource,
589
+ manifest: OpsDumpManifest,
590
+ plan: readonly OpsDumpTablePlan[],
591
+ part: number,
592
+ ): Promise<OpsDumpLine[] | OpsDataRefusal> {
593
+ const mismatch: OpsDataRefusal = { refused: 'part_mismatch', table: null }
594
+ const described = manifest.parts[part - 1]
595
+ if (described === undefined) {
596
+ return mismatch
597
+ }
598
+ // Every part but the last is exactly the fixed size, so part n starts at (n - 1) parts.
599
+ const body = await r2Get(bucket, OPS_DUMP_KEY({ ...source, consistent: true }, 'data'), {
600
+ range: { offset: (part - 1) * OPS_DUMP_PART_SIZE, length: described.size },
601
+ })
602
+ const bytes = body === null ? null : await bodyBytes(body)
603
+ if (bytes === null || bytes.length !== described.size || (await sha256OfBytes(bytes)) !== described.sha256) {
604
+ return mismatch
605
+ }
606
+ const text = await gunzipWithin(bytes, OPS_DUMP_PART_MAX_RAW_BYTES)
607
+ const raw = text?.split('\n')
608
+ // Each line ends with its newline, so the text ends with one.
609
+ if (raw === undefined || raw.pop() !== '' || raw.length > OPS_DUMP_PART_MAX_ROWS) {
610
+ return mismatch
611
+ }
612
+ const tables = new Map(plan.map((table) => [table.name, table]))
613
+ const lines: OpsDumpLine[] = []
614
+ for (const encoded of raw) {
615
+ const line = decodeOpsDumpLine(encoded)
616
+ const table = line === null ? undefined : tables.get(line.table)
617
+ if (line === null || table === undefined) {
618
+ return mismatch
619
+ }
620
+ if (line.row.length !== table.columns.length + 1 || line.row[0]?.type !== 'integer') {
621
+ return { refused: 'part_mismatch', table: table.name }
622
+ }
623
+ lines.push(line)
624
+ }
625
+ return lines
626
+ }
627
+
628
+ /** `bytes` as one gzip member, or `null` when it does not decompress, decompresses past `bound`, or is not UTF-8. */
629
+ async function gunzipWithin(bytes: Uint8Array, bound: number): Promise<string | null> {
630
+ const reader = new Blob([bytes.slice()]).stream().pipeThrough(new DecompressionStream('gzip')).getReader()
631
+ const chunks: Uint8Array[] = []
632
+ let total = 0
633
+ try {
634
+ for (let read = await reader.read(); !read.done; read = await reader.read()) {
635
+ total += read.value.byteLength
636
+ if (total > bound) {
637
+ await reader.cancel()
638
+ return null
639
+ }
640
+ chunks.push(read.value)
641
+ }
642
+ } catch {
643
+ return null
644
+ }
645
+ const raw = new Uint8Array(total)
646
+ let offset = 0
647
+ for (const chunk of chunks) {
648
+ raw.set(chunk, offset)
649
+ offset += chunk.byteLength
650
+ }
651
+ try {
652
+ return new TextDecoder('utf-8', { fatal: true }).decode(raw)
653
+ } catch {
654
+ return null
655
+ }
656
+ }
657
+
658
+ async function sha256OfBytes(bytes: Uint8Array): Promise<string> {
659
+ const digest = await crypto.subtle.digest('SHA-256', bytes.slice())
660
+ return [...new Uint8Array(digest)].map((byte) => byte.toString(16).padStart(2, '0')).join('')
661
+ }
662
+
663
+ /** The manifest at the backup's key, or `null` when it is missing, too large, does not parse or names another dump. */
664
+ async function readManifest(bucket: unknown, source: OpsDumpSource): Promise<OpsDumpManifest | null> {
665
+ const body = await r2Get(bucket, OPS_DUMP_KEY({ ...source, consistent: true }, 'manifest'))
666
+ if (body === null) {
667
+ return null
668
+ }
669
+ const size: unknown = typeof body === 'object' ? Reflect.get(body, 'size') : undefined
670
+ if (typeof size === 'number' && size > OPS_DUMP_MANIFEST_MAX_BYTES) {
671
+ return null
672
+ }
673
+ const text = method(body, 'text')
674
+ if (text === null) {
675
+ throw new Error('R2 answered the manifest without its body')
676
+ }
677
+ const content: unknown = await text()
678
+ const manifest = typeof content === 'string' ? readOpsDumpManifest(content) : null
679
+ return manifest !== null && manifest.object === source.object && manifest.dump === source.dump ? manifest : null
680
+ }
681
+
682
+ /** R2's `get` on the bucket, by method lookup at the binding's edge; `null` for a missing object. */
683
+ async function r2Get(bucket: unknown, key: string, options?: { readonly range: { readonly offset: number; readonly length: number } }) {
684
+ const get = method(bucket, 'get')
685
+ if (get === null) {
686
+ throw new Error("the FABRIKA_FILES binding has no get; a backup lies in the environment's bucket")
687
+ }
688
+ const body: unknown = await (options === undefined ? get(key) : get(key, options))
689
+ return body ?? null
690
+ }
691
+
692
+ async function bodyBytes(body: unknown): Promise<Uint8Array> {
693
+ const arrayBuffer = method(body, 'arrayBuffer')
694
+ if (arrayBuffer === null) {
695
+ throw new Error('R2 answered a dump part without its body')
696
+ }
697
+ const bytes: unknown = await arrayBuffer()
698
+ if (!(bytes instanceof ArrayBuffer)) {
699
+ throw new Error('R2 answered a dump part body that is not bytes')
700
+ }
701
+ return new Uint8Array(bytes)
702
+ }
703
+
704
+ /**
705
+ * Each bundle of the release cut to the manifest's ledger, which each must begin with exactly (by name and checksum);
706
+ * `null` for any difference, a bundle or a file the release lacks included.
707
+ */
708
+ function ledgerBundles(release: readonly SqliteMigrationBundle[], manifest: OpsDumpManifest): readonly SqliteMigrationBundle[] | null {
709
+ const names = new Set(release.map((bundle) => bundle.bundle))
710
+ if (manifest.migrations.some((migration) => !names.has(migration.bundle))) {
711
+ return null
712
+ }
713
+ const bundles: SqliteMigrationBundle[] = []
714
+ for (const bundle of release) {
715
+ const recorded = manifest.migrations.filter((migration) => migration.bundle === bundle.bundle).sort((left, right) =>
716
+ compareNames(left.name, right.name)
717
+ )
718
+ const files = [...bundle.migrations].sort((left, right) => compareNames(left.name, right.name))
719
+ const prefix = files.slice(0, recorded.length)
720
+ const same = prefix.length === recorded.length
721
+ && prefix.every((file, index) => file.name === recorded[index]?.name && sha256Hex(file.sql) === recorded[index]?.checksum)
722
+ if (!same) {
723
+ return null
724
+ }
725
+ bundles.push({ bundle: bundle.bundle, migrations: prefix })
726
+ }
727
+ return bundles
728
+ }
729
+
730
+ const SQLITE_SEQUENCE = 'sqlite_sequence'
731
+
732
+ /** `CREATE TRIGGER` as `sqlite_schema` stores it, the only DDL `complete` runs from the manifest. */
733
+ const TRIGGER_DDL = /^CREATE\s+(?:TEMP\s+|TEMPORARY\s+)?TRIGGER\s/i
734
+
735
+ /** Whether a dump holds `name`: no SQLite internal table but `sqlite_sequence`, and none of Cloudflare's `_cf_` ones. */
736
+ function dumped(name: string): boolean {
737
+ return !name.startsWith('_cf_') && (!name.startsWith('sqlite_') || name === SQLITE_SEQUENCE)
738
+ }
739
+
740
+ /** Whether the object holds any table but Cloudflare's own. */
741
+ function holdsTables(storage: SqliteStorage): boolean {
742
+ return storage.sql.exec(`SELECT name FROM sqlite_schema WHERE type = 'table'`).toArray().some((row) => !text(row['name']).startsWith('_cf_'))
743
+ }
744
+
745
+ /** Every `sqlite_schema` entry with SQL a dump's DDL lists, in no particular order. */
746
+ function readDdl(storage: SqliteStorage): OpsDumpSchemaEntry[] {
747
+ return storage.sql.exec('SELECT type, name, tbl_name, sql FROM sqlite_schema WHERE sql IS NOT NULL').toArray().flatMap((row) => {
748
+ const name = text(row['name'])
749
+ const table = text(row['tbl_name'])
750
+ const type = row['type']
751
+ if (!dumped(name) || !dumped(table) || (type !== 'table' && type !== 'index' && type !== 'trigger' && type !== 'view')) {
752
+ return []
753
+ }
754
+ return [{ type, name, table, sql: text(row['sql']) }]
755
+ })
756
+ }
757
+
758
+ function sameDdl(left: readonly OpsDumpSchemaEntry[], right: readonly OpsDumpSchemaEntry[]): boolean {
759
+ const canonical = (entries: readonly OpsDumpSchemaEntry[]) =>
760
+ entries.map((entry) => JSON.stringify([entry.type, entry.name, entry.table, entry.sql])).sort(compareNames).join('\n')
761
+ return left.length === right.length && canonical(left) === canonical(right)
762
+ }
763
+
764
+ /**
765
+ * The schema `planOpsDump` reads, from the PRAGMAs workerd admits, of every table a dump holds. An `INITIALLY DEFERRED`
766
+ * clause is read from the table's SQL outside its quotes and comments, and marks every foreign key of its table; the
767
+ * names a `CHECK` reads come from the same SQL (`checkNamesOf`).
768
+ */
769
+ export function describeDumpSchema(storage: SqliteStorage): OpsDumpSchema {
770
+ const tables: OpsDumpSchemaTable[] = []
771
+ for (const row of storage.sql.exec('PRAGMA table_list').toArray()) {
772
+ const name = text(row['name'])
773
+ const type = text(row['type'])
774
+ if (text(row['schema']) !== 'main' || type === 'view' || !dumped(name)) {
775
+ continue
776
+ }
777
+ const virtual = type === 'virtual'
778
+ const sql = storage.sql.exec(`SELECT sql FROM sqlite_schema WHERE type = 'table' AND name = ?`, name).toArray()[0]?.['sql']
779
+ const clauses = typeof sql === 'string' ? unquotedSql(sql) : null
780
+ tables.push({
781
+ name,
782
+ virtual,
783
+ withoutRowid: flag(row['wr']),
784
+ checkNames: typeof sql === 'string' ? checkNamesOf(sql) : null,
785
+ // A virtual table is refused whatever it holds, and its module may not be loaded to describe it.
786
+ columns: virtual ? [] : describeColumns(storage, name),
787
+ indexes: virtual ? [] : describeIndexes(storage, name),
788
+ foreignKeys: virtual ? [] : describeForeignKeys(storage, name, clauses === null || /\binitially\s+deferred\b/i.test(clauses)),
789
+ })
790
+ }
791
+ return { tables }
792
+ }
793
+
794
+ function describeColumns(storage: SqliteStorage, table: string): OpsDumpSchemaColumn[] {
795
+ return storage.sql.exec(`PRAGMA table_xinfo(${literal(table)})`).toArray().map((row) => {
796
+ const hidden = integer(row['hidden'])
797
+ return {
798
+ name: text(row['name']),
799
+ type: text(row['type']),
800
+ notNull: flag(row['notnull']),
801
+ primaryKey: integer(row['pk']),
802
+ generated: hidden === 2 || hidden === 3,
803
+ }
804
+ })
805
+ }
806
+
807
+ function describeIndexes(storage: SqliteStorage, table: string): OpsDumpSchemaIndex[] {
808
+ return storage.sql.exec(`PRAGMA index_list(${literal(table)})`).toArray().map((row) => {
809
+ const origin = row['origin']
810
+ if (origin !== 'c' && origin !== 'u' && origin !== 'pk') {
811
+ throw new Error('PRAGMA index_list answered an origin SQLite does not write')
812
+ }
813
+ const columns = storage.sql.exec(`PRAGMA index_xinfo(${literal(text(row['name']))})`).toArray()
814
+ .filter((column) => flag(column['key']))
815
+ .sort((left, right) => integer(left['seqno']) - integer(right['seqno']))
816
+ .map((column) => typeof column['name'] === 'string' ? column['name'] : null)
817
+ return { unique: flag(row['unique']), origin, partial: flag(row['partial']), columns }
818
+ })
819
+ }
820
+
821
+ function describeForeignKeys(storage: SqliteStorage, table: string, initiallyDeferred: boolean): OpsDumpSchemaForeignKey[] {
822
+ const keys = new Map<number, { parent: string; rows: { seq: number; from: string; to: string | null }[] }>()
823
+ for (const row of storage.sql.exec(`PRAGMA foreign_key_list(${literal(table)})`).toArray()) {
824
+ const id = integer(row['id'])
825
+ const to = row['to']
826
+ const entry = { seq: integer(row['seq']), from: text(row['from']), to: typeof to === 'string' ? to : null }
827
+ const key = keys.get(id)
828
+ if (key === undefined) {
829
+ keys.set(id, { parent: text(row['table']), rows: [entry] })
830
+ } else {
831
+ key.rows.push(entry)
832
+ }
833
+ }
834
+ return [...keys.entries()].sort(([left], [right]) => left - right).map(([, key]) => {
835
+ const rows = key.rows.sort((left, right) => left.seq - right.seq)
836
+ const references = rows.flatMap((row) => row.to === null ? [] : [row.to])
837
+ if (references.length !== 0 && references.length !== rows.length) {
838
+ throw new Error('PRAGMA foreign_key_list named the parent columns of a key only in part')
839
+ }
840
+ return { parent: key.parent, columns: rows.map((row) => row.from), references: references.length === 0 ? null : references, initiallyDeferred }
841
+ })
842
+ }
843
+
844
+ type SqlToken =
845
+ | { readonly kind: 'name'; readonly text: string; readonly quoted: boolean }
846
+ | { readonly kind: 'open' | 'close' | 'other' }
847
+
848
+ const WORD_CHAR = /[A-Za-z0-9_$\u0080-\uffff]/
849
+
850
+ /**
851
+ * The names the `CHECK` clauses of a table's `sql` read: every word and quoted identifier between a `CHECK` keyword's
852
+ * parentheses, ASCII-lowercase, keywords and numbers included, so it lists at least every column a `CHECK` reads. A
853
+ * string literal and a comment name nothing; a double-quoted string, which SQLite may read as either, counts as a name.
854
+ * `null` when the text does not scan (an unclosed quote, comment or parenthesis), which counts as naming every column.
855
+ */
856
+ export function checkNamesOf(sql: string): string[] | null {
857
+ const tokens = sqlTokens(sql)
858
+ if (tokens === null) return null
859
+ const names: string[] = []
860
+ for (let index = 0; index < tokens.length; index++) {
861
+ const token = tokens[index]
862
+ if (token?.kind !== 'name' || token.quoted || asciiLower(token.text) !== 'check' || tokens[index + 1]?.kind !== 'open') continue
863
+ let depth = 0
864
+ let end = index + 1
865
+ for (; end < tokens.length; end++) {
866
+ const inner = tokens[end]
867
+ if (inner?.kind === 'open') depth++
868
+ else if (inner?.kind === 'close' && --depth === 0) break
869
+ else if (inner?.kind === 'name') names.push(asciiLower(inner.text))
870
+ }
871
+ if (end === tokens.length) return null
872
+ index = end
873
+ }
874
+ return names
875
+ }
876
+
877
+ /** `sql` as words, quoted identifiers and parentheses; string literals, comments and whitespace dropped. */
878
+ function sqlTokens(sql: string): SqlToken[] | null {
879
+ const tokens: SqlToken[] = []
880
+ let index = 0
881
+ while (index < sql.length) {
882
+ const char = sql.charAt(index)
883
+ if (sql.startsWith('--', index)) {
884
+ const end = sql.indexOf('\n', index)
885
+ index = end < 0 ? sql.length : end + 1
886
+ } else if (sql.startsWith('/*', index)) {
887
+ const end = sql.indexOf('*/', index + 2)
888
+ if (end < 0) return null
889
+ index = end + 2
890
+ } else if (char === "'" || char === '"' || char === '`' || char === '[') {
891
+ const close = char === '[' ? ']' : char
892
+ let end = index + 1
893
+ let text = ''
894
+ for (;;) {
895
+ const next = sql.indexOf(close, end)
896
+ if (next < 0) return null
897
+ text += sql.slice(end, next)
898
+ if (close !== ']' && sql.charAt(next + 1) === close) {
899
+ text += close
900
+ end = next + 2
901
+ continue
902
+ }
903
+ end = next + 1
904
+ break
905
+ }
906
+ if (char !== "'") tokens.push({ kind: 'name', text, quoted: true })
907
+ index = end
908
+ } else if (WORD_CHAR.test(char)) {
909
+ let end = index + 1
910
+ while (end < sql.length && WORD_CHAR.test(sql.charAt(end))) end++
911
+ tokens.push({ kind: 'name', text: sql.slice(index, end), quoted: false })
912
+ index = end
913
+ } else {
914
+ if (char === '(') tokens.push({ kind: 'open' })
915
+ else if (char === ')') tokens.push({ kind: 'close' })
916
+ else if (!/\s/.test(char)) tokens.push({ kind: 'other' })
917
+ index++
918
+ }
919
+ }
920
+ return tokens
921
+ }
922
+
923
+ /** SQLite compares identifiers case-insensitively in ASCII only. */
924
+ function asciiLower(value: string): string {
925
+ return value.replace(/[A-Z]/g, (letter) => letter.toLowerCase())
926
+ }
927
+
928
+ /** `sql` with its string literals, quoted identifiers and comments blanked, so a keyword search finds only clauses. */
929
+ function unquotedSql(sql: string): string {
930
+ let out = ''
931
+ let index = 0
932
+ while (index < sql.length) {
933
+ const char = sql.charAt(index)
934
+ const close = char === "'" ? "'" : char === '"' ? '"' : char === '`' ? '`' : char === '[' ? ']' : null
935
+ if (close !== null) {
936
+ let end = index + 1
937
+ for (;;) {
938
+ end = sql.indexOf(close, end)
939
+ if (end < 0 || close === ']' || sql.charAt(end + 1) !== close) break
940
+ end += 2
941
+ }
942
+ index = end < 0 ? sql.length : end + 1
943
+ out += ' '
944
+ } else if (sql.startsWith('--', index)) {
945
+ const end = sql.indexOf('\n', index)
946
+ index = end < 0 ? sql.length : end + 1
947
+ out += ' '
948
+ } else if (sql.startsWith('/*', index)) {
949
+ const end = sql.indexOf('*/', index + 2)
950
+ index = end < 0 ? sql.length : end + 2
951
+ out += ' '
952
+ } else {
953
+ out += char
954
+ index++
955
+ }
956
+ }
957
+ return out
958
+ }
959
+
960
+ /** The import marker, parsed strictly; `null` when the object holds none. A marker that does not parse is a corrupt object. */
961
+ function readMarker(kv: ImportKv): ImportMarker | null {
962
+ const raw = kv.get(IMPORT_MARKER_KEY)
963
+ if (raw === undefined) {
964
+ return null
965
+ }
966
+ const value: unknown = typeof raw === 'string' ? parseJson(raw) : undefined
967
+ const source: unknown = isRecord(value) ? value['source'] : undefined
968
+ const verified = isRecord(value) ? readVerification(value['verified']) : undefined
969
+ if (
970
+ !isRecord(value) || !isRecord(source) || Object.keys(value).length !== 4 || Object.keys(source).length !== 2
971
+ || typeof source['object'] !== 'string' || typeof source['dump'] !== 'string' || typeof value['begun'] !== 'boolean'
972
+ || typeof value['loaded'] !== 'number' || !Number.isSafeInteger(value['loaded']) || value['loaded'] < 0 || verified === undefined
973
+ ) {
974
+ throw new Error('the import marker in KV storage does not parse')
975
+ }
976
+ return { source: { object: source['object'], dump: source['dump'] }, begun: value['begun'], loaded: value['loaded'], verified }
977
+ }
978
+
979
+ /** The marker's verification; `undefined` when it does not parse. */
980
+ function readVerification(value: unknown): Verification | null | undefined {
981
+ if (value === null) {
982
+ return null
983
+ }
984
+ if (!isRecord(value) || Object.keys(value).length !== 2) {
985
+ return undefined
986
+ }
987
+ const next = value['next']
988
+ const replay = value['replay']
989
+ if (next !== null && typeof next !== 'string') {
990
+ return undefined
991
+ }
992
+ if (replay === null) {
993
+ return { next, replay: null }
994
+ }
995
+ const cursor = isRecord(replay) ? replay['cursor'] : undefined
996
+ const ack = isRecord(replay) && Object.keys(replay).length === 2 ? parseOpsImportVerifyAck(replay['ack']) : null
997
+ return typeof cursor === 'string' && ack !== null ? { next, replay: { cursor, ack } } : undefined
998
+ }
999
+
1000
+ function writeMarker(kv: ImportKv, marker: ImportMarker): void {
1001
+ kv.put(IMPORT_MARKER_KEY, JSON.stringify(marker))
1002
+ }
1003
+
1004
+ function readLoadedPart(kv: ImportKv, part: number): OpsImportPartAck | null {
1005
+ const raw = kv.get(loadedPartKey(part))
1006
+ if (raw === undefined) {
1007
+ return null
1008
+ }
1009
+ const ack = typeof raw === 'string' ? parseOpsImportPartAck(parseJson(raw)) : null
1010
+ if (ack === null || ack.part !== part) {
1011
+ throw new Error('a loaded part recorded in KV storage does not parse')
1012
+ }
1013
+ return ack
1014
+ }
1015
+
1016
+ function readLinkedPart(kv: ImportKv, part: number): OpsImportLinkAck | null {
1017
+ const raw = kv.get(linkedPartKey(part))
1018
+ if (raw === undefined) {
1019
+ return null
1020
+ }
1021
+ const ack = typeof raw === 'string' ? parseOpsImportLinkAck(parseJson(raw)) : null
1022
+ if (ack === null || ack.part !== part) {
1023
+ throw new Error('a link pass recorded in KV storage does not parse')
1024
+ }
1025
+ return ack
1026
+ }
1027
+
1028
+ /** The acknowledgement `complete` answered for `source`, `null` when it never completed. */
1029
+ function readCompleted(kv: ImportKv, source: OpsDumpSource): OpsImportCompleteAck | null {
1030
+ const raw = kv.get(completedKey(source))
1031
+ if (raw === undefined) {
1032
+ return null
1033
+ }
1034
+ const ack = typeof raw === 'string' ? parseOpsImportCompleteAck(parseJson(raw)) : null
1035
+ if (ack === null || !sameSource(ack.source, source)) {
1036
+ throw new Error('a completed import recorded in KV storage does not parse')
1037
+ }
1038
+ return ack
1039
+ }
1040
+
1041
+ function sameSource(left: OpsDumpSource, right: OpsDumpSource): boolean {
1042
+ return left.object === right.object && left.dump === right.dump
1043
+ }
1044
+
1045
+ function refused(reason: OpsDataRefusalReason, table: string | null = null): Response {
1046
+ const refusal: OpsDataRefusal = { refused: reason, table }
1047
+ return jsonResponse(refusal, OPS_DATA_REFUSED_STATUS)
1048
+ }
1049
+
1050
+ /** The parameters of `bindings`; `sql.exec` binds bytes as an `ArrayBuffer`. */
1051
+ function params(bindings: readonly OpsDumpBinding[]): SqliteValue[] {
1052
+ return bindings.map(({ param }) => param instanceof Uint8Array ? param.slice().buffer : param)
1053
+ }
1054
+
1055
+ function quote(identifier: string): string {
1056
+ return `"${identifier.replaceAll('"', '""')}"`
1057
+ }
1058
+
1059
+ function literal(value: string): string {
1060
+ return `'${value.replaceAll("'", "''")}'`
1061
+ }
1062
+
1063
+ /** Code-unit order, which no locale changes; the migration runner orders files the same way. */
1064
+ function compareNames(left: string, right: string): number {
1065
+ return left < right ? -1 : left > right ? 1 : 0
1066
+ }
1067
+
1068
+ function parseJson(text: string): unknown {
1069
+ try {
1070
+ return JSON.parse(text)
1071
+ } catch {
1072
+ return undefined
1073
+ }
1074
+ }
1075
+
1076
+ function isRecord(value: unknown): value is Record<string, unknown> {
1077
+ return typeof value === 'object' && value !== null && !Array.isArray(value)
1078
+ }
1079
+
1080
+ // SQLite's own answers: a value of another type is a broken runtime, never input to refuse.
1081
+ function text(value: SqliteValue | undefined): string {
1082
+ if (typeof value !== 'string') {
1083
+ throw new Error('SQLite answered a schema name or SQL that is not text')
1084
+ }
1085
+ return value
1086
+ }
1087
+
1088
+ function integer(value: SqliteValue | undefined): number {
1089
+ if (typeof value !== 'number' || !Number.isSafeInteger(value)) {
1090
+ throw new Error('SQLite answered a schema number that is not an integer')
1091
+ }
1092
+ return value
1093
+ }
1094
+
1095
+ function flag(value: SqliteValue | undefined): boolean {
1096
+ return integer(value) !== 0
1097
+ }