@fabrika/app 0.9.0-dev.4 → 0.9.0-dev.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. package/package.json +9 -9
  2. package/skill/SKILL.md +41 -14
  3. package/skill/references/app-setup.md +43 -12
  4. package/skill/references/dev.md +2 -1
  5. package/skill/references/endpoint.md +30 -24
  6. package/skill/references/errors.md +40 -35
  7. package/skill/references/files.md +4 -2
  8. package/skill/references/jobs.md +21 -4
  9. package/skill/references/table-and-migration.md +19 -2
  10. package/skill/references/tests.md +40 -1
  11. package/skill/references/webhooks-links-connectors.md +1 -1
  12. package/src/build-recipe.ts +198 -0
  13. package/src/build.ts +60 -4
  14. package/src/bun.ts +181 -13
  15. package/src/cloudflare/application.ts +822 -0
  16. package/src/cloudflare/bindings.ts +159 -0
  17. package/src/cloudflare/export.ts +641 -0
  18. package/src/cloudflare/import.ts +1021 -0
  19. package/src/cloudflare/maintenance.ts +518 -0
  20. package/src/cloudflare/names.ts +16 -0
  21. package/src/cloudflare/ops.ts +375 -0
  22. package/src/cloudflare/r2-files.ts +172 -0
  23. package/src/cloudflare/restore.ts +167 -0
  24. package/src/cloudflare/tracker.ts +192 -0
  25. package/src/cloudflare.ts +100 -146
  26. package/src/contract/app-id.ts +5 -0
  27. package/src/contract/artifact-schema.ts +9 -1
  28. package/src/contract/build.ts +40 -2
  29. package/src/contract/compile.ts +1 -0
  30. package/src/contract/context.ts +35 -9
  31. package/src/contract/declaration.ts +2 -0
  32. package/src/contract/defaults.ts +22 -2
  33. package/src/contract/define.ts +29 -3
  34. package/src/contract/kit.ts +4 -1
  35. package/src/contract/platform.ts +3 -2
  36. package/src/contract/vocabulary.ts +26 -8
  37. package/src/dev/launcher.ts +50 -4
  38. package/src/disk-assets.ts +8 -3
  39. package/src/files/archive.ts +2 -2
  40. package/src/files/job-names.ts +12 -0
  41. package/src/files/kinds.ts +73 -31
  42. package/src/files/scan.ts +3 -4
  43. package/src/files/streams.ts +3 -1
  44. package/src/implement/anonymous.ts +12 -11
  45. package/src/implement/boot.ts +75 -6
  46. package/src/implement/bounded.ts +21 -2
  47. package/src/implement/capability.ts +15 -3
  48. package/src/implement/dispatch.ts +419 -177
  49. package/src/implement/failures.ts +17 -1
  50. package/src/implement/faults.ts +1 -0
  51. package/src/implement/index.ts +1 -0
  52. package/src/implement/invocation.ts +58 -10
  53. package/src/implement/principal.ts +25 -2
  54. package/src/implement/terminals.ts +93 -0
  55. package/src/implement/workload.ts +42 -16
  56. package/src/jobs/mail.ts +8 -0
  57. package/src/jobs/report.ts +5 -1
  58. package/src/jobs/runner.ts +672 -113
  59. package/src/jobs/schedule.ts +7 -1
  60. package/src/jobs/scrub.ts +14 -1
  61. package/src/jobs/types.ts +6 -0
  62. package/src/spa-fallback.ts +12 -0
  63. package/src/testing/endpoint-access.ts +307 -152
  64. package/src/testing/isolation.ts +8 -1
  65. package/src/testing/kit.ts +80 -2
  66. package/src/testing/proxy.ts +15 -10
  67. package/types/src/build-recipe.d.ts +70 -0
  68. package/types/src/build.d.ts +19 -1
  69. package/types/src/bun.d.ts +34 -1
  70. package/types/src/cloudflare/application.d.ts +73 -0
  71. package/types/src/cloudflare/bindings.d.ts +33 -0
  72. package/types/src/cloudflare/export.d.ts +16 -0
  73. package/types/src/cloudflare/import.d.ts +51 -0
  74. package/types/src/cloudflare/maintenance.d.ts +117 -0
  75. package/types/src/cloudflare/names.d.ts +11 -0
  76. package/types/src/cloudflare/ops.d.ts +81 -0
  77. package/types/src/cloudflare/r2-files.d.ts +9 -0
  78. package/types/src/cloudflare/restore.d.ts +9 -0
  79. package/types/src/cloudflare/tracker.d.ts +29 -0
  80. package/types/src/cloudflare.d.ts +12 -18
  81. package/types/src/contract/app-id.d.ts +2 -0
  82. package/types/src/contract/artifact-schema.d.ts +2 -0
  83. package/types/src/contract/context.d.ts +35 -11
  84. package/types/src/contract/declaration.d.ts +4 -0
  85. package/types/src/contract/defaults.d.ts +19 -1
  86. package/types/src/contract/define.d.ts +20 -3
  87. package/types/src/contract/kit.d.ts +2 -1
  88. package/types/src/contract/platform.d.ts +2 -2
  89. package/types/src/contract/vocabulary.d.ts +9 -7
  90. package/types/src/disk-assets.d.ts +4 -1
  91. package/types/src/files/job-names.d.ts +9 -0
  92. package/types/src/files/kinds.d.ts +21 -14
  93. package/types/src/files/scan.d.ts +1 -3
  94. package/types/src/implement/anonymous.d.ts +2 -4
  95. package/types/src/implement/boot.d.ts +17 -1
  96. package/types/src/implement/bounded.d.ts +11 -1
  97. package/types/src/implement/capability.d.ts +4 -2
  98. package/types/src/implement/dispatch.d.ts +34 -1
  99. package/types/src/implement/failures.d.ts +5 -0
  100. package/types/src/implement/faults.d.ts +1 -0
  101. package/types/src/implement/index.d.ts +1 -0
  102. package/types/src/implement/invocation.d.ts +22 -7
  103. package/types/src/implement/principal.d.ts +6 -1
  104. package/types/src/implement/terminals.d.ts +33 -0
  105. package/types/src/implement/workload.d.ts +5 -0
  106. package/types/src/jobs/runner.d.ts +66 -2
  107. package/types/src/jobs/schedule.d.ts +2 -0
  108. package/types/src/jobs/types.d.ts +6 -0
  109. package/types/src/spa-fallback.d.ts +2 -0
  110. package/types/src/testing/endpoint-access.d.ts +37 -5
  111. package/types/src/testing/isolation.d.ts +1 -1
  112. package/types/src/testing/kit.d.ts +22 -1
  113. package/types/src/testing/proxy.d.ts +4 -4
@@ -0,0 +1,641 @@
1
+ // One step of a dump of the object's database (ADR-0070 §2): the ops operation `export`, served before the application
2
+ // boots. It reads `sqlite_schema` and the tables directly, so it also dumps a database that cannot boot. Control's
3
+ // operation record holds the upload id, the cursor and the parts; the object keeps only a consistent dump's begun state.
4
+
5
+ import {
6
+ encodeOpsDumpLine,
7
+ foldOpsTableHash,
8
+ OPS_DATA_REFUSED_STATUS,
9
+ OPS_DUMP_FORMAT,
10
+ OPS_DUMP_KEY,
11
+ OPS_DUMP_PAGE_ROWS,
12
+ OPS_DUMP_PART_MAX_RAW_BYTES,
13
+ OPS_DUMP_PART_MAX_ROWS,
14
+ OPS_DUMP_PART_SIZE,
15
+ OPS_EMPTY_TABLE_HASH,
16
+ OPS_STALE_STATUS,
17
+ type OpsDataRefusal,
18
+ opsDumpColumnProjection,
19
+ type OpsDumpCursor,
20
+ type OpsDumpSchemaEntry,
21
+ type OpsDumpTable,
22
+ type OpsDumpTablePlan,
23
+ type OpsDumpValue,
24
+ opsDumpValueBinding,
25
+ type OpsExportBegin,
26
+ type OpsExportComplete,
27
+ type OpsExportPart,
28
+ type OpsExportRequest,
29
+ type OpsExportScope,
30
+ type OpsPublishedState,
31
+ type OpsStaleRefusal,
32
+ parseOpsExportBeginAck,
33
+ parseOpsExportCompleteAck,
34
+ parseOpsExportPartAck,
35
+ planOpsDump,
36
+ readOpsDumpValue,
37
+ sha256Hex,
38
+ } from '@fabrika/auth-core'
39
+ import type { SqliteValue } from '@fabrika/db/sqlite'
40
+ import { crc32, deflateRawSync } from 'node:zlib'
41
+ import appPackage from '../../package.json' with { type: 'json' }
42
+ import { jsonResponse } from '../errors.js'
43
+ import { method } from './bindings.js'
44
+ import { describeDumpSchema, importInProgress } from './import.js'
45
+ import type { MaintenanceBarrier } from './maintenance.js'
46
+ import type { OpsTarget } from './ops.js'
47
+
48
+ /** One step of a dump of the object's database (ADR-0070). */
49
+ export async function serveExport(target: OpsTarget, barrier: MaintenanceBarrier, step: OpsExportRequest): Promise<Response> {
50
+ return await exportStep(target, barrier, step)
51
+ }
52
+
53
+ /** What an export step reads: the object's storage, bucket and bookmarks, and the digest of the contract it was deployed with. */
54
+ export interface ExportSource extends Pick<OpsTarget, 'storage' | 'bucket' | 'recovery'> {
55
+ /** `FABRIKA_CONTRACT_DIGEST`, which the manifest records. */
56
+ readonly contractDigest: string | null
57
+ }
58
+
59
+ type Storage = ExportSource['storage']
60
+
61
+ /**
62
+ * One step of a dump. A diagnostic step runs in any mode. A consistent one runs in the barrier's turn, only at the
63
+ * published generation, never while the object is `open` or holds an import marker, so no transition and no migration
64
+ * falls inside it; past `begin` it refuses a database whose state or stretch is not the one its `begin` saw.
65
+ */
66
+ export async function exportStep(source: ExportSource, barrier: MaintenanceBarrier, step: OpsExportRequest): Promise<Response> {
67
+ if (!step.consistent) {
68
+ return await run(source, step)
69
+ }
70
+ const { generation } = step
71
+ if (generation === null) {
72
+ throw new Error('a consistent export step carries its generation; the wire parser requires it')
73
+ }
74
+ const result = await barrier.step(generation, async (mode) => {
75
+ const { frozenSince } = barrier
76
+ if (mode === 'open' || frozenSince === null) {
77
+ return { value: dataRefusal({ refused: 'wrong_mode', table: null }), holdUntil: null }
78
+ }
79
+ return { value: await runConsistent(source, step, frozenSince), holdUntil: null }
80
+ })
81
+ return result.kind === 'ran' ? result.value : staleRefusal(result.published)
82
+ }
83
+
84
+ /** Runs `step`; `state` is what a consistent dump's first `begin` records, `null` for a diagnostic one. */
85
+ async function run(source: ExportSource, step: OpsExportRequest, state: string | null = null): Promise<Response> {
86
+ switch (step.step) {
87
+ case 'begin':
88
+ return await begin(source, step, state)
89
+ case 'part':
90
+ return await part(source, step)
91
+ case 'complete':
92
+ return await complete(source, step)
93
+ }
94
+ }
95
+
96
+ /**
97
+ * The first `begin` of a dump records the state it saw with its upload and bookmark, once; a retried one replays that
98
+ * record, and every later step must carry its upload at that state, in the same stretch of modes that write nothing
99
+ * (`frozenSince`), at whatever generation of it. A migration runs inside a stretch, so the stretch alone would not show
100
+ * one between two steps. A storage without the synchronous KV holds no import.
101
+ */
102
+ async function runConsistent(source: ExportSource, step: OpsExportRequest, frozenSince: number): Promise<Response> {
103
+ const { kv } = source.storage
104
+ if (kv !== undefined && importInProgress({ kv })) {
105
+ return dataRefusal({ refused: 'wrong_mode', table: null })
106
+ }
107
+ const state = dumpState(source.storage, frozenSince)
108
+ const begun = await readBegun(source.bucket, step)
109
+ if (step.step === 'begin') {
110
+ return begun === null ? await run(source, step, state) : replayed(begun, state)
111
+ }
112
+ if (begun?.state !== state || begun.uploadId !== step.uploadId || (step.step === 'complete' && step.bookmark !== begun.bookmark)) {
113
+ return dataRefusal({ refused: 'ledger_mismatch', table: null })
114
+ }
115
+ return await run(source, step)
116
+ }
117
+
118
+ /** A retried `begin`: the first one's answer at the state it recorded, refused at any other. */
119
+ function replayed(begun: Begun, state: string): Response {
120
+ if (begun.state !== state) {
121
+ return dataRefusal({ refused: 'ledger_mismatch', table: null })
122
+ }
123
+ return answer(parseOpsExportBeginAck({ step: 'begin', uploadId: begun.uploadId, bookmark: begun.bookmark }), 'begin')
124
+ }
125
+
126
+ function staleRefusal(published: OpsPublishedState | null): Response {
127
+ const stale: OpsStaleRefusal = { refused: 'stale_generation', published }
128
+ return jsonResponse(stale, OPS_STALE_STATUS)
129
+ }
130
+
131
+ function dataRefusal(refusal: OpsDataRefusal): Response {
132
+ return jsonResponse(refusal, OPS_DATA_REFUSED_STATUS)
133
+ }
134
+
135
+ /**
136
+ * Plans the dump, starts the multipart upload of its data and answers the object's bookmark. A consistent dump records
137
+ * its `state` with both, only if nothing is recorded yet.
138
+ */
139
+ async function begin(source: ExportSource, step: OpsExportBegin, state: string | null): Promise<Response> {
140
+ const plan = planOpsDump(describeDumpSchema(source.storage))
141
+ if ('refused' in plan) {
142
+ return dataRefusal(plan)
143
+ }
144
+ if (source.recovery === null) {
145
+ throw new Error("the object's storage has no bookmarks; a Durable Object's storage has them")
146
+ }
147
+ const bookmark = await source.recovery.getCurrentBookmark()
148
+ const upload: unknown = await call(source.bucket, 'createMultipartUpload')(dataKey(step), { httpMetadata: { contentType: 'application/gzip' } })
149
+ const ack = parseOpsExportBeginAck({
150
+ step: 'begin',
151
+ uploadId: typeof upload === 'object' && upload !== null ? Reflect.get(upload, 'uploadId') : undefined,
152
+ bookmark,
153
+ })
154
+ if (ack === null || state === null) {
155
+ return answer(ack, 'begin')
156
+ }
157
+ const begun: Begun = { state, uploadId: ack.uploadId, bookmark: ack.bookmark }
158
+ const options = { httpMetadata: { contentType: 'application/json' }, onlyIf: { etagDoesNotMatch: '*' } }
159
+ if ((await call(source.bucket, 'put')(begunKey(step), JSON.stringify(begun), options)) === null) {
160
+ // Another begin recorded the dump first; its upload is the dump's, and this one is left to the lifecycle.
161
+ const recorded = await readBegun(source.bucket, step)
162
+ if (recorded === null) {
163
+ throw new Error("R2 refused to record a dump's begin, yet holds no record of it")
164
+ }
165
+ return replayed(recorded, state)
166
+ }
167
+ return answer(ack, 'begin')
168
+ }
169
+
170
+ /** Fills part `n` from its cursor: whole rows while every bound holds, one gzip member padded to the fixed size but the last. */
171
+ async function part(source: ExportSource, step: OpsExportPart): Promise<Response> {
172
+ const plan = planOpsDump(describeDumpSchema(source.storage))
173
+ if ('refused' in plan) {
174
+ return dataRefusal(plan)
175
+ }
176
+ const [first] = plan.tables
177
+ const start = step.cursor ?? (first === undefined ? null : fresh(first.name))
178
+ const index = start === null ? plan.tables.length : plan.tables.findIndex((table) => table.name === start.table)
179
+ if (start !== null && index < 0) {
180
+ return dataRefusal({ refused: 'schema_mismatch', table: start.table })
181
+ }
182
+ const collected = collect(source.storage, plan.tables.slice(index), start)
183
+ if ('refused' in collected) {
184
+ return dataRefusal(collected)
185
+ }
186
+ const filled = fill(collected)
187
+ if ('refused' in filled) {
188
+ return dataRefusal(filled)
189
+ }
190
+ return await uploadPart(source, step, filled, start === null ? { finished: [], next: null } : settle(start, collected, filled.rows))
191
+ }
192
+
193
+ /** Completes the upload with the parts Control recorded, writes the manifest beside the data and answers it. */
194
+ async function complete(source: ExportSource, step: OpsExportComplete): Promise<Response> {
195
+ const { storage } = source
196
+ const plan = planOpsDump(describeDumpSchema(storage))
197
+ if ('refused' in plan) {
198
+ return dataRefusal(plan)
199
+ }
200
+ const mismatch = plan.tables.find((table, index) => step.tables[index]?.name !== table.name)
201
+ if (mismatch !== undefined || step.tables.length !== plan.tables.length) {
202
+ return dataRefusal({ refused: 'schema_mismatch', table: mismatch?.name ?? null })
203
+ }
204
+ const ddl = schemaEntries(storage)
205
+ const migrations = migrationLedger(storage)
206
+ const key = dataKey(step)
207
+ const upload: unknown = await call(source.bucket, 'resumeMultipartUpload')(key, step.uploadId)
208
+ try {
209
+ await call(upload, 'complete')(step.parts.map((uploaded) => ({ partNumber: uploaded.part, etag: uploaded.etag })))
210
+ } catch (error) {
211
+ // A retried complete whose first answer was lost finds the upload gone and the data in place.
212
+ const head: unknown = await call(source.bucket, 'head')(key)
213
+ const size = typeof head === 'object' && head !== null ? Reflect.get(head, 'size') : undefined
214
+ if (size !== step.parts.reduce((sum, uploaded) => sum + uploaded.size, 0)) {
215
+ throw error
216
+ }
217
+ }
218
+ const manifest = {
219
+ format: OPS_DUMP_FORMAT,
220
+ dump: step.dump,
221
+ object: step.object,
222
+ consistent: step.consistent,
223
+ createdAt: new Date().toISOString(),
224
+ bookmark: step.bookmark,
225
+ release: appPackage.version,
226
+ contractDigest: source.contractDigest,
227
+ ddl,
228
+ migrations,
229
+ tables: step.tables,
230
+ parts: step.parts.map(({ part, size, sha256 }) => ({ part, size, sha256 })),
231
+ }
232
+ const ack = parseOpsExportCompleteAck({ step: 'complete', manifest })
233
+ if (ack === null) {
234
+ throw new Error("the dump's manifest does not parse as the wire's manifest")
235
+ }
236
+ await call(source.bucket, 'put')(OPS_DUMP_KEY(step, 'manifest'), JSON.stringify(ack.manifest), { httpMetadata: { contentType: 'application/json' } })
237
+ return jsonResponse(ack, 200)
238
+ }
239
+
240
+ async function uploadPart(source: ExportSource, step: OpsExportPart, part: Filled, settled: Settled): Promise<Response> {
241
+ const member = gzipMember(part)
242
+ const digest = new Uint8Array(await crypto.subtle.digest('SHA-256', member))
243
+ const sha256 = Array.from(digest, (byte) => byte.toString(16).padStart(2, '0')).join('')
244
+ const upload: unknown = await call(source.bucket, 'resumeMultipartUpload')(dataKey(step), step.uploadId)
245
+ // R2 keeps one part per number, so a retried step replaces the part it uploaded before.
246
+ const uploaded: unknown = await call(upload, 'uploadPart')(step.part, member)
247
+ const etag = typeof uploaded === 'object' && uploaded !== null ? Reflect.get(uploaded, 'etag') : undefined
248
+ return answer(
249
+ parseOpsExportPartAck({
250
+ step: 'part',
251
+ uploaded: { part: step.part, size: member.byteLength, sha256, etag },
252
+ tables: settled.finished,
253
+ next: settled.next,
254
+ }),
255
+ 'part',
256
+ )
257
+ }
258
+
259
+ function dataKey(scope: OpsExportScope): string {
260
+ return OPS_DUMP_KEY(scope, 'data')
261
+ }
262
+
263
+ /** What a consistent dump's first `begin` saw and answered: the digest of the database's state, its upload and bookmark. */
264
+ interface Begun {
265
+ readonly state: string
266
+ readonly uploadId: string
267
+ readonly bookmark: string
268
+ }
269
+
270
+ /** `begun` in the dump's own prefix, which the bucket's lifecycle expires with the dump. */
271
+ function begunKey(scope: OpsExportScope): string {
272
+ const manifest = OPS_DUMP_KEY(scope, 'manifest')
273
+ return `${manifest.slice(0, manifest.lastIndexOf('/'))}/begun`
274
+ }
275
+
276
+ /** What the dump's first `begin` recorded, `null` when nothing is recorded; a record that does not parse fails the step. */
277
+ async function readBegun(bucket: unknown, scope: OpsExportScope): Promise<Begun | null> {
278
+ const stored: unknown = await call(bucket, 'get')(begunKey(scope))
279
+ if (stored === null) {
280
+ return null
281
+ }
282
+ const body: unknown = await call(stored, 'text')()
283
+ const begun = typeof body === 'string' ? parseBegun(body) : null
284
+ if (begun === null) {
285
+ throw new Error("the dump's begin record does not parse")
286
+ }
287
+ return begun
288
+ }
289
+
290
+ function parseBegun(text: string): Begun | null {
291
+ let value: unknown
292
+ try {
293
+ value = JSON.parse(text)
294
+ } catch {
295
+ return null
296
+ }
297
+ if (typeof value !== 'object' || value === null || Array.isArray(value) || Object.keys(value).length !== 3) {
298
+ return null
299
+ }
300
+ const state: unknown = Reflect.get(value, 'state')
301
+ const uploadId: unknown = Reflect.get(value, 'uploadId')
302
+ const bookmark: unknown = Reflect.get(value, 'bookmark')
303
+ return typeof state === 'string' && typeof uploadId === 'string' && typeof bookmark === 'string' ? { state, uploadId, bookmark } : null
304
+ }
305
+
306
+ function answer<T>(ack: T | null, step: string): Response {
307
+ if (ack === null) {
308
+ throw new Error(`the export ${step} answer does not parse as the wire's answer`)
309
+ }
310
+ return jsonResponse(ack, 200)
311
+ }
312
+
313
+ function call(target: unknown, name: string): (...args: unknown[]) => unknown {
314
+ const found = method(target, name)
315
+ if (found === null) {
316
+ throw new Error(`the FABRIKA_FILES binding has no ${name}; dumps live in the environment's bucket`)
317
+ }
318
+ return found
319
+ }
320
+
321
+ // Reading: keyset pages of whole rows within the part's bounds.
322
+
323
+ function fresh(table: string): OpsDumpCursor {
324
+ return { table, after: null, rows: 0, hash: OPS_EMPTY_TABLE_HASH }
325
+ }
326
+
327
+ /** A row read for the part: its table, its rowid, and its line with the newline, where its canonical encoding starts at `rowStart`. */
328
+ interface ReadRow {
329
+ readonly kind: 'row'
330
+ readonly table: string
331
+ readonly rowid: string
332
+ readonly line: Uint8Array
333
+ readonly rowStart: number
334
+ }
335
+
336
+ /** A table read to its end. */
337
+ interface TableEnd {
338
+ readonly kind: 'end'
339
+ readonly table: string
340
+ }
341
+
342
+ interface Collected {
343
+ /** Rows and table ends in reading order: a table's end follows its last row. */
344
+ readonly items: readonly (ReadRow | TableEnd)[]
345
+ /** The table of the row the part could not take, `null` once every table was read. */
346
+ readonly breaker: string | null
347
+ }
348
+
349
+ /** `JSON.stringify` adds at most this to a value's text: the quotes, the tag and the separator. */
350
+ const VALUE_OVERHEAD = 8
351
+ /** A value that is no text or blob is at most this long: `-1.7976931348623157e+308`, `-9223372036854775808`, `null`. */
352
+ const SCALAR_BYTES = 32
353
+
354
+ /**
355
+ * Reads whole rows of `tables`, from the cursor's position in the first, while the part holds at most
356
+ * `OPS_DUMP_PART_MAX_ROWS` rows and its lines at most `OPS_DUMP_PART_MAX_RAW_BYTES`. A page first asks how long each
357
+ * row's line is at most, then reads the rows that fit and the one after them, so a page of large rows never sits in
358
+ * memory whole.
359
+ */
360
+ function collect(storage: Storage, tables: readonly OpsDumpTablePlan[], start: OpsDumpCursor | null): Collected | OpsDataRefusal {
361
+ const items: (ReadRow | TableEnd)[] = []
362
+ const encoder = new TextEncoder()
363
+ let rows = 0
364
+ let raw = 0
365
+ for (const table of tables) {
366
+ // A line is `{"table":"<name>","row":<row>}` (ADR-0070 §3), all ASCII: its row starts where an empty row's `[]}` does.
367
+ const rowStart = encodeOpsDumpLine({ table: table.name, row: [] }).length - '[]}'.length
368
+ let after = table.name === start?.table ? start.after : null
369
+ for (;;) {
370
+ const bounds = rowBounds(storage, table, after)
371
+ let fitting = 0
372
+ let budget = OPS_DUMP_PART_MAX_RAW_BYTES - raw
373
+ for (const bound of bounds) {
374
+ if (rows + fitting >= OPS_DUMP_PART_MAX_ROWS || bound > budget) {
375
+ break
376
+ }
377
+ budget -= bound
378
+ fitting++
379
+ }
380
+ const page = bounds.length === 0 ? [] : readRows(storage, table, after, Math.min(fitting + 1, bounds.length))
381
+ for (const { rowid, values } of page) {
382
+ const line = encoder.encode(`${encodeOpsDumpLine({ table: table.name, row: values })}\n`)
383
+ if (rows + 1 > OPS_DUMP_PART_MAX_ROWS || raw + line.byteLength > OPS_DUMP_PART_MAX_RAW_BYTES) {
384
+ return rows === 0 ? { refused: 'row_too_large', table: table.name } : { items, breaker: table.name }
385
+ }
386
+ items.push({ kind: 'row', table: table.name, rowid, line, rowStart })
387
+ after = rowid
388
+ rows++
389
+ raw += line.byteLength
390
+ }
391
+ if (page.length === bounds.length && bounds.length < OPS_DUMP_PAGE_ROWS) {
392
+ items.push({ kind: 'end', table: table.name })
393
+ break
394
+ }
395
+ }
396
+ }
397
+ return { items, breaker: null }
398
+ }
399
+
400
+ /** The keyset condition: the rowid after the cursor's, written back as the integer it was read as. */
401
+ function keysetCondition(table: OpsDumpTablePlan, after: string | null): { readonly sql: string; readonly bindings: SqliteValue[] } {
402
+ if (after === null) {
403
+ return { sql: '', bindings: [] }
404
+ }
405
+ const binding = opsDumpValueBinding({ type: 'integer', value: after })
406
+ if (typeof binding.param !== 'string') {
407
+ throw new Error('a rowid binds as its decimal text')
408
+ }
409
+ return { sql: `WHERE ${quoted(table.keyset)} > ${binding.sql}`, bindings: [binding.param] }
410
+ }
411
+
412
+ /** For each of the next rows of a page, at most how many bytes its line takes. */
413
+ function rowBounds(storage: Storage, table: OpsDumpTablePlan, after: string | null): number[] {
414
+ const sizes = table.columns.map((column) => {
415
+ const c = quoted(column)
416
+ return `CASE typeof(${c}) WHEN 'text' THEN 2 * length(CAST(${c} AS BLOB)) WHEN 'blob' THEN 2 * length(${c}) ELSE ${SCALAR_BYTES} END`
417
+ })
418
+ const fixed = `{"table":"${table.name}","row":[]}\n`.length + (table.columns.length + 1) * VALUE_OVERHEAD + SCALAR_BYTES
419
+ const condition = keysetCondition(table, after)
420
+ const query = `SELECT ${[String(fixed), ...sizes].join(' + ')} AS bound FROM ${source(table)} ${condition.sql}
421
+ ORDER BY ${quoted(table.keyset)} LIMIT ${OPS_DUMP_PAGE_ROWS}`
422
+ return storage.sql.exec(query, ...condition.bindings).toArray().map((row) => {
423
+ const bound = row['bound']
424
+ if (typeof bound !== 'number') {
425
+ throw new Error("a row's bound reads as a number")
426
+ }
427
+ return bound
428
+ })
429
+ }
430
+
431
+ interface ReadValues {
432
+ readonly rowid: string
433
+ readonly values: readonly OpsDumpValue[]
434
+ }
435
+
436
+ /**
437
+ * The next `limit` rows: each its rowid as decimal text, then the rowid and every column through the wire's projection.
438
+ * The column-name list of the common table expression names every result column, which an expression alone does not.
439
+ */
440
+ function readRows(storage: Storage, table: OpsDumpTablePlan, after: string | null, limit: number): ReadValues[] {
441
+ const keyset = quoted(table.keyset)
442
+ const names = ['k', 'r', ...table.columns.flatMap((_, index) => [`t${index}`, `v${index}`])]
443
+ const projections = [keyset, `CAST(${keyset} AS TEXT)`, ...table.columns.map(opsDumpColumnProjection)]
444
+ const condition = keysetCondition(table, after)
445
+ const query = `WITH page(${names.join(', ')}) AS (SELECT ${projections.join(', ')} FROM ${source(table)} ${condition.sql}
446
+ ORDER BY ${keyset} LIMIT ${limit}) SELECT ${names.slice(1).join(', ')} FROM page ORDER BY k`
447
+ return storage.sql.exec(query, ...condition.bindings).toArray().map((row) => {
448
+ const rowid = readOpsDumpValue('integer', row['r'])
449
+ if (rowid === null || rowid.type !== 'integer') {
450
+ throw new Error('a rowid reads as an integer')
451
+ }
452
+ return { rowid: rowid.value, values: [rowid, ...table.columns.map((_, index) => readOpsDumpValue(row[`t${index}`], row[`v${index}`]))] }
453
+ })
454
+ }
455
+
456
+ /** The tables a part finished, with their counts and hashes, and the cursor where the next part starts. */
457
+ interface Settled {
458
+ readonly finished: readonly OpsDumpTable[]
459
+ readonly next: OpsDumpCursor | null
460
+ }
461
+
462
+ /** Folds the part's first `count` rows into their tables' hashes from `start`; only the rows the part keeps are hashed. */
463
+ function settle(start: OpsDumpCursor, collected: Collected, count: number): Settled {
464
+ const decoder = new TextDecoder()
465
+ const finished: OpsDumpTable[] = []
466
+ const at = (cursor: OpsDumpCursor, table: string) => cursor.table === table ? cursor : fresh(table)
467
+ let cursor = start
468
+ let kept = 0
469
+ for (const item of collected.items) {
470
+ if (item.kind === 'end') {
471
+ const { rows, hash } = at(cursor, item.table)
472
+ finished.push({ name: item.table, rows, hash })
473
+ } else if (kept === count) {
474
+ return { finished, next: at(cursor, item.table) }
475
+ } else {
476
+ const row = decoder.decode(item.line.subarray(item.rowStart, item.line.byteLength - '}\n'.length))
477
+ const before = at(cursor, item.table)
478
+ cursor = { table: item.table, after: item.rowid, rows: before.rows + 1, hash: foldOpsTableHash(before.hash, [row]) }
479
+ kept++
480
+ }
481
+ }
482
+ return { finished, next: collected.breaker === null ? null : at(cursor, collected.breaker) }
483
+ }
484
+
485
+ // Writing: one gzip member per part.
486
+
487
+ /** The first `rows` lines of a part and their deflate stream; `last` when nothing follows them. */
488
+ interface Filled {
489
+ readonly rows: number
490
+ readonly raw: Uint8Array
491
+ readonly deflate: Uint8Array
492
+ readonly last: boolean
493
+ }
494
+
495
+ /**
496
+ * The rows that fit one part compressed: all of them when they do, else the longest prefix that does. A row that does
497
+ * not fit alone is refused.
498
+ */
499
+ function fill(collected: Collected): Filled | OpsDataRefusal {
500
+ const lines = collected.items.flatMap((item) => item.kind === 'row' ? [item.line] : [])
501
+ const all = compressed(lines, lines.length, collected.breaker === null)
502
+ if (memberSize(all) <= OPS_DUMP_PART_SIZE) {
503
+ return all
504
+ }
505
+ // The compressed size grows with the prefix: the longest prefix that fits is found by interpolation and halving.
506
+ const target = OPS_DUMP_PART_SIZE - PADDED_OVERHEAD
507
+ let fits: Filled | null = null
508
+ let breaksRows = lines.length
509
+ let breaksSize = all.deflate.byteLength
510
+ let halve = false
511
+ while (breaksRows - (fits?.rows ?? 0) > 1) {
512
+ const fitsRows = fits?.rows ?? 0
513
+ const fitsSize = fits?.deflate.byteLength ?? 0
514
+ const width = breaksRows - fitsRows
515
+ const interpolated = fitsRows + Math.floor((width * (target - fitsSize)) / Math.max(1, breaksSize - fitsSize))
516
+ const guess = halve ? fitsRows + Math.floor(width / 2) : Math.min(breaksRows - 1, Math.max(fitsRows + 1, interpolated))
517
+ const candidate = compressed(lines, guess, false)
518
+ if (candidate.deflate.byteLength <= target) {
519
+ fits = candidate
520
+ } else {
521
+ breaksRows = guess
522
+ breaksSize = candidate.deflate.byteLength
523
+ }
524
+ halve = breaksRows - (fits?.rows ?? 0) > width / 2
525
+ }
526
+ if (fits === null) {
527
+ const first = collected.items.find((item) => item.kind === 'row')
528
+ return { refused: 'row_too_large', table: first?.table ?? null }
529
+ }
530
+ return fits
531
+ }
532
+
533
+ function compressed(lines: readonly Uint8Array[], rows: number, last: boolean): Filled {
534
+ const raw = new Uint8Array(lines.slice(0, rows).reduce((sum, line) => sum + line.byteLength, 0))
535
+ let offset = 0
536
+ for (const line of lines.slice(0, rows)) {
537
+ raw.set(line, offset)
538
+ offset += line.byteLength
539
+ }
540
+ return { rows, raw, deflate: deflateRawSync(raw), last }
541
+ }
542
+
543
+ /** The gzip header (RFC 1952) without a name or a time, and the trailer; a padded member adds its comment's terminator. */
544
+ const GZIP_HEADER_BYTES = 10
545
+ const GZIP_TRAILER_BYTES = 8
546
+ const PADDED_OVERHEAD = GZIP_HEADER_BYTES + 1 + GZIP_TRAILER_BYTES
547
+ const FCOMMENT = 0x10
548
+
549
+ function memberSize(part: Filled): number {
550
+ return (part.last ? GZIP_HEADER_BYTES + GZIP_TRAILER_BYTES : PADDED_OVERHEAD) + part.deflate.byteLength
551
+ }
552
+
553
+ /** One gzip member of the part; every part but the last is padded to exactly the part size with spaces in its comment. */
554
+ function gzipMember(part: Filled): Uint8Array<ArrayBuffer> {
555
+ const member = new Uint8Array(part.last ? memberSize(part) : OPS_DUMP_PART_SIZE)
556
+ member.set([0x1F, 0x8B, 0x08, part.last ? 0 : FCOMMENT, 0, 0, 0, 0, 0, 0xFF])
557
+ let offset = GZIP_HEADER_BYTES
558
+ if (!part.last) {
559
+ const comment = OPS_DUMP_PART_SIZE - memberSize(part)
560
+ member.fill(0x20, offset, offset + comment)
561
+ offset += comment + 1
562
+ }
563
+ member.set(part.deflate, offset)
564
+ offset += part.deflate.byteLength
565
+ const trailer = new DataView(member.buffer, offset, GZIP_TRAILER_BYTES)
566
+ trailer.setUint32(0, crc32(part.raw), true)
567
+ trailer.setUint32(4, part.raw.byteLength >>> 0, true)
568
+ return member
569
+ }
570
+
571
+ // The schema and the migration ledger: what `sqlite_schema` and the ledger's two tables say, for the manifest and for
572
+ // the state a consistent dump begins at. The wire's schema rule reads the importer's description of the PRAGMAs.
573
+
574
+ /** SQLite's own tables but `sqlite_sequence`, and Cloudflare's `_cf_` tables, are no table of the dump. */
575
+ function dumped(name: string): boolean {
576
+ return name === 'sqlite_sequence' || !(name.startsWith('sqlite_') || name.startsWith('_cf_'))
577
+ }
578
+
579
+ function rowsOf(storage: Storage, query: string, ...bindings: SqliteValue[]): Record<string, SqliteValue>[] {
580
+ return storage.sql.exec(query, ...bindings).toArray()
581
+ }
582
+
583
+ function text(value: SqliteValue | undefined): string {
584
+ if (typeof value !== 'string') {
585
+ throw new Error('the schema answered a value that is not text')
586
+ }
587
+ return value
588
+ }
589
+
590
+ const SCHEMA_TYPES: readonly OpsDumpSchemaEntry['type'][] = ['table', 'index', 'trigger', 'view']
591
+
592
+ /** Every `sqlite_schema` entry with SQL of a dumped table, ordered by type and name, which no history changes. */
593
+ function schemaEntries(storage: Storage): OpsDumpSchemaEntry[] {
594
+ const entries: OpsDumpSchemaEntry[] = []
595
+ for (const row of rowsOf(storage, 'SELECT type, name, tbl_name, sql FROM main.sqlite_schema WHERE sql IS NOT NULL')) {
596
+ const type = SCHEMA_TYPES.find((candidate) => candidate === row['type'])
597
+ const name = text(row['name'])
598
+ const table = text(row['tbl_name'])
599
+ if (type !== undefined && dumped(name) && dumped(table)) {
600
+ entries.push({ type, name, table, sql: text(row['sql']) })
601
+ }
602
+ }
603
+ const order = (entry: OpsDumpSchemaEntry) => `${entry.type}\n${entry.name}`
604
+ return entries.sort((left, right) => order(left) < order(right) ? -1 : order(left) > order(right) ? 1 : 0)
605
+ }
606
+
607
+ /** The ledger `@fabrika/db/sqlite-migrate` keeps for both bundles; none in a database restored to before it existed. */
608
+ function migrationLedger(storage: Storage): Record<string, SqliteValue>[] {
609
+ return holdsTable(storage, 'fabrika_migrations')
610
+ ? rowsOf(storage, 'SELECT bundle, name, checksum, applied_at AS appliedAt FROM main.fabrika_migrations ORDER BY bundle, name')
611
+ : []
612
+ }
613
+
614
+ /** The digest each bundle's last applied file recorded, which the runner may write without applying a file. */
615
+ function bundleDigests(storage: Storage): Record<string, SqliteValue>[] {
616
+ return holdsTable(storage, 'fabrika_migration_bundles')
617
+ ? rowsOf(storage, 'SELECT bundle, digest FROM main.fabrika_migration_bundles ORDER BY bundle')
618
+ : []
619
+ }
620
+
621
+ function holdsTable(storage: Storage, name: string): boolean {
622
+ return rowsOf(storage, `SELECT 1 FROM main.sqlite_schema WHERE type = 'table' AND name = ?`, name).length > 0
623
+ }
624
+
625
+ /**
626
+ * The digest of what a consistent dump's steps must all see: the stretch of modes that write nothing, the DDL and both
627
+ * ledger tables. Inside a stretch a migration is what writes unannounced, and the runner records each of its writes in
628
+ * one of them; an import step or a restore's `arm` starts a new stretch.
629
+ */
630
+ function dumpState(storage: Storage, frozenSince: number): string {
631
+ return sha256Hex(JSON.stringify([frozenSince, schemaEntries(storage), migrationLedger(storage), bundleDigests(storage)]))
632
+ }
633
+
634
+ /** A table as a query reads it: qualified, so no common table expression of the query shadows it. */
635
+ function source(table: OpsDumpTablePlan): string {
636
+ return `main.${quoted(table.name)}`
637
+ }
638
+
639
+ function quoted(name: string): string {
640
+ return `"${name.replaceAll('"', '""')}"`
641
+ }