@fabrika/app 0.9.0-cloudflare.1 → 0.9.0-cloudflare.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. package/package.json +9 -9
  2. package/skill/SKILL.md +21 -17
  3. package/skill/references/app-setup.md +42 -11
  4. package/skill/references/endpoint.md +29 -22
  5. package/skill/references/errors.md +32 -31
  6. package/skill/references/jobs.md +22 -3
  7. package/skill/references/tests.md +29 -1
  8. package/src/build.ts +6 -1
  9. package/src/bun.ts +168 -5
  10. package/src/cloudflare/application.ts +546 -76
  11. package/src/cloudflare/bindings.ts +26 -5
  12. package/src/cloudflare/export.ts +621 -0
  13. package/src/cloudflare/import.ts +1097 -0
  14. package/src/cloudflare/maintenance.ts +518 -0
  15. package/src/cloudflare/names.ts +6 -0
  16. package/src/cloudflare/ops.ts +147 -18
  17. package/src/cloudflare/restore.ts +167 -0
  18. package/src/cloudflare/tracker.ts +192 -0
  19. package/src/cloudflare.ts +3 -2
  20. package/src/contract/artifact-schema.ts +13 -2
  21. package/src/contract/build.ts +38 -11
  22. package/src/contract/context.ts +26 -9
  23. package/src/contract/defaults.ts +22 -2
  24. package/src/contract/define.ts +1 -1
  25. package/src/contract/group.ts +10 -1
  26. package/src/contract/kit.ts +2 -1
  27. package/src/disk-assets.ts +8 -3
  28. package/src/files/kinds.ts +54 -24
  29. package/src/implement/anonymous.ts +1 -10
  30. package/src/implement/boot.ts +30 -12
  31. package/src/implement/capability.ts +3 -0
  32. package/src/implement/dispatch.ts +175 -37
  33. package/src/implement/egress.ts +367 -8
  34. package/src/implement/invocation.ts +60 -19
  35. package/src/implement/workload.ts +18 -3
  36. package/src/jobs/runner.ts +646 -98
  37. package/src/spa-fallback.ts +12 -0
  38. package/src/testing/endpoint-access.ts +135 -23
  39. package/src/testing/isolation.ts +8 -1
  40. package/src/testing/kit.ts +60 -0
  41. package/types/src/build.d.ts +1 -1
  42. package/types/src/bun.d.ts +34 -1
  43. package/types/src/cloudflare/application.d.ts +36 -5
  44. package/types/src/cloudflare/bindings.d.ts +6 -0
  45. package/types/src/cloudflare/export.d.ts +16 -0
  46. package/types/src/cloudflare/import.d.ts +58 -0
  47. package/types/src/cloudflare/maintenance.d.ts +117 -0
  48. package/types/src/cloudflare/names.d.ts +5 -0
  49. package/types/src/cloudflare/ops.d.ts +39 -4
  50. package/types/src/cloudflare/restore.d.ts +9 -0
  51. package/types/src/cloudflare/tracker.d.ts +29 -0
  52. package/types/src/cloudflare.d.ts +2 -1
  53. package/types/src/contract/artifact-schema.d.ts +2 -0
  54. package/types/src/contract/context.d.ts +24 -9
  55. package/types/src/contract/defaults.d.ts +19 -1
  56. package/types/src/contract/define.d.ts +1 -1
  57. package/types/src/contract/group.d.ts +9 -1
  58. package/types/src/contract/kit.d.ts +2 -2
  59. package/types/src/disk-assets.d.ts +4 -1
  60. package/types/src/files/kinds.d.ts +17 -6
  61. package/types/src/implement/anonymous.d.ts +0 -4
  62. package/types/src/implement/boot.d.ts +17 -0
  63. package/types/src/implement/capability.d.ts +2 -0
  64. package/types/src/implement/dispatch.d.ts +33 -1
  65. package/types/src/implement/egress.d.ts +36 -1
  66. package/types/src/implement/invocation.d.ts +20 -10
  67. package/types/src/implement/workload.d.ts +5 -0
  68. package/types/src/jobs/runner.d.ts +63 -2
  69. package/types/src/spa-fallback.d.ts +2 -0
  70. package/types/src/testing/endpoint-access.d.ts +25 -1
  71. package/types/src/testing/isolation.d.ts +1 -1
  72. package/types/src/testing/kit.d.ts +16 -0
@@ -9,6 +9,7 @@ import type { IamKeySource } from '../implement/boot.js'
9
9
  import { CONNECT_BINDING_PREFIX, type ConnectFetch } from '../implement/connect.js'
10
10
  import type { SpaAssets } from '../implement/dispatch.js'
11
11
  import type { WorkloadTokenSource } from '../implement/workload.js'
12
+ import { isPageRoute } from '../spa-fallback.js'
12
13
 
13
14
  /** The method `name` of a binding, bound to it, or `null` when it has none. */
14
15
  export function method(target: unknown, name: string): ((...args: unknown[]) => unknown) | null {
@@ -96,18 +97,38 @@ export function bindingWorkloadTokens(binding: unknown): WorkloadTokenSource | u
96
97
  }
97
98
  }
98
99
 
100
+ /**
101
+ * The SPA and link mounts' files from the Workers Assets binding, whose directory mirrors the request paths: the mount
102
+ * `/admin` reads `admin/…` of it, the root mount the rest (the Cloudflare deploy stages each mount's `dir` there). A page
103
+ * route that names no file (`isPageRoute`) gets the mount's `index.html`, as `diskAssets` serves it: Workers Assets
104
+ * answers `<prefix>/` with it. A missing asset stays 404.
105
+ */
99
106
  export function bindingAssets(binding: unknown): SpaAssets | undefined {
100
107
  const fetchAsset = method(binding, 'fetch')
101
108
  if (fetchAsset === null) {
102
109
  return undefined
103
110
  }
111
+ const fetched = async (request: Request): Promise<Response> => {
112
+ const response: unknown = await fetchAsset(request)
113
+ if (!(response instanceof Response)) {
114
+ throw new Error('the ASSETS binding answered something that is not a Response')
115
+ }
116
+ return response
117
+ }
104
118
  return {
105
- async serve(_mount, request) {
106
- const response: unknown = await fetchAsset(request)
107
- if (!(response instanceof Response)) {
108
- throw new Error('the ASSETS binding answered something that is not a Response')
119
+ async serve(mount, request) {
120
+ const response = await fetched(request)
121
+ if (response.status !== 404 || !isPageRoute(request)) {
122
+ return response
123
+ }
124
+ const index = new URL(request.url)
125
+ index.pathname = mount.prefix === '/' ? '/' : `${mount.prefix}/`
126
+ index.search = ''
127
+ if (index.pathname === new URL(request.url).pathname) {
128
+ return response
109
129
  }
110
- return response
130
+ await response.body?.cancel()
131
+ return await fetched(new Request(index, { method: request.method, headers: request.headers }))
111
132
  },
112
133
  }
113
134
  }
@@ -0,0 +1,621 @@
1
+ // One step of a dump of the object's database (ADR-0070 §2): the ops operation `export`, served before the application
2
+ // boots. It reads `sqlite_schema` and the tables directly, so it also dumps a database that cannot boot. Control's
3
+ // operation record holds the upload id, the cursor and the parts; the object keeps only a consistent dump's begun state.
4
+
5
+ import {
6
+ encodeOpsDumpLine,
7
+ foldOpsTableHash,
8
+ OPS_DATA_REFUSED_STATUS,
9
+ OPS_DUMP_FORMAT,
10
+ OPS_DUMP_KEY,
11
+ OPS_DUMP_PAGE_ROWS,
12
+ OPS_DUMP_PART_MAX_RAW_BYTES,
13
+ OPS_DUMP_PART_MAX_ROWS,
14
+ OPS_DUMP_PART_SIZE,
15
+ OPS_EMPTY_TABLE_HASH,
16
+ OPS_STALE_STATUS,
17
+ type OpsDataRefusal,
18
+ type OpsDumpCursor,
19
+ type OpsDumpPageRow,
20
+ type OpsDumpSchemaEntry,
21
+ type OpsDumpTable,
22
+ type OpsDumpTablePlan,
23
+ opsDumpValueBinding,
24
+ type OpsExportBegin,
25
+ type OpsExportComplete,
26
+ type OpsExportPart,
27
+ type OpsExportRequest,
28
+ type OpsExportScope,
29
+ type OpsPublishedState,
30
+ type OpsStaleRefusal,
31
+ parseOpsExportBeginAck,
32
+ parseOpsExportCompleteAck,
33
+ parseOpsExportPartAck,
34
+ planOpsDump,
35
+ readOpsDumpPage,
36
+ sha256Hex,
37
+ } from '@fabrika/auth-core'
38
+ import type { SqliteValue } from '@fabrika/db/sqlite'
39
+ import { crc32, deflateRawSync } from 'node:zlib'
40
+ import appPackage from '../../package.json' with { type: 'json' }
41
+ import { jsonResponse } from '../errors.js'
42
+ import { method } from './bindings.js'
43
+ import { describeDumpSchema, importInProgress } from './import.js'
44
+ import type { MaintenanceBarrier } from './maintenance.js'
45
+ import type { OpsTarget } from './ops.js'
46
+
47
+ /** One step of a dump of the object's database (ADR-0070). */
48
+ export async function serveExport(target: OpsTarget, barrier: MaintenanceBarrier, step: OpsExportRequest): Promise<Response> {
49
+ return await exportStep(target, barrier, step)
50
+ }
51
+
52
+ /** What an export step reads: the object's storage, bucket and bookmarks, and the digest of the contract it was deployed with. */
53
+ export interface ExportSource extends Pick<OpsTarget, 'storage' | 'bucket' | 'recovery'> {
54
+ /** `FABRIKA_CONTRACT_DIGEST`, which the manifest records. */
55
+ readonly contractDigest: string | null
56
+ }
57
+
58
+ type Storage = ExportSource['storage']
59
+
60
+ /**
61
+ * One step of a dump. A diagnostic step runs in any mode. A consistent one runs in the barrier's turn, only at the
62
+ * published generation, never while the object is `open` or holds an import marker, so no transition and no migration
63
+ * falls inside it; past `begin` it refuses a database whose state or stretch is not the one its `begin` saw.
64
+ */
65
+ export async function exportStep(source: ExportSource, barrier: MaintenanceBarrier, step: OpsExportRequest): Promise<Response> {
66
+ if (!step.consistent) {
67
+ return await run(source, step)
68
+ }
69
+ const { generation } = step
70
+ if (generation === null) {
71
+ throw new Error('a consistent export step carries its generation; the wire parser requires it')
72
+ }
73
+ const result = await barrier.step(generation, async (mode) => {
74
+ const { frozenSince } = barrier
75
+ if (mode === 'open' || frozenSince === null) {
76
+ return { value: dataRefusal({ refused: 'wrong_mode', table: null }), holdUntil: null }
77
+ }
78
+ return { value: await runConsistent(source, step, frozenSince), holdUntil: null }
79
+ })
80
+ return result.kind === 'ran' ? result.value : staleRefusal(result.published)
81
+ }
82
+
83
+ /** Runs `step`; `state` is what a consistent dump's first `begin` records, `null` for a diagnostic one. */
84
+ async function run(source: ExportSource, step: OpsExportRequest, state: string | null = null): Promise<Response> {
85
+ switch (step.step) {
86
+ case 'begin':
87
+ return await begin(source, step, state)
88
+ case 'part':
89
+ return await part(source, step)
90
+ case 'complete':
91
+ return await complete(source, step)
92
+ }
93
+ }
94
+
95
+ /**
96
+ * The first `begin` of a dump records the state it saw with its upload and bookmark, once; a retried one replays that
97
+ * record, and every later step must carry its upload at that state, in the same stretch of modes that write nothing
98
+ * (`frozenSince`), at whatever generation of it. A migration runs inside a stretch, so the stretch alone would not show
99
+ * one between two steps. A storage without the synchronous KV holds no import.
100
+ */
101
+ async function runConsistent(source: ExportSource, step: OpsExportRequest, frozenSince: number): Promise<Response> {
102
+ const { kv } = source.storage
103
+ if (kv !== undefined && importInProgress({ kv })) {
104
+ return dataRefusal({ refused: 'wrong_mode', table: null })
105
+ }
106
+ const state = dumpState(source.storage, frozenSince)
107
+ const begun = await readBegun(source.bucket, step)
108
+ if (step.step === 'begin') {
109
+ return begun === null ? await run(source, step, state) : replayed(begun, state)
110
+ }
111
+ if (begun?.state !== state || begun.uploadId !== step.uploadId || (step.step === 'complete' && step.bookmark !== begun.bookmark)) {
112
+ return dataRefusal({ refused: 'ledger_mismatch', table: null })
113
+ }
114
+ return await run(source, step)
115
+ }
116
+
117
+ /** A retried `begin`: the first one's answer at the state it recorded, refused at any other. */
118
+ function replayed(begun: Begun, state: string): Response {
119
+ if (begun.state !== state) {
120
+ return dataRefusal({ refused: 'ledger_mismatch', table: null })
121
+ }
122
+ return answer(parseOpsExportBeginAck({ step: 'begin', uploadId: begun.uploadId, bookmark: begun.bookmark }), 'begin')
123
+ }
124
+
125
+ function staleRefusal(published: OpsPublishedState | null): Response {
126
+ const stale: OpsStaleRefusal = { refused: 'stale_generation', published }
127
+ return jsonResponse(stale, OPS_STALE_STATUS)
128
+ }
129
+
130
+ function dataRefusal(refusal: OpsDataRefusal): Response {
131
+ return jsonResponse(refusal, OPS_DATA_REFUSED_STATUS)
132
+ }
133
+
134
+ /**
135
+ * Plans the dump, starts the multipart upload of its data and answers the object's bookmark. A consistent dump records
136
+ * its `state` with both, only if nothing is recorded yet.
137
+ */
138
+ async function begin(source: ExportSource, step: OpsExportBegin, state: string | null): Promise<Response> {
139
+ const plan = planOpsDump(describeDumpSchema(source.storage))
140
+ if ('refused' in plan) {
141
+ return dataRefusal(plan)
142
+ }
143
+ if (source.recovery === null) {
144
+ throw new Error("the object's storage has no bookmarks; a Durable Object's storage has them")
145
+ }
146
+ const bookmark = await source.recovery.getCurrentBookmark()
147
+ const upload: unknown = await call(source.bucket, 'createMultipartUpload')(dataKey(step), { httpMetadata: { contentType: 'application/gzip' } })
148
+ const ack = parseOpsExportBeginAck({
149
+ step: 'begin',
150
+ uploadId: typeof upload === 'object' && upload !== null ? Reflect.get(upload, 'uploadId') : undefined,
151
+ bookmark,
152
+ })
153
+ if (ack === null || state === null) {
154
+ return answer(ack, 'begin')
155
+ }
156
+ const begun: Begun = { state, uploadId: ack.uploadId, bookmark: ack.bookmark }
157
+ const options = { httpMetadata: { contentType: 'application/json' }, onlyIf: { etagDoesNotMatch: '*' } }
158
+ if ((await call(source.bucket, 'put')(begunKey(step), JSON.stringify(begun), options)) === null) {
159
+ // Another begin recorded the dump first; its upload is the dump's, and this one is left to the lifecycle.
160
+ const recorded = await readBegun(source.bucket, step)
161
+ if (recorded === null) {
162
+ throw new Error("R2 refused to record a dump's begin, yet holds no record of it")
163
+ }
164
+ return replayed(recorded, state)
165
+ }
166
+ return answer(ack, 'begin')
167
+ }
168
+
169
+ /** Fills part `n` from its cursor: whole rows while every bound holds, one gzip member padded to the fixed size but the last. */
170
+ async function part(source: ExportSource, step: OpsExportPart): Promise<Response> {
171
+ const plan = planOpsDump(describeDumpSchema(source.storage))
172
+ if ('refused' in plan) {
173
+ return dataRefusal(plan)
174
+ }
175
+ const [first] = plan.tables
176
+ const start = step.cursor ?? (first === undefined ? null : fresh(first.name))
177
+ const index = start === null ? plan.tables.length : plan.tables.findIndex((table) => table.name === start.table)
178
+ if (start !== null && index < 0) {
179
+ return dataRefusal({ refused: 'schema_mismatch', table: start.table })
180
+ }
181
+ const collected = collect(source.storage, plan.tables.slice(index), start)
182
+ if ('refused' in collected) {
183
+ return dataRefusal(collected)
184
+ }
185
+ const filled = fill(collected)
186
+ if ('refused' in filled) {
187
+ return dataRefusal(filled)
188
+ }
189
+ return await uploadPart(source, step, filled, start === null ? { finished: [], next: null } : settle(start, collected, filled.rows))
190
+ }
191
+
192
+ /** Completes the upload with the parts Control recorded, writes the manifest beside the data and answers it. */
193
+ async function complete(source: ExportSource, step: OpsExportComplete): Promise<Response> {
194
+ const { storage } = source
195
+ const plan = planOpsDump(describeDumpSchema(storage))
196
+ if ('refused' in plan) {
197
+ return dataRefusal(plan)
198
+ }
199
+ const mismatch = plan.tables.find((table, index) => step.tables[index]?.name !== table.name)
200
+ if (mismatch !== undefined || step.tables.length !== plan.tables.length) {
201
+ return dataRefusal({ refused: 'schema_mismatch', table: mismatch?.name ?? null })
202
+ }
203
+ const ddl = schemaEntries(storage)
204
+ const migrations = migrationLedger(storage)
205
+ const key = dataKey(step)
206
+ const upload: unknown = await call(source.bucket, 'resumeMultipartUpload')(key, step.uploadId)
207
+ try {
208
+ await call(upload, 'complete')(step.parts.map((uploaded) => ({ partNumber: uploaded.part, etag: uploaded.etag })))
209
+ } catch (error) {
210
+ // A retried complete whose first answer was lost finds the upload gone and the data in place.
211
+ const head: unknown = await call(source.bucket, 'head')(key)
212
+ const size = typeof head === 'object' && head !== null ? Reflect.get(head, 'size') : undefined
213
+ if (size !== step.parts.reduce((sum, uploaded) => sum + uploaded.size, 0)) {
214
+ throw error
215
+ }
216
+ }
217
+ const manifest = {
218
+ format: OPS_DUMP_FORMAT,
219
+ dump: step.dump,
220
+ object: step.object,
221
+ consistent: step.consistent,
222
+ createdAt: new Date().toISOString(),
223
+ bookmark: step.bookmark,
224
+ release: appPackage.version,
225
+ contractDigest: source.contractDigest,
226
+ ddl,
227
+ migrations,
228
+ tables: step.tables,
229
+ parts: step.parts.map(({ part, size, sha256 }) => ({ part, size, sha256 })),
230
+ }
231
+ const ack = parseOpsExportCompleteAck({ step: 'complete', manifest })
232
+ if (ack === null) {
233
+ throw new Error("the dump's manifest does not parse as the wire's manifest")
234
+ }
235
+ await call(source.bucket, 'put')(OPS_DUMP_KEY(step, 'manifest'), JSON.stringify(ack.manifest), { httpMetadata: { contentType: 'application/json' } })
236
+ return jsonResponse(ack, 200)
237
+ }
238
+
239
+ async function uploadPart(source: ExportSource, step: OpsExportPart, part: Filled, settled: Settled): Promise<Response> {
240
+ const member = gzipMember(part)
241
+ const digest = new Uint8Array(await crypto.subtle.digest('SHA-256', member))
242
+ const sha256 = Array.from(digest, (byte) => byte.toString(16).padStart(2, '0')).join('')
243
+ const upload: unknown = await call(source.bucket, 'resumeMultipartUpload')(dataKey(step), step.uploadId)
244
+ // R2 keeps one part per number, so a retried step replaces the part it uploaded before.
245
+ const uploaded: unknown = await call(upload, 'uploadPart')(step.part, member)
246
+ const etag = typeof uploaded === 'object' && uploaded !== null ? Reflect.get(uploaded, 'etag') : undefined
247
+ return answer(
248
+ parseOpsExportPartAck({
249
+ step: 'part',
250
+ uploaded: { part: step.part, size: member.byteLength, sha256, etag },
251
+ tables: settled.finished,
252
+ next: settled.next,
253
+ }),
254
+ 'part',
255
+ )
256
+ }
257
+
258
+ function dataKey(scope: OpsExportScope): string {
259
+ return OPS_DUMP_KEY(scope, 'data')
260
+ }
261
+
262
+ /** What a consistent dump's first `begin` saw and answered: the digest of the database's state, its upload and bookmark. */
263
+ interface Begun {
264
+ readonly state: string
265
+ readonly uploadId: string
266
+ readonly bookmark: string
267
+ }
268
+
269
+ /** `begun` in the dump's own prefix, which the bucket's lifecycle expires with the dump. */
270
+ function begunKey(scope: OpsExportScope): string {
271
+ const manifest = OPS_DUMP_KEY(scope, 'manifest')
272
+ return `${manifest.slice(0, manifest.lastIndexOf('/'))}/begun`
273
+ }
274
+
275
+ /** What the dump's first `begin` recorded, `null` when nothing is recorded; a record that does not parse fails the step. */
276
+ async function readBegun(bucket: unknown, scope: OpsExportScope): Promise<Begun | null> {
277
+ const stored: unknown = await call(bucket, 'get')(begunKey(scope))
278
+ if (stored === null) {
279
+ return null
280
+ }
281
+ const body: unknown = await call(stored, 'text')()
282
+ const begun = typeof body === 'string' ? parseBegun(body) : null
283
+ if (begun === null) {
284
+ throw new Error("the dump's begin record does not parse")
285
+ }
286
+ return begun
287
+ }
288
+
289
+ function parseBegun(text: string): Begun | null {
290
+ let value: unknown
291
+ try {
292
+ value = JSON.parse(text)
293
+ } catch {
294
+ return null
295
+ }
296
+ if (typeof value !== 'object' || value === null || Array.isArray(value) || Object.keys(value).length !== 3) {
297
+ return null
298
+ }
299
+ const state: unknown = Reflect.get(value, 'state')
300
+ const uploadId: unknown = Reflect.get(value, 'uploadId')
301
+ const bookmark: unknown = Reflect.get(value, 'bookmark')
302
+ return typeof state === 'string' && typeof uploadId === 'string' && typeof bookmark === 'string' ? { state, uploadId, bookmark } : null
303
+ }
304
+
305
+ function answer<T>(ack: T | null, step: string): Response {
306
+ if (ack === null) {
307
+ throw new Error(`the export ${step} answer does not parse as the wire's answer`)
308
+ }
309
+ return jsonResponse(ack, 200)
310
+ }
311
+
312
+ function call(target: unknown, name: string): (...args: unknown[]) => unknown {
313
+ const found = method(target, name)
314
+ if (found === null) {
315
+ throw new Error(`the FABRIKA_FILES binding has no ${name}; dumps live in the environment's bucket`)
316
+ }
317
+ return found
318
+ }
319
+
320
+ // Reading: keyset pages of whole rows within the part's bounds.
321
+
322
+ function fresh(table: string): OpsDumpCursor {
323
+ return { table, after: null, rows: 0, hash: OPS_EMPTY_TABLE_HASH }
324
+ }
325
+
326
+ /** A row read for the part: its table, its rowid, and its line with the newline, where its canonical encoding starts at `rowStart`. */
327
+ interface ReadRow {
328
+ readonly kind: 'row'
329
+ readonly table: string
330
+ readonly rowid: string
331
+ readonly line: Uint8Array
332
+ readonly rowStart: number
333
+ }
334
+
335
+ /** A table read to its end. */
336
+ interface TableEnd {
337
+ readonly kind: 'end'
338
+ readonly table: string
339
+ }
340
+
341
+ interface Collected {
342
+ /** Rows and table ends in reading order: a table's end follows its last row. */
343
+ readonly items: readonly (ReadRow | TableEnd)[]
344
+ /** The table of the row the part could not take, `null` once every table was read. */
345
+ readonly breaker: string | null
346
+ }
347
+
348
+ /** `JSON.stringify` adds at most this to a value's text: the quotes, the tag and the separator. */
349
+ const VALUE_OVERHEAD = 8
350
+ /** A value that is no text or blob is at most this long: `-1.7976931348623157e+308`, `-9223372036854775808`, `null`. */
351
+ const SCALAR_BYTES = 32
352
+
353
+ /**
354
+ * Reads whole rows of `tables`, from the cursor's position in the first, while the part holds at most
355
+ * `OPS_DUMP_PART_MAX_ROWS` rows and its lines at most `OPS_DUMP_PART_MAX_RAW_BYTES`. A page first asks how long each
356
+ * row's line is at most, then reads the rows that fit and the one after them, so a page of large rows never sits in
357
+ * memory whole.
358
+ */
359
+ function collect(storage: Storage, tables: readonly OpsDumpTablePlan[], start: OpsDumpCursor | null): Collected | OpsDataRefusal {
360
+ const items: (ReadRow | TableEnd)[] = []
361
+ const encoder = new TextEncoder()
362
+ let rows = 0
363
+ let raw = 0
364
+ for (const table of tables) {
365
+ // A line is `{"table":"<name>","row":<row>}` (ADR-0070 §3), all ASCII: its row starts where an empty row's `[]}` does.
366
+ const rowStart = encodeOpsDumpLine({ table: table.name, row: [] }).length - '[]}'.length
367
+ let after = table.name === start?.table ? start.after : null
368
+ for (;;) {
369
+ const bounds = rowBounds(storage, table, after)
370
+ let fitting = 0
371
+ let budget = OPS_DUMP_PART_MAX_RAW_BYTES - raw
372
+ for (const bound of bounds) {
373
+ if (rows + fitting >= OPS_DUMP_PART_MAX_ROWS || bound > budget) {
374
+ break
375
+ }
376
+ budget -= bound
377
+ fitting++
378
+ }
379
+ const page = bounds.length === 0 ? [] : readRows(storage, table, after, Math.min(fitting + 1, bounds.length))
380
+ for (const { rowid, values } of page) {
381
+ const line = encoder.encode(`${encodeOpsDumpLine({ table: table.name, row: values })}\n`)
382
+ if (rows + 1 > OPS_DUMP_PART_MAX_ROWS || raw + line.byteLength > OPS_DUMP_PART_MAX_RAW_BYTES) {
383
+ return rows === 0 ? { refused: 'row_too_large', table: table.name } : { items, breaker: table.name }
384
+ }
385
+ items.push({ kind: 'row', table: table.name, rowid, line, rowStart })
386
+ after = rowid
387
+ rows++
388
+ raw += line.byteLength
389
+ }
390
+ if (page.length === bounds.length && bounds.length < OPS_DUMP_PAGE_ROWS) {
391
+ items.push({ kind: 'end', table: table.name })
392
+ break
393
+ }
394
+ }
395
+ }
396
+ return { items, breaker: null }
397
+ }
398
+
399
+ /** The keyset condition: the rowid after the cursor's, written back as the integer it was read as. */
400
+ function keysetCondition(table: OpsDumpTablePlan, after: string | null): { readonly sql: string; readonly bindings: SqliteValue[] } {
401
+ if (after === null) {
402
+ return { sql: '', bindings: [] }
403
+ }
404
+ const binding = opsDumpValueBinding({ type: 'integer', value: after })
405
+ if (typeof binding.param !== 'string') {
406
+ throw new Error('a rowid binds as its decimal text')
407
+ }
408
+ return { sql: `WHERE ${quoted(table.keyset)} > ${binding.sql}`, bindings: [binding.param] }
409
+ }
410
+
411
+ /** For each of the next rows of a page, at most how many bytes its line takes. */
412
+ function rowBounds(storage: Storage, table: OpsDumpTablePlan, after: string | null): number[] {
413
+ const sizes = table.columns.map((column) => {
414
+ const c = quoted(column)
415
+ return `CASE typeof(${c}) WHEN 'text' THEN 2 * length(CAST(${c} AS BLOB)) WHEN 'blob' THEN 2 * length(${c}) ELSE ${SCALAR_BYTES} END`
416
+ })
417
+ const fixed = `{"table":"${table.name}","row":[]}\n`.length + (table.columns.length + 1) * VALUE_OVERHEAD + SCALAR_BYTES
418
+ const condition = keysetCondition(table, after)
419
+ const query = `SELECT ${[String(fixed), ...sizes].join(' + ')} AS bound FROM ${source(table)} ${condition.sql}
420
+ ORDER BY ${quoted(table.keyset)} LIMIT ${OPS_DUMP_PAGE_ROWS}`
421
+ return storage.sql.exec(query, ...condition.bindings).toArray().map((row) => {
422
+ const bound = row['bound']
423
+ if (typeof bound !== 'number') {
424
+ throw new Error("a row's bound reads as a number")
425
+ }
426
+ return bound
427
+ })
428
+ }
429
+
430
+ /** The next `limit` rows: each its rowid as decimal text, then the rowid and every column through the wire's projection. */
431
+ function readRows(storage: Storage, table: OpsDumpTablePlan, after: string | null, limit: number): OpsDumpPageRow[] {
432
+ const condition = keysetCondition(table, after)
433
+ return readOpsDumpPage((query) => storage.sql.exec(query, ...condition.bindings).toArray(), table, condition.sql, limit)
434
+ }
435
+
436
+ /** The tables a part finished, with their counts and hashes, and the cursor where the next part starts. */
437
+ interface Settled {
438
+ readonly finished: readonly OpsDumpTable[]
439
+ readonly next: OpsDumpCursor | null
440
+ }
441
+
442
+ /** Folds the part's first `count` rows into their tables' hashes from `start`; only the rows the part keeps are hashed. */
443
+ function settle(start: OpsDumpCursor, collected: Collected, count: number): Settled {
444
+ const decoder = new TextDecoder()
445
+ const finished: OpsDumpTable[] = []
446
+ const at = (cursor: OpsDumpCursor, table: string) => cursor.table === table ? cursor : fresh(table)
447
+ let cursor = start
448
+ let kept = 0
449
+ for (const item of collected.items) {
450
+ if (item.kind === 'end') {
451
+ const { rows, hash } = at(cursor, item.table)
452
+ finished.push({ name: item.table, rows, hash })
453
+ } else if (kept === count) {
454
+ return { finished, next: at(cursor, item.table) }
455
+ } else {
456
+ const row = decoder.decode(item.line.subarray(item.rowStart, item.line.byteLength - '}\n'.length))
457
+ const before = at(cursor, item.table)
458
+ cursor = { table: item.table, after: item.rowid, rows: before.rows + 1, hash: foldOpsTableHash(before.hash, [row]) }
459
+ kept++
460
+ }
461
+ }
462
+ return { finished, next: collected.breaker === null ? null : at(cursor, collected.breaker) }
463
+ }
464
+
465
+ // Writing: one gzip member per part.
466
+
467
+ /** The first `rows` lines of a part and their deflate stream; `last` when nothing follows them. */
468
+ interface Filled {
469
+ readonly rows: number
470
+ readonly raw: Uint8Array
471
+ readonly deflate: Uint8Array
472
+ readonly last: boolean
473
+ }
474
+
475
+ /**
476
+ * The rows that fit one part compressed: all of them when they do, else the longest prefix that does. A row that does
477
+ * not fit alone is refused.
478
+ */
479
+ function fill(collected: Collected): Filled | OpsDataRefusal {
480
+ const lines = collected.items.flatMap((item) => item.kind === 'row' ? [item.line] : [])
481
+ const all = compressed(lines, lines.length, collected.breaker === null)
482
+ if (memberSize(all) <= OPS_DUMP_PART_SIZE) {
483
+ return all
484
+ }
485
+ // The compressed size grows with the prefix: the longest prefix that fits is found by interpolation and halving.
486
+ const target = OPS_DUMP_PART_SIZE - PADDED_OVERHEAD
487
+ let fits: Filled | null = null
488
+ let breaksRows = lines.length
489
+ let breaksSize = all.deflate.byteLength
490
+ let halve = false
491
+ while (breaksRows - (fits?.rows ?? 0) > 1) {
492
+ const fitsRows = fits?.rows ?? 0
493
+ const fitsSize = fits?.deflate.byteLength ?? 0
494
+ const width = breaksRows - fitsRows
495
+ const interpolated = fitsRows + Math.floor((width * (target - fitsSize)) / Math.max(1, breaksSize - fitsSize))
496
+ const guess = halve ? fitsRows + Math.floor(width / 2) : Math.min(breaksRows - 1, Math.max(fitsRows + 1, interpolated))
497
+ const candidate = compressed(lines, guess, false)
498
+ if (candidate.deflate.byteLength <= target) {
499
+ fits = candidate
500
+ } else {
501
+ breaksRows = guess
502
+ breaksSize = candidate.deflate.byteLength
503
+ }
504
+ halve = breaksRows - (fits?.rows ?? 0) > width / 2
505
+ }
506
+ if (fits === null) {
507
+ const first = collected.items.find((item) => item.kind === 'row')
508
+ return { refused: 'row_too_large', table: first?.table ?? null }
509
+ }
510
+ return fits
511
+ }
512
+
513
+ function compressed(lines: readonly Uint8Array[], rows: number, last: boolean): Filled {
514
+ const raw = new Uint8Array(lines.slice(0, rows).reduce((sum, line) => sum + line.byteLength, 0))
515
+ let offset = 0
516
+ for (const line of lines.slice(0, rows)) {
517
+ raw.set(line, offset)
518
+ offset += line.byteLength
519
+ }
520
+ return { rows, raw, deflate: deflateRawSync(raw), last }
521
+ }
522
+
523
+ /** The gzip header (RFC 1952) without a name or a time, and the trailer; a padded member adds its comment's terminator. */
524
+ const GZIP_HEADER_BYTES = 10
525
+ const GZIP_TRAILER_BYTES = 8
526
+ const PADDED_OVERHEAD = GZIP_HEADER_BYTES + 1 + GZIP_TRAILER_BYTES
527
+ const FCOMMENT = 0x10
528
+
529
+ function memberSize(part: Filled): number {
530
+ return (part.last ? GZIP_HEADER_BYTES + GZIP_TRAILER_BYTES : PADDED_OVERHEAD) + part.deflate.byteLength
531
+ }
532
+
533
+ /** One gzip member of the part; every part but the last is padded to exactly the part size with spaces in its comment. */
534
+ function gzipMember(part: Filled): Uint8Array<ArrayBuffer> {
535
+ const member = new Uint8Array(part.last ? memberSize(part) : OPS_DUMP_PART_SIZE)
536
+ member.set([0x1F, 0x8B, 0x08, part.last ? 0 : FCOMMENT, 0, 0, 0, 0, 0, 0xFF])
537
+ let offset = GZIP_HEADER_BYTES
538
+ if (!part.last) {
539
+ const comment = OPS_DUMP_PART_SIZE - memberSize(part)
540
+ member.fill(0x20, offset, offset + comment)
541
+ offset += comment + 1
542
+ }
543
+ member.set(part.deflate, offset)
544
+ offset += part.deflate.byteLength
545
+ const trailer = new DataView(member.buffer, offset, GZIP_TRAILER_BYTES)
546
+ trailer.setUint32(0, crc32(part.raw), true)
547
+ trailer.setUint32(4, part.raw.byteLength >>> 0, true)
548
+ return member
549
+ }
550
+
551
+ // The schema and the migration ledger: what `sqlite_schema` and the ledger's two tables say, for the manifest and for
552
+ // the state a consistent dump begins at. The wire's schema rule reads the importer's description of the PRAGMAs.
553
+
554
+ /** SQLite's own tables but `sqlite_sequence`, and Cloudflare's `_cf_` tables, are no table of the dump. */
555
+ function dumped(name: string): boolean {
556
+ return name === 'sqlite_sequence' || !(name.startsWith('sqlite_') || name.startsWith('_cf_'))
557
+ }
558
+
559
+ function rowsOf(storage: Storage, query: string, ...bindings: SqliteValue[]): Record<string, SqliteValue>[] {
560
+ return storage.sql.exec(query, ...bindings).toArray()
561
+ }
562
+
563
+ function text(value: SqliteValue | undefined): string {
564
+ if (typeof value !== 'string') {
565
+ throw new Error('the schema answered a value that is not text')
566
+ }
567
+ return value
568
+ }
569
+
570
+ const SCHEMA_TYPES: readonly OpsDumpSchemaEntry['type'][] = ['table', 'index', 'trigger', 'view']
571
+
572
+ /** Every `sqlite_schema` entry with SQL of a dumped table, ordered by type and name, which no history changes. */
573
+ function schemaEntries(storage: Storage): OpsDumpSchemaEntry[] {
574
+ const entries: OpsDumpSchemaEntry[] = []
575
+ for (const row of rowsOf(storage, 'SELECT type, name, tbl_name, sql FROM main.sqlite_schema WHERE sql IS NOT NULL')) {
576
+ const type = SCHEMA_TYPES.find((candidate) => candidate === row['type'])
577
+ const name = text(row['name'])
578
+ const table = text(row['tbl_name'])
579
+ if (type !== undefined && dumped(name) && dumped(table)) {
580
+ entries.push({ type, name, table, sql: text(row['sql']) })
581
+ }
582
+ }
583
+ const order = (entry: OpsDumpSchemaEntry) => `${entry.type}\n${entry.name}`
584
+ return entries.sort((left, right) => order(left) < order(right) ? -1 : order(left) > order(right) ? 1 : 0)
585
+ }
586
+
587
+ /** The ledger `@fabrika/db/sqlite-migrate` keeps for both bundles; none in a database restored to before it existed. */
588
+ function migrationLedger(storage: Storage): Record<string, SqliteValue>[] {
589
+ return holdsTable(storage, 'fabrika_migrations')
590
+ ? rowsOf(storage, 'SELECT bundle, name, checksum, applied_at AS appliedAt FROM main.fabrika_migrations ORDER BY bundle, name')
591
+ : []
592
+ }
593
+
594
+ /** The digest each bundle's last applied file recorded, which the runner may write without applying a file. */
595
+ function bundleDigests(storage: Storage): Record<string, SqliteValue>[] {
596
+ return holdsTable(storage, 'fabrika_migration_bundles')
597
+ ? rowsOf(storage, 'SELECT bundle, digest FROM main.fabrika_migration_bundles ORDER BY bundle')
598
+ : []
599
+ }
600
+
601
+ function holdsTable(storage: Storage, name: string): boolean {
602
+ return rowsOf(storage, `SELECT 1 FROM main.sqlite_schema WHERE type = 'table' AND name = ?`, name).length > 0
603
+ }
604
+
605
+ /**
606
+ * The digest of what a consistent dump's steps must all see: the stretch of modes that write nothing, the DDL and both
607
+ * ledger tables. Inside a stretch a migration is what writes unannounced, and the runner records each of its writes in
608
+ * one of them; an import step or a restore's `arm` starts a new stretch.
609
+ */
610
+ function dumpState(storage: Storage, frozenSince: number): string {
611
+ return sha256Hex(JSON.stringify([frozenSince, schemaEntries(storage), migrationLedger(storage), bundleDigests(storage)]))
612
+ }
613
+
614
+ /** A table as a query reads it: qualified, so no common table expression of the query shadows it. */
615
+ function source(table: OpsDumpTablePlan): string {
616
+ return `main.${quoted(table.name)}`
617
+ }
618
+
619
+ function quoted(name: string): string {
620
+ return `"${name.replaceAll('"', '""')}"`
621
+ }