@fabrika/app 0.9.0-dev.1 → 0.9.0-dev.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (222) hide show
  1. package/package.json +32 -26
  2. package/skill/SKILL.md +46 -15
  3. package/skill/references/app-setup.md +43 -12
  4. package/skill/references/dev.md +5 -2
  5. package/skill/references/endpoint.md +39 -28
  6. package/skill/references/errors.md +50 -37
  7. package/skill/references/files.md +8 -4
  8. package/skill/references/jobs.md +39 -7
  9. package/skill/references/table-and-migration.md +47 -3
  10. package/skill/references/tests.md +73 -16
  11. package/skill/references/webhooks-links-connectors.md +1 -1
  12. package/src/build-recipe.ts +198 -0
  13. package/src/build.ts +60 -4
  14. package/src/bun.ts +181 -13
  15. package/src/cloudflare/application.ts +822 -0
  16. package/src/cloudflare/bindings.ts +159 -0
  17. package/src/cloudflare/export.ts +621 -0
  18. package/src/cloudflare/import.ts +888 -0
  19. package/src/cloudflare/maintenance.ts +518 -0
  20. package/src/cloudflare/names.ts +16 -0
  21. package/src/cloudflare/ops.ts +375 -0
  22. package/src/cloudflare/r2-files.ts +172 -0
  23. package/src/cloudflare/restore.ts +167 -0
  24. package/src/cloudflare/tracker.ts +192 -0
  25. package/src/cloudflare.ts +100 -146
  26. package/src/contract/app-id.ts +5 -0
  27. package/src/contract/artifact-schema.ts +9 -1
  28. package/src/contract/build.ts +40 -2
  29. package/src/contract/capability-cases.ts +30 -0
  30. package/src/contract/compile.ts +1 -0
  31. package/src/contract/containers.ts +19 -0
  32. package/src/contract/context.ts +35 -9
  33. package/src/contract/declaration.ts +2 -0
  34. package/src/contract/defaults.ts +27 -4
  35. package/src/contract/define.ts +29 -3
  36. package/src/contract/index.ts +1 -0
  37. package/src/contract/kit.ts +4 -1
  38. package/src/contract/platform.ts +3 -2
  39. package/src/contract/vocabulary.ts +26 -8
  40. package/src/dev/changes.ts +5 -43
  41. package/src/dev/contract-digest.ts +17 -0
  42. package/src/dev/index.ts +1 -1
  43. package/src/dev/launcher.ts +197 -41
  44. package/src/dev/output.ts +13 -3
  45. package/src/dev/wire.ts +11 -1
  46. package/src/disk-assets.ts +8 -3
  47. package/src/files/archive.ts +2 -2
  48. package/src/files/handle.ts +2 -1
  49. package/src/files/job-names.ts +12 -0
  50. package/src/files/kinds.ts +74 -32
  51. package/src/files/scan.ts +3 -4
  52. package/src/files/streams.ts +3 -1
  53. package/src/implement/anonymous.ts +12 -11
  54. package/src/implement/boot.ts +84 -27
  55. package/src/implement/bounded.ts +21 -2
  56. package/src/implement/capability.ts +16 -4
  57. package/src/implement/dispatch.ts +423 -178
  58. package/src/implement/failures.ts +59 -4
  59. package/src/implement/faults.ts +5 -1
  60. package/src/implement/index.ts +1 -0
  61. package/src/implement/invocation.ts +62 -11
  62. package/src/implement/principal.ts +25 -2
  63. package/src/implement/terminals.ts +93 -0
  64. package/src/implement/tokens.ts +4 -2
  65. package/src/implement/types.ts +22 -4
  66. package/src/implement/workload.ts +42 -16
  67. package/src/jobs/mail.ts +8 -0
  68. package/src/jobs/report.ts +5 -1
  69. package/src/jobs/runner.ts +702 -126
  70. package/src/jobs/schedule.ts +7 -1
  71. package/src/jobs/scrub.ts +14 -1
  72. package/src/jobs/types.ts +6 -0
  73. package/src/spa-fallback.ts +12 -0
  74. package/src/testing/endpoint-access.ts +493 -202
  75. package/src/testing/index.ts +1 -1
  76. package/src/testing/isolation.ts +8 -1
  77. package/src/testing/kit.ts +86 -4
  78. package/src/testing/policy-isolation.ts +2 -2
  79. package/src/testing/proxy.ts +15 -10
  80. package/types/src/agent-skill.d.ts +29 -0
  81. package/types/src/build-recipe.d.ts +70 -0
  82. package/types/src/build.d.ts +29 -0
  83. package/types/src/bun.d.ts +99 -0
  84. package/types/src/client/caller.d.ts +19 -0
  85. package/types/src/client/contract.d.ts +131 -0
  86. package/types/src/client/error-reporting.d.ts +66 -0
  87. package/types/src/client/index.d.ts +4 -0
  88. package/types/src/client/rpc.d.ts +14 -0
  89. package/types/src/client/share.d.ts +2 -0
  90. package/types/src/client/transport.d.ts +42 -0
  91. package/types/src/cloudflare/application.d.ts +73 -0
  92. package/types/src/cloudflare/bindings.d.ts +33 -0
  93. package/types/src/cloudflare/export.d.ts +16 -0
  94. package/types/src/cloudflare/import.d.ts +47 -0
  95. package/types/src/cloudflare/maintenance.d.ts +117 -0
  96. package/types/src/cloudflare/names.d.ts +11 -0
  97. package/types/src/cloudflare/ops.d.ts +81 -0
  98. package/types/src/cloudflare/r2-files.d.ts +9 -0
  99. package/types/src/cloudflare/restore.d.ts +9 -0
  100. package/types/src/cloudflare/tracker.d.ts +29 -0
  101. package/types/src/cloudflare.d.ts +43 -0
  102. package/types/src/contract/app-id.d.ts +2 -0
  103. package/types/src/contract/artifact-schema.d.ts +10 -0
  104. package/types/src/contract/attachment.d.ts +41 -0
  105. package/types/src/contract/build.d.ts +12 -0
  106. package/types/src/contract/capability-cases.d.ts +9 -0
  107. package/types/src/contract/compile.d.ts +11 -0
  108. package/types/src/contract/connect.d.ts +51 -0
  109. package/types/src/contract/containers.d.ts +3 -0
  110. package/types/src/contract/context.d.ts +525 -0
  111. package/types/src/contract/declaration.d.ts +307 -0
  112. package/types/src/contract/defaults.d.ts +228 -0
  113. package/types/src/contract/define.d.ts +376 -0
  114. package/types/src/contract/file.d.ts +44 -0
  115. package/types/src/contract/group.d.ts +274 -0
  116. package/types/src/contract/handled.d.ts +18 -0
  117. package/types/src/contract/index.d.ts +20 -0
  118. package/types/src/contract/kit.d.ts +211 -0
  119. package/types/src/contract/platform.d.ts +9 -0
  120. package/types/src/contract/problems.d.ts +14 -0
  121. package/types/src/contract/redirect.d.ts +25 -0
  122. package/types/src/contract/schema-from-shape.d.ts +10 -0
  123. package/types/src/contract/schema-shape.d.ts +5 -0
  124. package/types/src/contract/snapshot-source.d.ts +15 -0
  125. package/types/src/contract/token.d.ts +58 -0
  126. package/types/src/contract/vocabulary.d.ts +32 -0
  127. package/types/src/contract/webhook-binding.d.ts +24 -0
  128. package/types/src/contract/webhook-scheme.d.ts +23 -0
  129. package/types/src/dev/changes.d.ts +7 -0
  130. package/types/src/dev/contract-digest.d.ts +1 -0
  131. package/types/src/dev/index.d.ts +2 -0
  132. package/types/src/dev/launcher.d.ts +32 -0
  133. package/types/src/dev/output.d.ts +28 -0
  134. package/types/src/dev/wire.d.ts +126 -0
  135. package/types/src/disk-assets.d.ts +6 -0
  136. package/types/src/errors.d.ts +46 -0
  137. package/types/src/fetch.d.ts +7 -0
  138. package/types/src/files/archive.d.ts +26 -0
  139. package/types/src/files/clamscan.d.ts +32 -0
  140. package/types/src/files/handle.d.ts +107 -0
  141. package/types/src/files/job-names.d.ts +9 -0
  142. package/types/src/files/kinds.d.ts +112 -0
  143. package/types/src/files/scan.d.ts +78 -0
  144. package/types/src/files/sniff.d.ts +7 -0
  145. package/types/src/files/streams.d.ts +17 -0
  146. package/types/src/files/treatment.d.ts +18 -0
  147. package/types/src/from-row.d.ts +20 -0
  148. package/types/src/honeypot.d.ts +2 -0
  149. package/types/src/implement/anonymous.d.ts +24 -0
  150. package/types/src/implement/attachment.d.ts +14 -0
  151. package/types/src/implement/boot.d.ts +128 -0
  152. package/types/src/implement/bounded.d.ts +32 -0
  153. package/types/src/implement/browser-config.d.ts +19 -0
  154. package/types/src/implement/capability.d.ts +142 -0
  155. package/types/src/implement/challenge.d.ts +75 -0
  156. package/types/src/implement/client-address.d.ts +11 -0
  157. package/types/src/implement/connect.d.ts +67 -0
  158. package/types/src/implement/crashes.d.ts +31 -0
  159. package/types/src/implement/dispatch.d.ts +251 -0
  160. package/types/src/implement/egress.d.ts +14 -0
  161. package/types/src/implement/environment.d.ts +14 -0
  162. package/types/src/implement/failures.d.ts +58 -0
  163. package/types/src/implement/faults.d.ts +48 -0
  164. package/types/src/implement/headers.d.ts +7 -0
  165. package/types/src/implement/implement.d.ts +58 -0
  166. package/types/src/implement/index.d.ts +25 -0
  167. package/types/src/implement/invocation.d.ts +136 -0
  168. package/types/src/implement/membership.d.ts +201 -0
  169. package/types/src/implement/multipart.d.ts +36 -0
  170. package/types/src/implement/params.d.ts +7 -0
  171. package/types/src/implement/principal.d.ts +57 -0
  172. package/types/src/implement/project.d.ts +11 -0
  173. package/types/src/implement/public.d.ts +61 -0
  174. package/types/src/implement/serve.d.ts +25 -0
  175. package/types/src/implement/terminals.d.ts +33 -0
  176. package/types/src/implement/tokens.d.ts +49 -0
  177. package/types/src/implement/types.d.ts +263 -0
  178. package/types/src/implement/urls.d.ts +15 -0
  179. package/types/src/implement/verifiers.d.ts +19 -0
  180. package/types/src/implement/webhook.d.ts +55 -0
  181. package/types/src/implement/workload.d.ts +61 -0
  182. package/types/src/implement/xlsx.d.ts +6 -0
  183. package/types/src/index.d.ts +12 -0
  184. package/types/src/jobs/exception.d.ts +40 -0
  185. package/types/src/jobs/mail.d.ts +67 -0
  186. package/types/src/jobs/report.d.ts +56 -0
  187. package/types/src/jobs/runner.d.ts +218 -0
  188. package/types/src/jobs/schedule.d.ts +17 -0
  189. package/types/src/jobs/scrub.d.ts +12 -0
  190. package/types/src/jobs/types.d.ts +98 -0
  191. package/types/src/react/caller.d.ts +30 -0
  192. package/types/src/react/errors.d.ts +40 -0
  193. package/types/src/react/hooks.d.ts +110 -0
  194. package/types/src/react/index.d.ts +5 -0
  195. package/types/src/react/provider.d.ts +58 -0
  196. package/types/src/react/store.d.ts +68 -0
  197. package/types/src/react/testing.d.ts +28 -0
  198. package/types/src/react/types.d.ts +33 -0
  199. package/types/src/rpc/builder.d.ts +52 -0
  200. package/types/src/rpc/contract.d.ts +23 -0
  201. package/types/src/rpc/dispatcher.d.ts +11 -0
  202. package/types/src/rpc/types.d.ts +62 -0
  203. package/types/src/service/define.d.ts +42 -0
  204. package/types/src/service/index.d.ts +3 -0
  205. package/types/src/service/serve.d.ts +28 -0
  206. package/types/src/spa-fallback.d.ts +2 -0
  207. package/types/src/standard-schema.d.ts +55 -0
  208. package/types/src/testing/caller.d.ts +95 -0
  209. package/types/src/testing/challenge.d.ts +11 -0
  210. package/types/src/testing/dev.d.ts +97 -0
  211. package/types/src/testing/endpoint-access.d.ts +132 -0
  212. package/types/src/testing/index.d.ts +12 -0
  213. package/types/src/testing/isolation.d.ts +3 -0
  214. package/types/src/testing/kit.d.ts +207 -0
  215. package/types/src/testing/membership.d.ts +132 -0
  216. package/types/src/testing/policy-isolation.d.ts +113 -0
  217. package/types/src/testing/preload.d.ts +1 -0
  218. package/types/src/testing/proxy.d.ts +40 -0
  219. package/types/src/testing/runner.d.ts +8 -0
  220. package/types/src/testing/sample-file.d.ts +5 -0
  221. package/types/src/testing/status.d.ts +4 -0
  222. package/types/src/undumpable.d.ts +5 -0
@@ -0,0 +1,888 @@
1
+ // One step of loading a consistent backup into a fresh object (ADR-0070 §6): the ops operation `import`, served before
2
+ // the application boots. Control drives the steps and holds their state; the object keeps only its import marker and
3
+ // the steps' receipts in KV storage, which every step that writes updates in the same `transactionSync` as its rows.
4
+ // Every step runs in the maintenance barrier's turn, so no transition interleaves with it.
5
+
6
+ import {
7
+ decodeOpsDumpLine,
8
+ describeOpsDumpSchema,
9
+ encodeOpsDumpRow,
10
+ foldOpsTableHash,
11
+ OPS_DATA_REFUSED_STATUS,
12
+ OPS_DUMP_KEY,
13
+ OPS_DUMP_MANIFEST_MAX_BYTES,
14
+ OPS_DUMP_PAGE_ROWS,
15
+ OPS_DUMP_PART_MAX_RAW_BYTES,
16
+ OPS_DUMP_PART_MAX_ROWS,
17
+ OPS_DUMP_PART_SIZE,
18
+ OPS_EMPTY_TABLE_HASH,
19
+ OPS_STALE_STATUS,
20
+ type OpsDataRefusal,
21
+ type OpsDataRefusalReason,
22
+ type OpsDumpBinding,
23
+ type OpsDumpCursor,
24
+ type OpsDumpLine,
25
+ type OpsDumpManifest,
26
+ opsDumpRowidSql,
27
+ type OpsDumpSchema,
28
+ type OpsDumpSchemaEntry,
29
+ type OpsDumpSource,
30
+ type OpsDumpTable,
31
+ type OpsDumpTablePlan,
32
+ type OpsDumpValue,
33
+ opsDumpValueBinding,
34
+ type OpsImportBeginAck,
35
+ type OpsImportCompleteAck,
36
+ type OpsImportDiscardAck,
37
+ type OpsImportLinkAck,
38
+ type OpsImportPartAck,
39
+ type OpsImportRequest,
40
+ type OpsImportVerifyAck,
41
+ type OpsStaleRefusal,
42
+ parseOpsImportCompleteAck,
43
+ parseOpsImportLinkAck,
44
+ parseOpsImportPartAck,
45
+ parseOpsImportVerifyAck,
46
+ planOpsDump,
47
+ readOpsDumpManifest,
48
+ readOpsDumpPage,
49
+ sha256Hex,
50
+ } from '@fabrika/auth-core'
51
+ import type { SqliteStorage, SqliteValue } from '@fabrika/db/sqlite'
52
+ import { migrateSqlite, type SqliteMigrationBundle } from '@fabrika/db/sqlite-migrate'
53
+ import { jsonResponse } from '../errors.js'
54
+ import { method } from './bindings.js'
55
+ import type { MaintenanceBarrier, TurnStep } from './maintenance.js'
56
+ import type { OpsTarget } from './ops.js'
57
+
58
+ /**
59
+ * One step of loading a dump into a fresh object (ADR-0070); `serving` is the object of ordinary traffic. A runtime
60
+ * whose storage has no synchronous KV or `deleteAll` does not serve the operation.
61
+ */
62
+ export async function serveImport(target: OpsTarget, barrier: MaintenanceBarrier, step: OpsImportRequest, serving: string): Promise<Response> {
63
+ const { storage } = target
64
+ const { kv, deleteAll } = storage
65
+ if (kv === undefined || deleteAll === undefined) {
66
+ return jsonResponse({ refused: 'not_implemented' }, 501)
67
+ }
68
+ const importing: ImportStorage = {
69
+ sql: storage.sql,
70
+ transaction: (closure) => storage.transaction(closure),
71
+ transactionSync: (closure) => storage.transactionSync(closure),
72
+ kv,
73
+ deleteAll: () => deleteAll.call(storage),
74
+ }
75
+ return await importStep({ storage: importing, bucket: target.bucket, migrations: target.migrations, serving }, barrier, step)
76
+ }
77
+
78
+ /** workerd's synchronous KV storage of a SQLite-backed object: it lives in the same database, so it commits with a transaction. */
79
+ export interface ImportKv {
80
+ get(key: string): unknown
81
+ put(key: string, value: unknown): void
82
+ delete(key: string): boolean
83
+ }
84
+
85
+ /** What the importer needs of the object's storage; a SQLite-backed `DurableObjectStorage` fits it. */
86
+ export interface ImportStorage extends SqliteStorage {
87
+ readonly kv: ImportKv
88
+ /** Wipes the object's database, its KV storage and its alarm. */
89
+ deleteAll(): Promise<void>
90
+ }
91
+
92
+ /** The object an import step loads into. */
93
+ export interface ImportTarget {
94
+ readonly storage: ImportStorage
95
+ /** The environment's bucket (the `FABRIKA_FILES` R2 binding), where the backup lies. */
96
+ readonly bucket: unknown
97
+ /** The release's migration bundles in the order the object applies them: the framework's, then the application's. */
98
+ readonly migrations: readonly SqliteMigrationBundle[]
99
+ /** The object of ordinary traffic (`FABRIKA_APPLICATION_OBJECT`), which `discard` never wipes. */
100
+ readonly serving: string
101
+ }
102
+
103
+ /** The KV key of the import marker. Invariant: an object that holds it never boots and never leaves `inspect`. */
104
+ export const IMPORT_MARKER_KEY = 'fabrika:import'
105
+
106
+ /** Whether the object holds an import marker, so it must neither boot nor leave `inspect`. */
107
+ export function importInProgress(storage: { readonly kv: ImportKv }): boolean {
108
+ return storage.kv.get(IMPORT_MARKER_KEY) !== undefined
109
+ }
110
+
111
+ /** The KV key under which part `part` records its acknowledgement once loaded, so a retry loads nothing twice. */
112
+ function loadedPartKey(part: number): string {
113
+ return `${IMPORT_MARKER_KEY}:part:${part}`
114
+ }
115
+
116
+ /** The KV key under which part `part`'s link pass records its acknowledgement. */
117
+ function linkedPartKey(part: number): string {
118
+ return `${IMPORT_MARKER_KEY}:link:${part}`
119
+ }
120
+
121
+ /**
122
+ * The KV key of the receipt `complete` leaves for `source`, so a retried `complete` answers the same. It blocks nothing
123
+ * and stays for the object's life: `begin` takes only a fresh object, so an object completes one import.
124
+ */
125
+ function completedKey(source: OpsDumpSource): string {
126
+ return `${IMPORT_MARKER_KEY}:complete:${source.object}/${source.dump}`
127
+ }
128
+
129
+ /**
130
+ * The marker: the backup the object is loading, whether `begin` cleared the target, how many parts are loaded, and the
131
+ * verification of what is loaded now: `null` for none, else the cursor its next step must carry (`null` once every
132
+ * table matched the manifest) and the last step's cursor with its answer, which a retry of that step gets again. A
133
+ * part or a link pass that writes resets it.
134
+ */
135
+ interface ImportMarker {
136
+ readonly source: OpsDumpSource
137
+ readonly begun: boolean
138
+ readonly loaded: number
139
+ readonly verified: Verification | null
140
+ }
141
+
142
+ interface Verification {
143
+ readonly next: string | null
144
+ /** `null` after a step that started the verification, which a retry starts again. */
145
+ readonly replay: { readonly cursor: string; readonly ack: OpsImportVerifyAck } | null
146
+ }
147
+
148
+ /**
149
+ * One step of loading the consistent backup `step.source` into the fresh object `target` (ADR-0070 §6), in the
150
+ * barrier's turn: ADR-0069's stale refusal unless `step.generation` is the published one, then `serving_object` on the
151
+ * object of ordinary traffic, then `wrong_mode` unless `inspect`. The turn is held across the step's R2 reads, so the
152
+ * generation and mode it was admitted under still hold when it commits.
153
+ */
154
+ export async function importStep(target: ImportTarget, barrier: MaintenanceBarrier, step: OpsImportRequest): Promise<Response> {
155
+ const result = await barrier.step(step.generation, async (mode): Promise<TurnStep<Response>> => {
156
+ if (step.object === target.serving) {
157
+ return answered(refused('serving_object'))
158
+ }
159
+ if (mode !== 'inspect') {
160
+ return answered(refused('wrong_mode'))
161
+ }
162
+ if (step.step !== 'verify') {
163
+ await barrier.restrictedWrite()
164
+ }
165
+ return answered(await admittedStep(target, step))
166
+ })
167
+ if (result.kind === 'ran') {
168
+ return result.value
169
+ }
170
+ const refusal: OpsStaleRefusal = { refused: 'stale_generation', published: result.published }
171
+ return jsonResponse(refusal, OPS_STALE_STATUS)
172
+ }
173
+
174
+ async function admittedStep(target: ImportTarget, step: OpsImportRequest): Promise<Response> {
175
+ const { kv } = target.storage
176
+ if (step.step === 'discard') {
177
+ return await discard(target)
178
+ }
179
+ if (step.step === 'begin') {
180
+ return await begin(target, step.source)
181
+ }
182
+ if (importing(kv, step.source) === null) {
183
+ const completed = step.step === 'complete' ? readCompleted(kv, step.source) : null
184
+ return completed === null ? refused('no_import') : jsonResponse(completed, 200)
185
+ }
186
+ const manifest = await readManifest(target.bucket, step.source)
187
+ if (manifest === null) {
188
+ return refused('invalid_manifest')
189
+ }
190
+ if (step.step === 'complete') {
191
+ return complete(target.storage, step.source, manifest)
192
+ }
193
+ const plan = currentPlan(target.storage, manifest)
194
+ if ('refused' in plan) {
195
+ return jsonResponse(plan, OPS_DATA_REFUSED_STATUS)
196
+ }
197
+ switch (step.step) {
198
+ case 'part':
199
+ return await loadPart(target, step.source, manifest, plan, step.part)
200
+ case 'link':
201
+ return await linkPart(target, step.source, manifest, plan, step.part)
202
+ case 'verify':
203
+ return verify(target.storage, step.source, manifest, plan, step.cursor)
204
+ }
205
+ }
206
+
207
+ function answered(response: Response): TurnStep<Response> {
208
+ return { value: response, holdUntil: null }
209
+ }
210
+
211
+ /** The marker, when it names `source` and records `begin` done; `null` otherwise. */
212
+ function importing(kv: ImportKv, source: OpsDumpSource): ImportMarker | null {
213
+ const marker = readMarker(kv)
214
+ return marker?.begun === true && sameSource(marker.source, source) ? marker : null
215
+ }
216
+
217
+ /**
218
+ * Verifies the manifest, checks the target is fresh, writes the marker, migrates the target to exactly the manifest's
219
+ * ledger, checks the schema and its plan, then clears the target in one transaction that records `begin` done. A retry
220
+ * under the same backup resumes after what the marker records.
221
+ */
222
+ async function begin(target: ImportTarget, source: OpsDumpSource): Promise<Response> {
223
+ const { storage } = target
224
+ const manifest = await readManifest(target.bucket, source)
225
+ if (manifest === null) {
226
+ return refused('invalid_manifest')
227
+ }
228
+ if (!manifest.consistent) {
229
+ return refused('fuzzy_dump')
230
+ }
231
+ const marker = readMarker(storage.kv)
232
+ if (marker !== null && !sameSource(marker.source, source)) {
233
+ return refused('not_fresh')
234
+ }
235
+ if (marker?.begun === true) {
236
+ return beginAck(manifest)
237
+ }
238
+ if (marker === null && holdsTables(storage)) {
239
+ return refused('not_fresh')
240
+ }
241
+ const bundles = ledgerBundles(target.migrations, manifest)
242
+ if (bundles === null) {
243
+ return refused('ledger_mismatch')
244
+ }
245
+ if (marker === null) {
246
+ writeMarker(storage.kv, { source, begun: false, loaded: 0, verified: null })
247
+ }
248
+ // A database the framework never migrated has no ledger, and gets none.
249
+ if (manifest.migrations.length > 0) {
250
+ await migrateSqlite(
251
+ { exec: (query, ...bindings) => storage.sql.exec(query, ...bindings), transactionSync: (fn) => storage.transactionSync(fn) },
252
+ bundles,
253
+ )
254
+ }
255
+ if (!sameDdl(readDdl(storage), manifest.ddl)) {
256
+ return refused('schema_mismatch')
257
+ }
258
+ const plan = currentPlan(storage, manifest)
259
+ if ('refused' in plan) {
260
+ return jsonResponse(plan, OPS_DATA_REFUSED_STATUS)
261
+ }
262
+ storage.transactionSync(() => {
263
+ for (const entry of readDdl(storage)) {
264
+ if (entry.type === 'trigger') {
265
+ storage.sql.exec(`DROP TRIGGER ${quote(entry.name)}`)
266
+ }
267
+ }
268
+ // Every deferred column takes a temporary NULL (the schema rule), so no row of its cycle references another one after this.
269
+ for (const table of plan) {
270
+ if (table.deferred.length > 0) {
271
+ storage.sql.exec(`UPDATE ${quote(table.name)} SET ${table.deferred.map((column) => `${quote(column)} = NULL`).join(', ')}`)
272
+ }
273
+ }
274
+ for (const table of [...plan].reverse()) {
275
+ storage.sql.exec(`DELETE FROM ${quote(table.name)}`)
276
+ }
277
+ writeMarker(storage.kv, { source, begun: true, loaded: 0, verified: null })
278
+ })
279
+ return beginAck(manifest)
280
+ }
281
+
282
+ function beginAck(manifest: OpsDumpManifest): Response {
283
+ const ack: OpsImportBeginAck = { step: 'begin', parts: manifest.parts.length }
284
+ return jsonResponse(ack, 200)
285
+ }
286
+
287
+ /**
288
+ * Inserts part `part`'s rows with their rowids, every value through the wire's binding and every deferred column as
289
+ * NULL, in one transaction that reads the marker and the part's receipt again, records the part and resets the
290
+ * verification.
291
+ */
292
+ async function loadPart(target: ImportTarget, source: OpsDumpSource, manifest: OpsDumpManifest, plan: readonly OpsDumpTablePlan[], part: number) {
293
+ const { storage } = target
294
+ const recorded = readLoadedPart(storage.kv, part)
295
+ if (recorded !== null) {
296
+ return jsonResponse(recorded, 200)
297
+ }
298
+ const lines = await readPart(target.bucket, source, manifest, plan, part)
299
+ if (!Array.isArray(lines)) {
300
+ return jsonResponse(lines, OPS_DATA_REFUSED_STATUS)
301
+ }
302
+ const tables = new Map(plan.map((table) => [table.name, table]))
303
+ const ack = storage.transactionSync((): OpsImportPartAck | OpsDataRefusal => {
304
+ const marker = importing(storage.kv, source)
305
+ if (marker === null) {
306
+ return { refused: 'no_import', table: null }
307
+ }
308
+ const again = readLoadedPart(storage.kv, part)
309
+ if (again !== null) {
310
+ return again
311
+ }
312
+ let deferred = 0
313
+ for (const { table: name, row } of lines) {
314
+ const table = planned(tables, name)
315
+ if (insertRow(storage, table, row)) {
316
+ deferred++
317
+ }
318
+ }
319
+ const loaded: OpsImportPartAck = { step: 'part', part, rows: lines.length, deferred }
320
+ storage.kv.put(loadedPartKey(part), JSON.stringify(loaded))
321
+ writeMarker(storage.kv, { ...marker, loaded: marker.loaded + 1, verified: null })
322
+ return loaded
323
+ })
324
+ return jsonResponse(ack, 'refused' in ack ? OPS_DATA_REFUSED_STATUS : 200)
325
+ }
326
+
327
+ /** Writes one row; answers whether it has a deferred key for the link pass to set. */
328
+ function insertRow(storage: SqliteStorage, table: OpsDumpTablePlan, row: readonly OpsDumpValue[]): boolean {
329
+ const [rowid, ...values] = row
330
+ // Inline, so a row of a table at workerd's column limit binds no more parameters than its columns.
331
+ const key = opsDumpRowidSql(rowid ?? null)
332
+ const columns: string[] = [table.keyset]
333
+ const keys: string[] = [key]
334
+ const bindings: OpsDumpBinding[] = []
335
+ let waits = false
336
+ for (const [index, column] of table.columns.entries()) {
337
+ const value = values[index] ?? null
338
+ // An INTEGER PRIMARY KEY column is the rowid itself, written once.
339
+ if (column === table.keyset) {
340
+ continue
341
+ }
342
+ const deferred = table.deferred.includes(column)
343
+ waits ||= deferred && value !== null
344
+ const binding = opsDumpValueBinding(deferred ? null : value)
345
+ columns.push(column)
346
+ keys.push(binding.sql)
347
+ bindings.push(binding)
348
+ }
349
+ if (table.name === SQLITE_SEQUENCE) {
350
+ // Loading an AUTOINCREMENT table writes its own counter row; the source's row replaces it, whichever table loads first.
351
+ const name = opsDumpValueBinding(values[0] ?? null)
352
+ storage.sql.exec(`DELETE FROM ${SQLITE_SEQUENCE} WHERE rowid = ${key} OR name = ${name.sql}`, ...params([name]))
353
+ }
354
+ storage.sql.exec(
355
+ `INSERT INTO ${quote(table.name)} (${columns.map(quote).join(', ')}) VALUES (${keys.join(', ')})`,
356
+ ...params(bindings),
357
+ )
358
+ return waits
359
+ }
360
+
361
+ /**
362
+ * Sets the deferred keys of part `part`'s rows by their rowids, once every part is loaded, in one transaction that
363
+ * reads the marker and the pass's receipt again, records the pass and resets the verification.
364
+ */
365
+ async function linkPart(target: ImportTarget, source: OpsDumpSource, manifest: OpsDumpManifest, plan: readonly OpsDumpTablePlan[], part: number) {
366
+ const { storage } = target
367
+ const recorded = readLinkedPart(storage.kv, part)
368
+ if (recorded !== null) {
369
+ return jsonResponse(recorded, 200)
370
+ }
371
+ if (importing(storage.kv, source)?.loaded !== manifest.parts.length) {
372
+ return refused('part_mismatch')
373
+ }
374
+ const lines = await readPart(target.bucket, source, manifest, plan, part)
375
+ if (!Array.isArray(lines)) {
376
+ return jsonResponse(lines, OPS_DATA_REFUSED_STATUS)
377
+ }
378
+ const tables = new Map(plan.map((table) => [table.name, table]))
379
+ const ack = storage.transactionSync((): OpsImportLinkAck | OpsDataRefusal => {
380
+ const marker = importing(storage.kv, source)
381
+ if (marker === null) {
382
+ return { refused: 'no_import', table: null }
383
+ }
384
+ if (marker.loaded !== manifest.parts.length) {
385
+ return { refused: 'part_mismatch', table: null }
386
+ }
387
+ const again = readLinkedPart(storage.kv, part)
388
+ if (again !== null) {
389
+ return again
390
+ }
391
+ let linked = 0
392
+ for (const { table: name, row } of lines) {
393
+ const table = planned(tables, name)
394
+ const [rowid, ...values] = row
395
+ const deferred = table.columns.flatMap((column, index) => table.deferred.includes(column) ? [{ column, value: values[index] ?? null }] : [])
396
+ if (deferred.every(({ value }) => value === null)) {
397
+ continue
398
+ }
399
+ const sets = deferred.map(({ column, value }) => ({ column, binding: opsDumpValueBinding(value) }))
400
+ storage.sql.exec(
401
+ `UPDATE ${quote(table.name)} SET ${sets.map(({ column, binding }) => `${quote(column)} = ${binding.sql}`).join(', ')} WHERE ${
402
+ quote(table.keyset)
403
+ } = ${opsDumpRowidSql(rowid ?? null)}`,
404
+ ...params(sets.map(({ binding }) => binding)),
405
+ )
406
+ linked++
407
+ }
408
+ const done: OpsImportLinkAck = { step: 'link', part, rows: linked }
409
+ storage.kv.put(linkedPartKey(part), JSON.stringify(done))
410
+ writeMarker(storage.kv, { ...marker, verified: null })
411
+ return done
412
+ })
413
+ return jsonResponse(ack, 'refused' in ack ? OPS_DATA_REFUSED_STATUS : 200)
414
+ }
415
+
416
+ /**
417
+ * Reads the loaded tables back from `cursor` with the producer's projection and fold, at most a part's row bound per
418
+ * step, and compares each finished table with the manifest. A verification starts at `null`; any other cursor must be
419
+ * the one the marker records this verification handed out last, or the one its last step was sent, whose answer it
420
+ * repeats. It writes no row: only its progress, into the marker, which a mismatch clears.
421
+ */
422
+ function verify(
423
+ storage: ImportStorage,
424
+ source: OpsDumpSource,
425
+ manifest: OpsDumpManifest,
426
+ plan: readonly OpsDumpTablePlan[],
427
+ cursor: OpsDumpCursor | null,
428
+ ): Response {
429
+ const marker = importing(storage.kv, source)
430
+ if (marker === null) {
431
+ return refused('no_import')
432
+ }
433
+ const mismatch = (table: string | null): Response => {
434
+ writeMarker(storage.kv, { ...marker, verified: null })
435
+ return refused('verify_mismatch', table)
436
+ }
437
+ if (cursor !== null) {
438
+ const key = cursorKey(cursor)
439
+ const replay = marker.verified?.replay ?? null
440
+ // Its answer was lost: what is loaded has not changed since, or the verification would be reset.
441
+ if (replay?.cursor === key) {
442
+ return jsonResponse(replay.ack, 200)
443
+ }
444
+ if ((marker.verified?.next ?? null) !== key) {
445
+ return mismatch(cursor.table)
446
+ }
447
+ }
448
+ const tables = new Map(plan.map((table) => [table.name, table]))
449
+ const order = manifest.tables
450
+ let index = cursor === null ? 0 : order.findIndex((table) => table.name === cursor.table)
451
+ if (index < 0) {
452
+ return mismatch(cursor?.table ?? null)
453
+ }
454
+ const startOf = (table: OpsDumpTable | undefined): OpsDumpCursor | null =>
455
+ table === undefined ? null : { table: table.name, after: null, rows: 0, hash: OPS_EMPTY_TABLE_HASH }
456
+ let current = cursor ?? startOf(order[0])
457
+ const finished: OpsDumpTable[] = []
458
+ let budget = OPS_DUMP_PART_MAX_ROWS
459
+ while (current !== null && budget > 0) {
460
+ const expected = order[index]
461
+ if (expected === undefined) {
462
+ throw new Error('a verify cursor past the last table of the manifest')
463
+ }
464
+ const page = readPage(storage, planned(tables, expected.name), current.after)
465
+ const last = page.at(-1)?.[0]
466
+ current = {
467
+ table: expected.name,
468
+ after: last?.type === 'integer' ? last.value : current.after,
469
+ rows: current.rows + page.length,
470
+ hash: foldOpsTableHash(current.hash, page.map((row) => encodeOpsDumpRow(row))),
471
+ }
472
+ budget -= page.length
473
+ if (current.rows > expected.rows) {
474
+ return mismatch(expected.name)
475
+ }
476
+ if (page.length < OPS_DUMP_PAGE_ROWS) {
477
+ if (current.rows !== expected.rows || current.hash !== expected.hash) {
478
+ return mismatch(expected.name)
479
+ }
480
+ finished.push(expected)
481
+ index++
482
+ current = startOf(order[index])
483
+ }
484
+ }
485
+ const ack: OpsImportVerifyAck = { step: 'verify', tables: finished, next: current }
486
+ const next = current === null ? null : cursorKey(current)
487
+ writeMarker(storage.kv, { ...marker, verified: { next, replay: cursor === null ? null : { cursor: cursorKey(cursor), ack } } })
488
+ return jsonResponse(ack, 200)
489
+ }
490
+
491
+ /** A cursor as the marker records it: its fields in a fixed order, whatever order a request named them in. */
492
+ function cursorKey(cursor: OpsDumpCursor): string {
493
+ return JSON.stringify([cursor.table, cursor.after, cursor.rows, cursor.hash])
494
+ }
495
+
496
+ /** One keyset page of `table` after the rowid `after`: each row its rowid, then its columns, as the producer reads them. */
497
+ function readPage(storage: SqliteStorage, table: OpsDumpTablePlan, after: string | null): (readonly OpsDumpValue[])[] {
498
+ const condition = after === null ? '' : `WHERE ${quote(table.keyset)} > CAST(? AS INTEGER)`
499
+ const query = (sql: string) => after === null ? storage.sql.exec(sql).toArray() : storage.sql.exec(sql, after).toArray()
500
+ return readOpsDumpPage(query, table, condition, OPS_DUMP_PAGE_ROWS).map((row) => row.values)
501
+ }
502
+
503
+ /**
504
+ * Once every part is loaded, every link pass a part needs ran, and a verification of what is loaded now matched every
505
+ * table: recreates the triggers from the manifest's DDL, compares the whole schema again, clears the marker and the
506
+ * steps' receipts, and leaves the completion's receipt.
507
+ */
508
+ function complete(storage: ImportStorage, source: OpsDumpSource, manifest: OpsDumpManifest): Response {
509
+ const answer = storage.transactionSync((): OpsImportCompleteAck | OpsDataRefusal => {
510
+ const marker = importing(storage.kv, source)
511
+ if (marker === null) {
512
+ return { refused: 'no_import', table: null }
513
+ }
514
+ const loaded = manifest.parts.map((part) => readLoadedPart(storage.kv, part.part))
515
+ const unlinked = loaded.some((ack) => ack === null || (ack.deferred > 0 && readLinkedPart(storage.kv, ack.part) === null))
516
+ if (marker.loaded !== manifest.parts.length || unlinked) {
517
+ return { refused: 'part_mismatch', table: null }
518
+ }
519
+ if (marker.verified === null || marker.verified.next !== null) {
520
+ return { refused: 'verify_mismatch', table: null }
521
+ }
522
+ const present = new Set(readDdl(storage).filter((entry) => entry.type === 'trigger').map((entry) => entry.name))
523
+ for (const entry of manifest.ddl) {
524
+ if (entry.type !== 'trigger' || present.has(entry.name)) {
525
+ continue
526
+ }
527
+ if (!TRIGGER_DDL.test(entry.sql)) {
528
+ return { refused: 'schema_mismatch', table: null }
529
+ }
530
+ storage.sql.exec(entry.sql)
531
+ }
532
+ if (!sameDdl(readDdl(storage), manifest.ddl)) {
533
+ return { refused: 'schema_mismatch', table: null }
534
+ }
535
+ for (const part of manifest.parts) {
536
+ storage.kv.delete(loadedPartKey(part.part))
537
+ storage.kv.delete(linkedPartKey(part.part))
538
+ }
539
+ storage.kv.delete(IMPORT_MARKER_KEY)
540
+ const ack: OpsImportCompleteAck = { step: 'complete', source: { object: source.object, dump: source.dump } }
541
+ storage.kv.put(completedKey(source), JSON.stringify(ack))
542
+ return ack
543
+ })
544
+ return jsonResponse(answer, 'refused' in answer ? OPS_DATA_REFUSED_STATUS : 200)
545
+ }
546
+
547
+ /** Wipes an `inspect` object that holds an import marker; the caller refused the object of ordinary traffic. */
548
+ async function discard(target: ImportTarget): Promise<Response> {
549
+ if (readMarker(target.storage.kv) === null) {
550
+ return refused('no_import')
551
+ }
552
+ await target.storage.deleteAll()
553
+ const ack: OpsImportDiscardAck = { step: 'discard' }
554
+ return jsonResponse(ack, 200)
555
+ }
556
+
557
+ /** The plan of the target's schema, which must list the manifest's tables in the manifest's order. */
558
+ function currentPlan(storage: SqliteStorage, manifest: OpsDumpManifest): readonly OpsDumpTablePlan[] | OpsDataRefusal {
559
+ const plan = planOpsDump(describeDumpSchema(storage))
560
+ if ('refused' in plan) {
561
+ return plan
562
+ }
563
+ const names = plan.tables.map((table) => table.name)
564
+ const expected = manifest.tables.map((table) => table.name)
565
+ if (names.length !== expected.length || names.some((name, index) => name !== expected[index])) {
566
+ return { refused: 'schema_mismatch', table: null }
567
+ }
568
+ return plan.tables
569
+ }
570
+
571
+ function planned(tables: ReadonlyMap<string, OpsDumpTablePlan>, name: string): OpsDumpTablePlan {
572
+ const table = tables.get(name)
573
+ if (table === undefined) {
574
+ throw new Error('a dump line names a table the plan does not hold')
575
+ }
576
+ return table
577
+ }
578
+
579
+ /**
580
+ * Part `part`'s lines: its byte range read and checked against the manifest's size and SHA-256, decompressed up to the
581
+ * raw bound, at most the row bound, each line in the wire's encoding and fitting its table's plan.
582
+ */
583
+ async function readPart(
584
+ bucket: unknown,
585
+ source: OpsDumpSource,
586
+ manifest: OpsDumpManifest,
587
+ plan: readonly OpsDumpTablePlan[],
588
+ part: number,
589
+ ): Promise<OpsDumpLine[] | OpsDataRefusal> {
590
+ const mismatch: OpsDataRefusal = { refused: 'part_mismatch', table: null }
591
+ const described = manifest.parts[part - 1]
592
+ if (described === undefined) {
593
+ return mismatch
594
+ }
595
+ // Every part but the last is exactly the fixed size, so part n starts at (n - 1) parts.
596
+ const body = await r2Get(bucket, OPS_DUMP_KEY({ ...source, consistent: true }, 'data'), {
597
+ range: { offset: (part - 1) * OPS_DUMP_PART_SIZE, length: described.size },
598
+ })
599
+ const bytes = body === null ? null : await bodyBytes(body)
600
+ if (bytes === null || bytes.length !== described.size || (await sha256OfBytes(bytes)) !== described.sha256) {
601
+ return mismatch
602
+ }
603
+ const text = await gunzipWithin(bytes, OPS_DUMP_PART_MAX_RAW_BYTES)
604
+ const raw = text?.split('\n')
605
+ // Each line ends with its newline, so the text ends with one.
606
+ if (raw === undefined || raw.pop() !== '' || raw.length > OPS_DUMP_PART_MAX_ROWS) {
607
+ return mismatch
608
+ }
609
+ const tables = new Map(plan.map((table) => [table.name, table]))
610
+ const lines: OpsDumpLine[] = []
611
+ for (const encoded of raw) {
612
+ const line = decodeOpsDumpLine(encoded)
613
+ const table = line === null ? undefined : tables.get(line.table)
614
+ if (line === null || table === undefined) {
615
+ return mismatch
616
+ }
617
+ if (line.row.length !== table.columns.length + 1 || line.row[0]?.type !== 'integer') {
618
+ return { refused: 'part_mismatch', table: table.name }
619
+ }
620
+ lines.push(line)
621
+ }
622
+ return lines
623
+ }
624
+
625
+ /** `bytes` as one gzip member, or `null` when it does not decompress, decompresses past `bound`, or is not UTF-8. */
626
+ async function gunzipWithin(bytes: Uint8Array, bound: number): Promise<string | null> {
627
+ const reader = new Blob([bytes.slice()]).stream().pipeThrough(new DecompressionStream('gzip')).getReader()
628
+ const chunks: Uint8Array[] = []
629
+ let total = 0
630
+ try {
631
+ for (let read = await reader.read(); !read.done; read = await reader.read()) {
632
+ total += read.value.byteLength
633
+ if (total > bound) {
634
+ await reader.cancel()
635
+ return null
636
+ }
637
+ chunks.push(read.value)
638
+ }
639
+ } catch {
640
+ return null
641
+ }
642
+ const raw = new Uint8Array(total)
643
+ let offset = 0
644
+ for (const chunk of chunks) {
645
+ raw.set(chunk, offset)
646
+ offset += chunk.byteLength
647
+ }
648
+ try {
649
+ return new TextDecoder('utf-8', { fatal: true }).decode(raw)
650
+ } catch {
651
+ return null
652
+ }
653
+ }
654
+
655
+ async function sha256OfBytes(bytes: Uint8Array): Promise<string> {
656
+ const digest = await crypto.subtle.digest('SHA-256', bytes.slice())
657
+ return [...new Uint8Array(digest)].map((byte) => byte.toString(16).padStart(2, '0')).join('')
658
+ }
659
+
660
+ /** The manifest at the backup's key, or `null` when it is missing, too large, does not parse or names another dump. */
661
+ async function readManifest(bucket: unknown, source: OpsDumpSource): Promise<OpsDumpManifest | null> {
662
+ const body = await r2Get(bucket, OPS_DUMP_KEY({ ...source, consistent: true }, 'manifest'))
663
+ if (body === null) {
664
+ return null
665
+ }
666
+ const size: unknown = typeof body === 'object' ? Reflect.get(body, 'size') : undefined
667
+ if (typeof size === 'number' && size > OPS_DUMP_MANIFEST_MAX_BYTES) {
668
+ return null
669
+ }
670
+ const text = method(body, 'text')
671
+ if (text === null) {
672
+ throw new Error('R2 answered the manifest without its body')
673
+ }
674
+ const content: unknown = await text()
675
+ const manifest = typeof content === 'string' ? readOpsDumpManifest(content) : null
676
+ return manifest !== null && manifest.object === source.object && manifest.dump === source.dump ? manifest : null
677
+ }
678
+
679
+ /** R2's `get` on the bucket, by method lookup at the binding's edge; `null` for a missing object. */
680
+ async function r2Get(bucket: unknown, key: string, options?: { readonly range: { readonly offset: number; readonly length: number } }) {
681
+ const get = method(bucket, 'get')
682
+ if (get === null) {
683
+ throw new Error("the FABRIKA_FILES binding has no get; a backup lies in the environment's bucket")
684
+ }
685
+ const body: unknown = await (options === undefined ? get(key) : get(key, options))
686
+ return body ?? null
687
+ }
688
+
689
+ async function bodyBytes(body: unknown): Promise<Uint8Array> {
690
+ const arrayBuffer = method(body, 'arrayBuffer')
691
+ if (arrayBuffer === null) {
692
+ throw new Error('R2 answered a dump part without its body')
693
+ }
694
+ const bytes: unknown = await arrayBuffer()
695
+ if (!(bytes instanceof ArrayBuffer)) {
696
+ throw new Error('R2 answered a dump part body that is not bytes')
697
+ }
698
+ return new Uint8Array(bytes)
699
+ }
700
+
701
+ /**
702
+ * Each bundle of the release cut to the manifest's ledger, which each must begin with exactly (by name and checksum);
703
+ * `null` for any difference, a bundle or a file the release lacks included.
704
+ */
705
+ function ledgerBundles(release: readonly SqliteMigrationBundle[], manifest: OpsDumpManifest): readonly SqliteMigrationBundle[] | null {
706
+ const names = new Set(release.map((bundle) => bundle.bundle))
707
+ if (manifest.migrations.some((migration) => !names.has(migration.bundle))) {
708
+ return null
709
+ }
710
+ const bundles: SqliteMigrationBundle[] = []
711
+ for (const bundle of release) {
712
+ const recorded = manifest.migrations.filter((migration) => migration.bundle === bundle.bundle).sort((left, right) =>
713
+ compareNames(left.name, right.name)
714
+ )
715
+ const files = [...bundle.migrations].sort((left, right) => compareNames(left.name, right.name))
716
+ const prefix = files.slice(0, recorded.length)
717
+ const same = prefix.length === recorded.length
718
+ && prefix.every((file, index) => file.name === recorded[index]?.name && sha256Hex(file.sql) === recorded[index]?.checksum)
719
+ if (!same) {
720
+ return null
721
+ }
722
+ bundles.push({ bundle: bundle.bundle, migrations: prefix })
723
+ }
724
+ return bundles
725
+ }
726
+
727
+ const SQLITE_SEQUENCE = 'sqlite_sequence'
728
+
729
+ /** `CREATE TRIGGER` as `sqlite_schema` stores it, the only DDL `complete` runs from the manifest. */
730
+ const TRIGGER_DDL = /^CREATE\s+(?:TEMP\s+|TEMPORARY\s+)?TRIGGER\s/i
731
+
732
+ /** Whether a dump holds `name`: no SQLite internal table but `sqlite_sequence`, and none of Cloudflare's `_cf_` ones. */
733
+ function dumped(name: string): boolean {
734
+ return !name.startsWith('_cf_') && (!name.startsWith('sqlite_') || name === SQLITE_SEQUENCE)
735
+ }
736
+
737
+ /** Whether the object holds any table but Cloudflare's own. */
738
+ function holdsTables(storage: SqliteStorage): boolean {
739
+ return storage.sql.exec(`SELECT name FROM sqlite_schema WHERE type = 'table'`).toArray().some((row) => !text(row['name']).startsWith('_cf_'))
740
+ }
741
+
742
+ /** Every `sqlite_schema` entry with SQL a dump's DDL lists, in no particular order. */
743
+ function readDdl(storage: SqliteStorage): OpsDumpSchemaEntry[] {
744
+ return storage.sql.exec('SELECT type, name, tbl_name, sql FROM sqlite_schema WHERE sql IS NOT NULL').toArray().flatMap((row) => {
745
+ const name = text(row['name'])
746
+ const table = text(row['tbl_name'])
747
+ const type = row['type']
748
+ if (!dumped(name) || !dumped(table) || (type !== 'table' && type !== 'index' && type !== 'trigger' && type !== 'view')) {
749
+ return []
750
+ }
751
+ return [{ type, name, table, sql: text(row['sql']) }]
752
+ })
753
+ }
754
+
755
+ function sameDdl(left: readonly OpsDumpSchemaEntry[], right: readonly OpsDumpSchemaEntry[]): boolean {
756
+ const canonical = (entries: readonly OpsDumpSchemaEntry[]) =>
757
+ entries.map((entry) => JSON.stringify([entry.type, entry.name, entry.table, entry.sql])).sort(compareNames).join('\n')
758
+ return left.length === right.length && canonical(left) === canonical(right)
759
+ }
760
+
761
+ /** The schema `planOpsDump` reads, of every table a dump holds (`describeOpsDumpSchema`). */
762
+ export function describeDumpSchema(storage: SqliteStorage): OpsDumpSchema {
763
+ return describeOpsDumpSchema((query, ...bindings) => storage.sql.exec(query, ...bindings).toArray())
764
+ }
765
+
766
+ /** The import marker, parsed strictly; `null` when the object holds none. A marker that does not parse is a corrupt object. */
767
+ function readMarker(kv: ImportKv): ImportMarker | null {
768
+ const raw = kv.get(IMPORT_MARKER_KEY)
769
+ if (raw === undefined) {
770
+ return null
771
+ }
772
+ const value: unknown = typeof raw === 'string' ? parseJson(raw) : undefined
773
+ const source: unknown = isRecord(value) ? value['source'] : undefined
774
+ const verified = isRecord(value) ? readVerification(value['verified']) : undefined
775
+ if (
776
+ !isRecord(value) || !isRecord(source) || Object.keys(value).length !== 4 || Object.keys(source).length !== 2
777
+ || typeof source['object'] !== 'string' || typeof source['dump'] !== 'string' || typeof value['begun'] !== 'boolean'
778
+ || typeof value['loaded'] !== 'number' || !Number.isSafeInteger(value['loaded']) || value['loaded'] < 0 || verified === undefined
779
+ ) {
780
+ throw new Error('the import marker in KV storage does not parse')
781
+ }
782
+ return { source: { object: source['object'], dump: source['dump'] }, begun: value['begun'], loaded: value['loaded'], verified }
783
+ }
784
+
785
+ /** The marker's verification; `undefined` when it does not parse. */
786
+ function readVerification(value: unknown): Verification | null | undefined {
787
+ if (value === null) {
788
+ return null
789
+ }
790
+ if (!isRecord(value) || Object.keys(value).length !== 2) {
791
+ return undefined
792
+ }
793
+ const next = value['next']
794
+ const replay = value['replay']
795
+ if (next !== null && typeof next !== 'string') {
796
+ return undefined
797
+ }
798
+ if (replay === null) {
799
+ return { next, replay: null }
800
+ }
801
+ const cursor = isRecord(replay) ? replay['cursor'] : undefined
802
+ const ack = isRecord(replay) && Object.keys(replay).length === 2 ? parseOpsImportVerifyAck(replay['ack']) : null
803
+ return typeof cursor === 'string' && ack !== null ? { next, replay: { cursor, ack } } : undefined
804
+ }
805
+
806
+ function writeMarker(kv: ImportKv, marker: ImportMarker): void {
807
+ kv.put(IMPORT_MARKER_KEY, JSON.stringify(marker))
808
+ }
809
+
810
+ function readLoadedPart(kv: ImportKv, part: number): OpsImportPartAck | null {
811
+ const raw = kv.get(loadedPartKey(part))
812
+ if (raw === undefined) {
813
+ return null
814
+ }
815
+ const ack = typeof raw === 'string' ? parseOpsImportPartAck(parseJson(raw)) : null
816
+ if (ack === null || ack.part !== part) {
817
+ throw new Error('a loaded part recorded in KV storage does not parse')
818
+ }
819
+ return ack
820
+ }
821
+
822
+ function readLinkedPart(kv: ImportKv, part: number): OpsImportLinkAck | null {
823
+ const raw = kv.get(linkedPartKey(part))
824
+ if (raw === undefined) {
825
+ return null
826
+ }
827
+ const ack = typeof raw === 'string' ? parseOpsImportLinkAck(parseJson(raw)) : null
828
+ if (ack === null || ack.part !== part) {
829
+ throw new Error('a link pass recorded in KV storage does not parse')
830
+ }
831
+ return ack
832
+ }
833
+
834
+ /** The acknowledgement `complete` answered for `source`, `null` when it never completed. */
835
+ function readCompleted(kv: ImportKv, source: OpsDumpSource): OpsImportCompleteAck | null {
836
+ const raw = kv.get(completedKey(source))
837
+ if (raw === undefined) {
838
+ return null
839
+ }
840
+ const ack = typeof raw === 'string' ? parseOpsImportCompleteAck(parseJson(raw)) : null
841
+ if (ack === null || !sameSource(ack.source, source)) {
842
+ throw new Error('a completed import recorded in KV storage does not parse')
843
+ }
844
+ return ack
845
+ }
846
+
847
+ function sameSource(left: OpsDumpSource, right: OpsDumpSource): boolean {
848
+ return left.object === right.object && left.dump === right.dump
849
+ }
850
+
851
+ function refused(reason: OpsDataRefusalReason, table: string | null = null): Response {
852
+ const refusal: OpsDataRefusal = { refused: reason, table }
853
+ return jsonResponse(refusal, OPS_DATA_REFUSED_STATUS)
854
+ }
855
+
856
+ /** The parameters of `bindings`; `sql.exec` binds bytes as an `ArrayBuffer`. */
857
+ function params(bindings: readonly OpsDumpBinding[]): SqliteValue[] {
858
+ return bindings.map(({ param }) => param instanceof Uint8Array ? param.slice().buffer : param)
859
+ }
860
+
861
+ function quote(identifier: string): string {
862
+ return `"${identifier.replaceAll('"', '""')}"`
863
+ }
864
+
865
+ /** Code-unit order, which no locale changes; the migration runner orders files the same way. */
866
+ function compareNames(left: string, right: string): number {
867
+ return left < right ? -1 : left > right ? 1 : 0
868
+ }
869
+
870
+ function parseJson(text: string): unknown {
871
+ try {
872
+ return JSON.parse(text)
873
+ } catch {
874
+ return undefined
875
+ }
876
+ }
877
+
878
+ function isRecord(value: unknown): value is Record<string, unknown> {
879
+ return typeof value === 'object' && value !== null && !Array.isArray(value)
880
+ }
881
+
882
+ // SQLite's own answers: a value of another type is a broken runtime, never input to refuse.
883
+ function text(value: SqliteValue | undefined): string {
884
+ if (typeof value !== 'string') {
885
+ throw new Error('SQLite answered a schema name or SQL that is not text')
886
+ }
887
+ return value
888
+ }