@fabrika/app 0.9.0-dev.4 → 0.9.0-dev.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +9 -9
- package/skill/SKILL.md +41 -14
- package/skill/references/app-setup.md +43 -12
- package/skill/references/dev.md +2 -1
- package/skill/references/endpoint.md +30 -24
- package/skill/references/errors.md +40 -35
- package/skill/references/files.md +4 -2
- package/skill/references/jobs.md +21 -4
- package/skill/references/table-and-migration.md +19 -2
- package/skill/references/tests.md +40 -1
- package/skill/references/webhooks-links-connectors.md +1 -1
- package/src/build-recipe.ts +198 -0
- package/src/build.ts +60 -4
- package/src/bun.ts +181 -13
- package/src/cloudflare/application.ts +822 -0
- package/src/cloudflare/bindings.ts +159 -0
- package/src/cloudflare/export.ts +641 -0
- package/src/cloudflare/import.ts +1021 -0
- package/src/cloudflare/maintenance.ts +518 -0
- package/src/cloudflare/names.ts +16 -0
- package/src/cloudflare/ops.ts +375 -0
- package/src/cloudflare/r2-files.ts +172 -0
- package/src/cloudflare/restore.ts +167 -0
- package/src/cloudflare/tracker.ts +192 -0
- package/src/cloudflare.ts +100 -146
- package/src/contract/app-id.ts +5 -0
- package/src/contract/artifact-schema.ts +9 -1
- package/src/contract/build.ts +40 -2
- package/src/contract/compile.ts +1 -0
- package/src/contract/context.ts +35 -9
- package/src/contract/declaration.ts +2 -0
- package/src/contract/defaults.ts +22 -2
- package/src/contract/define.ts +29 -3
- package/src/contract/kit.ts +4 -1
- package/src/contract/platform.ts +3 -2
- package/src/contract/vocabulary.ts +26 -8
- package/src/dev/launcher.ts +50 -4
- package/src/disk-assets.ts +8 -3
- package/src/files/archive.ts +2 -2
- package/src/files/job-names.ts +12 -0
- package/src/files/kinds.ts +73 -31
- package/src/files/scan.ts +3 -4
- package/src/files/streams.ts +3 -1
- package/src/implement/anonymous.ts +12 -11
- package/src/implement/boot.ts +75 -6
- package/src/implement/bounded.ts +21 -2
- package/src/implement/capability.ts +15 -3
- package/src/implement/dispatch.ts +419 -177
- package/src/implement/failures.ts +17 -1
- package/src/implement/faults.ts +1 -0
- package/src/implement/index.ts +1 -0
- package/src/implement/invocation.ts +58 -10
- package/src/implement/principal.ts +25 -2
- package/src/implement/terminals.ts +93 -0
- package/src/implement/workload.ts +42 -16
- package/src/jobs/mail.ts +8 -0
- package/src/jobs/report.ts +5 -1
- package/src/jobs/runner.ts +672 -113
- package/src/jobs/schedule.ts +7 -1
- package/src/jobs/scrub.ts +14 -1
- package/src/jobs/types.ts +6 -0
- package/src/spa-fallback.ts +12 -0
- package/src/testing/endpoint-access.ts +307 -152
- package/src/testing/isolation.ts +8 -1
- package/src/testing/kit.ts +80 -2
- package/src/testing/proxy.ts +15 -10
- package/types/src/build-recipe.d.ts +70 -0
- package/types/src/build.d.ts +19 -1
- package/types/src/bun.d.ts +34 -1
- package/types/src/cloudflare/application.d.ts +73 -0
- package/types/src/cloudflare/bindings.d.ts +33 -0
- package/types/src/cloudflare/export.d.ts +16 -0
- package/types/src/cloudflare/import.d.ts +51 -0
- package/types/src/cloudflare/maintenance.d.ts +117 -0
- package/types/src/cloudflare/names.d.ts +11 -0
- package/types/src/cloudflare/ops.d.ts +81 -0
- package/types/src/cloudflare/r2-files.d.ts +9 -0
- package/types/src/cloudflare/restore.d.ts +9 -0
- package/types/src/cloudflare/tracker.d.ts +29 -0
- package/types/src/cloudflare.d.ts +12 -18
- package/types/src/contract/app-id.d.ts +2 -0
- package/types/src/contract/artifact-schema.d.ts +2 -0
- package/types/src/contract/context.d.ts +35 -11
- package/types/src/contract/declaration.d.ts +4 -0
- package/types/src/contract/defaults.d.ts +19 -1
- package/types/src/contract/define.d.ts +20 -3
- package/types/src/contract/kit.d.ts +2 -1
- package/types/src/contract/platform.d.ts +2 -2
- package/types/src/contract/vocabulary.d.ts +9 -7
- package/types/src/disk-assets.d.ts +4 -1
- package/types/src/files/job-names.d.ts +9 -0
- package/types/src/files/kinds.d.ts +21 -14
- package/types/src/files/scan.d.ts +1 -3
- package/types/src/implement/anonymous.d.ts +2 -4
- package/types/src/implement/boot.d.ts +17 -1
- package/types/src/implement/bounded.d.ts +11 -1
- package/types/src/implement/capability.d.ts +4 -2
- package/types/src/implement/dispatch.d.ts +34 -1
- package/types/src/implement/failures.d.ts +5 -0
- package/types/src/implement/faults.d.ts +1 -0
- package/types/src/implement/index.d.ts +1 -0
- package/types/src/implement/invocation.d.ts +22 -7
- package/types/src/implement/principal.d.ts +6 -1
- package/types/src/implement/terminals.d.ts +33 -0
- package/types/src/implement/workload.d.ts +5 -0
- package/types/src/jobs/runner.d.ts +66 -2
- package/types/src/jobs/schedule.d.ts +2 -0
- package/types/src/jobs/types.d.ts +6 -0
- package/types/src/spa-fallback.d.ts +2 -0
- package/types/src/testing/endpoint-access.d.ts +37 -5
- package/types/src/testing/isolation.d.ts +1 -1
- package/types/src/testing/kit.d.ts +22 -1
- package/types/src/testing/proxy.d.ts +4 -4
|
@@ -0,0 +1,1021 @@
|
|
|
1
|
+
// One step of loading a consistent backup into a fresh object (ADR-0070 §6): the ops operation `import`, served before
|
|
2
|
+
// the application boots. Control drives the steps and holds their state; the object keeps only its import marker and
|
|
3
|
+
// the steps' receipts in KV storage, which every step that writes updates in the same `transactionSync` as its rows.
|
|
4
|
+
// Every step runs in the maintenance barrier's turn, so no transition interleaves with it.
|
|
5
|
+
|
|
6
|
+
import {
|
|
7
|
+
decodeOpsDumpLine,
|
|
8
|
+
encodeOpsDumpRow,
|
|
9
|
+
foldOpsTableHash,
|
|
10
|
+
OPS_DATA_REFUSED_STATUS,
|
|
11
|
+
OPS_DUMP_KEY,
|
|
12
|
+
OPS_DUMP_MANIFEST_MAX_BYTES,
|
|
13
|
+
OPS_DUMP_PAGE_ROWS,
|
|
14
|
+
OPS_DUMP_PART_MAX_RAW_BYTES,
|
|
15
|
+
OPS_DUMP_PART_MAX_ROWS,
|
|
16
|
+
OPS_DUMP_PART_SIZE,
|
|
17
|
+
OPS_EMPTY_TABLE_HASH,
|
|
18
|
+
OPS_STALE_STATUS,
|
|
19
|
+
type OpsDataRefusal,
|
|
20
|
+
type OpsDataRefusalReason,
|
|
21
|
+
type OpsDumpBinding,
|
|
22
|
+
opsDumpColumnProjection,
|
|
23
|
+
type OpsDumpCursor,
|
|
24
|
+
type OpsDumpLine,
|
|
25
|
+
type OpsDumpManifest,
|
|
26
|
+
type OpsDumpSchema,
|
|
27
|
+
type OpsDumpSchemaColumn,
|
|
28
|
+
type OpsDumpSchemaEntry,
|
|
29
|
+
type OpsDumpSchemaForeignKey,
|
|
30
|
+
type OpsDumpSchemaIndex,
|
|
31
|
+
type OpsDumpSchemaTable,
|
|
32
|
+
type OpsDumpSource,
|
|
33
|
+
type OpsDumpTable,
|
|
34
|
+
type OpsDumpTablePlan,
|
|
35
|
+
type OpsDumpValue,
|
|
36
|
+
opsDumpValueBinding,
|
|
37
|
+
type OpsImportBeginAck,
|
|
38
|
+
type OpsImportCompleteAck,
|
|
39
|
+
type OpsImportDiscardAck,
|
|
40
|
+
type OpsImportLinkAck,
|
|
41
|
+
type OpsImportPartAck,
|
|
42
|
+
type OpsImportRequest,
|
|
43
|
+
type OpsImportVerifyAck,
|
|
44
|
+
type OpsStaleRefusal,
|
|
45
|
+
parseOpsImportCompleteAck,
|
|
46
|
+
parseOpsImportLinkAck,
|
|
47
|
+
parseOpsImportPartAck,
|
|
48
|
+
parseOpsImportVerifyAck,
|
|
49
|
+
planOpsDump,
|
|
50
|
+
readOpsDumpManifest,
|
|
51
|
+
readOpsDumpValue,
|
|
52
|
+
sha256Hex,
|
|
53
|
+
} from '@fabrika/auth-core'
|
|
54
|
+
import type { SqliteStorage, SqliteValue } from '@fabrika/db/sqlite'
|
|
55
|
+
import { migrateSqlite, type SqliteMigrationBundle } from '@fabrika/db/sqlite-migrate'
|
|
56
|
+
import { jsonResponse } from '../errors.js'
|
|
57
|
+
import { method } from './bindings.js'
|
|
58
|
+
import type { MaintenanceBarrier, TurnStep } from './maintenance.js'
|
|
59
|
+
import type { OpsTarget } from './ops.js'
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* One step of loading a dump into a fresh object (ADR-0070); `serving` is the object of ordinary traffic. A runtime
|
|
63
|
+
* whose storage has no synchronous KV or `deleteAll` does not serve the operation.
|
|
64
|
+
*/
|
|
65
|
+
export async function serveImport(target: OpsTarget, barrier: MaintenanceBarrier, step: OpsImportRequest, serving: string): Promise<Response> {
|
|
66
|
+
const { storage } = target
|
|
67
|
+
const { kv, deleteAll } = storage
|
|
68
|
+
if (kv === undefined || deleteAll === undefined) {
|
|
69
|
+
return jsonResponse({ refused: 'not_implemented' }, 501)
|
|
70
|
+
}
|
|
71
|
+
const importing: ImportStorage = {
|
|
72
|
+
sql: storage.sql,
|
|
73
|
+
transaction: (closure) => storage.transaction(closure),
|
|
74
|
+
transactionSync: (closure) => storage.transactionSync(closure),
|
|
75
|
+
kv,
|
|
76
|
+
deleteAll: () => deleteAll.call(storage),
|
|
77
|
+
}
|
|
78
|
+
return await importStep({ storage: importing, bucket: target.bucket, migrations: target.migrations, serving }, barrier, step)
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/** workerd's synchronous KV storage of a SQLite-backed object: it lives in the same database, so it commits with a transaction. */
|
|
82
|
+
export interface ImportKv {
|
|
83
|
+
get(key: string): unknown
|
|
84
|
+
put(key: string, value: unknown): void
|
|
85
|
+
delete(key: string): boolean
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/** What the importer needs of the object's storage; a SQLite-backed `DurableObjectStorage` fits it. */
|
|
89
|
+
export interface ImportStorage extends SqliteStorage {
|
|
90
|
+
readonly kv: ImportKv
|
|
91
|
+
/** Wipes the object's database, its KV storage and its alarm. */
|
|
92
|
+
deleteAll(): Promise<void>
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/** The object an import step loads into. */
|
|
96
|
+
export interface ImportTarget {
|
|
97
|
+
readonly storage: ImportStorage
|
|
98
|
+
/** The environment's bucket (the `FABRIKA_FILES` R2 binding), where the backup lies. */
|
|
99
|
+
readonly bucket: unknown
|
|
100
|
+
/** The release's migration bundles in the order the object applies them: the framework's, then the application's. */
|
|
101
|
+
readonly migrations: readonly SqliteMigrationBundle[]
|
|
102
|
+
/** The object of ordinary traffic (`FABRIKA_APPLICATION_OBJECT`), which `discard` never wipes. */
|
|
103
|
+
readonly serving: string
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/** The KV key of the import marker. Invariant: an object that holds it never boots and never leaves `inspect`. */
|
|
107
|
+
export const IMPORT_MARKER_KEY = 'fabrika:import'
|
|
108
|
+
|
|
109
|
+
/** Whether the object holds an import marker, so it must neither boot nor leave `inspect`. */
|
|
110
|
+
export function importInProgress(storage: { readonly kv: ImportKv }): boolean {
|
|
111
|
+
return storage.kv.get(IMPORT_MARKER_KEY) !== undefined
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/** The KV key under which part `part` records its acknowledgement once loaded, so a retry loads nothing twice. */
|
|
115
|
+
function loadedPartKey(part: number): string {
|
|
116
|
+
return `${IMPORT_MARKER_KEY}:part:${part}`
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
/** The KV key under which part `part`'s link pass records its acknowledgement. */
|
|
120
|
+
function linkedPartKey(part: number): string {
|
|
121
|
+
return `${IMPORT_MARKER_KEY}:link:${part}`
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/**
|
|
125
|
+
* The KV key of the receipt `complete` leaves for `source`, so a retried `complete` answers the same. It blocks nothing
|
|
126
|
+
* and stays for the object's life: `begin` takes only a fresh object, so an object completes one import.
|
|
127
|
+
*/
|
|
128
|
+
function completedKey(source: OpsDumpSource): string {
|
|
129
|
+
return `${IMPORT_MARKER_KEY}:complete:${source.object}/${source.dump}`
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* The marker: the backup the object is loading, whether `begin` cleared the target, how many parts are loaded, and the
|
|
134
|
+
* verification of what is loaded now: `null` for none, else the cursor its next step must carry (`null` once every
|
|
135
|
+
* table matched the manifest) and the last step's cursor with its answer, which a retry of that step gets again. A
|
|
136
|
+
* part or a link pass that writes resets it.
|
|
137
|
+
*/
|
|
138
|
+
interface ImportMarker {
|
|
139
|
+
readonly source: OpsDumpSource
|
|
140
|
+
readonly begun: boolean
|
|
141
|
+
readonly loaded: number
|
|
142
|
+
readonly verified: Verification | null
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
interface Verification {
|
|
146
|
+
readonly next: string | null
|
|
147
|
+
/** `null` after a step that started the verification, which a retry starts again. */
|
|
148
|
+
readonly replay: { readonly cursor: string; readonly ack: OpsImportVerifyAck } | null
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/**
|
|
152
|
+
* One step of loading the consistent backup `step.source` into the fresh object `target` (ADR-0070 §6), in the
|
|
153
|
+
* barrier's turn: ADR-0069's stale refusal unless `step.generation` is the published one, then `serving_object` on the
|
|
154
|
+
* object of ordinary traffic, then `wrong_mode` unless `inspect`. The turn is held across the step's R2 reads, so the
|
|
155
|
+
* generation and mode it was admitted under still hold when it commits.
|
|
156
|
+
*/
|
|
157
|
+
export async function importStep(target: ImportTarget, barrier: MaintenanceBarrier, step: OpsImportRequest): Promise<Response> {
|
|
158
|
+
const result = await barrier.step(step.generation, async (mode): Promise<TurnStep<Response>> => {
|
|
159
|
+
if (step.object === target.serving) {
|
|
160
|
+
return answered(refused('serving_object'))
|
|
161
|
+
}
|
|
162
|
+
if (mode !== 'inspect') {
|
|
163
|
+
return answered(refused('wrong_mode'))
|
|
164
|
+
}
|
|
165
|
+
if (step.step !== 'verify') {
|
|
166
|
+
await barrier.restrictedWrite()
|
|
167
|
+
}
|
|
168
|
+
return answered(await admittedStep(target, step))
|
|
169
|
+
})
|
|
170
|
+
if (result.kind === 'ran') {
|
|
171
|
+
return result.value
|
|
172
|
+
}
|
|
173
|
+
const refusal: OpsStaleRefusal = { refused: 'stale_generation', published: result.published }
|
|
174
|
+
return jsonResponse(refusal, OPS_STALE_STATUS)
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
async function admittedStep(target: ImportTarget, step: OpsImportRequest): Promise<Response> {
|
|
178
|
+
const { kv } = target.storage
|
|
179
|
+
if (step.step === 'discard') {
|
|
180
|
+
return await discard(target)
|
|
181
|
+
}
|
|
182
|
+
if (step.step === 'begin') {
|
|
183
|
+
return await begin(target, step.source)
|
|
184
|
+
}
|
|
185
|
+
if (importing(kv, step.source) === null) {
|
|
186
|
+
const completed = step.step === 'complete' ? readCompleted(kv, step.source) : null
|
|
187
|
+
return completed === null ? refused('no_import') : jsonResponse(completed, 200)
|
|
188
|
+
}
|
|
189
|
+
const manifest = await readManifest(target.bucket, step.source)
|
|
190
|
+
if (manifest === null) {
|
|
191
|
+
return refused('invalid_manifest')
|
|
192
|
+
}
|
|
193
|
+
if (step.step === 'complete') {
|
|
194
|
+
return complete(target.storage, step.source, manifest)
|
|
195
|
+
}
|
|
196
|
+
const plan = currentPlan(target.storage, manifest)
|
|
197
|
+
if ('refused' in plan) {
|
|
198
|
+
return jsonResponse(plan, OPS_DATA_REFUSED_STATUS)
|
|
199
|
+
}
|
|
200
|
+
switch (step.step) {
|
|
201
|
+
case 'part':
|
|
202
|
+
return await loadPart(target, step.source, manifest, plan, step.part)
|
|
203
|
+
case 'link':
|
|
204
|
+
return await linkPart(target, step.source, manifest, plan, step.part)
|
|
205
|
+
case 'verify':
|
|
206
|
+
return verify(target.storage, step.source, manifest, plan, step.cursor)
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
function answered(response: Response): TurnStep<Response> {
|
|
211
|
+
return { value: response, holdUntil: null }
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
/** The marker, when it names `source` and records `begin` done; `null` otherwise. */
|
|
215
|
+
function importing(kv: ImportKv, source: OpsDumpSource): ImportMarker | null {
|
|
216
|
+
const marker = readMarker(kv)
|
|
217
|
+
return marker?.begun === true && sameSource(marker.source, source) ? marker : null
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
/**
|
|
221
|
+
* Verifies the manifest, checks the target is fresh, writes the marker, migrates the target to exactly the manifest's
|
|
222
|
+
* ledger, checks the schema and its plan, then clears the target in one transaction that records `begin` done. A retry
|
|
223
|
+
* under the same backup resumes after what the marker records.
|
|
224
|
+
*/
|
|
225
|
+
async function begin(target: ImportTarget, source: OpsDumpSource): Promise<Response> {
|
|
226
|
+
const { storage } = target
|
|
227
|
+
const manifest = await readManifest(target.bucket, source)
|
|
228
|
+
if (manifest === null) {
|
|
229
|
+
return refused('invalid_manifest')
|
|
230
|
+
}
|
|
231
|
+
if (!manifest.consistent) {
|
|
232
|
+
return refused('fuzzy_dump')
|
|
233
|
+
}
|
|
234
|
+
const marker = readMarker(storage.kv)
|
|
235
|
+
if (marker !== null && !sameSource(marker.source, source)) {
|
|
236
|
+
return refused('not_fresh')
|
|
237
|
+
}
|
|
238
|
+
if (marker?.begun === true) {
|
|
239
|
+
return beginAck(manifest)
|
|
240
|
+
}
|
|
241
|
+
if (marker === null && holdsTables(storage)) {
|
|
242
|
+
return refused('not_fresh')
|
|
243
|
+
}
|
|
244
|
+
const bundles = ledgerBundles(target.migrations, manifest)
|
|
245
|
+
if (bundles === null) {
|
|
246
|
+
return refused('ledger_mismatch')
|
|
247
|
+
}
|
|
248
|
+
if (marker === null) {
|
|
249
|
+
writeMarker(storage.kv, { source, begun: false, loaded: 0, verified: null })
|
|
250
|
+
}
|
|
251
|
+
// A database the framework never migrated has no ledger, and gets none.
|
|
252
|
+
if (manifest.migrations.length > 0) {
|
|
253
|
+
await migrateSqlite(
|
|
254
|
+
{ exec: (query, ...bindings) => storage.sql.exec(query, ...bindings), transactionSync: (fn) => storage.transactionSync(fn) },
|
|
255
|
+
bundles,
|
|
256
|
+
)
|
|
257
|
+
}
|
|
258
|
+
if (!sameDdl(readDdl(storage), manifest.ddl)) {
|
|
259
|
+
return refused('schema_mismatch')
|
|
260
|
+
}
|
|
261
|
+
const plan = currentPlan(storage, manifest)
|
|
262
|
+
if ('refused' in plan) {
|
|
263
|
+
return jsonResponse(plan, OPS_DATA_REFUSED_STATUS)
|
|
264
|
+
}
|
|
265
|
+
storage.transactionSync(() => {
|
|
266
|
+
for (const entry of readDdl(storage)) {
|
|
267
|
+
if (entry.type === 'trigger') {
|
|
268
|
+
storage.sql.exec(`DROP TRIGGER ${quote(entry.name)}`)
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
// Every deferred column takes a temporary NULL (the schema rule), so no row of its cycle references another one after this.
|
|
272
|
+
for (const table of plan) {
|
|
273
|
+
if (table.deferred.length > 0) {
|
|
274
|
+
storage.sql.exec(`UPDATE ${quote(table.name)} SET ${table.deferred.map((column) => `${quote(column)} = NULL`).join(', ')}`)
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
for (const table of [...plan].reverse()) {
|
|
278
|
+
storage.sql.exec(`DELETE FROM ${quote(table.name)}`)
|
|
279
|
+
}
|
|
280
|
+
writeMarker(storage.kv, { source, begun: true, loaded: 0, verified: null })
|
|
281
|
+
})
|
|
282
|
+
return beginAck(manifest)
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
function beginAck(manifest: OpsDumpManifest): Response {
|
|
286
|
+
const ack: OpsImportBeginAck = { step: 'begin', parts: manifest.parts.length }
|
|
287
|
+
return jsonResponse(ack, 200)
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
/**
|
|
291
|
+
* Inserts part `part`'s rows with their rowids, every value through the wire's binding and every deferred column as
|
|
292
|
+
* NULL, in one transaction that reads the marker and the part's receipt again, records the part and resets the
|
|
293
|
+
* verification.
|
|
294
|
+
*/
|
|
295
|
+
async function loadPart(target: ImportTarget, source: OpsDumpSource, manifest: OpsDumpManifest, plan: readonly OpsDumpTablePlan[], part: number) {
|
|
296
|
+
const { storage } = target
|
|
297
|
+
const recorded = readLoadedPart(storage.kv, part)
|
|
298
|
+
if (recorded !== null) {
|
|
299
|
+
return jsonResponse(recorded, 200)
|
|
300
|
+
}
|
|
301
|
+
const lines = await readPart(target.bucket, source, manifest, plan, part)
|
|
302
|
+
if (!Array.isArray(lines)) {
|
|
303
|
+
return jsonResponse(lines, OPS_DATA_REFUSED_STATUS)
|
|
304
|
+
}
|
|
305
|
+
const tables = new Map(plan.map((table) => [table.name, table]))
|
|
306
|
+
const ack = storage.transactionSync((): OpsImportPartAck | OpsDataRefusal => {
|
|
307
|
+
const marker = importing(storage.kv, source)
|
|
308
|
+
if (marker === null) {
|
|
309
|
+
return { refused: 'no_import', table: null }
|
|
310
|
+
}
|
|
311
|
+
const again = readLoadedPart(storage.kv, part)
|
|
312
|
+
if (again !== null) {
|
|
313
|
+
return again
|
|
314
|
+
}
|
|
315
|
+
let deferred = 0
|
|
316
|
+
for (const { table: name, row } of lines) {
|
|
317
|
+
const table = planned(tables, name)
|
|
318
|
+
if (insertRow(storage, table, row)) {
|
|
319
|
+
deferred++
|
|
320
|
+
}
|
|
321
|
+
}
|
|
322
|
+
const loaded: OpsImportPartAck = { step: 'part', part, rows: lines.length, deferred }
|
|
323
|
+
storage.kv.put(loadedPartKey(part), JSON.stringify(loaded))
|
|
324
|
+
writeMarker(storage.kv, { ...marker, loaded: marker.loaded + 1, verified: null })
|
|
325
|
+
return loaded
|
|
326
|
+
})
|
|
327
|
+
return jsonResponse(ack, 'refused' in ack ? OPS_DATA_REFUSED_STATUS : 200)
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
/** Writes one row; answers whether it has a deferred key for the link pass to set. */
|
|
331
|
+
function insertRow(storage: SqliteStorage, table: OpsDumpTablePlan, row: readonly OpsDumpValue[]): boolean {
|
|
332
|
+
const [rowid, ...values] = row
|
|
333
|
+
const key = opsDumpValueBinding(rowid ?? null)
|
|
334
|
+
const columns: string[] = [table.keyset]
|
|
335
|
+
const bindings = [key]
|
|
336
|
+
let waits = false
|
|
337
|
+
for (const [index, column] of table.columns.entries()) {
|
|
338
|
+
const value = values[index] ?? null
|
|
339
|
+
// An INTEGER PRIMARY KEY column is the rowid itself, written once.
|
|
340
|
+
if (column === table.keyset) {
|
|
341
|
+
continue
|
|
342
|
+
}
|
|
343
|
+
const deferred = table.deferred.includes(column)
|
|
344
|
+
waits ||= deferred && value !== null
|
|
345
|
+
columns.push(column)
|
|
346
|
+
bindings.push(opsDumpValueBinding(deferred ? null : value))
|
|
347
|
+
}
|
|
348
|
+
if (table.name === SQLITE_SEQUENCE) {
|
|
349
|
+
// Loading an AUTOINCREMENT table writes its own counter row; the source's row replaces it, whichever table loads first.
|
|
350
|
+
const name = opsDumpValueBinding(values[0] ?? null)
|
|
351
|
+
storage.sql.exec(`DELETE FROM ${SQLITE_SEQUENCE} WHERE rowid = ${key.sql} OR name = ${name.sql}`, ...params([key, name]))
|
|
352
|
+
}
|
|
353
|
+
storage.sql.exec(
|
|
354
|
+
`INSERT INTO ${quote(table.name)} (${columns.map(quote).join(', ')}) VALUES (${bindings.map((binding) => binding.sql).join(', ')})`,
|
|
355
|
+
...params(bindings),
|
|
356
|
+
)
|
|
357
|
+
return waits
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
/**
|
|
361
|
+
* Sets the deferred keys of part `part`'s rows by their rowids, once every part is loaded, in one transaction that
|
|
362
|
+
* reads the marker and the pass's receipt again, records the pass and resets the verification.
|
|
363
|
+
*/
|
|
364
|
+
async function linkPart(target: ImportTarget, source: OpsDumpSource, manifest: OpsDumpManifest, plan: readonly OpsDumpTablePlan[], part: number) {
|
|
365
|
+
const { storage } = target
|
|
366
|
+
const recorded = readLinkedPart(storage.kv, part)
|
|
367
|
+
if (recorded !== null) {
|
|
368
|
+
return jsonResponse(recorded, 200)
|
|
369
|
+
}
|
|
370
|
+
if (importing(storage.kv, source)?.loaded !== manifest.parts.length) {
|
|
371
|
+
return refused('part_mismatch')
|
|
372
|
+
}
|
|
373
|
+
const lines = await readPart(target.bucket, source, manifest, plan, part)
|
|
374
|
+
if (!Array.isArray(lines)) {
|
|
375
|
+
return jsonResponse(lines, OPS_DATA_REFUSED_STATUS)
|
|
376
|
+
}
|
|
377
|
+
const tables = new Map(plan.map((table) => [table.name, table]))
|
|
378
|
+
const ack = storage.transactionSync((): OpsImportLinkAck | OpsDataRefusal => {
|
|
379
|
+
const marker = importing(storage.kv, source)
|
|
380
|
+
if (marker === null) {
|
|
381
|
+
return { refused: 'no_import', table: null }
|
|
382
|
+
}
|
|
383
|
+
if (marker.loaded !== manifest.parts.length) {
|
|
384
|
+
return { refused: 'part_mismatch', table: null }
|
|
385
|
+
}
|
|
386
|
+
const again = readLinkedPart(storage.kv, part)
|
|
387
|
+
if (again !== null) {
|
|
388
|
+
return again
|
|
389
|
+
}
|
|
390
|
+
let linked = 0
|
|
391
|
+
for (const { table: name, row } of lines) {
|
|
392
|
+
const table = planned(tables, name)
|
|
393
|
+
const [rowid, ...values] = row
|
|
394
|
+
const deferred = table.columns.flatMap((column, index) => table.deferred.includes(column) ? [{ column, value: values[index] ?? null }] : [])
|
|
395
|
+
if (deferred.every(({ value }) => value === null)) {
|
|
396
|
+
continue
|
|
397
|
+
}
|
|
398
|
+
const sets = deferred.map(({ column, value }) => ({ column, binding: opsDumpValueBinding(value) }))
|
|
399
|
+
const key = opsDumpValueBinding(rowid ?? null)
|
|
400
|
+
storage.sql.exec(
|
|
401
|
+
`UPDATE ${quote(table.name)} SET ${sets.map(({ column, binding }) => `${quote(column)} = ${binding.sql}`).join(', ')} WHERE ${
|
|
402
|
+
quote(table.keyset)
|
|
403
|
+
} = ${key.sql}`,
|
|
404
|
+
...params([...sets.map(({ binding }) => binding), key]),
|
|
405
|
+
)
|
|
406
|
+
linked++
|
|
407
|
+
}
|
|
408
|
+
const done: OpsImportLinkAck = { step: 'link', part, rows: linked }
|
|
409
|
+
storage.kv.put(linkedPartKey(part), JSON.stringify(done))
|
|
410
|
+
writeMarker(storage.kv, { ...marker, verified: null })
|
|
411
|
+
return done
|
|
412
|
+
})
|
|
413
|
+
return jsonResponse(ack, 'refused' in ack ? OPS_DATA_REFUSED_STATUS : 200)
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
/**
|
|
417
|
+
* Reads the loaded tables back from `cursor` with the producer's projection and fold, at most a part's row bound per
|
|
418
|
+
* step, and compares each finished table with the manifest. A verification starts at `null`; any other cursor must be
|
|
419
|
+
* the one the marker records this verification handed out last, or the one its last step was sent, whose answer it
|
|
420
|
+
* repeats. It writes no row: only its progress, into the marker, which a mismatch clears.
|
|
421
|
+
*/
|
|
422
|
+
function verify(
|
|
423
|
+
storage: ImportStorage,
|
|
424
|
+
source: OpsDumpSource,
|
|
425
|
+
manifest: OpsDumpManifest,
|
|
426
|
+
plan: readonly OpsDumpTablePlan[],
|
|
427
|
+
cursor: OpsDumpCursor | null,
|
|
428
|
+
): Response {
|
|
429
|
+
const marker = importing(storage.kv, source)
|
|
430
|
+
if (marker === null) {
|
|
431
|
+
return refused('no_import')
|
|
432
|
+
}
|
|
433
|
+
const mismatch = (table: string | null): Response => {
|
|
434
|
+
writeMarker(storage.kv, { ...marker, verified: null })
|
|
435
|
+
return refused('verify_mismatch', table)
|
|
436
|
+
}
|
|
437
|
+
if (cursor !== null) {
|
|
438
|
+
const key = cursorKey(cursor)
|
|
439
|
+
const replay = marker.verified?.replay ?? null
|
|
440
|
+
// Its answer was lost: what is loaded has not changed since, or the verification would be reset.
|
|
441
|
+
if (replay?.cursor === key) {
|
|
442
|
+
return jsonResponse(replay.ack, 200)
|
|
443
|
+
}
|
|
444
|
+
if ((marker.verified?.next ?? null) !== key) {
|
|
445
|
+
return mismatch(cursor.table)
|
|
446
|
+
}
|
|
447
|
+
}
|
|
448
|
+
const tables = new Map(plan.map((table) => [table.name, table]))
|
|
449
|
+
const order = manifest.tables
|
|
450
|
+
let index = cursor === null ? 0 : order.findIndex((table) => table.name === cursor.table)
|
|
451
|
+
if (index < 0) {
|
|
452
|
+
return mismatch(cursor?.table ?? null)
|
|
453
|
+
}
|
|
454
|
+
const startOf = (table: OpsDumpTable | undefined): OpsDumpCursor | null =>
|
|
455
|
+
table === undefined ? null : { table: table.name, after: null, rows: 0, hash: OPS_EMPTY_TABLE_HASH }
|
|
456
|
+
let current = cursor ?? startOf(order[0])
|
|
457
|
+
const finished: OpsDumpTable[] = []
|
|
458
|
+
let budget = OPS_DUMP_PART_MAX_ROWS
|
|
459
|
+
while (current !== null && budget > 0) {
|
|
460
|
+
const expected = order[index]
|
|
461
|
+
if (expected === undefined) {
|
|
462
|
+
throw new Error('a verify cursor past the last table of the manifest')
|
|
463
|
+
}
|
|
464
|
+
const page = readPage(storage, planned(tables, expected.name), current.after)
|
|
465
|
+
const last = page.at(-1)?.[0]
|
|
466
|
+
current = {
|
|
467
|
+
table: expected.name,
|
|
468
|
+
after: last?.type === 'integer' ? last.value : current.after,
|
|
469
|
+
rows: current.rows + page.length,
|
|
470
|
+
hash: foldOpsTableHash(current.hash, page.map((row) => encodeOpsDumpRow(row))),
|
|
471
|
+
}
|
|
472
|
+
budget -= page.length
|
|
473
|
+
if (current.rows > expected.rows) {
|
|
474
|
+
return mismatch(expected.name)
|
|
475
|
+
}
|
|
476
|
+
if (page.length < OPS_DUMP_PAGE_ROWS) {
|
|
477
|
+
if (current.rows !== expected.rows || current.hash !== expected.hash) {
|
|
478
|
+
return mismatch(expected.name)
|
|
479
|
+
}
|
|
480
|
+
finished.push(expected)
|
|
481
|
+
index++
|
|
482
|
+
current = startOf(order[index])
|
|
483
|
+
}
|
|
484
|
+
}
|
|
485
|
+
const ack: OpsImportVerifyAck = { step: 'verify', tables: finished, next: current }
|
|
486
|
+
const next = current === null ? null : cursorKey(current)
|
|
487
|
+
writeMarker(storage.kv, { ...marker, verified: { next, replay: cursor === null ? null : { cursor: cursorKey(cursor), ack } } })
|
|
488
|
+
return jsonResponse(ack, 200)
|
|
489
|
+
}
|
|
490
|
+
|
|
491
|
+
/** A cursor as the marker records it: its fields in a fixed order, whatever order a request named them in. */
|
|
492
|
+
function cursorKey(cursor: OpsDumpCursor): string {
|
|
493
|
+
return JSON.stringify([cursor.table, cursor.after, cursor.rows, cursor.hash])
|
|
494
|
+
}
|
|
495
|
+
|
|
496
|
+
/** One keyset page of `table` after the rowid `after`: each row its rowid, then its columns, as the producer reads them. */
|
|
497
|
+
function readPage(storage: SqliteStorage, table: OpsDumpTablePlan, after: string | null): (readonly OpsDumpValue[])[] {
|
|
498
|
+
const keyset = quote(table.keyset)
|
|
499
|
+
// The CTE names each projected column, so no two result columns share a name whatever the table's columns are called.
|
|
500
|
+
const names = table.columns.flatMap((_, index) => [`t${index}`, `v${index}`])
|
|
501
|
+
const select = [keyset, `CAST(${keyset} AS TEXT)`, ...table.columns.map((column) => opsDumpColumnProjection(column))].join(', ')
|
|
502
|
+
const where = after === null ? '' : ` WHERE ${keyset} > CAST(? AS INTEGER)`
|
|
503
|
+
// Qualified, so a table named `page` is not the CTE itself.
|
|
504
|
+
const query =
|
|
505
|
+
`WITH page(r, k, ${names.join(', ')}) AS (SELECT ${select} FROM main.${quote(table.name)}${where} ORDER BY ${keyset} LIMIT ${OPS_DUMP_PAGE_ROWS}) `
|
|
506
|
+
+ `SELECT k, ${names.join(', ')} FROM page ORDER BY r`
|
|
507
|
+
const rows = after === null ? storage.sql.exec(query).toArray() : storage.sql.exec(query, after).toArray()
|
|
508
|
+
return rows.map((row) => [
|
|
509
|
+
readOpsDumpValue('integer', row['k']),
|
|
510
|
+
...table.columns.map((_, index) => readOpsDumpValue(row[`t${index}`], row[`v${index}`])),
|
|
511
|
+
])
|
|
512
|
+
}
|
|
513
|
+
|
|
514
|
+
/**
|
|
515
|
+
* Once every part is loaded, every link pass a part needs ran, and a verification of what is loaded now matched every
|
|
516
|
+
* table: recreates the triggers from the manifest's DDL, compares the whole schema again, clears the marker and the
|
|
517
|
+
* steps' receipts, and leaves the completion's receipt.
|
|
518
|
+
*/
|
|
519
|
+
function complete(storage: ImportStorage, source: OpsDumpSource, manifest: OpsDumpManifest): Response {
|
|
520
|
+
const answer = storage.transactionSync((): OpsImportCompleteAck | OpsDataRefusal => {
|
|
521
|
+
const marker = importing(storage.kv, source)
|
|
522
|
+
if (marker === null) {
|
|
523
|
+
return { refused: 'no_import', table: null }
|
|
524
|
+
}
|
|
525
|
+
const loaded = manifest.parts.map((part) => readLoadedPart(storage.kv, part.part))
|
|
526
|
+
const unlinked = loaded.some((ack) => ack === null || (ack.deferred > 0 && readLinkedPart(storage.kv, ack.part) === null))
|
|
527
|
+
if (marker.loaded !== manifest.parts.length || unlinked) {
|
|
528
|
+
return { refused: 'part_mismatch', table: null }
|
|
529
|
+
}
|
|
530
|
+
if (marker.verified === null || marker.verified.next !== null) {
|
|
531
|
+
return { refused: 'verify_mismatch', table: null }
|
|
532
|
+
}
|
|
533
|
+
const present = new Set(readDdl(storage).filter((entry) => entry.type === 'trigger').map((entry) => entry.name))
|
|
534
|
+
for (const entry of manifest.ddl) {
|
|
535
|
+
if (entry.type !== 'trigger' || present.has(entry.name)) {
|
|
536
|
+
continue
|
|
537
|
+
}
|
|
538
|
+
if (!TRIGGER_DDL.test(entry.sql)) {
|
|
539
|
+
return { refused: 'schema_mismatch', table: null }
|
|
540
|
+
}
|
|
541
|
+
storage.sql.exec(entry.sql)
|
|
542
|
+
}
|
|
543
|
+
if (!sameDdl(readDdl(storage), manifest.ddl)) {
|
|
544
|
+
return { refused: 'schema_mismatch', table: null }
|
|
545
|
+
}
|
|
546
|
+
for (const part of manifest.parts) {
|
|
547
|
+
storage.kv.delete(loadedPartKey(part.part))
|
|
548
|
+
storage.kv.delete(linkedPartKey(part.part))
|
|
549
|
+
}
|
|
550
|
+
storage.kv.delete(IMPORT_MARKER_KEY)
|
|
551
|
+
const ack: OpsImportCompleteAck = { step: 'complete', source: { object: source.object, dump: source.dump } }
|
|
552
|
+
storage.kv.put(completedKey(source), JSON.stringify(ack))
|
|
553
|
+
return ack
|
|
554
|
+
})
|
|
555
|
+
return jsonResponse(answer, 'refused' in answer ? OPS_DATA_REFUSED_STATUS : 200)
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
/** Wipes an `inspect` object that holds an import marker; the caller refused the object of ordinary traffic. */
|
|
559
|
+
async function discard(target: ImportTarget): Promise<Response> {
|
|
560
|
+
if (readMarker(target.storage.kv) === null) {
|
|
561
|
+
return refused('no_import')
|
|
562
|
+
}
|
|
563
|
+
await target.storage.deleteAll()
|
|
564
|
+
const ack: OpsImportDiscardAck = { step: 'discard' }
|
|
565
|
+
return jsonResponse(ack, 200)
|
|
566
|
+
}
|
|
567
|
+
|
|
568
|
+
/** The plan of the target's schema, which must list the manifest's tables in the manifest's order. */
|
|
569
|
+
function currentPlan(storage: SqliteStorage, manifest: OpsDumpManifest): readonly OpsDumpTablePlan[] | OpsDataRefusal {
|
|
570
|
+
const plan = planOpsDump(describeDumpSchema(storage))
|
|
571
|
+
if ('refused' in plan) {
|
|
572
|
+
return plan
|
|
573
|
+
}
|
|
574
|
+
const names = plan.tables.map((table) => table.name)
|
|
575
|
+
const expected = manifest.tables.map((table) => table.name)
|
|
576
|
+
if (names.length !== expected.length || names.some((name, index) => name !== expected[index])) {
|
|
577
|
+
return { refused: 'schema_mismatch', table: null }
|
|
578
|
+
}
|
|
579
|
+
return plan.tables
|
|
580
|
+
}
|
|
581
|
+
|
|
582
|
+
function planned(tables: ReadonlyMap<string, OpsDumpTablePlan>, name: string): OpsDumpTablePlan {
|
|
583
|
+
const table = tables.get(name)
|
|
584
|
+
if (table === undefined) {
|
|
585
|
+
throw new Error('a dump line names a table the plan does not hold')
|
|
586
|
+
}
|
|
587
|
+
return table
|
|
588
|
+
}
|
|
589
|
+
|
|
590
|
+
/**
|
|
591
|
+
* Part `part`'s lines: its byte range read and checked against the manifest's size and SHA-256, decompressed up to the
|
|
592
|
+
* raw bound, at most the row bound, each line in the wire's encoding and fitting its table's plan.
|
|
593
|
+
*/
|
|
594
|
+
async function readPart(
|
|
595
|
+
bucket: unknown,
|
|
596
|
+
source: OpsDumpSource,
|
|
597
|
+
manifest: OpsDumpManifest,
|
|
598
|
+
plan: readonly OpsDumpTablePlan[],
|
|
599
|
+
part: number,
|
|
600
|
+
): Promise<OpsDumpLine[] | OpsDataRefusal> {
|
|
601
|
+
const mismatch: OpsDataRefusal = { refused: 'part_mismatch', table: null }
|
|
602
|
+
const described = manifest.parts[part - 1]
|
|
603
|
+
if (described === undefined) {
|
|
604
|
+
return mismatch
|
|
605
|
+
}
|
|
606
|
+
// Every part but the last is exactly the fixed size, so part n starts at (n - 1) parts.
|
|
607
|
+
const body = await r2Get(bucket, OPS_DUMP_KEY({ ...source, consistent: true }, 'data'), {
|
|
608
|
+
range: { offset: (part - 1) * OPS_DUMP_PART_SIZE, length: described.size },
|
|
609
|
+
})
|
|
610
|
+
const bytes = body === null ? null : await bodyBytes(body)
|
|
611
|
+
if (bytes === null || bytes.length !== described.size || (await sha256OfBytes(bytes)) !== described.sha256) {
|
|
612
|
+
return mismatch
|
|
613
|
+
}
|
|
614
|
+
const text = await gunzipWithin(bytes, OPS_DUMP_PART_MAX_RAW_BYTES)
|
|
615
|
+
const raw = text?.split('\n')
|
|
616
|
+
// Each line ends with its newline, so the text ends with one.
|
|
617
|
+
if (raw === undefined || raw.pop() !== '' || raw.length > OPS_DUMP_PART_MAX_ROWS) {
|
|
618
|
+
return mismatch
|
|
619
|
+
}
|
|
620
|
+
const tables = new Map(plan.map((table) => [table.name, table]))
|
|
621
|
+
const lines: OpsDumpLine[] = []
|
|
622
|
+
for (const encoded of raw) {
|
|
623
|
+
const line = decodeOpsDumpLine(encoded)
|
|
624
|
+
const table = line === null ? undefined : tables.get(line.table)
|
|
625
|
+
if (line === null || table === undefined) {
|
|
626
|
+
return mismatch
|
|
627
|
+
}
|
|
628
|
+
if (line.row.length !== table.columns.length + 1 || line.row[0]?.type !== 'integer') {
|
|
629
|
+
return { refused: 'part_mismatch', table: table.name }
|
|
630
|
+
}
|
|
631
|
+
lines.push(line)
|
|
632
|
+
}
|
|
633
|
+
return lines
|
|
634
|
+
}
|
|
635
|
+
|
|
636
|
+
/** `bytes` as one gzip member, or `null` when it does not decompress, decompresses past `bound`, or is not UTF-8. */
|
|
637
|
+
async function gunzipWithin(bytes: Uint8Array, bound: number): Promise<string | null> {
|
|
638
|
+
const reader = new Blob([bytes.slice()]).stream().pipeThrough(new DecompressionStream('gzip')).getReader()
|
|
639
|
+
const chunks: Uint8Array[] = []
|
|
640
|
+
let total = 0
|
|
641
|
+
try {
|
|
642
|
+
for (let read = await reader.read(); !read.done; read = await reader.read()) {
|
|
643
|
+
total += read.value.byteLength
|
|
644
|
+
if (total > bound) {
|
|
645
|
+
await reader.cancel()
|
|
646
|
+
return null
|
|
647
|
+
}
|
|
648
|
+
chunks.push(read.value)
|
|
649
|
+
}
|
|
650
|
+
} catch {
|
|
651
|
+
return null
|
|
652
|
+
}
|
|
653
|
+
const raw = new Uint8Array(total)
|
|
654
|
+
let offset = 0
|
|
655
|
+
for (const chunk of chunks) {
|
|
656
|
+
raw.set(chunk, offset)
|
|
657
|
+
offset += chunk.byteLength
|
|
658
|
+
}
|
|
659
|
+
try {
|
|
660
|
+
return new TextDecoder('utf-8', { fatal: true }).decode(raw)
|
|
661
|
+
} catch {
|
|
662
|
+
return null
|
|
663
|
+
}
|
|
664
|
+
}
|
|
665
|
+
|
|
666
|
+
async function sha256OfBytes(bytes: Uint8Array): Promise<string> {
|
|
667
|
+
const digest = await crypto.subtle.digest('SHA-256', bytes.slice())
|
|
668
|
+
return [...new Uint8Array(digest)].map((byte) => byte.toString(16).padStart(2, '0')).join('')
|
|
669
|
+
}
|
|
670
|
+
|
|
671
|
+
/** The manifest at the backup's key, or `null` when it is missing, too large, does not parse or names another dump. */
|
|
672
|
+
async function readManifest(bucket: unknown, source: OpsDumpSource): Promise<OpsDumpManifest | null> {
|
|
673
|
+
const body = await r2Get(bucket, OPS_DUMP_KEY({ ...source, consistent: true }, 'manifest'))
|
|
674
|
+
if (body === null) {
|
|
675
|
+
return null
|
|
676
|
+
}
|
|
677
|
+
const size: unknown = typeof body === 'object' ? Reflect.get(body, 'size') : undefined
|
|
678
|
+
if (typeof size === 'number' && size > OPS_DUMP_MANIFEST_MAX_BYTES) {
|
|
679
|
+
return null
|
|
680
|
+
}
|
|
681
|
+
const text = method(body, 'text')
|
|
682
|
+
if (text === null) {
|
|
683
|
+
throw new Error('R2 answered the manifest without its body')
|
|
684
|
+
}
|
|
685
|
+
const content: unknown = await text()
|
|
686
|
+
const manifest = typeof content === 'string' ? readOpsDumpManifest(content) : null
|
|
687
|
+
return manifest !== null && manifest.object === source.object && manifest.dump === source.dump ? manifest : null
|
|
688
|
+
}
|
|
689
|
+
|
|
690
|
+
/** R2's `get` on the bucket, by method lookup at the binding's edge; `null` for a missing object. */
|
|
691
|
+
async function r2Get(bucket: unknown, key: string, options?: { readonly range: { readonly offset: number; readonly length: number } }) {
|
|
692
|
+
const get = method(bucket, 'get')
|
|
693
|
+
if (get === null) {
|
|
694
|
+
throw new Error("the FABRIKA_FILES binding has no get; a backup lies in the environment's bucket")
|
|
695
|
+
}
|
|
696
|
+
const body: unknown = await (options === undefined ? get(key) : get(key, options))
|
|
697
|
+
return body ?? null
|
|
698
|
+
}
|
|
699
|
+
|
|
700
|
+
async function bodyBytes(body: unknown): Promise<Uint8Array> {
|
|
701
|
+
const arrayBuffer = method(body, 'arrayBuffer')
|
|
702
|
+
if (arrayBuffer === null) {
|
|
703
|
+
throw new Error('R2 answered a dump part without its body')
|
|
704
|
+
}
|
|
705
|
+
const bytes: unknown = await arrayBuffer()
|
|
706
|
+
if (!(bytes instanceof ArrayBuffer)) {
|
|
707
|
+
throw new Error('R2 answered a dump part body that is not bytes')
|
|
708
|
+
}
|
|
709
|
+
return new Uint8Array(bytes)
|
|
710
|
+
}
|
|
711
|
+
|
|
712
|
+
/**
|
|
713
|
+
* Each bundle of the release cut to the manifest's ledger, which each must begin with exactly (by name and checksum);
|
|
714
|
+
* `null` for any difference, a bundle or a file the release lacks included.
|
|
715
|
+
*/
|
|
716
|
+
function ledgerBundles(release: readonly SqliteMigrationBundle[], manifest: OpsDumpManifest): readonly SqliteMigrationBundle[] | null {
|
|
717
|
+
const names = new Set(release.map((bundle) => bundle.bundle))
|
|
718
|
+
if (manifest.migrations.some((migration) => !names.has(migration.bundle))) {
|
|
719
|
+
return null
|
|
720
|
+
}
|
|
721
|
+
const bundles: SqliteMigrationBundle[] = []
|
|
722
|
+
for (const bundle of release) {
|
|
723
|
+
const recorded = manifest.migrations.filter((migration) => migration.bundle === bundle.bundle).sort((left, right) =>
|
|
724
|
+
compareNames(left.name, right.name)
|
|
725
|
+
)
|
|
726
|
+
const files = [...bundle.migrations].sort((left, right) => compareNames(left.name, right.name))
|
|
727
|
+
const prefix = files.slice(0, recorded.length)
|
|
728
|
+
const same = prefix.length === recorded.length
|
|
729
|
+
&& prefix.every((file, index) => file.name === recorded[index]?.name && sha256Hex(file.sql) === recorded[index]?.checksum)
|
|
730
|
+
if (!same) {
|
|
731
|
+
return null
|
|
732
|
+
}
|
|
733
|
+
bundles.push({ bundle: bundle.bundle, migrations: prefix })
|
|
734
|
+
}
|
|
735
|
+
return bundles
|
|
736
|
+
}
|
|
737
|
+
|
|
738
|
+
const SQLITE_SEQUENCE = 'sqlite_sequence'
|
|
739
|
+
|
|
740
|
+
/** `CREATE TRIGGER` as `sqlite_schema` stores it, the only DDL `complete` runs from the manifest. */
|
|
741
|
+
const TRIGGER_DDL = /^CREATE\s+(?:TEMP\s+|TEMPORARY\s+)?TRIGGER\s/i
|
|
742
|
+
|
|
743
|
+
/** Whether a dump holds `name`: no SQLite internal table but `sqlite_sequence`, and none of Cloudflare's `_cf_` ones. */
|
|
744
|
+
function dumped(name: string): boolean {
|
|
745
|
+
return !name.startsWith('_cf_') && (!name.startsWith('sqlite_') || name === SQLITE_SEQUENCE)
|
|
746
|
+
}
|
|
747
|
+
|
|
748
|
+
/** Whether the object holds any table but Cloudflare's own. */
|
|
749
|
+
function holdsTables(storage: SqliteStorage): boolean {
|
|
750
|
+
return storage.sql.exec(`SELECT name FROM sqlite_schema WHERE type = 'table'`).toArray().some((row) => !text(row['name']).startsWith('_cf_'))
|
|
751
|
+
}
|
|
752
|
+
|
|
753
|
+
/** Every `sqlite_schema` entry with SQL a dump's DDL lists, in no particular order. */
|
|
754
|
+
function readDdl(storage: SqliteStorage): OpsDumpSchemaEntry[] {
|
|
755
|
+
return storage.sql.exec('SELECT type, name, tbl_name, sql FROM sqlite_schema WHERE sql IS NOT NULL').toArray().flatMap((row) => {
|
|
756
|
+
const name = text(row['name'])
|
|
757
|
+
const table = text(row['tbl_name'])
|
|
758
|
+
const type = row['type']
|
|
759
|
+
if (!dumped(name) || !dumped(table) || (type !== 'table' && type !== 'index' && type !== 'trigger' && type !== 'view')) {
|
|
760
|
+
return []
|
|
761
|
+
}
|
|
762
|
+
return [{ type, name, table, sql: text(row['sql']) }]
|
|
763
|
+
})
|
|
764
|
+
}
|
|
765
|
+
|
|
766
|
+
function sameDdl(left: readonly OpsDumpSchemaEntry[], right: readonly OpsDumpSchemaEntry[]): boolean {
|
|
767
|
+
const canonical = (entries: readonly OpsDumpSchemaEntry[]) =>
|
|
768
|
+
entries.map((entry) => JSON.stringify([entry.type, entry.name, entry.table, entry.sql])).sort(compareNames).join('\n')
|
|
769
|
+
return left.length === right.length && canonical(left) === canonical(right)
|
|
770
|
+
}
|
|
771
|
+
|
|
772
|
+
/**
|
|
773
|
+
* The schema `planOpsDump` reads, from the PRAGMAs workerd admits, of every table a dump holds. A `CHECK` or an
|
|
774
|
+
* `INITIALLY DEFERRED` clause is read from the table's SQL outside its quotes and comments; one such clause marks every
|
|
775
|
+
* foreign key of its table.
|
|
776
|
+
*/
|
|
777
|
+
export function describeDumpSchema(storage: SqliteStorage): OpsDumpSchema {
|
|
778
|
+
const tables: OpsDumpSchemaTable[] = []
|
|
779
|
+
for (const row of storage.sql.exec('PRAGMA table_list').toArray()) {
|
|
780
|
+
const name = text(row['name'])
|
|
781
|
+
const type = text(row['type'])
|
|
782
|
+
if (text(row['schema']) !== 'main' || type === 'view' || !dumped(name)) {
|
|
783
|
+
continue
|
|
784
|
+
}
|
|
785
|
+
const virtual = type === 'virtual'
|
|
786
|
+
const sql = storage.sql.exec(`SELECT sql FROM sqlite_schema WHERE type = 'table' AND name = ?`, name).toArray()[0]?.['sql']
|
|
787
|
+
const clauses = typeof sql === 'string' ? unquotedSql(sql) : null
|
|
788
|
+
tables.push({
|
|
789
|
+
name,
|
|
790
|
+
virtual,
|
|
791
|
+
withoutRowid: flag(row['wr']),
|
|
792
|
+
checked: clauses === null || /\bcheck\b/i.test(clauses),
|
|
793
|
+
// A virtual table is refused whatever it holds, and its module may not be loaded to describe it.
|
|
794
|
+
columns: virtual ? [] : describeColumns(storage, name),
|
|
795
|
+
indexes: virtual ? [] : describeIndexes(storage, name),
|
|
796
|
+
foreignKeys: virtual ? [] : describeForeignKeys(storage, name, clauses === null || /\binitially\s+deferred\b/i.test(clauses)),
|
|
797
|
+
})
|
|
798
|
+
}
|
|
799
|
+
return { tables }
|
|
800
|
+
}
|
|
801
|
+
|
|
802
|
+
function describeColumns(storage: SqliteStorage, table: string): OpsDumpSchemaColumn[] {
|
|
803
|
+
return storage.sql.exec(`PRAGMA table_xinfo(${literal(table)})`).toArray().map((row) => {
|
|
804
|
+
const hidden = integer(row['hidden'])
|
|
805
|
+
return {
|
|
806
|
+
name: text(row['name']),
|
|
807
|
+
type: text(row['type']),
|
|
808
|
+
notNull: flag(row['notnull']),
|
|
809
|
+
primaryKey: integer(row['pk']),
|
|
810
|
+
generated: hidden === 2 || hidden === 3,
|
|
811
|
+
}
|
|
812
|
+
})
|
|
813
|
+
}
|
|
814
|
+
|
|
815
|
+
function describeIndexes(storage: SqliteStorage, table: string): OpsDumpSchemaIndex[] {
|
|
816
|
+
return storage.sql.exec(`PRAGMA index_list(${literal(table)})`).toArray().map((row) => {
|
|
817
|
+
const origin = row['origin']
|
|
818
|
+
if (origin !== 'c' && origin !== 'u' && origin !== 'pk') {
|
|
819
|
+
throw new Error('PRAGMA index_list answered an origin SQLite does not write')
|
|
820
|
+
}
|
|
821
|
+
const columns = storage.sql.exec(`PRAGMA index_xinfo(${literal(text(row['name']))})`).toArray()
|
|
822
|
+
.filter((column) => flag(column['key']))
|
|
823
|
+
.sort((left, right) => integer(left['seqno']) - integer(right['seqno']))
|
|
824
|
+
.map((column) => typeof column['name'] === 'string' ? column['name'] : null)
|
|
825
|
+
return { unique: flag(row['unique']), origin, partial: flag(row['partial']), columns }
|
|
826
|
+
})
|
|
827
|
+
}
|
|
828
|
+
|
|
829
|
+
function describeForeignKeys(storage: SqliteStorage, table: string, initiallyDeferred: boolean): OpsDumpSchemaForeignKey[] {
|
|
830
|
+
const keys = new Map<number, { parent: string; rows: { seq: number; from: string; to: string | null }[] }>()
|
|
831
|
+
for (const row of storage.sql.exec(`PRAGMA foreign_key_list(${literal(table)})`).toArray()) {
|
|
832
|
+
const id = integer(row['id'])
|
|
833
|
+
const to = row['to']
|
|
834
|
+
const entry = { seq: integer(row['seq']), from: text(row['from']), to: typeof to === 'string' ? to : null }
|
|
835
|
+
const key = keys.get(id)
|
|
836
|
+
if (key === undefined) {
|
|
837
|
+
keys.set(id, { parent: text(row['table']), rows: [entry] })
|
|
838
|
+
} else {
|
|
839
|
+
key.rows.push(entry)
|
|
840
|
+
}
|
|
841
|
+
}
|
|
842
|
+
return [...keys.entries()].sort(([left], [right]) => left - right).map(([, key]) => {
|
|
843
|
+
const rows = key.rows.sort((left, right) => left.seq - right.seq)
|
|
844
|
+
const references = rows.flatMap((row) => row.to === null ? [] : [row.to])
|
|
845
|
+
if (references.length !== 0 && references.length !== rows.length) {
|
|
846
|
+
throw new Error('PRAGMA foreign_key_list named the parent columns of a key only in part')
|
|
847
|
+
}
|
|
848
|
+
return { parent: key.parent, columns: rows.map((row) => row.from), references: references.length === 0 ? null : references, initiallyDeferred }
|
|
849
|
+
})
|
|
850
|
+
}
|
|
851
|
+
|
|
852
|
+
/** `sql` with its string literals, quoted identifiers and comments blanked, so a keyword search finds only clauses. */
|
|
853
|
+
function unquotedSql(sql: string): string {
|
|
854
|
+
let out = ''
|
|
855
|
+
let index = 0
|
|
856
|
+
while (index < sql.length) {
|
|
857
|
+
const char = sql.charAt(index)
|
|
858
|
+
const close = char === "'" ? "'" : char === '"' ? '"' : char === '`' ? '`' : char === '[' ? ']' : null
|
|
859
|
+
if (close !== null) {
|
|
860
|
+
let end = index + 1
|
|
861
|
+
for (;;) {
|
|
862
|
+
end = sql.indexOf(close, end)
|
|
863
|
+
if (end < 0 || close === ']' || sql.charAt(end + 1) !== close) break
|
|
864
|
+
end += 2
|
|
865
|
+
}
|
|
866
|
+
index = end < 0 ? sql.length : end + 1
|
|
867
|
+
out += ' '
|
|
868
|
+
} else if (sql.startsWith('--', index)) {
|
|
869
|
+
const end = sql.indexOf('\n', index)
|
|
870
|
+
index = end < 0 ? sql.length : end + 1
|
|
871
|
+
out += ' '
|
|
872
|
+
} else if (sql.startsWith('/*', index)) {
|
|
873
|
+
const end = sql.indexOf('*/', index + 2)
|
|
874
|
+
index = end < 0 ? sql.length : end + 2
|
|
875
|
+
out += ' '
|
|
876
|
+
} else {
|
|
877
|
+
out += char
|
|
878
|
+
index++
|
|
879
|
+
}
|
|
880
|
+
}
|
|
881
|
+
return out
|
|
882
|
+
}
|
|
883
|
+
|
|
884
|
+
/** The import marker, parsed strictly; `null` when the object holds none. A marker that does not parse is a corrupt object. */
|
|
885
|
+
function readMarker(kv: ImportKv): ImportMarker | null {
|
|
886
|
+
const raw = kv.get(IMPORT_MARKER_KEY)
|
|
887
|
+
if (raw === undefined) {
|
|
888
|
+
return null
|
|
889
|
+
}
|
|
890
|
+
const value: unknown = typeof raw === 'string' ? parseJson(raw) : undefined
|
|
891
|
+
const source: unknown = isRecord(value) ? value['source'] : undefined
|
|
892
|
+
const verified = isRecord(value) ? readVerification(value['verified']) : undefined
|
|
893
|
+
if (
|
|
894
|
+
!isRecord(value) || !isRecord(source) || Object.keys(value).length !== 4 || Object.keys(source).length !== 2
|
|
895
|
+
|| typeof source['object'] !== 'string' || typeof source['dump'] !== 'string' || typeof value['begun'] !== 'boolean'
|
|
896
|
+
|| typeof value['loaded'] !== 'number' || !Number.isSafeInteger(value['loaded']) || value['loaded'] < 0 || verified === undefined
|
|
897
|
+
) {
|
|
898
|
+
throw new Error('the import marker in KV storage does not parse')
|
|
899
|
+
}
|
|
900
|
+
return { source: { object: source['object'], dump: source['dump'] }, begun: value['begun'], loaded: value['loaded'], verified }
|
|
901
|
+
}
|
|
902
|
+
|
|
903
|
+
/** The marker's verification; `undefined` when it does not parse. */
|
|
904
|
+
function readVerification(value: unknown): Verification | null | undefined {
|
|
905
|
+
if (value === null) {
|
|
906
|
+
return null
|
|
907
|
+
}
|
|
908
|
+
if (!isRecord(value) || Object.keys(value).length !== 2) {
|
|
909
|
+
return undefined
|
|
910
|
+
}
|
|
911
|
+
const next = value['next']
|
|
912
|
+
const replay = value['replay']
|
|
913
|
+
if (next !== null && typeof next !== 'string') {
|
|
914
|
+
return undefined
|
|
915
|
+
}
|
|
916
|
+
if (replay === null) {
|
|
917
|
+
return { next, replay: null }
|
|
918
|
+
}
|
|
919
|
+
const cursor = isRecord(replay) ? replay['cursor'] : undefined
|
|
920
|
+
const ack = isRecord(replay) && Object.keys(replay).length === 2 ? parseOpsImportVerifyAck(replay['ack']) : null
|
|
921
|
+
return typeof cursor === 'string' && ack !== null ? { next, replay: { cursor, ack } } : undefined
|
|
922
|
+
}
|
|
923
|
+
|
|
924
|
+
function writeMarker(kv: ImportKv, marker: ImportMarker): void {
|
|
925
|
+
kv.put(IMPORT_MARKER_KEY, JSON.stringify(marker))
|
|
926
|
+
}
|
|
927
|
+
|
|
928
|
+
function readLoadedPart(kv: ImportKv, part: number): OpsImportPartAck | null {
|
|
929
|
+
const raw = kv.get(loadedPartKey(part))
|
|
930
|
+
if (raw === undefined) {
|
|
931
|
+
return null
|
|
932
|
+
}
|
|
933
|
+
const ack = typeof raw === 'string' ? parseOpsImportPartAck(parseJson(raw)) : null
|
|
934
|
+
if (ack === null || ack.part !== part) {
|
|
935
|
+
throw new Error('a loaded part recorded in KV storage does not parse')
|
|
936
|
+
}
|
|
937
|
+
return ack
|
|
938
|
+
}
|
|
939
|
+
|
|
940
|
+
function readLinkedPart(kv: ImportKv, part: number): OpsImportLinkAck | null {
|
|
941
|
+
const raw = kv.get(linkedPartKey(part))
|
|
942
|
+
if (raw === undefined) {
|
|
943
|
+
return null
|
|
944
|
+
}
|
|
945
|
+
const ack = typeof raw === 'string' ? parseOpsImportLinkAck(parseJson(raw)) : null
|
|
946
|
+
if (ack === null || ack.part !== part) {
|
|
947
|
+
throw new Error('a link pass recorded in KV storage does not parse')
|
|
948
|
+
}
|
|
949
|
+
return ack
|
|
950
|
+
}
|
|
951
|
+
|
|
952
|
+
/** The acknowledgement `complete` answered for `source`, `null` when it never completed. */
|
|
953
|
+
function readCompleted(kv: ImportKv, source: OpsDumpSource): OpsImportCompleteAck | null {
|
|
954
|
+
const raw = kv.get(completedKey(source))
|
|
955
|
+
if (raw === undefined) {
|
|
956
|
+
return null
|
|
957
|
+
}
|
|
958
|
+
const ack = typeof raw === 'string' ? parseOpsImportCompleteAck(parseJson(raw)) : null
|
|
959
|
+
if (ack === null || !sameSource(ack.source, source)) {
|
|
960
|
+
throw new Error('a completed import recorded in KV storage does not parse')
|
|
961
|
+
}
|
|
962
|
+
return ack
|
|
963
|
+
}
|
|
964
|
+
|
|
965
|
+
function sameSource(left: OpsDumpSource, right: OpsDumpSource): boolean {
|
|
966
|
+
return left.object === right.object && left.dump === right.dump
|
|
967
|
+
}
|
|
968
|
+
|
|
969
|
+
function refused(reason: OpsDataRefusalReason, table: string | null = null): Response {
|
|
970
|
+
const refusal: OpsDataRefusal = { refused: reason, table }
|
|
971
|
+
return jsonResponse(refusal, OPS_DATA_REFUSED_STATUS)
|
|
972
|
+
}
|
|
973
|
+
|
|
974
|
+
/** The parameters of `bindings`; `sql.exec` binds bytes as an `ArrayBuffer`. */
|
|
975
|
+
function params(bindings: readonly OpsDumpBinding[]): SqliteValue[] {
|
|
976
|
+
return bindings.map(({ param }) => param instanceof Uint8Array ? param.slice().buffer : param)
|
|
977
|
+
}
|
|
978
|
+
|
|
979
|
+
function quote(identifier: string): string {
|
|
980
|
+
return `"${identifier.replaceAll('"', '""')}"`
|
|
981
|
+
}
|
|
982
|
+
|
|
983
|
+
function literal(value: string): string {
|
|
984
|
+
return `'${value.replaceAll("'", "''")}'`
|
|
985
|
+
}
|
|
986
|
+
|
|
987
|
+
/** Code-unit order, which no locale changes; the migration runner orders files the same way. */
|
|
988
|
+
function compareNames(left: string, right: string): number {
|
|
989
|
+
return left < right ? -1 : left > right ? 1 : 0
|
|
990
|
+
}
|
|
991
|
+
|
|
992
|
+
function parseJson(text: string): unknown {
|
|
993
|
+
try {
|
|
994
|
+
return JSON.parse(text)
|
|
995
|
+
} catch {
|
|
996
|
+
return undefined
|
|
997
|
+
}
|
|
998
|
+
}
|
|
999
|
+
|
|
1000
|
+
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
1001
|
+
return typeof value === 'object' && value !== null && !Array.isArray(value)
|
|
1002
|
+
}
|
|
1003
|
+
|
|
1004
|
+
// SQLite's own answers: a value of another type is a broken runtime, never input to refuse.
|
|
1005
|
+
function text(value: SqliteValue | undefined): string {
|
|
1006
|
+
if (typeof value !== 'string') {
|
|
1007
|
+
throw new Error('SQLite answered a schema name or SQL that is not text')
|
|
1008
|
+
}
|
|
1009
|
+
return value
|
|
1010
|
+
}
|
|
1011
|
+
|
|
1012
|
+
function integer(value: SqliteValue | undefined): number {
|
|
1013
|
+
if (typeof value !== 'number' || !Number.isSafeInteger(value)) {
|
|
1014
|
+
throw new Error('SQLite answered a schema number that is not an integer')
|
|
1015
|
+
}
|
|
1016
|
+
return value
|
|
1017
|
+
}
|
|
1018
|
+
|
|
1019
|
+
function flag(value: SqliteValue | undefined): boolean {
|
|
1020
|
+
return integer(value) !== 0
|
|
1021
|
+
}
|