@fabrika/app 0.9.0-dev.3 → 0.9.0-dev.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +14 -9
- package/skill/SKILL.md +41 -14
- package/skill/references/app-setup.md +43 -12
- package/skill/references/dev.md +2 -1
- package/skill/references/endpoint.md +30 -24
- package/skill/references/errors.md +40 -35
- package/skill/references/files.md +4 -2
- package/skill/references/jobs.md +21 -4
- package/skill/references/table-and-migration.md +19 -2
- package/skill/references/tests.md +40 -1
- package/skill/references/webhooks-links-connectors.md +1 -1
- package/src/build-recipe.ts +198 -0
- package/src/build.ts +60 -4
- package/src/bun.ts +181 -13
- package/src/cloudflare/application.ts +822 -0
- package/src/cloudflare/bindings.ts +159 -0
- package/src/cloudflare/export.ts +641 -0
- package/src/cloudflare/import.ts +1021 -0
- package/src/cloudflare/maintenance.ts +518 -0
- package/src/cloudflare/names.ts +16 -0
- package/src/cloudflare/ops.ts +375 -0
- package/src/cloudflare/r2-files.ts +172 -0
- package/src/cloudflare/restore.ts +167 -0
- package/src/cloudflare/tracker.ts +192 -0
- package/src/cloudflare.ts +100 -146
- package/src/contract/app-id.ts +5 -0
- package/src/contract/artifact-schema.ts +9 -1
- package/src/contract/build.ts +40 -2
- package/src/contract/compile.ts +1 -0
- package/src/contract/context.ts +35 -9
- package/src/contract/declaration.ts +2 -0
- package/src/contract/defaults.ts +22 -2
- package/src/contract/define.ts +29 -3
- package/src/contract/kit.ts +4 -1
- package/src/contract/platform.ts +3 -2
- package/src/contract/vocabulary.ts +26 -8
- package/src/dev/launcher.ts +84 -13
- package/src/dev/output.ts +13 -3
- package/src/disk-assets.ts +8 -3
- package/src/files/archive.ts +2 -2
- package/src/files/job-names.ts +12 -0
- package/src/files/kinds.ts +73 -31
- package/src/files/scan.ts +3 -4
- package/src/files/streams.ts +3 -1
- package/src/implement/anonymous.ts +12 -11
- package/src/implement/boot.ts +75 -6
- package/src/implement/bounded.ts +21 -2
- package/src/implement/capability.ts +15 -3
- package/src/implement/dispatch.ts +419 -177
- package/src/implement/failures.ts +17 -1
- package/src/implement/faults.ts +1 -0
- package/src/implement/index.ts +1 -0
- package/src/implement/invocation.ts +58 -10
- package/src/implement/principal.ts +25 -2
- package/src/implement/terminals.ts +93 -0
- package/src/implement/workload.ts +42 -16
- package/src/jobs/mail.ts +8 -0
- package/src/jobs/report.ts +5 -1
- package/src/jobs/runner.ts +672 -113
- package/src/jobs/schedule.ts +7 -1
- package/src/jobs/scrub.ts +14 -1
- package/src/jobs/types.ts +6 -0
- package/src/spa-fallback.ts +12 -0
- package/src/testing/endpoint-access.ts +307 -152
- package/src/testing/isolation.ts +8 -1
- package/src/testing/kit.ts +80 -2
- package/src/testing/proxy.ts +15 -10
- package/types/src/build-recipe.d.ts +70 -0
- package/types/src/build.d.ts +19 -1
- package/types/src/bun.d.ts +34 -1
- package/types/src/cloudflare/application.d.ts +73 -0
- package/types/src/cloudflare/bindings.d.ts +33 -0
- package/types/src/cloudflare/export.d.ts +16 -0
- package/types/src/cloudflare/import.d.ts +51 -0
- package/types/src/cloudflare/maintenance.d.ts +117 -0
- package/types/src/cloudflare/names.d.ts +11 -0
- package/types/src/cloudflare/ops.d.ts +81 -0
- package/types/src/cloudflare/r2-files.d.ts +9 -0
- package/types/src/cloudflare/restore.d.ts +9 -0
- package/types/src/cloudflare/tracker.d.ts +29 -0
- package/types/src/cloudflare.d.ts +12 -18
- package/types/src/contract/app-id.d.ts +2 -0
- package/types/src/contract/artifact-schema.d.ts +2 -0
- package/types/src/contract/context.d.ts +35 -11
- package/types/src/contract/declaration.d.ts +4 -0
- package/types/src/contract/defaults.d.ts +19 -1
- package/types/src/contract/define.d.ts +20 -3
- package/types/src/contract/kit.d.ts +2 -1
- package/types/src/contract/platform.d.ts +2 -2
- package/types/src/contract/vocabulary.d.ts +9 -7
- package/types/src/dev/output.d.ts +8 -2
- package/types/src/disk-assets.d.ts +4 -1
- package/types/src/files/job-names.d.ts +9 -0
- package/types/src/files/kinds.d.ts +21 -14
- package/types/src/files/scan.d.ts +1 -3
- package/types/src/implement/anonymous.d.ts +2 -4
- package/types/src/implement/boot.d.ts +17 -1
- package/types/src/implement/bounded.d.ts +11 -1
- package/types/src/implement/capability.d.ts +4 -2
- package/types/src/implement/dispatch.d.ts +34 -1
- package/types/src/implement/failures.d.ts +5 -0
- package/types/src/implement/faults.d.ts +1 -0
- package/types/src/implement/index.d.ts +1 -0
- package/types/src/implement/invocation.d.ts +22 -7
- package/types/src/implement/principal.d.ts +6 -1
- package/types/src/implement/terminals.d.ts +33 -0
- package/types/src/implement/workload.d.ts +5 -0
- package/types/src/jobs/runner.d.ts +66 -2
- package/types/src/jobs/schedule.d.ts +2 -0
- package/types/src/jobs/types.d.ts +6 -0
- package/types/src/spa-fallback.d.ts +2 -0
- package/types/src/testing/endpoint-access.d.ts +37 -5
- package/types/src/testing/isolation.d.ts +1 -1
- package/types/src/testing/kit.d.ts +22 -1
- package/types/src/testing/proxy.d.ts +4 -4
|
@@ -0,0 +1,641 @@
|
|
|
1
|
+
// One step of a dump of the object's database (ADR-0070 §2): the ops operation `export`, served before the application
|
|
2
|
+
// boots. It reads `sqlite_schema` and the tables directly, so it also dumps a database that cannot boot. Control's
|
|
3
|
+
// operation record holds the upload id, the cursor and the parts; the object keeps only a consistent dump's begun state.
|
|
4
|
+
|
|
5
|
+
import {
|
|
6
|
+
encodeOpsDumpLine,
|
|
7
|
+
foldOpsTableHash,
|
|
8
|
+
OPS_DATA_REFUSED_STATUS,
|
|
9
|
+
OPS_DUMP_FORMAT,
|
|
10
|
+
OPS_DUMP_KEY,
|
|
11
|
+
OPS_DUMP_PAGE_ROWS,
|
|
12
|
+
OPS_DUMP_PART_MAX_RAW_BYTES,
|
|
13
|
+
OPS_DUMP_PART_MAX_ROWS,
|
|
14
|
+
OPS_DUMP_PART_SIZE,
|
|
15
|
+
OPS_EMPTY_TABLE_HASH,
|
|
16
|
+
OPS_STALE_STATUS,
|
|
17
|
+
type OpsDataRefusal,
|
|
18
|
+
opsDumpColumnProjection,
|
|
19
|
+
type OpsDumpCursor,
|
|
20
|
+
type OpsDumpSchemaEntry,
|
|
21
|
+
type OpsDumpTable,
|
|
22
|
+
type OpsDumpTablePlan,
|
|
23
|
+
type OpsDumpValue,
|
|
24
|
+
opsDumpValueBinding,
|
|
25
|
+
type OpsExportBegin,
|
|
26
|
+
type OpsExportComplete,
|
|
27
|
+
type OpsExportPart,
|
|
28
|
+
type OpsExportRequest,
|
|
29
|
+
type OpsExportScope,
|
|
30
|
+
type OpsPublishedState,
|
|
31
|
+
type OpsStaleRefusal,
|
|
32
|
+
parseOpsExportBeginAck,
|
|
33
|
+
parseOpsExportCompleteAck,
|
|
34
|
+
parseOpsExportPartAck,
|
|
35
|
+
planOpsDump,
|
|
36
|
+
readOpsDumpValue,
|
|
37
|
+
sha256Hex,
|
|
38
|
+
} from '@fabrika/auth-core'
|
|
39
|
+
import type { SqliteValue } from '@fabrika/db/sqlite'
|
|
40
|
+
import { crc32, deflateRawSync } from 'node:zlib'
|
|
41
|
+
import appPackage from '../../package.json' with { type: 'json' }
|
|
42
|
+
import { jsonResponse } from '../errors.js'
|
|
43
|
+
import { method } from './bindings.js'
|
|
44
|
+
import { describeDumpSchema, importInProgress } from './import.js'
|
|
45
|
+
import type { MaintenanceBarrier } from './maintenance.js'
|
|
46
|
+
import type { OpsTarget } from './ops.js'
|
|
47
|
+
|
|
48
|
+
/** One step of a dump of the object's database (ADR-0070). */
|
|
49
|
+
export async function serveExport(target: OpsTarget, barrier: MaintenanceBarrier, step: OpsExportRequest): Promise<Response> {
|
|
50
|
+
return await exportStep(target, barrier, step)
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/** What an export step reads: the object's storage, bucket and bookmarks, and the digest of the contract it was deployed with. */
|
|
54
|
+
export interface ExportSource extends Pick<OpsTarget, 'storage' | 'bucket' | 'recovery'> {
|
|
55
|
+
/** `FABRIKA_CONTRACT_DIGEST`, which the manifest records. */
|
|
56
|
+
readonly contractDigest: string | null
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
type Storage = ExportSource['storage']
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* One step of a dump. A diagnostic step runs in any mode. A consistent one runs in the barrier's turn, only at the
|
|
63
|
+
* published generation, never while the object is `open` or holds an import marker, so no transition and no migration
|
|
64
|
+
* falls inside it; past `begin` it refuses a database whose state or stretch is not the one its `begin` saw.
|
|
65
|
+
*/
|
|
66
|
+
export async function exportStep(source: ExportSource, barrier: MaintenanceBarrier, step: OpsExportRequest): Promise<Response> {
|
|
67
|
+
if (!step.consistent) {
|
|
68
|
+
return await run(source, step)
|
|
69
|
+
}
|
|
70
|
+
const { generation } = step
|
|
71
|
+
if (generation === null) {
|
|
72
|
+
throw new Error('a consistent export step carries its generation; the wire parser requires it')
|
|
73
|
+
}
|
|
74
|
+
const result = await barrier.step(generation, async (mode) => {
|
|
75
|
+
const { frozenSince } = barrier
|
|
76
|
+
if (mode === 'open' || frozenSince === null) {
|
|
77
|
+
return { value: dataRefusal({ refused: 'wrong_mode', table: null }), holdUntil: null }
|
|
78
|
+
}
|
|
79
|
+
return { value: await runConsistent(source, step, frozenSince), holdUntil: null }
|
|
80
|
+
})
|
|
81
|
+
return result.kind === 'ran' ? result.value : staleRefusal(result.published)
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/** Runs `step`; `state` is what a consistent dump's first `begin` records, `null` for a diagnostic one. */
|
|
85
|
+
async function run(source: ExportSource, step: OpsExportRequest, state: string | null = null): Promise<Response> {
|
|
86
|
+
switch (step.step) {
|
|
87
|
+
case 'begin':
|
|
88
|
+
return await begin(source, step, state)
|
|
89
|
+
case 'part':
|
|
90
|
+
return await part(source, step)
|
|
91
|
+
case 'complete':
|
|
92
|
+
return await complete(source, step)
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* The first `begin` of a dump records the state it saw with its upload and bookmark, once; a retried one replays that
|
|
98
|
+
* record, and every later step must carry its upload at that state, in the same stretch of modes that write nothing
|
|
99
|
+
* (`frozenSince`), at whatever generation of it. A migration runs inside a stretch, so the stretch alone would not show
|
|
100
|
+
* one between two steps. A storage without the synchronous KV holds no import.
|
|
101
|
+
*/
|
|
102
|
+
async function runConsistent(source: ExportSource, step: OpsExportRequest, frozenSince: number): Promise<Response> {
|
|
103
|
+
const { kv } = source.storage
|
|
104
|
+
if (kv !== undefined && importInProgress({ kv })) {
|
|
105
|
+
return dataRefusal({ refused: 'wrong_mode', table: null })
|
|
106
|
+
}
|
|
107
|
+
const state = dumpState(source.storage, frozenSince)
|
|
108
|
+
const begun = await readBegun(source.bucket, step)
|
|
109
|
+
if (step.step === 'begin') {
|
|
110
|
+
return begun === null ? await run(source, step, state) : replayed(begun, state)
|
|
111
|
+
}
|
|
112
|
+
if (begun?.state !== state || begun.uploadId !== step.uploadId || (step.step === 'complete' && step.bookmark !== begun.bookmark)) {
|
|
113
|
+
return dataRefusal({ refused: 'ledger_mismatch', table: null })
|
|
114
|
+
}
|
|
115
|
+
return await run(source, step)
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/** A retried `begin`: the first one's answer at the state it recorded, refused at any other. */
|
|
119
|
+
function replayed(begun: Begun, state: string): Response {
|
|
120
|
+
if (begun.state !== state) {
|
|
121
|
+
return dataRefusal({ refused: 'ledger_mismatch', table: null })
|
|
122
|
+
}
|
|
123
|
+
return answer(parseOpsExportBeginAck({ step: 'begin', uploadId: begun.uploadId, bookmark: begun.bookmark }), 'begin')
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
function staleRefusal(published: OpsPublishedState | null): Response {
|
|
127
|
+
const stale: OpsStaleRefusal = { refused: 'stale_generation', published }
|
|
128
|
+
return jsonResponse(stale, OPS_STALE_STATUS)
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
function dataRefusal(refusal: OpsDataRefusal): Response {
|
|
132
|
+
return jsonResponse(refusal, OPS_DATA_REFUSED_STATUS)
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/**
|
|
136
|
+
* Plans the dump, starts the multipart upload of its data and answers the object's bookmark. A consistent dump records
|
|
137
|
+
* its `state` with both, only if nothing is recorded yet.
|
|
138
|
+
*/
|
|
139
|
+
async function begin(source: ExportSource, step: OpsExportBegin, state: string | null): Promise<Response> {
|
|
140
|
+
const plan = planOpsDump(describeDumpSchema(source.storage))
|
|
141
|
+
if ('refused' in plan) {
|
|
142
|
+
return dataRefusal(plan)
|
|
143
|
+
}
|
|
144
|
+
if (source.recovery === null) {
|
|
145
|
+
throw new Error("the object's storage has no bookmarks; a Durable Object's storage has them")
|
|
146
|
+
}
|
|
147
|
+
const bookmark = await source.recovery.getCurrentBookmark()
|
|
148
|
+
const upload: unknown = await call(source.bucket, 'createMultipartUpload')(dataKey(step), { httpMetadata: { contentType: 'application/gzip' } })
|
|
149
|
+
const ack = parseOpsExportBeginAck({
|
|
150
|
+
step: 'begin',
|
|
151
|
+
uploadId: typeof upload === 'object' && upload !== null ? Reflect.get(upload, 'uploadId') : undefined,
|
|
152
|
+
bookmark,
|
|
153
|
+
})
|
|
154
|
+
if (ack === null || state === null) {
|
|
155
|
+
return answer(ack, 'begin')
|
|
156
|
+
}
|
|
157
|
+
const begun: Begun = { state, uploadId: ack.uploadId, bookmark: ack.bookmark }
|
|
158
|
+
const options = { httpMetadata: { contentType: 'application/json' }, onlyIf: { etagDoesNotMatch: '*' } }
|
|
159
|
+
if ((await call(source.bucket, 'put')(begunKey(step), JSON.stringify(begun), options)) === null) {
|
|
160
|
+
// Another begin recorded the dump first; its upload is the dump's, and this one is left to the lifecycle.
|
|
161
|
+
const recorded = await readBegun(source.bucket, step)
|
|
162
|
+
if (recorded === null) {
|
|
163
|
+
throw new Error("R2 refused to record a dump's begin, yet holds no record of it")
|
|
164
|
+
}
|
|
165
|
+
return replayed(recorded, state)
|
|
166
|
+
}
|
|
167
|
+
return answer(ack, 'begin')
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/** Fills part `n` from its cursor: whole rows while every bound holds, one gzip member padded to the fixed size but the last. */
|
|
171
|
+
async function part(source: ExportSource, step: OpsExportPart): Promise<Response> {
|
|
172
|
+
const plan = planOpsDump(describeDumpSchema(source.storage))
|
|
173
|
+
if ('refused' in plan) {
|
|
174
|
+
return dataRefusal(plan)
|
|
175
|
+
}
|
|
176
|
+
const [first] = plan.tables
|
|
177
|
+
const start = step.cursor ?? (first === undefined ? null : fresh(first.name))
|
|
178
|
+
const index = start === null ? plan.tables.length : plan.tables.findIndex((table) => table.name === start.table)
|
|
179
|
+
if (start !== null && index < 0) {
|
|
180
|
+
return dataRefusal({ refused: 'schema_mismatch', table: start.table })
|
|
181
|
+
}
|
|
182
|
+
const collected = collect(source.storage, plan.tables.slice(index), start)
|
|
183
|
+
if ('refused' in collected) {
|
|
184
|
+
return dataRefusal(collected)
|
|
185
|
+
}
|
|
186
|
+
const filled = fill(collected)
|
|
187
|
+
if ('refused' in filled) {
|
|
188
|
+
return dataRefusal(filled)
|
|
189
|
+
}
|
|
190
|
+
return await uploadPart(source, step, filled, start === null ? { finished: [], next: null } : settle(start, collected, filled.rows))
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
/** Completes the upload with the parts Control recorded, writes the manifest beside the data and answers it. */
|
|
194
|
+
async function complete(source: ExportSource, step: OpsExportComplete): Promise<Response> {
|
|
195
|
+
const { storage } = source
|
|
196
|
+
const plan = planOpsDump(describeDumpSchema(storage))
|
|
197
|
+
if ('refused' in plan) {
|
|
198
|
+
return dataRefusal(plan)
|
|
199
|
+
}
|
|
200
|
+
const mismatch = plan.tables.find((table, index) => step.tables[index]?.name !== table.name)
|
|
201
|
+
if (mismatch !== undefined || step.tables.length !== plan.tables.length) {
|
|
202
|
+
return dataRefusal({ refused: 'schema_mismatch', table: mismatch?.name ?? null })
|
|
203
|
+
}
|
|
204
|
+
const ddl = schemaEntries(storage)
|
|
205
|
+
const migrations = migrationLedger(storage)
|
|
206
|
+
const key = dataKey(step)
|
|
207
|
+
const upload: unknown = await call(source.bucket, 'resumeMultipartUpload')(key, step.uploadId)
|
|
208
|
+
try {
|
|
209
|
+
await call(upload, 'complete')(step.parts.map((uploaded) => ({ partNumber: uploaded.part, etag: uploaded.etag })))
|
|
210
|
+
} catch (error) {
|
|
211
|
+
// A retried complete whose first answer was lost finds the upload gone and the data in place.
|
|
212
|
+
const head: unknown = await call(source.bucket, 'head')(key)
|
|
213
|
+
const size = typeof head === 'object' && head !== null ? Reflect.get(head, 'size') : undefined
|
|
214
|
+
if (size !== step.parts.reduce((sum, uploaded) => sum + uploaded.size, 0)) {
|
|
215
|
+
throw error
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
const manifest = {
|
|
219
|
+
format: OPS_DUMP_FORMAT,
|
|
220
|
+
dump: step.dump,
|
|
221
|
+
object: step.object,
|
|
222
|
+
consistent: step.consistent,
|
|
223
|
+
createdAt: new Date().toISOString(),
|
|
224
|
+
bookmark: step.bookmark,
|
|
225
|
+
release: appPackage.version,
|
|
226
|
+
contractDigest: source.contractDigest,
|
|
227
|
+
ddl,
|
|
228
|
+
migrations,
|
|
229
|
+
tables: step.tables,
|
|
230
|
+
parts: step.parts.map(({ part, size, sha256 }) => ({ part, size, sha256 })),
|
|
231
|
+
}
|
|
232
|
+
const ack = parseOpsExportCompleteAck({ step: 'complete', manifest })
|
|
233
|
+
if (ack === null) {
|
|
234
|
+
throw new Error("the dump's manifest does not parse as the wire's manifest")
|
|
235
|
+
}
|
|
236
|
+
await call(source.bucket, 'put')(OPS_DUMP_KEY(step, 'manifest'), JSON.stringify(ack.manifest), { httpMetadata: { contentType: 'application/json' } })
|
|
237
|
+
return jsonResponse(ack, 200)
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
async function uploadPart(source: ExportSource, step: OpsExportPart, part: Filled, settled: Settled): Promise<Response> {
|
|
241
|
+
const member = gzipMember(part)
|
|
242
|
+
const digest = new Uint8Array(await crypto.subtle.digest('SHA-256', member))
|
|
243
|
+
const sha256 = Array.from(digest, (byte) => byte.toString(16).padStart(2, '0')).join('')
|
|
244
|
+
const upload: unknown = await call(source.bucket, 'resumeMultipartUpload')(dataKey(step), step.uploadId)
|
|
245
|
+
// R2 keeps one part per number, so a retried step replaces the part it uploaded before.
|
|
246
|
+
const uploaded: unknown = await call(upload, 'uploadPart')(step.part, member)
|
|
247
|
+
const etag = typeof uploaded === 'object' && uploaded !== null ? Reflect.get(uploaded, 'etag') : undefined
|
|
248
|
+
return answer(
|
|
249
|
+
parseOpsExportPartAck({
|
|
250
|
+
step: 'part',
|
|
251
|
+
uploaded: { part: step.part, size: member.byteLength, sha256, etag },
|
|
252
|
+
tables: settled.finished,
|
|
253
|
+
next: settled.next,
|
|
254
|
+
}),
|
|
255
|
+
'part',
|
|
256
|
+
)
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
function dataKey(scope: OpsExportScope): string {
|
|
260
|
+
return OPS_DUMP_KEY(scope, 'data')
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
/** What a consistent dump's first `begin` saw and answered: the digest of the database's state, its upload and bookmark. */
|
|
264
|
+
interface Begun {
|
|
265
|
+
readonly state: string
|
|
266
|
+
readonly uploadId: string
|
|
267
|
+
readonly bookmark: string
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
/** `begun` in the dump's own prefix, which the bucket's lifecycle expires with the dump. */
|
|
271
|
+
function begunKey(scope: OpsExportScope): string {
|
|
272
|
+
const manifest = OPS_DUMP_KEY(scope, 'manifest')
|
|
273
|
+
return `${manifest.slice(0, manifest.lastIndexOf('/'))}/begun`
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
/** What the dump's first `begin` recorded, `null` when nothing is recorded; a record that does not parse fails the step. */
|
|
277
|
+
async function readBegun(bucket: unknown, scope: OpsExportScope): Promise<Begun | null> {
|
|
278
|
+
const stored: unknown = await call(bucket, 'get')(begunKey(scope))
|
|
279
|
+
if (stored === null) {
|
|
280
|
+
return null
|
|
281
|
+
}
|
|
282
|
+
const body: unknown = await call(stored, 'text')()
|
|
283
|
+
const begun = typeof body === 'string' ? parseBegun(body) : null
|
|
284
|
+
if (begun === null) {
|
|
285
|
+
throw new Error("the dump's begin record does not parse")
|
|
286
|
+
}
|
|
287
|
+
return begun
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
function parseBegun(text: string): Begun | null {
|
|
291
|
+
let value: unknown
|
|
292
|
+
try {
|
|
293
|
+
value = JSON.parse(text)
|
|
294
|
+
} catch {
|
|
295
|
+
return null
|
|
296
|
+
}
|
|
297
|
+
if (typeof value !== 'object' || value === null || Array.isArray(value) || Object.keys(value).length !== 3) {
|
|
298
|
+
return null
|
|
299
|
+
}
|
|
300
|
+
const state: unknown = Reflect.get(value, 'state')
|
|
301
|
+
const uploadId: unknown = Reflect.get(value, 'uploadId')
|
|
302
|
+
const bookmark: unknown = Reflect.get(value, 'bookmark')
|
|
303
|
+
return typeof state === 'string' && typeof uploadId === 'string' && typeof bookmark === 'string' ? { state, uploadId, bookmark } : null
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
function answer<T>(ack: T | null, step: string): Response {
|
|
307
|
+
if (ack === null) {
|
|
308
|
+
throw new Error(`the export ${step} answer does not parse as the wire's answer`)
|
|
309
|
+
}
|
|
310
|
+
return jsonResponse(ack, 200)
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
function call(target: unknown, name: string): (...args: unknown[]) => unknown {
|
|
314
|
+
const found = method(target, name)
|
|
315
|
+
if (found === null) {
|
|
316
|
+
throw new Error(`the FABRIKA_FILES binding has no ${name}; dumps live in the environment's bucket`)
|
|
317
|
+
}
|
|
318
|
+
return found
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
// Reading: keyset pages of whole rows within the part's bounds.
|
|
322
|
+
|
|
323
|
+
function fresh(table: string): OpsDumpCursor {
|
|
324
|
+
return { table, after: null, rows: 0, hash: OPS_EMPTY_TABLE_HASH }
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
/** A row read for the part: its table, its rowid, and its line with the newline, where its canonical encoding starts at `rowStart`. */
|
|
328
|
+
interface ReadRow {
|
|
329
|
+
readonly kind: 'row'
|
|
330
|
+
readonly table: string
|
|
331
|
+
readonly rowid: string
|
|
332
|
+
readonly line: Uint8Array
|
|
333
|
+
readonly rowStart: number
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
/** A table read to its end. */
|
|
337
|
+
interface TableEnd {
|
|
338
|
+
readonly kind: 'end'
|
|
339
|
+
readonly table: string
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
interface Collected {
|
|
343
|
+
/** Rows and table ends in reading order: a table's end follows its last row. */
|
|
344
|
+
readonly items: readonly (ReadRow | TableEnd)[]
|
|
345
|
+
/** The table of the row the part could not take, `null` once every table was read. */
|
|
346
|
+
readonly breaker: string | null
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
/** `JSON.stringify` adds at most this to a value's text: the quotes, the tag and the separator. */
|
|
350
|
+
const VALUE_OVERHEAD = 8
|
|
351
|
+
/** A value that is no text or blob is at most this long: `-1.7976931348623157e+308`, `-9223372036854775808`, `null`. */
|
|
352
|
+
const SCALAR_BYTES = 32
|
|
353
|
+
|
|
354
|
+
/**
|
|
355
|
+
* Reads whole rows of `tables`, from the cursor's position in the first, while the part holds at most
|
|
356
|
+
* `OPS_DUMP_PART_MAX_ROWS` rows and its lines at most `OPS_DUMP_PART_MAX_RAW_BYTES`. A page first asks how long each
|
|
357
|
+
* row's line is at most, then reads the rows that fit and the one after them, so a page of large rows never sits in
|
|
358
|
+
* memory whole.
|
|
359
|
+
*/
|
|
360
|
+
function collect(storage: Storage, tables: readonly OpsDumpTablePlan[], start: OpsDumpCursor | null): Collected | OpsDataRefusal {
|
|
361
|
+
const items: (ReadRow | TableEnd)[] = []
|
|
362
|
+
const encoder = new TextEncoder()
|
|
363
|
+
let rows = 0
|
|
364
|
+
let raw = 0
|
|
365
|
+
for (const table of tables) {
|
|
366
|
+
// A line is `{"table":"<name>","row":<row>}` (ADR-0070 §3), all ASCII: its row starts where an empty row's `[]}` does.
|
|
367
|
+
const rowStart = encodeOpsDumpLine({ table: table.name, row: [] }).length - '[]}'.length
|
|
368
|
+
let after = table.name === start?.table ? start.after : null
|
|
369
|
+
for (;;) {
|
|
370
|
+
const bounds = rowBounds(storage, table, after)
|
|
371
|
+
let fitting = 0
|
|
372
|
+
let budget = OPS_DUMP_PART_MAX_RAW_BYTES - raw
|
|
373
|
+
for (const bound of bounds) {
|
|
374
|
+
if (rows + fitting >= OPS_DUMP_PART_MAX_ROWS || bound > budget) {
|
|
375
|
+
break
|
|
376
|
+
}
|
|
377
|
+
budget -= bound
|
|
378
|
+
fitting++
|
|
379
|
+
}
|
|
380
|
+
const page = bounds.length === 0 ? [] : readRows(storage, table, after, Math.min(fitting + 1, bounds.length))
|
|
381
|
+
for (const { rowid, values } of page) {
|
|
382
|
+
const line = encoder.encode(`${encodeOpsDumpLine({ table: table.name, row: values })}\n`)
|
|
383
|
+
if (rows + 1 > OPS_DUMP_PART_MAX_ROWS || raw + line.byteLength > OPS_DUMP_PART_MAX_RAW_BYTES) {
|
|
384
|
+
return rows === 0 ? { refused: 'row_too_large', table: table.name } : { items, breaker: table.name }
|
|
385
|
+
}
|
|
386
|
+
items.push({ kind: 'row', table: table.name, rowid, line, rowStart })
|
|
387
|
+
after = rowid
|
|
388
|
+
rows++
|
|
389
|
+
raw += line.byteLength
|
|
390
|
+
}
|
|
391
|
+
if (page.length === bounds.length && bounds.length < OPS_DUMP_PAGE_ROWS) {
|
|
392
|
+
items.push({ kind: 'end', table: table.name })
|
|
393
|
+
break
|
|
394
|
+
}
|
|
395
|
+
}
|
|
396
|
+
}
|
|
397
|
+
return { items, breaker: null }
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
/** The keyset condition: the rowid after the cursor's, written back as the integer it was read as. */
|
|
401
|
+
function keysetCondition(table: OpsDumpTablePlan, after: string | null): { readonly sql: string; readonly bindings: SqliteValue[] } {
|
|
402
|
+
if (after === null) {
|
|
403
|
+
return { sql: '', bindings: [] }
|
|
404
|
+
}
|
|
405
|
+
const binding = opsDumpValueBinding({ type: 'integer', value: after })
|
|
406
|
+
if (typeof binding.param !== 'string') {
|
|
407
|
+
throw new Error('a rowid binds as its decimal text')
|
|
408
|
+
}
|
|
409
|
+
return { sql: `WHERE ${quoted(table.keyset)} > ${binding.sql}`, bindings: [binding.param] }
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
/** For each of the next rows of a page, at most how many bytes its line takes. */
|
|
413
|
+
function rowBounds(storage: Storage, table: OpsDumpTablePlan, after: string | null): number[] {
|
|
414
|
+
const sizes = table.columns.map((column) => {
|
|
415
|
+
const c = quoted(column)
|
|
416
|
+
return `CASE typeof(${c}) WHEN 'text' THEN 2 * length(CAST(${c} AS BLOB)) WHEN 'blob' THEN 2 * length(${c}) ELSE ${SCALAR_BYTES} END`
|
|
417
|
+
})
|
|
418
|
+
const fixed = `{"table":"${table.name}","row":[]}\n`.length + (table.columns.length + 1) * VALUE_OVERHEAD + SCALAR_BYTES
|
|
419
|
+
const condition = keysetCondition(table, after)
|
|
420
|
+
const query = `SELECT ${[String(fixed), ...sizes].join(' + ')} AS bound FROM ${source(table)} ${condition.sql}
|
|
421
|
+
ORDER BY ${quoted(table.keyset)} LIMIT ${OPS_DUMP_PAGE_ROWS}`
|
|
422
|
+
return storage.sql.exec(query, ...condition.bindings).toArray().map((row) => {
|
|
423
|
+
const bound = row['bound']
|
|
424
|
+
if (typeof bound !== 'number') {
|
|
425
|
+
throw new Error("a row's bound reads as a number")
|
|
426
|
+
}
|
|
427
|
+
return bound
|
|
428
|
+
})
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
interface ReadValues {
|
|
432
|
+
readonly rowid: string
|
|
433
|
+
readonly values: readonly OpsDumpValue[]
|
|
434
|
+
}
|
|
435
|
+
|
|
436
|
+
/**
|
|
437
|
+
* The next `limit` rows: each its rowid as decimal text, then the rowid and every column through the wire's projection.
|
|
438
|
+
* The column-name list of the common table expression names every result column, which an expression alone does not.
|
|
439
|
+
*/
|
|
440
|
+
function readRows(storage: Storage, table: OpsDumpTablePlan, after: string | null, limit: number): ReadValues[] {
|
|
441
|
+
const keyset = quoted(table.keyset)
|
|
442
|
+
const names = ['k', 'r', ...table.columns.flatMap((_, index) => [`t${index}`, `v${index}`])]
|
|
443
|
+
const projections = [keyset, `CAST(${keyset} AS TEXT)`, ...table.columns.map(opsDumpColumnProjection)]
|
|
444
|
+
const condition = keysetCondition(table, after)
|
|
445
|
+
const query = `WITH page(${names.join(', ')}) AS (SELECT ${projections.join(', ')} FROM ${source(table)} ${condition.sql}
|
|
446
|
+
ORDER BY ${keyset} LIMIT ${limit}) SELECT ${names.slice(1).join(', ')} FROM page ORDER BY k`
|
|
447
|
+
return storage.sql.exec(query, ...condition.bindings).toArray().map((row) => {
|
|
448
|
+
const rowid = readOpsDumpValue('integer', row['r'])
|
|
449
|
+
if (rowid === null || rowid.type !== 'integer') {
|
|
450
|
+
throw new Error('a rowid reads as an integer')
|
|
451
|
+
}
|
|
452
|
+
return { rowid: rowid.value, values: [rowid, ...table.columns.map((_, index) => readOpsDumpValue(row[`t${index}`], row[`v${index}`]))] }
|
|
453
|
+
})
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
/** The tables a part finished, with their counts and hashes, and the cursor where the next part starts. */
|
|
457
|
+
interface Settled {
|
|
458
|
+
readonly finished: readonly OpsDumpTable[]
|
|
459
|
+
readonly next: OpsDumpCursor | null
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
/** Folds the part's first `count` rows into their tables' hashes from `start`; only the rows the part keeps are hashed. */
|
|
463
|
+
function settle(start: OpsDumpCursor, collected: Collected, count: number): Settled {
|
|
464
|
+
const decoder = new TextDecoder()
|
|
465
|
+
const finished: OpsDumpTable[] = []
|
|
466
|
+
const at = (cursor: OpsDumpCursor, table: string) => cursor.table === table ? cursor : fresh(table)
|
|
467
|
+
let cursor = start
|
|
468
|
+
let kept = 0
|
|
469
|
+
for (const item of collected.items) {
|
|
470
|
+
if (item.kind === 'end') {
|
|
471
|
+
const { rows, hash } = at(cursor, item.table)
|
|
472
|
+
finished.push({ name: item.table, rows, hash })
|
|
473
|
+
} else if (kept === count) {
|
|
474
|
+
return { finished, next: at(cursor, item.table) }
|
|
475
|
+
} else {
|
|
476
|
+
const row = decoder.decode(item.line.subarray(item.rowStart, item.line.byteLength - '}\n'.length))
|
|
477
|
+
const before = at(cursor, item.table)
|
|
478
|
+
cursor = { table: item.table, after: item.rowid, rows: before.rows + 1, hash: foldOpsTableHash(before.hash, [row]) }
|
|
479
|
+
kept++
|
|
480
|
+
}
|
|
481
|
+
}
|
|
482
|
+
return { finished, next: collected.breaker === null ? null : at(cursor, collected.breaker) }
|
|
483
|
+
}
|
|
484
|
+
|
|
485
|
+
// Writing: one gzip member per part.
|
|
486
|
+
|
|
487
|
+
/** The first `rows` lines of a part and their deflate stream; `last` when nothing follows them. */
|
|
488
|
+
interface Filled {
|
|
489
|
+
readonly rows: number
|
|
490
|
+
readonly raw: Uint8Array
|
|
491
|
+
readonly deflate: Uint8Array
|
|
492
|
+
readonly last: boolean
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
/**
|
|
496
|
+
* The rows that fit one part compressed: all of them when they do, else the longest prefix that does. A row that does
|
|
497
|
+
* not fit alone is refused.
|
|
498
|
+
*/
|
|
499
|
+
function fill(collected: Collected): Filled | OpsDataRefusal {
|
|
500
|
+
const lines = collected.items.flatMap((item) => item.kind === 'row' ? [item.line] : [])
|
|
501
|
+
const all = compressed(lines, lines.length, collected.breaker === null)
|
|
502
|
+
if (memberSize(all) <= OPS_DUMP_PART_SIZE) {
|
|
503
|
+
return all
|
|
504
|
+
}
|
|
505
|
+
// The compressed size grows with the prefix: the longest prefix that fits is found by interpolation and halving.
|
|
506
|
+
const target = OPS_DUMP_PART_SIZE - PADDED_OVERHEAD
|
|
507
|
+
let fits: Filled | null = null
|
|
508
|
+
let breaksRows = lines.length
|
|
509
|
+
let breaksSize = all.deflate.byteLength
|
|
510
|
+
let halve = false
|
|
511
|
+
while (breaksRows - (fits?.rows ?? 0) > 1) {
|
|
512
|
+
const fitsRows = fits?.rows ?? 0
|
|
513
|
+
const fitsSize = fits?.deflate.byteLength ?? 0
|
|
514
|
+
const width = breaksRows - fitsRows
|
|
515
|
+
const interpolated = fitsRows + Math.floor((width * (target - fitsSize)) / Math.max(1, breaksSize - fitsSize))
|
|
516
|
+
const guess = halve ? fitsRows + Math.floor(width / 2) : Math.min(breaksRows - 1, Math.max(fitsRows + 1, interpolated))
|
|
517
|
+
const candidate = compressed(lines, guess, false)
|
|
518
|
+
if (candidate.deflate.byteLength <= target) {
|
|
519
|
+
fits = candidate
|
|
520
|
+
} else {
|
|
521
|
+
breaksRows = guess
|
|
522
|
+
breaksSize = candidate.deflate.byteLength
|
|
523
|
+
}
|
|
524
|
+
halve = breaksRows - (fits?.rows ?? 0) > width / 2
|
|
525
|
+
}
|
|
526
|
+
if (fits === null) {
|
|
527
|
+
const first = collected.items.find((item) => item.kind === 'row')
|
|
528
|
+
return { refused: 'row_too_large', table: first?.table ?? null }
|
|
529
|
+
}
|
|
530
|
+
return fits
|
|
531
|
+
}
|
|
532
|
+
|
|
533
|
+
function compressed(lines: readonly Uint8Array[], rows: number, last: boolean): Filled {
|
|
534
|
+
const raw = new Uint8Array(lines.slice(0, rows).reduce((sum, line) => sum + line.byteLength, 0))
|
|
535
|
+
let offset = 0
|
|
536
|
+
for (const line of lines.slice(0, rows)) {
|
|
537
|
+
raw.set(line, offset)
|
|
538
|
+
offset += line.byteLength
|
|
539
|
+
}
|
|
540
|
+
return { rows, raw, deflate: deflateRawSync(raw), last }
|
|
541
|
+
}
|
|
542
|
+
|
|
543
|
+
/** The gzip header (RFC 1952) without a name or a time, and the trailer; a padded member adds its comment's terminator. */
|
|
544
|
+
const GZIP_HEADER_BYTES = 10
|
|
545
|
+
const GZIP_TRAILER_BYTES = 8
|
|
546
|
+
const PADDED_OVERHEAD = GZIP_HEADER_BYTES + 1 + GZIP_TRAILER_BYTES
|
|
547
|
+
const FCOMMENT = 0x10
|
|
548
|
+
|
|
549
|
+
function memberSize(part: Filled): number {
|
|
550
|
+
return (part.last ? GZIP_HEADER_BYTES + GZIP_TRAILER_BYTES : PADDED_OVERHEAD) + part.deflate.byteLength
|
|
551
|
+
}
|
|
552
|
+
|
|
553
|
+
/** One gzip member of the part; every part but the last is padded to exactly the part size with spaces in its comment. */
|
|
554
|
+
function gzipMember(part: Filled): Uint8Array<ArrayBuffer> {
|
|
555
|
+
const member = new Uint8Array(part.last ? memberSize(part) : OPS_DUMP_PART_SIZE)
|
|
556
|
+
member.set([0x1F, 0x8B, 0x08, part.last ? 0 : FCOMMENT, 0, 0, 0, 0, 0, 0xFF])
|
|
557
|
+
let offset = GZIP_HEADER_BYTES
|
|
558
|
+
if (!part.last) {
|
|
559
|
+
const comment = OPS_DUMP_PART_SIZE - memberSize(part)
|
|
560
|
+
member.fill(0x20, offset, offset + comment)
|
|
561
|
+
offset += comment + 1
|
|
562
|
+
}
|
|
563
|
+
member.set(part.deflate, offset)
|
|
564
|
+
offset += part.deflate.byteLength
|
|
565
|
+
const trailer = new DataView(member.buffer, offset, GZIP_TRAILER_BYTES)
|
|
566
|
+
trailer.setUint32(0, crc32(part.raw), true)
|
|
567
|
+
trailer.setUint32(4, part.raw.byteLength >>> 0, true)
|
|
568
|
+
return member
|
|
569
|
+
}
|
|
570
|
+
|
|
571
|
+
// The schema and the migration ledger: what `sqlite_schema` and the ledger's two tables say, for the manifest and for
|
|
572
|
+
// the state a consistent dump begins at. The wire's schema rule reads the importer's description of the PRAGMAs.
|
|
573
|
+
|
|
574
|
+
/** SQLite's own tables but `sqlite_sequence`, and Cloudflare's `_cf_` tables, are no table of the dump. */
|
|
575
|
+
function dumped(name: string): boolean {
|
|
576
|
+
return name === 'sqlite_sequence' || !(name.startsWith('sqlite_') || name.startsWith('_cf_'))
|
|
577
|
+
}
|
|
578
|
+
|
|
579
|
+
function rowsOf(storage: Storage, query: string, ...bindings: SqliteValue[]): Record<string, SqliteValue>[] {
|
|
580
|
+
return storage.sql.exec(query, ...bindings).toArray()
|
|
581
|
+
}
|
|
582
|
+
|
|
583
|
+
function text(value: SqliteValue | undefined): string {
|
|
584
|
+
if (typeof value !== 'string') {
|
|
585
|
+
throw new Error('the schema answered a value that is not text')
|
|
586
|
+
}
|
|
587
|
+
return value
|
|
588
|
+
}
|
|
589
|
+
|
|
590
|
+
const SCHEMA_TYPES: readonly OpsDumpSchemaEntry['type'][] = ['table', 'index', 'trigger', 'view']
|
|
591
|
+
|
|
592
|
+
/** Every `sqlite_schema` entry with SQL of a dumped table, ordered by type and name, which no history changes. */
|
|
593
|
+
function schemaEntries(storage: Storage): OpsDumpSchemaEntry[] {
|
|
594
|
+
const entries: OpsDumpSchemaEntry[] = []
|
|
595
|
+
for (const row of rowsOf(storage, 'SELECT type, name, tbl_name, sql FROM main.sqlite_schema WHERE sql IS NOT NULL')) {
|
|
596
|
+
const type = SCHEMA_TYPES.find((candidate) => candidate === row['type'])
|
|
597
|
+
const name = text(row['name'])
|
|
598
|
+
const table = text(row['tbl_name'])
|
|
599
|
+
if (type !== undefined && dumped(name) && dumped(table)) {
|
|
600
|
+
entries.push({ type, name, table, sql: text(row['sql']) })
|
|
601
|
+
}
|
|
602
|
+
}
|
|
603
|
+
const order = (entry: OpsDumpSchemaEntry) => `${entry.type}\n${entry.name}`
|
|
604
|
+
return entries.sort((left, right) => order(left) < order(right) ? -1 : order(left) > order(right) ? 1 : 0)
|
|
605
|
+
}
|
|
606
|
+
|
|
607
|
+
/** The ledger `@fabrika/db/sqlite-migrate` keeps for both bundles; none in a database restored to before it existed. */
|
|
608
|
+
function migrationLedger(storage: Storage): Record<string, SqliteValue>[] {
|
|
609
|
+
return holdsTable(storage, 'fabrika_migrations')
|
|
610
|
+
? rowsOf(storage, 'SELECT bundle, name, checksum, applied_at AS appliedAt FROM main.fabrika_migrations ORDER BY bundle, name')
|
|
611
|
+
: []
|
|
612
|
+
}
|
|
613
|
+
|
|
614
|
+
/** The digest each bundle's last applied file recorded, which the runner may write without applying a file. */
|
|
615
|
+
function bundleDigests(storage: Storage): Record<string, SqliteValue>[] {
|
|
616
|
+
return holdsTable(storage, 'fabrika_migration_bundles')
|
|
617
|
+
? rowsOf(storage, 'SELECT bundle, digest FROM main.fabrika_migration_bundles ORDER BY bundle')
|
|
618
|
+
: []
|
|
619
|
+
}
|
|
620
|
+
|
|
621
|
+
function holdsTable(storage: Storage, name: string): boolean {
|
|
622
|
+
return rowsOf(storage, `SELECT 1 FROM main.sqlite_schema WHERE type = 'table' AND name = ?`, name).length > 0
|
|
623
|
+
}
|
|
624
|
+
|
|
625
|
+
/**
|
|
626
|
+
* The digest of what a consistent dump's steps must all see: the stretch of modes that write nothing, the DDL and both
|
|
627
|
+
* ledger tables. Inside a stretch a migration is what writes unannounced, and the runner records each of its writes in
|
|
628
|
+
* one of them; an import step or a restore's `arm` starts a new stretch.
|
|
629
|
+
*/
|
|
630
|
+
function dumpState(storage: Storage, frozenSince: number): string {
|
|
631
|
+
return sha256Hex(JSON.stringify([frozenSince, schemaEntries(storage), migrationLedger(storage), bundleDigests(storage)]))
|
|
632
|
+
}
|
|
633
|
+
|
|
634
|
+
/** A table as a query reads it: qualified, so no common table expression of the query shadows it. */
|
|
635
|
+
function source(table: OpsDumpTablePlan): string {
|
|
636
|
+
return `main.${quoted(table.name)}`
|
|
637
|
+
}
|
|
638
|
+
|
|
639
|
+
function quoted(name: string): string {
|
|
640
|
+
return `"${name.replaceAll('"', '""')}"`
|
|
641
|
+
}
|