@runeditrun/rer 0.2.77 → 0.2.78
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -0
- package/cli/export.mjs +1 -1
- package/cli/portability-migrations.mjs +35 -14
- package/cli/portability-worker.mjs +2 -2
- package/cli/portability.mjs +2 -2
- package/global/CHANGELOG.md +8 -0
- package/global/README.md +32 -0
- package/modules/deploy-kit/owner-cli.mjs +15 -5
- package/modules/deploy-kit/provider.mjs +9 -4
- package/modules/import-kit/src/d1/README.md +1322 -0
- package/modules/import-kit/src/d1/apply.ts +115 -0
- package/modules/import-kit/src/d1/budget.ts +89 -0
- package/modules/import-kit/src/d1/compressed-plan.ts +63 -0
- package/modules/import-kit/src/d1/creation.ts +95 -0
- package/modules/import-kit/src/d1/custody-fence.ts +27 -0
- package/modules/import-kit/src/d1/custody-state-schema.ts +67 -0
- package/modules/import-kit/src/d1/database.ts +163 -0
- package/modules/import-kit/src/d1/entry-locator.ts +61 -0
- package/modules/import-kit/src/d1/erased-artifacts.ts +38 -0
- package/modules/import-kit/src/d1/erased-custody.ts +36 -0
- package/modules/import-kit/src/d1/erased-plan.ts +47 -0
- package/modules/import-kit/src/d1/erased-reversal.ts +19 -0
- package/modules/import-kit/src/d1/erasure-bounded.ts +22 -0
- package/modules/import-kit/src/d1/erasure-claims.ts +27 -0
- package/modules/import-kit/src/d1/erasure-compiler.ts +67 -0
- package/modules/import-kit/src/d1/erasure-disposition.ts +16 -0
- package/modules/import-kit/src/d1/erasure-fragments.ts +86 -0
- package/modules/import-kit/src/d1/erasure-index.ts +53 -0
- package/modules/import-kit/src/d1/erasure-pages.ts +62 -0
- package/modules/import-kit/src/d1/erasure-progress-schema.ts +8 -0
- package/modules/import-kit/src/d1/erasure-progress.ts +61 -0
- package/modules/import-kit/src/d1/erasure-related.ts +124 -0
- package/modules/import-kit/src/d1/erasure-report.ts +31 -0
- package/modules/import-kit/src/d1/erasure-schema.ts +24 -0
- package/modules/import-kit/src/d1/erasure-selection.ts +77 -0
- package/modules/import-kit/src/d1/erasure-snapshot.ts +15 -0
- package/modules/import-kit/src/d1/erasure-stage.ts +146 -0
- package/modules/import-kit/src/d1/erasure-step.ts +12 -0
- package/modules/import-kit/src/d1/erasure-superseded.ts +98 -0
- package/modules/import-kit/src/d1/erasure-values.ts +83 -0
- package/modules/import-kit/src/d1/erasure-writes.ts +46 -0
- package/modules/import-kit/src/d1/erasure.ts +170 -0
- package/modules/import-kit/src/d1/evolutions/0001-preserved-undo.ts +13 -0
- package/modules/import-kit/src/d1/expected-state.ts +91 -0
- package/modules/import-kit/src/d1/field-redaction.ts +86 -0
- package/modules/import-kit/src/d1/guards.ts +37 -0
- package/modules/import-kit/src/d1/history.ts +71 -0
- package/modules/import-kit/src/d1/index.ts +88 -0
- package/modules/import-kit/src/d1/integrity.ts +127 -0
- package/modules/import-kit/src/d1/inventory-snapshot.ts +84 -0
- package/modules/import-kit/src/d1/jobs.ts +97 -0
- package/modules/import-kit/src/d1/native-set.ts +48 -0
- package/modules/import-kit/src/d1/native-staging-admission.ts +19 -0
- package/modules/import-kit/src/d1/native-staging-context.ts +44 -0
- package/modules/import-kit/src/d1/native-staging-lifecycle.ts +33 -0
- package/modules/import-kit/src/d1/native-staging-offset.ts +74 -0
- package/modules/import-kit/src/d1/native-staging-publication.ts +18 -0
- package/modules/import-kit/src/d1/native-staging-rows.ts +72 -0
- package/modules/import-kit/src/d1/native-staging-schema.ts +28 -0
- package/modules/import-kit/src/d1/native-staging-state.ts +22 -0
- package/modules/import-kit/src/d1/native-staging.ts +64 -0
- package/modules/import-kit/src/d1/original-copies.ts +53 -0
- package/modules/import-kit/src/d1/original-storage.ts +56 -0
- package/modules/import-kit/src/d1/originals.ts +74 -0
- package/modules/import-kit/src/d1/plan-dictionary.ts +80 -0
- package/modules/import-kit/src/d1/plans.ts +121 -0
- package/modules/import-kit/src/d1/preparation-fingerprint.ts +57 -0
- package/modules/import-kit/src/d1/preparation-header.ts +33 -0
- package/modules/import-kit/src/d1/preparation-reader.ts +98 -0
- package/modules/import-kit/src/d1/preparation-schema.ts +16 -0
- package/modules/import-kit/src/d1/preparation-state.ts +56 -0
- package/modules/import-kit/src/d1/preparation-step-combine.ts +80 -0
- package/modules/import-kit/src/d1/preparation-step-control.ts +48 -0
- package/modules/import-kit/src/d1/preparation-step-hash.ts +131 -0
- package/modules/import-kit/src/d1/preparation-step-pages.ts +77 -0
- package/modules/import-kit/src/d1/preparation-steps-schema.ts +11 -0
- package/modules/import-kit/src/d1/preparation-steps.ts +102 -0
- package/modules/import-kit/src/d1/preparations.ts +87 -0
- package/modules/import-kit/src/d1/preserved-preview.ts +89 -0
- package/modules/import-kit/src/d1/preserved-reverse.ts +91 -0
- package/modules/import-kit/src/d1/provenance-set.ts +87 -0
- package/modules/import-kit/src/d1/publication-fence.ts +22 -0
- package/modules/import-kit/src/d1/publication-receipt-schema.ts +15 -0
- package/modules/import-kit/src/d1/published-stream.ts +11 -0
- package/modules/import-kit/src/d1/read-fence.ts +64 -0
- package/modules/import-kit/src/d1/read-results.ts +16 -0
- package/modules/import-kit/src/d1/redacted-custody.ts +76 -0
- package/modules/import-kit/src/d1/redaction-step.ts +125 -0
- package/modules/import-kit/src/d1/referenced-plans.ts +136 -0
- package/modules/import-kit/src/d1/retained-values.ts +35 -0
- package/modules/import-kit/src/d1/retirement-bounded.ts +71 -0
- package/modules/import-kit/src/d1/retirement-door.ts +105 -0
- package/modules/import-kit/src/d1/retirement-schema.ts +39 -0
- package/modules/import-kit/src/d1/retirement-scope.ts +94 -0
- package/modules/import-kit/src/d1/retirement-steps.ts +76 -0
- package/modules/import-kit/src/d1/retirement-tally.ts +74 -0
- package/modules/import-kit/src/d1/retirement.ts +136 -0
- package/modules/import-kit/src/d1/reversal-conversion-record.ts +19 -0
- package/modules/import-kit/src/d1/reversal-conversion-schema.ts +11 -0
- package/modules/import-kit/src/d1/reversal-conversion.ts +40 -0
- package/modules/import-kit/src/d1/reversal-fence-schema.ts +14 -0
- package/modules/import-kit/src/d1/reversal-fence.ts +106 -0
- package/modules/import-kit/src/d1/reversal-journal.ts +24 -0
- package/modules/import-kit/src/d1/reverse.ts +103 -0
- package/modules/import-kit/src/d1/reviewed-source.ts +23 -0
- package/modules/import-kit/src/d1/sets.ts +78 -0
- package/modules/import-kit/src/d1/staged-creation.ts +201 -0
- package/modules/import-kit/src/d1/staging-admission.ts +45 -0
- package/modules/import-kit/src/d1/staging-control.ts +43 -0
- package/modules/import-kit/src/d1/staging-erasure.ts +28 -0
- package/modules/import-kit/src/d1/staging-lifecycle.ts +49 -0
- package/modules/import-kit/src/d1/staging-plan.ts +15 -0
- package/modules/import-kit/src/d1/staging-privacy.ts +22 -0
- package/modules/import-kit/src/d1/staging-publication.ts +79 -0
- package/modules/import-kit/src/d1/staging-schema.ts +37 -0
- package/modules/import-kit/src/d1/staging-statements.ts +8 -0
- package/modules/import-kit/src/d1/staging-transport.ts +70 -0
- package/modules/import-kit/src/d1/stream-boundary.ts +11 -0
- package/modules/import-kit/src/d1/stream-create.ts +55 -0
- package/modules/import-kit/src/d1/stream-identity-admission.ts +37 -0
- package/modules/import-kit/src/d1/stream-input.ts +18 -0
- package/modules/import-kit/src/d1/stream-layout.ts +43 -0
- package/modules/import-kit/src/d1/stream-origin.ts +44 -0
- package/modules/import-kit/src/d1/stream-page-integrity.ts +18 -0
- package/modules/import-kit/src/d1/stream-partition.ts +50 -0
- package/modules/import-kit/src/d1/stream-plan.ts +133 -0
- package/modules/import-kit/src/d1/stream-receipt.ts +45 -0
- package/modules/import-kit/src/d1/stream-revision-schema.ts +35 -0
- package/modules/import-kit/src/d1/stream-snapshot.ts +26 -0
- package/modules/import-kit/src/d1/stream-stage.ts +79 -0
- package/modules/import-kit/src/d1/stream-state.ts +25 -0
- package/modules/import-kit/src/d1/stream-working-set.ts +30 -0
- package/modules/import-kit/src/d1/streams.ts +39 -0
- package/modules/import-kit/src/d1/target-guards.ts +9 -0
- package/modules/import-kit/src/d1/target-identities.ts +17 -0
- package/modules/import-kit/src/d1/transport.ts +10 -0
- package/modules/import-kit/src/d1/types.ts +162 -0
- package/modules/import-kit/src/d1/undo-observations.ts +39 -0
- package/modules/import-kit/src/d1/undo-report.ts +22 -0
- package/modules/import-kit/src/d1/undo-snapshot.ts +87 -0
- package/modules/import-kit/src/d1/undo-state.ts +84 -0
- package/modules/import-kit/src/d1/undo-types.ts +110 -0
- package/modules/import-kit/src/journal/authority.ts +12 -0
- package/modules/import-kit/src/plan/budget.ts +60 -0
- package/modules/import-kit/src/plan/index.ts +123 -0
- package/modules/import-kit/src/plan/size.ts +122 -0
- package/modules/import-kit/src/provenance/canonical-hash.ts +130 -0
- package/modules/import-kit/src/provenance/canonical-stream.ts +85 -0
- package/modules/import-kit/src/provenance/credential-names.ts +131 -0
- package/modules/import-kit/src/provenance/credentials.ts +317 -0
- package/modules/import-kit/src/provenance/identity.ts +51 -0
- package/modules/import-kit/src/provenance/json-snapshot.ts +21 -0
- package/modules/import-kit/src/provenance/resumable-sha256.ts +75 -0
- package/modules/import-kit/src/provenance/validate-json.ts +19 -0
- package/modules/import-kit/src/storage/originals.ts +87 -0
- package/modules/import-kit/src/storage/provenance.ts +64 -0
- package/modules/import-kit/src/storage/schema.ts +42 -0
- package/modules/import-kit/src/types.ts +47 -0
- package/modules/portability/README.md +98 -2
- package/modules/portability/backfills.mjs +42 -15
- package/modules/portability/ledgers.mjs +7 -1
- package/modules/portability/transient-tables.mjs +40 -0
- package/package.json +3 -1
- package/scripts/publish-check/reviewed.py +3 -1
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import { storedPlanSnapshot } from './plans.ts';
|
|
2
|
+
import { originalCopies, copyGuards } from './original-copies.ts';
|
|
3
|
+
import { Buffer } from 'node:buffer';
|
|
4
|
+
import type { ImportPlan } from '../types.ts';
|
|
5
|
+
import { originalsFor } from '../storage/originals.ts';
|
|
6
|
+
import type { D1Sql } from './types.ts';
|
|
7
|
+
import { guard, sql } from './database.ts';
|
|
8
|
+
import { exactRows, jsonChunks, PACKED_JSON_BYTES, SNAPSHOT_BYTES } from './sets.ts';
|
|
9
|
+
export const originalVersions = new WeakMap<ImportPlan, number>();
|
|
10
|
+
export function originalStorage(plan: ImportPlan, id: string, version: number) {
|
|
11
|
+
const chunks: any[] = [];
|
|
12
|
+
const rows = originalsFor(plan, id).map(o => {
|
|
13
|
+
if (version === 3) {
|
|
14
|
+
if (o.original.status === 'retained') {
|
|
15
|
+
const base64 = o.original.base64;
|
|
16
|
+
// Empty originals still have one exact chunk; every boundary is a
|
|
17
|
+
// complete base64 quantum. No provider binary conversion is required.
|
|
18
|
+
for (let start = 0, ordinal = 0; start < base64.length || ordinal === 0; start += SNAPSHOT_BYTES, ordinal++) chunks.push({ job_id: id, kind: o.kind, original_key: o.key, ordinal, bytes: base64.slice(start, start + SNAPSHOT_BYTES) });
|
|
19
|
+
}
|
|
20
|
+
return { job_id: id, kind: o.kind, original_key: o.key, name: o.name, sha256: o.sha256, size: o.size, bytes: null };
|
|
21
|
+
}
|
|
22
|
+
const bytes = o.original.status === 'retained' ? Buffer.from(o.original.base64, 'base64') : null;
|
|
23
|
+
if (version === 2 && bytes && bytes.length > SNAPSHOT_BYTES) for (let start = 0, ordinal = 0; start < bytes.length; start += SNAPSHOT_BYTES, ordinal++) chunks.push({ job_id: id, kind: o.kind, original_key: o.key, ordinal, bytes: bytes.subarray(start, start + SNAPSHOT_BYTES).toString('hex').toUpperCase() });
|
|
24
|
+
return { job_id: id, kind: o.kind, original_key: o.key, name: o.name, sha256: o.sha256, size: o.size, bytes: bytes && !(version === 2 && bytes.length > SNAPSHOT_BYTES) ? bytes.toString('hex').toUpperCase() : null };
|
|
25
|
+
});
|
|
26
|
+
return { rows, chunks };
|
|
27
|
+
}
|
|
28
|
+
/** `encoded` is the plan's stored form when it is not a loaded snapshot (creation). */
|
|
29
|
+
export function custodyGuards(plan: ImportPlan, id: string, encoded = storedPlanSnapshot(plan, id)): D1Sql[] {
|
|
30
|
+
const version = originalVersions.get(plan) ?? 3, { rows, chunks } = originalStorage(plan, id, version);
|
|
31
|
+
const guards = [guard(sql('EXISTS(SELECT 1 FROM ik_original_sets WHERE job_id=? AND version=?)', id, version))];
|
|
32
|
+
if (version === 3) {
|
|
33
|
+
const metadata = exactRows('ik_originals', ['job_id','kind','original_key','name','sha256','size','bytes'], rows, sql('job_id=?', id), PACKED_JSON_BYTES);
|
|
34
|
+
const count = guard(sql('(SELECT COUNT(*) FROM ik_d1_original_chunks WHERE job_id=?)=?', id, chunks.length));
|
|
35
|
+
// Canonical chunks hold each original literally: compare positions, never
|
|
36
|
+
// bytes. A compressed plan does not, so its original chunks compare exactly.
|
|
37
|
+
if (encoded.chunks.length && !encoded.compressed) return [...guards, ...metadata, count, ...copyGuards(originalCopies(encoded, chunks))];
|
|
38
|
+
return [...guards, ...metadata, ...exactRows('ik_d1_original_chunks', ['job_id','kind','original_key','ordinal','bytes'], chunks, sql('job_id=?', id), PACKED_JSON_BYTES)];
|
|
39
|
+
}
|
|
40
|
+
for (const [table, expected, columns] of [
|
|
41
|
+
['ik_originals', rows, ['job_id','kind','original_key','name','sha256','size']],
|
|
42
|
+
['ik_d1_original_chunks', chunks, ['job_id','kind','original_key','ordinal']],
|
|
43
|
+
] as const) {
|
|
44
|
+
guards.push(guard(sql(`(SELECT COUNT(*) FROM ${table} WHERE job_id=?)=?`, id, expected.length)));
|
|
45
|
+
for (const row of expected) {
|
|
46
|
+
// An old inline original can be near 2 MB; encode no giant hex snapshot.
|
|
47
|
+
if (row.bytes !== null && row.bytes.length > 2 * SNAPSHOT_BYTES) {
|
|
48
|
+
guards.push(...exactRows(table, [...columns], [row]));
|
|
49
|
+
const key = columns.filter(c => c === 'job_id' || c === 'kind' || c === 'original_key' || c === 'ordinal');
|
|
50
|
+
guards.push(guard(sql(`EXISTS(SELECT 1 FROM ${table} WHERE ${key.map(c => `${c}=?`).join(' AND ')} AND bytes IS ?)`, ...key.map(c => row[c]), Uint8Array.from(Buffer.from(row.bytes, 'hex')).buffer)));
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
for (const chunk of jsonChunks(expected.filter(r => r.bytes === null || r.bytes.length <= 2 * SNAPSHOT_BYTES))) guards.push(guard(sql(`NOT EXISTS(SELECT 1 FROM json_each(?) e WHERE NOT EXISTS(SELECT 1 FROM ${table} t WHERE ${columns.map(c => `t.${c} IS json_extract(e.value,'$.${c}')`).join(' AND ')} AND CASE WHEN t.bytes IS NULL THEN NULL ELSE hex(t.bytes) END IS json_extract(e.value,'$.bytes')))`, chunk)));
|
|
54
|
+
}
|
|
55
|
+
return guards;
|
|
56
|
+
}
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import type { InventorySnapshot } from './inventory-snapshot.ts';
|
|
2
|
+
import { requestContext } from './budget.ts';
|
|
3
|
+
import { originalStorage, originalVersions, custodyGuards } from './original-storage.ts';
|
|
4
|
+
import { Buffer } from 'node:buffer';
|
|
5
|
+
import type { ImportPlan, RetainedOriginal } from '../types.ts';
|
|
6
|
+
import { originalsFor } from '../storage/originals.ts';
|
|
7
|
+
import { sha256 } from '../provenance/identity.ts';
|
|
8
|
+
import type { D1ImportContext, D1Sql } from './types.ts';
|
|
9
|
+
import { sql, guard, authorize, readAtomic } from './database.ts';
|
|
10
|
+
import { stored, boundary } from './jobs.ts';
|
|
11
|
+
import { originalCopies, retainCopies } from './original-copies.ts';
|
|
12
|
+
import type { StoredPlan } from './plans.ts';
|
|
13
|
+
import { insertJsonRows } from './sets.ts';
|
|
14
|
+
import { nativeStagingScope } from './native-staging-context.ts';
|
|
15
|
+
|
|
16
|
+
export function retainOriginals(plan: ImportPlan, id: string, encoded: StoredPlan): D1Sql[] {
|
|
17
|
+
const { rows, chunks } = originalStorage(plan, id, 3);
|
|
18
|
+
return [sql('INSERT INTO ik_original_sets(job_id,version) VALUES(?,3)', id),
|
|
19
|
+
...insertJsonRows('ik_originals', ['job_id','kind','original_key','name','sha256','size','bytes'], rows),
|
|
20
|
+
// Canonical chunks already hold each original: copy it in SQL. A compressed
|
|
21
|
+
// plan does not, so its original chunks are bound once, here.
|
|
22
|
+
...(encoded.chunks.length && !encoded.compressed ? retainCopies(originalCopies(encoded, chunks)) : insertJsonRows('ik_d1_original_chunks', ['job_id','kind','original_key','ordinal','bytes'], chunks)),
|
|
23
|
+
// INSERT SELECT must not silently publish fewer rows if a source chunk
|
|
24
|
+
// disappears. A failing CHECK rolls the whole creation transaction back.
|
|
25
|
+
guard(sql('(SELECT COUNT(*) FROM ik_d1_original_chunks WHERE job_id=?)=?', id, chunks.length))];
|
|
26
|
+
}
|
|
27
|
+
export async function readOriginals(plan: ImportPlan, id: string, context: D1ImportContext, snapshot?: InventorySnapshot): Promise<RetainedOriginal[]> {
|
|
28
|
+
const expected = originalsFor(plan, id), db = context.storage.database;
|
|
29
|
+
// Validate metadata types with the existing version read, without spending an
|
|
30
|
+
// additional D1 request or passing a BLOB to SQLite's JSON encoder.
|
|
31
|
+
const version = snapshot ? snapshot.rows('ik_original_sets', 'job_id', [id])[0] : await db.prepare("SELECT version,EXISTS(SELECT 1 FROM ik_originals WHERE job_id=ik_original_sets.job_id AND typeof(name)<>'text') AS invalid_names FROM ik_original_sets WHERE job_id=?").bind(id).first();
|
|
32
|
+
if (version?.invalid_names) throw new Error('ORIGINAL_CUSTODY_MISMATCH');
|
|
33
|
+
// JSON encoding preserves embedded NULs across SQLite text drivers, including Node 22.
|
|
34
|
+
const results = snapshot ? snapshot.rows('ik_originals', 'job_id', [id]) : (await db.prepare('SELECT *,json_quote(name) AS name_json FROM ik_originals WHERE job_id=?').bind(id).all()).results;
|
|
35
|
+
const chunks = snapshot ? snapshot.rows('ik_d1_original_chunks', 'job_id', [id]).sort((a, b) => a.ordinal - b.ordinal) : await originalChunks(db, id);
|
|
36
|
+
if (![1,2,3,4].includes(version?.version) || results.length !== expected.length) throw new Error('ORIGINAL_CUSTODY_MISMATCH');
|
|
37
|
+
if (version!.version === 4 && expected.some(o => o.original.status !== 'unavailable')) throw new Error('ORIGINAL_CUSTODY_MISMATCH');
|
|
38
|
+
const encoded = originalStorage(plan, id, version!.version);
|
|
39
|
+
if (chunks.length !== encoded.chunks.length) throw new Error('ORIGINAL_CUSTODY_MISMATCH');
|
|
40
|
+
for (const o of expected) {
|
|
41
|
+
const row = results.find(r => r.kind === o.kind && r.original_key === o.key);
|
|
42
|
+
if (!row || (row.name_json === undefined ? row.name : JSON.parse(row.name_json)) !== o.name || row.sha256 !== o.sha256 || row.size !== o.size) throw new Error('ORIGINAL_CUSTODY_MISMATCH');
|
|
43
|
+
if (o.original.status === 'unavailable') { if (row.bytes !== null) throw new Error('ORIGINAL_CUSTODY_MISMATCH'); }
|
|
44
|
+
else {
|
|
45
|
+
const parts = chunks.filter(c => c.kind === o.kind && c.original_key === o.key);
|
|
46
|
+
const expectedParts = encoded.chunks.filter(c => c.kind === o.kind && c.original_key === o.key);
|
|
47
|
+
if (parts.length !== expectedParts.length || parts.some((c, i) => c.ordinal !== i || (version!.version === 3 ? c.bytes : Buffer.from(c.bytes).toString('hex').toUpperCase()) !== expectedParts[i].bytes)) throw new Error('ORIGINAL_CUSTODY_MISMATCH');
|
|
48
|
+
if (parts.length && row.bytes !== null) throw new Error('ORIGINAL_CUSTODY_MISMATCH');
|
|
49
|
+
if (!parts.length && row.bytes === null) throw new Error('ORIGINAL_CUSTODY_MISSING');
|
|
50
|
+
const bytes = parts.length ? version!.version === 3 ? Buffer.from(parts.map(c => c.bytes).join(''), 'base64') : Buffer.concat(parts.map(c => Buffer.from(c.bytes))) : Buffer.from(row.bytes);
|
|
51
|
+
if (bytes.length !== o.size || sha256(bytes) !== o.sha256 || bytes.toString('base64') !== o.original.base64) throw new Error('ORIGINAL_CUSTODY_MISMATCH');
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
originalVersions.set(plan, version!.version);
|
|
55
|
+
return expected;
|
|
56
|
+
}
|
|
57
|
+
/** Original chunks are at most 128 KiB of base64 each; one read returns a
|
|
58
|
+
* bounded page of them, never a whole multi-megabyte original. */
|
|
59
|
+
const ORIGINAL_READ_ROWS = 16;
|
|
60
|
+
async function originalChunks(db: D1ImportContext['storage']['database'], id: string): Promise<any[]> {
|
|
61
|
+
const rows: any[] = [];
|
|
62
|
+
for (;;) {
|
|
63
|
+
const { results } = await db.prepare(`SELECT * FROM ik_d1_original_chunks WHERE job_id=? ORDER BY kind,original_key,ordinal LIMIT ${ORIGINAL_READ_ROWS} OFFSET ?`).bind(id, rows.length).all();
|
|
64
|
+
rows.push(...results);
|
|
65
|
+
if (results.length < ORIGINAL_READ_ROWS) return rows;
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
export async function readD1ImportOriginals(id: string, context: D1ImportContext): Promise<RetainedOriginal[]> {
|
|
69
|
+
context = requestContext(context);
|
|
70
|
+
await authorize(context, 'read');
|
|
71
|
+
const { job, plan } = await stored(id, context), originals = await readOriginals(plan, id, context);
|
|
72
|
+
await readAtomic(context, [...boundary(job, plan), ...custodyGuards(plan, id)], nativeStagingScope(job, plan));
|
|
73
|
+
return originals;
|
|
74
|
+
}
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
import type { Json } from '../types.ts';
|
|
2
|
+
import { canonicalSize } from '../provenance/canonical-stream.ts';
|
|
3
|
+
/** Page-local lossless JSON dictionary. Tags distinguish every JSON type; object
|
|
4
|
+
* keys are data, including __proto__. No dictionary survives outside its page. */
|
|
5
|
+
export function dictionaryEncode(value: Json): Json {
|
|
6
|
+
const strings: string[] = [], indexes = new Map<string, number>();
|
|
7
|
+
const json: Json[] = [], jsonIndexes = new Map<string, number>();
|
|
8
|
+
const intern = (text: string) => { const prior=indexes.get(text);if(prior!==undefined)return prior;const id=strings.length;strings.push(text);indexes.set(text,id);return id; };
|
|
9
|
+
const encodedString = (text: string, newEntry: boolean): Json => {
|
|
10
|
+
if (text[0] === '{' || text[0] === '[') {
|
|
11
|
+
const prior = jsonIndexes.get(text);
|
|
12
|
+
if (prior !== undefined) return [3, prior];
|
|
13
|
+
let parsed: Json | undefined;
|
|
14
|
+
if (newEntry) try { const candidate = JSON.parse(text); if (JSON.stringify(candidate) === text) parsed = candidate; } catch {}
|
|
15
|
+
if (parsed !== undefined) {
|
|
16
|
+
const tree = encode(parsed, false), index = json.length;
|
|
17
|
+
json.push(tree); jsonIndexes.set(text, index); return [3, index];
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
return [0, intern(text)];
|
|
21
|
+
};
|
|
22
|
+
const encode=(item: Json, newEntry = true):Json=>{
|
|
23
|
+
if(typeof item==='string')return encodedString(item,newEntry);
|
|
24
|
+
if(Array.isArray(item))return [1,...Array.from({length:item.length},(_,index)=>encode(item[index],newEntry))];
|
|
25
|
+
if(item&&typeof item==='object'&&(Object.getPrototypeOf(item)===Object.prototype||Object.getPrototypeOf(item)===null))return [2,...Object.entries(item).flatMap(([key,entry])=>[intern(key),encode(entry,newEntry)])];
|
|
26
|
+
if(item===null||typeof item==='boolean'||typeof item==='number'&&Number.isFinite(item))return Object.is(item,-0)?0:item;
|
|
27
|
+
throw new Error('INVALID_JSON: durable values must be finite JSON');
|
|
28
|
+
};
|
|
29
|
+
// Canonically serialized JSON strings share their inner literals too. Their
|
|
30
|
+
// exact original text is reconstructed once, independently of occurrences.
|
|
31
|
+
// Encoded descendants are arrays and finite primitives. Keeping the only
|
|
32
|
+
// object in sorted-key order makes native JSON.stringify byte-canonical.
|
|
33
|
+
const tree=encode(value);return {json,strings,tree,version:2};
|
|
34
|
+
}
|
|
35
|
+
export function dictionaryDecode(value: any, maximumPayloadBytes = Number.MAX_SAFE_INTEGER): Json {
|
|
36
|
+
if (!Number.isSafeInteger(maximumPayloadBytes) || maximumPayloadBytes < 0) throw new Error('STORED_PLAN_CHANGED');
|
|
37
|
+
const current = value?.version === 2;
|
|
38
|
+
if(!value||Object.keys(value).sort().join(',')!==(current?'json,strings,tree,version':'strings,tree')||!Array.isArray(value.strings)||current&&!Array.isArray(value.json))throw new Error('STORED_PLAN_CHANGED');
|
|
39
|
+
const distinct = new Set<string>();
|
|
40
|
+
for(let index=0;index<value.strings.length;index++){const text=value.strings[index];if(typeof text!=='string'||distinct.has(text))throw new Error('STORED_PLAN_CHANGED');distinct.add(text);}
|
|
41
|
+
const used=new Set<number>();
|
|
42
|
+
const jsonStrings: string[] = [], usedJson = new Set<number>(), distinctJson = new Set<string>();
|
|
43
|
+
if(current){
|
|
44
|
+
const direct=new Set<number>();
|
|
45
|
+
const scan=(item:any):void=>{if(Array.isArray(item)){if(item[0]===3&&item.length===2){const index=item[1];if(!Number.isSafeInteger(index)||index<0||index>=value.json.length)throw new Error('STORED_PLAN_CHANGED');direct.add(index);}else for(let index=1;index<item.length;index++)scan(item[index]);}};
|
|
46
|
+
scan(value.tree);
|
|
47
|
+
// Every reconstructed string occurs in the payload itself. Intermediate
|
|
48
|
+
// nested text stays literal, so the complete table is bounded by payload
|
|
49
|
+
// bytes rather than the sum of arbitrary hidden reconstruction chains.
|
|
50
|
+
if(direct.size!==value.json.length)throw new Error('STORED_PLAN_CHANGED');
|
|
51
|
+
}
|
|
52
|
+
const string=(index:unknown)=>{if(!Number.isSafeInteger(index)||Number(index)<0||Number(index)>=value.strings.length)throw new Error('STORED_PLAN_CHANGED');used.add(Number(index));return value.strings[Number(index)] as string;};
|
|
53
|
+
const decode=(item:any):Json=>{
|
|
54
|
+
if(!Array.isArray(item)){if(item===null||typeof item==='boolean'||typeof item==='number'&&Number.isFinite(item))return item;throw new Error('STORED_PLAN_CHANGED');}
|
|
55
|
+
if(item[0]===0&&item.length===2)return string(item[1]);
|
|
56
|
+
if(current&&item[0]===3&&item.length===2){const index=item[1];if(!Number.isSafeInteger(index)||index<0||index>=jsonStrings.length)throw new Error('STORED_PLAN_CHANGED');usedJson.add(index);return jsonStrings[index];}
|
|
57
|
+
if(item[0]===1)return Array.from({length:item.length-1},(_,index)=>decode(item[index+1]));
|
|
58
|
+
if(item[0]!==2||item.length%2!==1)throw new Error('STORED_PLAN_CHANGED');
|
|
59
|
+
const result:Record<string,Json>={};
|
|
60
|
+
for(let i=1;i<item.length;i+=2){const key=string(item[i]);if(Object.hasOwn(result,key))throw new Error('STORED_PLAN_CHANGED');Object.defineProperty(result,key,{value:decode(item[i+1]),enumerable:true,writable:true,configurable:true});}
|
|
61
|
+
return result;
|
|
62
|
+
};
|
|
63
|
+
let reconstructedBytes=0;
|
|
64
|
+
if(current)for(let index=0;index<value.json.length;index++){
|
|
65
|
+
const parsed=decode(value.json[index]);
|
|
66
|
+
if(!parsed||typeof parsed!=='object')throw new Error('STORED_PLAN_CHANGED');
|
|
67
|
+
// A small reference graph can describe exponentially larger embedded
|
|
68
|
+
// text. Check its exact serialized size before allocating that string.
|
|
69
|
+
const size=canonicalSize(parsed);
|
|
70
|
+
if(size>maximumPayloadBytes-reconstructedBytes)throw new Error('STORED_PLAN_CHANGED');
|
|
71
|
+
reconstructedBytes+=size;
|
|
72
|
+
const text=JSON.stringify(parsed);
|
|
73
|
+
if(distinctJson.has(text))throw new Error('STORED_PLAN_CHANGED');
|
|
74
|
+
distinctJson.add(text);jsonStrings.push(text);
|
|
75
|
+
}
|
|
76
|
+
const decoded=decode(value.tree);
|
|
77
|
+
if(used.size!==value.strings.length||usedJson.size!==jsonStrings.length)throw new Error('STORED_PLAN_CHANGED');
|
|
78
|
+
if(current&&canonicalSize(decoded)>maximumPayloadBytes)throw new Error('STORED_PLAN_CHANGED');
|
|
79
|
+
return decoded;
|
|
80
|
+
}
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
import { retainedSql } from './retained-values.ts';
|
|
2
|
+
import { changedLazy, putRow } from './expected-state.ts';
|
|
3
|
+
import { compressedPlan, storedStreamPage } from './stream-plan.ts';
|
|
4
|
+
import { expandCompressedEnvelope } from './compressed-plan.ts';
|
|
5
|
+
import { Buffer } from 'node:buffer';
|
|
6
|
+
import { createHash } from 'node:crypto';
|
|
7
|
+
import type { ImportPlan } from '../types.ts';
|
|
8
|
+
import type { D1Database, D1Sql, D1PlanCodec } from './types.ts';
|
|
9
|
+
import { canonicalJson, sha256 } from '../provenance/identity.ts';
|
|
10
|
+
import { guard, sql } from './database.ts';
|
|
11
|
+
import { canonicalChunks, canonicalHash } from '../provenance/canonical-stream.ts';
|
|
12
|
+
import { jsonChunks, PACKED_JSON_BYTES, SNAPSHOT_BYTES } from './sets.ts';
|
|
13
|
+
import { entryLocator } from './entry-locator.ts';
|
|
14
|
+
|
|
15
|
+
export interface StoredPlan {
|
|
16
|
+
revision?: number; body: string; chunks: { job_id: string; ordinal: number; body: string }[];
|
|
17
|
+
/** The chunks hold a compressed codec envelope, not the canonical plan, so
|
|
18
|
+
* any original bytes are not literally inside them. */
|
|
19
|
+
compressed?: boolean;
|
|
20
|
+
/** The plan's custody revision (ik_d1_custody_revisions), read before any
|
|
21
|
+
* chunk of a large plan was. */
|
|
22
|
+
fence?: number;
|
|
23
|
+
}
|
|
24
|
+
/** A stored plan larger than this is fenced by its custody revision instead of
|
|
25
|
+
* resending every chunk in each guard, and is created through bounded staging. */
|
|
26
|
+
export const LARGE_PLAN_BYTES = 4 * 1024 * 1024;
|
|
27
|
+
const large = (envelope: any): boolean => envelope?.d1Plan === 1 && Number.isSafeInteger(envelope.size) && envelope.size > LARGE_PLAN_BYTES;
|
|
28
|
+
const snapshots = new WeakMap<ImportPlan, StoredPlan>();
|
|
29
|
+
// Legacy chunks may approach D1's 2 MB value ceiling. Three rows remain below
|
|
30
|
+
// the RPC ceiling even with JSON escaping and whole-result two-byte strings.
|
|
31
|
+
const PLAN_READ_ROWS = 3;
|
|
32
|
+
export function planChunkReads(body: string, fenced = false): number {
|
|
33
|
+
const envelope = JSON.parse(body);
|
|
34
|
+
// A fenced large plan's revision read precedes its chunk reads.
|
|
35
|
+
return envelope?.d1Plan === 1 ? Math.ceil((envelope.chunks + 1) / PLAN_READ_ROWS) + Number(fenced && large(envelope)) : 0;
|
|
36
|
+
}
|
|
37
|
+
export function encodePlan(plan: ImportPlan, id: string): StoredPlan {
|
|
38
|
+
const chunks: StoredPlan['chunks'] = [], hash = createHash('sha256'); let size = 0;
|
|
39
|
+
for (const body of canonicalChunks(plan, SNAPSHOT_BYTES)) { chunks.push({ job_id: id, ordinal: chunks.length, body }); size += Buffer.byteLength(body); hash.update(body); }
|
|
40
|
+
if (chunks.length === 1) return { body: chunks[0].body, chunks: [] };
|
|
41
|
+
const entries = entryLocator(plan, chunks), byte = ([chunk, offset]: number[]) => Buffer.from(chunks[chunk].body)[offset];
|
|
42
|
+
if (entries.starts.some(start => byte(start) !== 0x7b) || byte(entries.end) !== 0x5d) throw new Error('IMPORT_PLAN_LOCATOR');
|
|
43
|
+
return { body: canonicalJson({ d1Plan: 1, chunks: chunks.length, entries, size, sha256: hash.digest('hex') }), chunks };
|
|
44
|
+
}
|
|
45
|
+
/** The adapter's codec retains a normalized plan. A plan holding original bytes
|
|
46
|
+
* (an available artifact or an attachment) stays canonical, as erasure re-retains
|
|
47
|
+
* it, so its originals are copied from, and compared by position with, its chunks. */
|
|
48
|
+
export const creationCodec = (plan: ImportPlan, codec?: D1PlanCodec): D1PlanCodec | undefined =>
|
|
49
|
+
codec && !plan.source.artifacts.some(a => a.original?.status === 'available') && !plan.entries.some(e => e.effects.some(effect => effect.attachment !== undefined)) ? codec : undefined;
|
|
50
|
+
export function retainPlan(plan: ImportPlan, id: string, codec?: D1PlanCodec): { body: string; statements: D1Sql[]; encoded: StoredPlan } {
|
|
51
|
+
const encoded: StoredPlan = codec ? storedStreamPage(compressedPlan(plan, codec), id, codec) : encodePlan(plan, id);
|
|
52
|
+
if (codec && encoded.chunks.length) encoded.compressed = true;
|
|
53
|
+
const rows = encoded.chunks.map(({ ordinal, body }) => ({ ordinal, body }));
|
|
54
|
+
return { encoded, body: encoded.body, statements: jsonChunks(rows, PACKED_JSON_BYTES).map(body => changedLazy(retainedSql(["INSERT INTO ik_d1_plan_chunks(job_id,ordinal,body) SELECT owner.job_id,json_extract(value,'$.ordinal'),json_extract(value,'$.body') FROM json_each(",") CROSS JOIN (SELECT "," AS job_id) owner"],[body,id],[0]),()=>JSON.parse(body).map((row:any)=>putRow('ik_d1_plan_chunks',{job_id:id,...row})))) };
|
|
55
|
+
}
|
|
56
|
+
/** `fenced`: capture a large plan's custody revisions before reading its chunks,
|
|
57
|
+
* so its guards fence those revisions instead of resending the plan. Only for
|
|
58
|
+
* guards evaluated before the transaction's own writes (unbounded transport). */
|
|
59
|
+
export async function loadPlans(rows: any[], db: D1Database, codec?: D1PlanCodec, fence?:D1Sql, fenced = false): Promise<Map<string, ImportPlan>> {
|
|
60
|
+
const result = new Map<string, ImportPlan>();
|
|
61
|
+
const marked = rows.filter(r => JSON.parse(r.plan_json)?.d1Plan === 1);
|
|
62
|
+
const chunks: any[] = [], fences = new Map<string, number>();
|
|
63
|
+
// D1 serializes the entire result as one RPC string, not separate row values.
|
|
64
|
+
// Read one job at a time; the final hash/cardinality check rejects torn reads.
|
|
65
|
+
if (marked.length) {
|
|
66
|
+
for (const row of marked) {
|
|
67
|
+
const envelope = JSON.parse(row.plan_json);
|
|
68
|
+
if (!Number.isSafeInteger(envelope.chunks) || envelope.chunks < 1) throw new Error('STORED_PLAN_CHANGED');
|
|
69
|
+
if (fenced && large(envelope)) {
|
|
70
|
+
// Any later change to these chunks or the job's custody advances a revision.
|
|
71
|
+
const revision = await db.prepare('SELECT revision FROM ik_d1_custody_revisions WHERE job_id=?').bind(row.id).first();
|
|
72
|
+
if (revision && Number.isSafeInteger(revision.revision)) fences.set(row.id, revision.revision);
|
|
73
|
+
}
|
|
74
|
+
for (let offset = 0; offset <= envelope.chunks; offset += PLAN_READ_ROWS) {
|
|
75
|
+
const found = await db.prepare(`SELECT * FROM ik_d1_plan_chunks WHERE job_id IN (SELECT value FROM json_each(?))${fence?` AND (${fence.sql})`:''} ORDER BY job_id,ordinal LIMIT ${PLAN_READ_ROWS} OFFSET ${offset}`).bind(JSON.stringify([row.id]),...fence?.params??[]).all();
|
|
76
|
+
chunks.push(...found.results);
|
|
77
|
+
if (found.results.length < PLAN_READ_ROWS) break;
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
for (const row of rows) {
|
|
82
|
+
const parsed = JSON.parse(row.plan_json), parts = chunks.filter(c => c.job_id === row.id);
|
|
83
|
+
let plan: ImportPlan, compressed = false;
|
|
84
|
+
if (parsed?.d1Plan === 1) {
|
|
85
|
+
if (canonicalJson(parsed) !== row.plan_json || !['["chunks","d1Plan","sha256","size"]','["chunks","d1Plan","entries","sha256","size"]'].includes(canonicalJson(Object.keys(parsed).sort())) || !Number.isSafeInteger(parsed.size) || parsed.size < 0 || typeof parsed.sha256 !== 'string' || !/^[a-f0-9]{64}$/.test(parsed.sha256) || !Number.isSafeInteger(parsed.chunks) || parsed.chunks < 1 || parts.length !== parsed.chunks || parts.some((c, i) => c.ordinal !== i || typeof c.body !== 'string')) throw new Error('STORED_PLAN_CHANGED');
|
|
86
|
+
const body = parts.map(c => c.body).join('');
|
|
87
|
+
if (Buffer.byteLength(body) !== parsed.size || sha256(body) !== parsed.sha256) throw new Error('STORED_PLAN_CHANGED');
|
|
88
|
+
const payload = JSON.parse(body);
|
|
89
|
+
if ([2,3].includes(payload?.d1Plan)) { if (parsed.entries !== undefined) throw new Error('STORED_PLAN_CHANGED'); plan = expandCompressedEnvelope(body,payload,codec); compressed = true; }
|
|
90
|
+
else { if (canonicalHash(payload) !== parsed.sha256) throw new Error('STORED_PLAN_CHANGED'); plan = payload; }
|
|
91
|
+
// The entry locator a referenced read trusts must describe these exact chunks.
|
|
92
|
+
if (parsed.entries !== undefined && canonicalJson(entryLocator(plan, parts)) !== canonicalJson(parsed.entries)) throw new Error('STORED_PLAN_CHANGED');
|
|
93
|
+
} else plan = [2,3].includes(parsed?.d1Plan) ? expandCompressedEnvelope(row.plan_json,parsed,codec) : parsed;
|
|
94
|
+
const captured = fences.get(row.id);
|
|
95
|
+
snapshots.set(plan, { body: row.plan_json, chunks: parts, ...(row.custody_revision === undefined ? {} : {revision: row.custody_revision}), ...(compressed ? {compressed} : {}), ...(captured === undefined ? {} : {fence: captured}) }); result.set(row.id, plan);
|
|
96
|
+
}
|
|
97
|
+
return result;
|
|
98
|
+
}
|
|
99
|
+
export function planBody(plan: ImportPlan): string { return snapshots.get(plan)?.body ?? canonicalJson(plan); }
|
|
100
|
+
export function planGuards(plan: ImportPlan, id: string): D1Sql[] {
|
|
101
|
+
const snapshot = snapshots.get(plan);
|
|
102
|
+
if (!snapshot?.chunks.length) {
|
|
103
|
+
const predicate=retainedSql(['(SELECT COUNT(*) FROM ik_d1_plan_chunks WHERE job_id=',')=0'],[id],[]);
|
|
104
|
+
predicate.retainedValues!.privacyPlan=plan;predicate.retainedValues!.privacyJobId=id;
|
|
105
|
+
return [guard(predicate)];
|
|
106
|
+
}
|
|
107
|
+
const guards = [guard(sql('(SELECT count(*) FROM ik_d1_plan_chunks WHERE job_id=?)=?', id, snapshot.chunks.length))];
|
|
108
|
+
// The chunks read after this revision matched the envelope's digest and the
|
|
109
|
+
// fingerprint; while it holds, they are unchanged. Every chunk insert, update
|
|
110
|
+
// or delete and every plan_json update advances it.
|
|
111
|
+
if (snapshot.fence !== undefined) return [...guards, guard(sql('EXISTS(SELECT 1 FROM ik_d1_custody_revisions r JOIN ik_jobs j ON j.id=r.job_id WHERE r.job_id=? AND r.revision=? AND j.plan_json IS ?)', id, snapshot.fence, snapshot.body))];
|
|
112
|
+
for(const body of jsonChunks(snapshot.chunks, PACKED_JSON_BYTES)) {
|
|
113
|
+
const predicate=retainedSql(["NOT EXISTS(SELECT 1 FROM json_each(", ") e WHERE NOT EXISTS(SELECT 1 FROM ik_d1_plan_chunks c WHERE c.job_id IS json_extract(e.value,'$.job_id') AND c.ordinal IS json_extract(e.value,'$.ordinal') AND c.body IS json_extract(e.value,'$.body')))"], [body], [0]);
|
|
114
|
+
predicate.retainedValues!.privacyPlan=plan;
|
|
115
|
+
predicate.retainedValues!.privacyJobId=id;
|
|
116
|
+
guards.push(guard(predicate));
|
|
117
|
+
}
|
|
118
|
+
return guards;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
export function storedPlanSnapshot(plan: ImportPlan, id: string): StoredPlan { return snapshots.get(plan) ?? encodePlan(plan, id); }
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import { createCanonicalStreamHash } from '../provenance/canonical-hash.ts';
|
|
2
|
+
import { canonicalHash } from '../provenance/canonical-stream.ts';
|
|
3
|
+
import { originalsFor } from '../storage/originals.ts';
|
|
4
|
+
import { planIndex } from './erasure-index.ts';
|
|
5
|
+
import { assertStreamPage } from './stream-page-integrity.ts';
|
|
6
|
+
import { readPreparationPlans, preparationIndex } from './preparation-reader.ts';
|
|
7
|
+
import { preparationRead, type PreparationState } from './preparation-state.ts';
|
|
8
|
+
import { policyHash } from './undo-state.ts';
|
|
9
|
+
import { sql } from './database.ts';
|
|
10
|
+
import type { UnverifiedStreamPages } from './stream-plan.ts';
|
|
11
|
+
|
|
12
|
+
/** Same sorted object keys, canonical values and array order as planFingerprint.
|
|
13
|
+
* Only the two large arrays obtain each page across an awaited custody read. */
|
|
14
|
+
export async function verifyPreparationFingerprint(state:PreparationState,header:UnverifiedStreamPages['header'],pages:any[]):Promise<void>{
|
|
15
|
+
if(header.version!==1||header.destination!==state.stage.destination||header.namespace!==state.stage.namespace||header.mappingVersion!==state.stage.mapping_version||header.principalId!==state.stage.principal_id||header.counts.unique!==state.stage.total)throw new Error('PLAN_BINDING_MISMATCH');
|
|
16
|
+
const writer=createCanonicalStreamHash(),index=preparationIndex(state)[Symbol.asyncIterator]();
|
|
17
|
+
const counts={input:0,unique:0,duplicates:0,mapped:0,unmapped:0,rejected:0,effects:0,byKind:Object.create(null) as Record<string,number>};
|
|
18
|
+
const document={...header,entries:[],source:{...header.source,records:[]}};
|
|
19
|
+
const records=async(source:boolean)=>{
|
|
20
|
+
writer.token('[');let first=true;
|
|
21
|
+
for await(const {page,plan} of readPreparationPlans(state,pages)){
|
|
22
|
+
assertStreamPage({header},plan);
|
|
23
|
+
if(!source){
|
|
24
|
+
if(plan.entries.some(e=>e.effects.some(effect=>effect.attachment))||plan.source.artifacts.some(a=>a.original?.status==='available'))throw new Error('IMPORT_STREAM_NORMALIZED_SOURCE_REQUIRED');
|
|
25
|
+
for(const key of ['input','unique','duplicates','mapped','unmapped','rejected','effects'] as const)counts[key]+=plan.counts[key];
|
|
26
|
+
for(const [kind,count]of Object.entries(plan.counts.byKind))counts.byKind[kind]=(counts.byKind[kind]??0)+count;
|
|
27
|
+
const identities=planIndex(plan,page.job_id).map(row=>[page.ordinal,row.ordinal,row.source_identity,row.target_identity]);
|
|
28
|
+
if(identities.length!==page.identities||canonicalHash(identities)!==page.index_hash||policyHash(page.job_id,plan.fingerprint,state.stage.reversal_policy)!==page.policy_hash)throw new Error('IMPORT_STAGE_CONTENT_CHANGED');
|
|
29
|
+
for(const expected of identities){const actual=await index.next();if(actual.done||canonicalHash(expected)!==canonicalHash([actual.value.page_ordinal,actual.value.ordinal,actual.value.source_identity,actual.value.target_identity]))throw new Error('IMPORT_STAGE_CONTENT_CHANGED');}
|
|
30
|
+
}
|
|
31
|
+
for(const entry of plan.entries){if(!first)writer.token(',');first=false;writer.value(source?entry.source:entry);}
|
|
32
|
+
}
|
|
33
|
+
writer.token(']');
|
|
34
|
+
};
|
|
35
|
+
try{
|
|
36
|
+
writer.token('{');let first=true;
|
|
37
|
+
for(const key of Object.keys(document).sort()){
|
|
38
|
+
if(!first)writer.token(',');first=false;writer.value(key);writer.token(':');
|
|
39
|
+
if(key==='entries')await records(false);
|
|
40
|
+
else if(key==='source'){
|
|
41
|
+
writer.token('{');let firstSource=true;
|
|
42
|
+
for(const sourceKey of Object.keys(document.source).sort()){
|
|
43
|
+
if(!firstSource)writer.token(',');firstSource=false;writer.value(sourceKey);writer.token(':');
|
|
44
|
+
if(sourceKey==='records')await records(true);else writer.value((document.source as any)[sourceKey]);
|
|
45
|
+
}
|
|
46
|
+
writer.token('}');
|
|
47
|
+
}else writer.value((document as any)[key]);
|
|
48
|
+
}
|
|
49
|
+
writer.token('}');
|
|
50
|
+
if(!(await index.next()).done)throw new Error('IMPORT_STAGE_CONTENT_CHANGED');
|
|
51
|
+
if(writer.digest()!==state.stage.fingerprint)throw new Error('REVIEW_FINGERPRINT_MISMATCH');
|
|
52
|
+
}finally{writer.clear();await index.return?.(undefined);}
|
|
53
|
+
if(canonicalHash(counts)!==canonicalHash(header.counts))throw new Error('IMPORT_STREAM_COUNTS_CHANGED');
|
|
54
|
+
const expected=originalsFor({...header,source:{...header.source,records:[]},entries:[],fingerprint:state.stage.fingerprint},'').map(o=>[o.key,o.name,o.sha256,o.size]);
|
|
55
|
+
const stored=await preparationRead(state,sql('SELECT original_key,name,sha256,size FROM ik_d1_staging_originals WHERE stage_id=? ORDER BY rowid',state.stage.id));
|
|
56
|
+
if(canonicalHash(stored.map(o=>[o.original_key,o.name,o.sha256,o.size]))!==canonicalHash(expected))throw new Error('IMPORT_STAGE_CONTENT_CHANGED');
|
|
57
|
+
}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import { Buffer } from 'node:buffer';
|
|
2
|
+
import type { UnverifiedStreamPages } from './stream-plan.ts';
|
|
3
|
+
import type { D1ImportContext } from './types.ts';
|
|
4
|
+
import { canonicalChunks, canonicalHash, canonicalSize } from '../provenance/canonical-stream.ts';
|
|
5
|
+
import { sha256 } from '../provenance/identity.ts';
|
|
6
|
+
import { guard, sql } from './database.ts';
|
|
7
|
+
import { remaining, D1RequestLimitError } from './budget.ts';
|
|
8
|
+
import { StageWriter, bindingPredicate, stageClear, stagingAtomic, STAGING_ROWS_BYTES, type StageBinding } from './staging-transport.ts';
|
|
9
|
+
|
|
10
|
+
export function preparationHeader(header: UnverifiedStreamPages['header'], context: D1ImportContext) {
|
|
11
|
+
const header_hash = canonicalHash(header), header_size = canonicalSize(header);
|
|
12
|
+
let header_chunks = 0, bytes = 0, largest = 0;
|
|
13
|
+
for (const body of canonicalChunks(header, 64 * 1024)) {
|
|
14
|
+
const size = Buffer.byteLength(JSON.stringify([header_chunks++,body,sha256(body),Buffer.byteLength(body)])) + 1;
|
|
15
|
+
bytes += size; largest = Math.max(largest, size);
|
|
16
|
+
}
|
|
17
|
+
// Reserve the additional header writes and their guarded receipt. The shared
|
|
18
|
+
// stage admission still reserves all ordinary writes and publication.
|
|
19
|
+
const reserve = Math.ceil(bytes / (STAGING_ROWS_BYTES - largest - 2)) * 16 + 16;
|
|
20
|
+
if (!Number.isSafeInteger(reserve) || reserve > remaining(context)) throw new D1RequestLimitError();
|
|
21
|
+
return { reserve, async store(stage: StageBinding, revision: number) {
|
|
22
|
+
const writer = new StageWriter(context, stage, 'headers', ['ordinal','body','body_hash','body_size']); writer.revision = revision;
|
|
23
|
+
let ordinal = 0;
|
|
24
|
+
for (const body of canonicalChunks(header,64 * 1024)) await writer.append([ordinal++,body,sha256(body),Buffer.byteLength(body)]);
|
|
25
|
+
await writer.flush(); revision = writer.revision;
|
|
26
|
+
const values = [stage.id,stage.generation,context.storage.targets.planCodec?.id ?? null,header_hash,header_size,header_chunks];
|
|
27
|
+
await stagingAtomic(context,[guard(bindingPredicate(stage)),guard(stageClear(stage)),guard(sql('EXISTS(SELECT 1 FROM ik_d1_staging WHERE id=? AND revision=?)',stage.id,revision)),
|
|
28
|
+
sql('INSERT INTO ik_d1_preparation_receipts(stage_id,generation,codec_id,header_hash,header_size,header_chunks) VALUES(?,?,?,?,?,?)',...values),
|
|
29
|
+
guard(sql('EXISTS(SELECT 1 FROM ik_d1_preparation_receipts WHERE stage_id IS ? AND generation IS ? AND codec_id IS ? AND header_hash IS ? AND header_size IS ? AND header_chunks IS ?)',...values)),
|
|
30
|
+
guard(sql('EXISTS(SELECT 1 FROM ik_d1_staging WHERE id=? AND revision=?)',stage.id,revision+1))]);
|
|
31
|
+
return revision + 1;
|
|
32
|
+
} };
|
|
33
|
+
}
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
import { Buffer } from 'node:buffer';
|
|
2
|
+
import { createHash } from 'node:crypto';
|
|
3
|
+
import { canonicalHash } from '../provenance/canonical-stream.ts';
|
|
4
|
+
import { sha256 } from '../provenance/identity.ts';
|
|
5
|
+
import { planFingerprint } from '../plan/index.ts';
|
|
6
|
+
import { expandCompressedEnvelope } from './compressed-plan.ts';
|
|
7
|
+
import { sql } from './database.ts';
|
|
8
|
+
import { layoutHash } from './stream-state.ts';
|
|
9
|
+
import { preparationRead, type PreparationState } from './preparation-state.ts';
|
|
10
|
+
import type { ImportPlan } from '../types.ts';
|
|
11
|
+
import type { UnverifiedStreamPages } from './stream-plan.ts';
|
|
12
|
+
import type { D1PlanCodec } from './types.ts';
|
|
13
|
+
|
|
14
|
+
async function* chunks(state:PreparationState,headers=false):AsyncGenerator<any>{
|
|
15
|
+
let page=-1,ordinal=-1;
|
|
16
|
+
for(;;){
|
|
17
|
+
const rows=await preparationRead(state,headers
|
|
18
|
+
?sql('SELECT ordinal,body,body_hash,body_size FROM ik_d1_staging_headers WHERE stage_id=? AND ordinal>? ORDER BY ordinal LIMIT 32',state.stage.id,ordinal)
|
|
19
|
+
:sql('SELECT page_ordinal,ordinal,body,body_hash,body_size FROM ik_d1_staging_chunks WHERE stage_id=? AND (page_ordinal>? OR (page_ordinal=? AND ordinal>?)) ORDER BY page_ordinal,ordinal LIMIT 32',state.stage.id,page,page,ordinal));
|
|
20
|
+
if(!rows.length)return;
|
|
21
|
+
for(const row of rows){
|
|
22
|
+
if(typeof row.body!=='string'||Buffer.byteLength(row.body)!==row.body_size||sha256(row.body)!==row.body_hash)throw new Error('IMPORT_STAGE_CONTENT_CHANGED');
|
|
23
|
+
page=row.page_ordinal??-1;ordinal=row.ordinal;yield row;
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
export async function preparationHeaderRead(state:PreparationState):Promise<UnverifiedStreamPages['header']>{
|
|
29
|
+
const bodies:string[]=[],hash=createHash('sha256');let bytes=0,ordinal=0;
|
|
30
|
+
for await(const row of chunks(state,true)){
|
|
31
|
+
if(row.ordinal!==ordinal++)throw new Error('IMPORT_PREPARATION_CHANGED');
|
|
32
|
+
bodies.push(row.body);hash.update(row.body);bytes+=row.body_size;
|
|
33
|
+
}
|
|
34
|
+
if(ordinal!==state.receipt.header_chunks||bytes!==state.receipt.header_size||hash.digest('hex')!==state.receipt.header_hash)throw new Error('IMPORT_PREPARATION_CHANGED');
|
|
35
|
+
let body=bodies.join('');bodies.length=0;
|
|
36
|
+
const header=JSON.parse(body);body='';
|
|
37
|
+
if(canonicalHash(header)!==state.receipt.header_hash)throw new Error('IMPORT_PREPARATION_CHANGED');
|
|
38
|
+
return header;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export async function preparationPages(state:PreparationState){
|
|
42
|
+
const pages=await preparationRead(state,sql('SELECT ordinal,job_id,total FROM ik_d1_staging_pages WHERE stage_id=? ORDER BY ordinal',state.stage.id));
|
|
43
|
+
if(pages.length!==state.stage.pages||pages.some((p,i)=>p.ordinal!==i||!Number.isSafeInteger(p.total)||p.total<0)||pages.reduce((n,p)=>n+p.total,0)!==state.stage.total||layoutHash(pages)!==state.stage.layout_hash)throw new Error('IMPORT_STREAM_LAYOUT_CHANGED');
|
|
44
|
+
return pages;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** At most 32 stored chunks and one expanded plan are live. A second canonical
|
|
48
|
+
* source traversal starts a fresh reader; no expanded page cache spans pages. */
|
|
49
|
+
export async function* readPreparationPlans(state:PreparationState,pages:any[]):AsyncGenerator<{page:any;plan:ImportPlan}>{
|
|
50
|
+
const stored=chunks(state)[Symbol.asyncIterator]();
|
|
51
|
+
try {
|
|
52
|
+
let pageNumber=0;
|
|
53
|
+
for await(const page of pageMetadata(state)){
|
|
54
|
+
const expected=pages[pageNumber++];
|
|
55
|
+
if(!expected||page.ordinal!==expected.ordinal||page.job_id!==expected.job_id||page.total!==expected.total)throw new Error('IMPORT_STREAM_LAYOUT_CHANGED');
|
|
56
|
+
const parts:string[]=[];let bytes=0;
|
|
57
|
+
for(let ordinal=0;ordinal<page.chunks;ordinal++){
|
|
58
|
+
const next=await stored.next();
|
|
59
|
+
if(next.done||next.value.page_ordinal!==page.ordinal||next.value.ordinal!==ordinal)throw new Error('IMPORT_STAGE_CONTENT_CHANGED');
|
|
60
|
+
parts.push(next.value.body);bytes+=next.value.body_size;
|
|
61
|
+
}
|
|
62
|
+
yield {page,plan:storedPagePlan(page,parts,bytes,state.context.storage.targets.planCodec)};
|
|
63
|
+
}
|
|
64
|
+
if(pageNumber!==pages.length||!(await stored.next()).done)throw new Error('IMPORT_STAGE_CONTENT_CHANGED');
|
|
65
|
+
} finally { await stored.return?.(undefined); }
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/** One staged page's plan from its chunk bodies, checked against the page's
|
|
69
|
+
* manifest, size, digest, plan fingerprint and record count. */
|
|
70
|
+
export function storedPagePlan(page:any,parts:string[],bytes:number,codec?:D1PlanCodec):ImportPlan{
|
|
71
|
+
const body=parts.join('');parts.length=0;
|
|
72
|
+
if(bytes!==page.body_size||sha256(body)!==page.body_hash)throw new Error('IMPORT_STAGE_CONTENT_CHANGED');
|
|
73
|
+
const manifest=JSON.parse(page.plan_json);
|
|
74
|
+
if(canonicalHash(manifest)!==sha256(page.plan_json)||canonicalHash(manifest)!==canonicalHash({d1Plan:1,chunks:page.chunks,size:bytes,sha256:page.body_hash}))throw new Error('IMPORT_STAGE_CONTENT_CHANGED');
|
|
75
|
+
const envelope=JSON.parse(body);
|
|
76
|
+
const plan=([2,3].includes(envelope?.d1Plan)?expandCompressedEnvelope(body,envelope,codec):envelope) as ImportPlan;
|
|
77
|
+
if(![2,3].includes(envelope?.d1Plan)&&(page.inline_plan!==1||canonicalHash(plan)!==page.body_hash||planFingerprint(plan)!==plan.fingerprint))throw new Error('STORED_PLAN_CHANGED');
|
|
78
|
+
if(plan.fingerprint!==page.fingerprint||plan.entries.length!==page.total)throw new Error('IMPORT_STREAM_PAGE_CHANGED');
|
|
79
|
+
return plan;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
export async function* preparationIndex(state:PreparationState):AsyncGenerator<any>{
|
|
83
|
+
let cursor=0;
|
|
84
|
+
for(;;){
|
|
85
|
+
const rows=await preparationRead(state,sql('SELECT rowid AS cursor,page_ordinal,ordinal,source_identity,target_identity FROM ik_d1_staging_index WHERE stage_id=? AND rowid>? ORDER BY rowid LIMIT 4096',state.stage.id,cursor));
|
|
86
|
+
if(!rows.length)return;
|
|
87
|
+
for(const row of rows){cursor=row.cursor;yield row;}
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
async function* pageMetadata(state:PreparationState):AsyncGenerator<any>{
|
|
92
|
+
let ordinal=-1;
|
|
93
|
+
for(;;){
|
|
94
|
+
const rows=await preparationRead(state,sql('SELECT * FROM ik_d1_staging_pages WHERE stage_id=? AND ordinal>? ORDER BY ordinal LIMIT 128',state.stage.id,ordinal));
|
|
95
|
+
if(!rows.length)return;
|
|
96
|
+
for(const row of rows){ordinal=row.ordinal;yield row;}
|
|
97
|
+
}
|
|
98
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/** Additive custody for an acknowledged but unverified preparation. Neither
|
|
2
|
+
* table is a public job. Both disappear with their exact staging generation. */
|
|
3
|
+
export const d1PreparationSchema = [
|
|
4
|
+
'CREATE TABLE IF NOT EXISTS ik_d1_preparation_receipts (stage_id TEXT PRIMARY KEY REFERENCES ik_d1_staging(id) ON DELETE CASCADE, generation TEXT NOT NULL, codec_id TEXT, header_hash TEXT NOT NULL, header_size INTEGER NOT NULL, header_chunks INTEGER NOT NULL)',
|
|
5
|
+
'CREATE TABLE IF NOT EXISTS ik_d1_staging_headers (stage_id TEXT NOT NULL REFERENCES ik_d1_staging(id) ON DELETE CASCADE, ordinal INTEGER NOT NULL, body TEXT NOT NULL, body_hash TEXT NOT NULL, body_size INTEGER NOT NULL, PRIMARY KEY(stage_id,ordinal))',
|
|
6
|
+
...['INSERT','UPDATE','DELETE'].flatMap(event => {
|
|
7
|
+
const refs = event === 'INSERT' ? 'id=NEW.stage_id' : event === 'DELETE' ? 'id=OLD.stage_id' : 'id=OLD.stage_id OR id=NEW.stage_id';
|
|
8
|
+
return ['ik_d1_preparation_receipts','ik_d1_staging_headers'].map(table => `CREATE TRIGGER IF NOT EXISTS ${table}_${event.toLowerCase()} AFTER ${event} ON ${table} BEGIN UPDATE ik_d1_staging SET revision=revision+1,sealed_revision=NULL WHERE ${refs}; END`);
|
|
9
|
+
}),
|
|
10
|
+
"CREATE TRIGGER IF NOT EXISTS ik_d1_preparation_receipt_immutable BEFORE UPDATE ON ik_d1_preparation_receipts BEGIN SELECT RAISE(ABORT,'IMPORT_PREPARATION_CHANGED'); END",
|
|
11
|
+
"CREATE TRIGGER IF NOT EXISTS ik_d1_staging_headers_registered BEFORE INSERT ON ik_d1_staging_headers WHEN NOT EXISTS(SELECT 1 FROM ik_d1_staging WHERE id=NEW.stage_id AND registered_revision IS NOT NULL) BEGIN SELECT RAISE(ABORT,'IMPORT_STAGE_INDEX_REQUIRED'); END",
|
|
12
|
+
...['INSERT','UPDATE','DELETE'].map(event => {
|
|
13
|
+
const refs = event === 'INSERT' ? 'stage_id=NEW.stage_id' : event === 'DELETE' ? 'stage_id=OLD.stage_id' : 'stage_id=OLD.stage_id OR stage_id=NEW.stage_id';
|
|
14
|
+
return `CREATE TRIGGER IF NOT EXISTS ik_d1_preparation_privacy_${event.toLowerCase()} AFTER ${event} ON ik_d1_staging_index BEGIN DELETE FROM ik_d1_staging_headers WHERE ${refs}; END`;
|
|
15
|
+
}),
|
|
16
|
+
];
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import type { D1ImportContext, D1Sql } from './types.ts';
|
|
2
|
+
import { authorize, guard, prepare, readonlyChecks, sql } from './database.ts';
|
|
3
|
+
import { requestContext, remaining, D1RequestLimitError } from './budget.ts';
|
|
4
|
+
import { bindingPredicate, stageClear, stagingBatchBytes, STAGING_BATCH_BYTES, type StageBinding } from './staging-transport.ts';
|
|
5
|
+
import { exactRows } from './sets.ts';
|
|
6
|
+
|
|
7
|
+
export interface ImportPreparation {
|
|
8
|
+
id: string; state: 'preparing' | 'preparation-interrupted'; expectedFingerprint: string;
|
|
9
|
+
checkpoint: 0; total: number; error: string | null;
|
|
10
|
+
}
|
|
11
|
+
export type PreparationState = Awaited<ReturnType<typeof preparationState>>;
|
|
12
|
+
export const preparationStatus = (stage: StageBinding): ImportPreparation => ({ id:stage.id,state:'preparing',expectedFingerprint:stage.fingerprint,checkpoint:0,total:stage.total,error:null });
|
|
13
|
+
|
|
14
|
+
export async function preparationState(id: string, input: D1ImportContext, action:'read'|'create'='read') {
|
|
15
|
+
const context = requestContext(input); await authorize(context,action);
|
|
16
|
+
const db=context.storage.database;
|
|
17
|
+
const row=await db.prepare('SELECT * FROM ik_d1_staging WHERE id=?').bind(id).first();
|
|
18
|
+
if(!row || row.destination!==context.destination || row.namespace!==context.namespace || row.mapping_version!==context.mappingVersion || row.principal_id!==context.authorization.principalId) throw new Error('IMPORT_PREPARATION_NOT_FOUND');
|
|
19
|
+
const stage:StageBinding={id:row.id,generation:row.generation,destination:row.destination,namespace:row.namespace,mapping_version:row.mapping_version,principal_id:row.principal_id,fingerprint:row.fingerprint,operation_key:row.operation_key,total:row.total,pages:row.pages,layout_hash:row.layout_hash,reversal_policy:row.reversal_policy};
|
|
20
|
+
const receipt=await db.prepare('SELECT * FROM ik_d1_preparation_receipts WHERE stage_id=?').bind(id).first();
|
|
21
|
+
if(!receipt || receipt.generation!==stage.generation || row.registered_revision===null || row.sealed_revision!==row.revision) throw new Error('IMPORT_PREPARATION_INCOMPLETE');
|
|
22
|
+
if(receipt.codec_id!==(context.storage.targets.planCodec?.id??null)) throw new Error('IMPORT_PLAN_CODEC_REQUIRED');
|
|
23
|
+
if(context.reviewedFingerprint!==undefined && context.reviewedFingerprint!==stage.fingerprint) throw new Error('REVIEW_FINGERPRINT_MISMATCH');
|
|
24
|
+
const state={context,stage,revision:row.revision as number,receipt,action};
|
|
25
|
+
await preparationRead(state,sql('SELECT 1 AS present'));
|
|
26
|
+
return state;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/** Every awaited read has the same generation/revision/receipt and current
|
|
30
|
+
* authority on both sides. No stale read confers permission to publish. */
|
|
31
|
+
export async function preparationRead(state:{context:D1ImportContext;stage:StageBinding;revision:number;receipt:any;action:'read'|'create'}, query:D1Sql):Promise<any[]> {
|
|
32
|
+
const {context,stage,revision,receipt}=state;
|
|
33
|
+
const checks=readonlyChecks([guard(context.authorization.predicate(state.action,context.destination)),guard(bindingPredicate(stage)),guard(stageClear(stage)),
|
|
34
|
+
guard(sql('EXISTS(SELECT 1 FROM ik_d1_staging WHERE id=? AND revision=? AND sealed_revision=? AND registered_revision IS NOT NULL)',stage.id,revision,revision)),
|
|
35
|
+
...exactRows('ik_d1_preparation_receipts',Object.keys(receipt),[receipt],sql('stage_id=?',stage.id))]);
|
|
36
|
+
const statements=[...checks,query,...checks];
|
|
37
|
+
if(statements.length>remaining(context))throw new D1RequestLimitError();
|
|
38
|
+
if(stagingBatchBytes(statements)>STAGING_BATCH_BYTES)throw new Error('IMPORT_STAGE_TRANSPORT_LIMIT');
|
|
39
|
+
const results=await context.storage.database.batch(statements.map(statement=>prepare(context.storage.database,statement)));
|
|
40
|
+
const result=results[checks.length] as any,rows=Array.isArray(result)?result:result?.results;
|
|
41
|
+
if(!Array.isArray(rows))throw new Error('IMPORT_D1_READ_RECEIPT_REQUIRED');
|
|
42
|
+
return rows;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export async function readD1ImportPreparation(id:string,input:D1ImportContext):Promise<ImportPreparation>{
|
|
46
|
+
const context=requestContext(input);await authorize(context,'read');
|
|
47
|
+
const row=await context.storage.database.prepare('SELECT * FROM ik_d1_staging WHERE id=? AND destination=? AND namespace=? AND mapping_version=? AND principal_id=?').bind(id,context.destination,context.namespace,context.mappingVersion,context.authorization.principalId).first();
|
|
48
|
+
if(!row)throw new Error('IMPORT_PREPARATION_NOT_FOUND');
|
|
49
|
+
if(context.reviewedFingerprint!==undefined&&context.reviewedFingerprint!==row.fingerprint)throw new Error('REVIEW_FINGERPRINT_MISMATCH');
|
|
50
|
+
const receipt=await context.storage.database.prepare('SELECT generation FROM ik_d1_preparation_receipts WHERE stage_id=?').bind(id).first();
|
|
51
|
+
if(receipt&&receipt.generation===row.generation&&row.sealed_revision===row.revision&&row.registered_revision!==null)return preparationStatus((await preparationState(id,context)).stage);
|
|
52
|
+
const statements=readonlyChecks([guard(context.authorization.predicate('read',context.destination)),...exactRows('ik_d1_staging',Object.keys(row),[row],sql('id=?',id))]);
|
|
53
|
+
if(statements.length>remaining(context))throw new D1RequestLimitError();
|
|
54
|
+
await context.storage.database.batch(statements.map(statement=>prepare(context.storage.database,statement)));
|
|
55
|
+
return{id,state:'preparation-interrupted',expectedFingerprint:row.fingerprint,checkpoint:0,total:row.total,error:'Preparation is incomplete or has changed. Retry the reviewed files or cancel this preparation.'};
|
|
56
|
+
}
|