@runeditrun/rer 0.2.77 → 0.2.78
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -0
- package/cli/export.mjs +1 -1
- package/cli/portability-migrations.mjs +35 -14
- package/cli/portability-worker.mjs +2 -2
- package/cli/portability.mjs +2 -2
- package/global/CHANGELOG.md +8 -0
- package/global/README.md +32 -0
- package/modules/deploy-kit/owner-cli.mjs +15 -5
- package/modules/deploy-kit/provider.mjs +9 -4
- package/modules/import-kit/src/d1/README.md +1322 -0
- package/modules/import-kit/src/d1/apply.ts +115 -0
- package/modules/import-kit/src/d1/budget.ts +89 -0
- package/modules/import-kit/src/d1/compressed-plan.ts +63 -0
- package/modules/import-kit/src/d1/creation.ts +95 -0
- package/modules/import-kit/src/d1/custody-fence.ts +27 -0
- package/modules/import-kit/src/d1/custody-state-schema.ts +67 -0
- package/modules/import-kit/src/d1/database.ts +163 -0
- package/modules/import-kit/src/d1/entry-locator.ts +61 -0
- package/modules/import-kit/src/d1/erased-artifacts.ts +38 -0
- package/modules/import-kit/src/d1/erased-custody.ts +36 -0
- package/modules/import-kit/src/d1/erased-plan.ts +47 -0
- package/modules/import-kit/src/d1/erased-reversal.ts +19 -0
- package/modules/import-kit/src/d1/erasure-bounded.ts +22 -0
- package/modules/import-kit/src/d1/erasure-claims.ts +27 -0
- package/modules/import-kit/src/d1/erasure-compiler.ts +67 -0
- package/modules/import-kit/src/d1/erasure-disposition.ts +16 -0
- package/modules/import-kit/src/d1/erasure-fragments.ts +86 -0
- package/modules/import-kit/src/d1/erasure-index.ts +53 -0
- package/modules/import-kit/src/d1/erasure-pages.ts +62 -0
- package/modules/import-kit/src/d1/erasure-progress-schema.ts +8 -0
- package/modules/import-kit/src/d1/erasure-progress.ts +61 -0
- package/modules/import-kit/src/d1/erasure-related.ts +124 -0
- package/modules/import-kit/src/d1/erasure-report.ts +31 -0
- package/modules/import-kit/src/d1/erasure-schema.ts +24 -0
- package/modules/import-kit/src/d1/erasure-selection.ts +77 -0
- package/modules/import-kit/src/d1/erasure-snapshot.ts +15 -0
- package/modules/import-kit/src/d1/erasure-stage.ts +146 -0
- package/modules/import-kit/src/d1/erasure-step.ts +12 -0
- package/modules/import-kit/src/d1/erasure-superseded.ts +98 -0
- package/modules/import-kit/src/d1/erasure-values.ts +83 -0
- package/modules/import-kit/src/d1/erasure-writes.ts +46 -0
- package/modules/import-kit/src/d1/erasure.ts +170 -0
- package/modules/import-kit/src/d1/evolutions/0001-preserved-undo.ts +13 -0
- package/modules/import-kit/src/d1/expected-state.ts +91 -0
- package/modules/import-kit/src/d1/field-redaction.ts +86 -0
- package/modules/import-kit/src/d1/guards.ts +37 -0
- package/modules/import-kit/src/d1/history.ts +71 -0
- package/modules/import-kit/src/d1/index.ts +88 -0
- package/modules/import-kit/src/d1/integrity.ts +127 -0
- package/modules/import-kit/src/d1/inventory-snapshot.ts +84 -0
- package/modules/import-kit/src/d1/jobs.ts +97 -0
- package/modules/import-kit/src/d1/native-set.ts +48 -0
- package/modules/import-kit/src/d1/native-staging-admission.ts +19 -0
- package/modules/import-kit/src/d1/native-staging-context.ts +44 -0
- package/modules/import-kit/src/d1/native-staging-lifecycle.ts +33 -0
- package/modules/import-kit/src/d1/native-staging-offset.ts +74 -0
- package/modules/import-kit/src/d1/native-staging-publication.ts +18 -0
- package/modules/import-kit/src/d1/native-staging-rows.ts +72 -0
- package/modules/import-kit/src/d1/native-staging-schema.ts +28 -0
- package/modules/import-kit/src/d1/native-staging-state.ts +22 -0
- package/modules/import-kit/src/d1/native-staging.ts +64 -0
- package/modules/import-kit/src/d1/original-copies.ts +53 -0
- package/modules/import-kit/src/d1/original-storage.ts +56 -0
- package/modules/import-kit/src/d1/originals.ts +74 -0
- package/modules/import-kit/src/d1/plan-dictionary.ts +80 -0
- package/modules/import-kit/src/d1/plans.ts +121 -0
- package/modules/import-kit/src/d1/preparation-fingerprint.ts +57 -0
- package/modules/import-kit/src/d1/preparation-header.ts +33 -0
- package/modules/import-kit/src/d1/preparation-reader.ts +98 -0
- package/modules/import-kit/src/d1/preparation-schema.ts +16 -0
- package/modules/import-kit/src/d1/preparation-state.ts +56 -0
- package/modules/import-kit/src/d1/preparation-step-combine.ts +80 -0
- package/modules/import-kit/src/d1/preparation-step-control.ts +48 -0
- package/modules/import-kit/src/d1/preparation-step-hash.ts +131 -0
- package/modules/import-kit/src/d1/preparation-step-pages.ts +77 -0
- package/modules/import-kit/src/d1/preparation-steps-schema.ts +11 -0
- package/modules/import-kit/src/d1/preparation-steps.ts +102 -0
- package/modules/import-kit/src/d1/preparations.ts +87 -0
- package/modules/import-kit/src/d1/preserved-preview.ts +89 -0
- package/modules/import-kit/src/d1/preserved-reverse.ts +91 -0
- package/modules/import-kit/src/d1/provenance-set.ts +87 -0
- package/modules/import-kit/src/d1/publication-fence.ts +22 -0
- package/modules/import-kit/src/d1/publication-receipt-schema.ts +15 -0
- package/modules/import-kit/src/d1/published-stream.ts +11 -0
- package/modules/import-kit/src/d1/read-fence.ts +64 -0
- package/modules/import-kit/src/d1/read-results.ts +16 -0
- package/modules/import-kit/src/d1/redacted-custody.ts +76 -0
- package/modules/import-kit/src/d1/redaction-step.ts +125 -0
- package/modules/import-kit/src/d1/referenced-plans.ts +136 -0
- package/modules/import-kit/src/d1/retained-values.ts +35 -0
- package/modules/import-kit/src/d1/retirement-bounded.ts +71 -0
- package/modules/import-kit/src/d1/retirement-door.ts +105 -0
- package/modules/import-kit/src/d1/retirement-schema.ts +39 -0
- package/modules/import-kit/src/d1/retirement-scope.ts +94 -0
- package/modules/import-kit/src/d1/retirement-steps.ts +76 -0
- package/modules/import-kit/src/d1/retirement-tally.ts +74 -0
- package/modules/import-kit/src/d1/retirement.ts +136 -0
- package/modules/import-kit/src/d1/reversal-conversion-record.ts +19 -0
- package/modules/import-kit/src/d1/reversal-conversion-schema.ts +11 -0
- package/modules/import-kit/src/d1/reversal-conversion.ts +40 -0
- package/modules/import-kit/src/d1/reversal-fence-schema.ts +14 -0
- package/modules/import-kit/src/d1/reversal-fence.ts +106 -0
- package/modules/import-kit/src/d1/reversal-journal.ts +24 -0
- package/modules/import-kit/src/d1/reverse.ts +103 -0
- package/modules/import-kit/src/d1/reviewed-source.ts +23 -0
- package/modules/import-kit/src/d1/sets.ts +78 -0
- package/modules/import-kit/src/d1/staged-creation.ts +201 -0
- package/modules/import-kit/src/d1/staging-admission.ts +45 -0
- package/modules/import-kit/src/d1/staging-control.ts +43 -0
- package/modules/import-kit/src/d1/staging-erasure.ts +28 -0
- package/modules/import-kit/src/d1/staging-lifecycle.ts +49 -0
- package/modules/import-kit/src/d1/staging-plan.ts +15 -0
- package/modules/import-kit/src/d1/staging-privacy.ts +22 -0
- package/modules/import-kit/src/d1/staging-publication.ts +79 -0
- package/modules/import-kit/src/d1/staging-schema.ts +37 -0
- package/modules/import-kit/src/d1/staging-statements.ts +8 -0
- package/modules/import-kit/src/d1/staging-transport.ts +70 -0
- package/modules/import-kit/src/d1/stream-boundary.ts +11 -0
- package/modules/import-kit/src/d1/stream-create.ts +55 -0
- package/modules/import-kit/src/d1/stream-identity-admission.ts +37 -0
- package/modules/import-kit/src/d1/stream-input.ts +18 -0
- package/modules/import-kit/src/d1/stream-layout.ts +43 -0
- package/modules/import-kit/src/d1/stream-origin.ts +44 -0
- package/modules/import-kit/src/d1/stream-page-integrity.ts +18 -0
- package/modules/import-kit/src/d1/stream-partition.ts +50 -0
- package/modules/import-kit/src/d1/stream-plan.ts +133 -0
- package/modules/import-kit/src/d1/stream-receipt.ts +45 -0
- package/modules/import-kit/src/d1/stream-revision-schema.ts +35 -0
- package/modules/import-kit/src/d1/stream-snapshot.ts +26 -0
- package/modules/import-kit/src/d1/stream-stage.ts +79 -0
- package/modules/import-kit/src/d1/stream-state.ts +25 -0
- package/modules/import-kit/src/d1/stream-working-set.ts +30 -0
- package/modules/import-kit/src/d1/streams.ts +39 -0
- package/modules/import-kit/src/d1/target-guards.ts +9 -0
- package/modules/import-kit/src/d1/target-identities.ts +17 -0
- package/modules/import-kit/src/d1/transport.ts +10 -0
- package/modules/import-kit/src/d1/types.ts +162 -0
- package/modules/import-kit/src/d1/undo-observations.ts +39 -0
- package/modules/import-kit/src/d1/undo-report.ts +22 -0
- package/modules/import-kit/src/d1/undo-snapshot.ts +87 -0
- package/modules/import-kit/src/d1/undo-state.ts +84 -0
- package/modules/import-kit/src/d1/undo-types.ts +110 -0
- package/modules/import-kit/src/journal/authority.ts +12 -0
- package/modules/import-kit/src/plan/budget.ts +60 -0
- package/modules/import-kit/src/plan/index.ts +123 -0
- package/modules/import-kit/src/plan/size.ts +122 -0
- package/modules/import-kit/src/provenance/canonical-hash.ts +130 -0
- package/modules/import-kit/src/provenance/canonical-stream.ts +85 -0
- package/modules/import-kit/src/provenance/credential-names.ts +131 -0
- package/modules/import-kit/src/provenance/credentials.ts +317 -0
- package/modules/import-kit/src/provenance/identity.ts +51 -0
- package/modules/import-kit/src/provenance/json-snapshot.ts +21 -0
- package/modules/import-kit/src/provenance/resumable-sha256.ts +75 -0
- package/modules/import-kit/src/provenance/validate-json.ts +19 -0
- package/modules/import-kit/src/storage/originals.ts +87 -0
- package/modules/import-kit/src/storage/provenance.ts +64 -0
- package/modules/import-kit/src/storage/schema.ts +42 -0
- package/modules/import-kit/src/types.ts +47 -0
- package/modules/portability/README.md +98 -2
- package/modules/portability/backfills.mjs +42 -15
- package/modules/portability/ledgers.mjs +7 -1
- package/modules/portability/transient-tables.mjs +40 -0
- package/package.json +3 -1
- package/scripts/publish-check/reviewed.py +3 -1
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto';
|
|
2
|
+
import type { D1ImportContext } from './types.ts';
|
|
3
|
+
import { canonicalHash } from '../provenance/canonical-stream.ts';
|
|
4
|
+
import { guard, sql } from './database.ts';
|
|
5
|
+
import { exactRows } from './sets.ts';
|
|
6
|
+
import { layoutHash } from './stream-state.ts';
|
|
7
|
+
import { policyHash } from './undo-state.ts';
|
|
8
|
+
import { preparationHeader } from './preparation-header.ts';
|
|
9
|
+
import { reserveReturn } from './budget.ts';
|
|
10
|
+
import { beginStageWrites, stageRevisionWrites } from './staging-control.ts';
|
|
11
|
+
import { stagingAtomic, type StageBinding } from './staging-transport.ts';
|
|
12
|
+
import { controlUpdate, progress, stageRevisions, stagesFence, type ImportPreparationStep, type StepControl, type StepStage } from './preparation-step-control.ts';
|
|
13
|
+
import { mergedHeader, stageHeader } from './preparation-step-pages.ts';
|
|
14
|
+
|
|
15
|
+
/** The last step: copy every step generation into one sealed preparation under
|
|
16
|
+
* the verified fingerprint, in SQL. Pages take their global ordinals and new
|
|
17
|
+
* job ids (the first page's is the stream id), so their index and policy
|
|
18
|
+
* hashes are recomputed from the stored index; payload bytes are copied, never
|
|
19
|
+
* re-sent. The step generations go in the sealing transaction, and the control
|
|
20
|
+
* row then records the revision the fingerprint was verified for. */
|
|
21
|
+
export async function combineStep(context: D1ImportContext, row: StepControl): Promise<ImportPreparationStep> {
|
|
22
|
+
const db = context.storage.database, stages: StepStage[] = JSON.parse(row.stages), state = JSON.parse(row.hash_json);
|
|
23
|
+
const revisions = await stageRevisions(context, stages);
|
|
24
|
+
if (canonicalHash(state.revisions) !== canonicalHash(revisions)) throw new Error('IMPORT_PREPARATION_CHANGED');
|
|
25
|
+
const ids = JSON.stringify(stages.map(stage => stage.id));
|
|
26
|
+
const stored = (await db.prepare('SELECT stage_id,ordinal,total,fingerprint,identities,chunks FROM ik_d1_staging_pages WHERE stage_id IN (SELECT value FROM json_each(?))').bind(ids).all<any>()).results;
|
|
27
|
+
const mapping: [string, number, number, string, string, string][] = [], layout: { ordinal: number; job_id: string; total: number }[] = [];
|
|
28
|
+
let chunks = 0, identities = 0;
|
|
29
|
+
for (const stage of stages) {
|
|
30
|
+
const pages = stored.filter(page => page.stage_id === stage.id).sort((a, b) => a.ordinal - b.ordinal);
|
|
31
|
+
if (pages.length !== stage.pages) throw new Error('IMPORT_STREAM_LAYOUT_CHANGED');
|
|
32
|
+
const index = await stageRows(context, stage.id);
|
|
33
|
+
for (const page of pages) {
|
|
34
|
+
const ordinal = layout.length, job = ordinal === 0 ? row.id : randomUUID();
|
|
35
|
+
const rows = index.filter(item => item.page_ordinal === page.ordinal).map(item => [ordinal, item.ordinal, item.source_identity, item.target_identity]);
|
|
36
|
+
if (rows.length !== page.identities) throw new Error('IMPORT_STAGE_CONTENT_CHANGED');
|
|
37
|
+
mapping.push([stage.id, page.ordinal, ordinal, job, canonicalHash(rows), policyHash(job, page.fingerprint, row.reversal_policy)]);
|
|
38
|
+
layout.push({ ordinal, job_id: job, total: page.total });
|
|
39
|
+
chunks += page.chunks; identities += rows.length;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
const header = mergedHeader(await Promise.all(stages.map(stage => stageHeader(context, stage.id))));
|
|
43
|
+
const final: StageBinding = { id: row.id, generation: randomUUID(), destination: row.destination, namespace: row.namespace, mapping_version: row.mapping_version, principal_id: row.principal_id,
|
|
44
|
+
fingerprint: row.fingerprint, operation_key: row.operation_key, total: layout.reduce((n, page) => n + page.total, 0), pages: layout.length, layout_hash: layoutHash(layout), reversal_policy: row.reversal_policy };
|
|
45
|
+
if (final.total !== row.total) throw new Error('IMPORT_STREAM_COUNTS_CHANGED');
|
|
46
|
+
const originals = (await db.prepare('SELECT count(*) AS n FROM ik_d1_staging_originals WHERE stage_id=?').bind(stages[0].id).first<{ n: number }>())!.n;
|
|
47
|
+
const prior = await db.prepare('SELECT * FROM ik_d1_staging WHERE destination=? AND principal_id=? AND operation_key=?').bind(row.destination, row.principal_id, row.operation_key).first<any>();
|
|
48
|
+
if (prior && (prior.id !== row.id || prior.fingerprint !== row.fingerprint)) throw new Error('OPERATION_KEY_CONFLICT');
|
|
49
|
+
const map = JSON.stringify(mapping), at = (n: number) => `json_extract(m.value,'$[${n}]')`;
|
|
50
|
+
const head = preparationHeader(header, context);
|
|
51
|
+
const reserved = reserveReturn(context, head.reserve + 16);
|
|
52
|
+
// One transaction: the new generation, its pages and complete privacy index
|
|
53
|
+
// (registered before any payload), then its chunks and originals.
|
|
54
|
+
await stagingAtomic(reserved, [...stagesFence(revisions), ...exactRows('ik_d1_preparation_steps', Object.keys(row), [row]), ...beginStageWrites(final, prior),
|
|
55
|
+
sql(`INSERT INTO ik_d1_staging_pages(stage_id,ordinal,job_id,total,fingerprint,plan_json,body_hash,body_size,chunks,identities,index_hash,policy_hash,inline_plan) SELECT ?,${at(2)},${at(3)},p.total,p.fingerprint,p.plan_json,p.body_hash,p.body_size,p.chunks,p.identities,${at(4)},${at(5)},p.inline_plan FROM json_each(?) AS m JOIN ik_d1_staging_pages p ON p.stage_id=${at(0)} AND p.ordinal=${at(1)} ORDER BY m.key`, final.id, map),
|
|
56
|
+
sql(`INSERT INTO ik_d1_staging_index(stage_id,page_ordinal,ordinal,source_identity,target_identity) SELECT ?,${at(2)},i.ordinal,i.source_identity,i.target_identity FROM json_each(?) AS m JOIN ik_d1_staging_index i ON i.stage_id=${at(0)} AND i.page_ordinal=${at(1)} ORDER BY m.key,i.rowid`, final.id, map),
|
|
57
|
+
// A normalized stream names each source once and each native target once.
|
|
58
|
+
guard(sql("NOT EXISTS(SELECT 1 FROM ik_d1_staging_index WHERE stage_id=? GROUP BY source_identity HAVING count(DISTINCT page_ordinal||':'||ordinal)>1) AND NOT EXISTS(SELECT 1 FROM ik_d1_staging_index WHERE stage_id=? AND target_identity<>'' GROUP BY target_identity HAVING count(*)>1)", final.id, final.id)),
|
|
59
|
+
...stageRevisionWrites(final, layout.length + identities, false),
|
|
60
|
+
sql(`INSERT INTO ik_d1_staging_chunks(stage_id,page_ordinal,ordinal,body,body_hash,body_size) SELECT ?,${at(2)},c.ordinal,c.body,c.body_hash,c.body_size FROM json_each(?) AS m JOIN ik_d1_staging_chunks c ON c.stage_id=${at(0)} AND c.page_ordinal=${at(1)} ORDER BY m.key,c.ordinal`, final.id, map),
|
|
61
|
+
sql('INSERT INTO ik_d1_staging_originals(stage_id,original_key,name,sha256,size) SELECT ?,original_key,name,sha256,size FROM ik_d1_staging_originals WHERE stage_id=? ORDER BY rowid', final.id, stages[0].id),
|
|
62
|
+
// Every slice came from the same artifacts, so each holds the same originals.
|
|
63
|
+
guard(sql("NOT EXISTS(SELECT 1 FROM json_each(?) AS s WHERE (SELECT count(*) FROM ik_d1_staging_originals o WHERE o.stage_id=s.value)<>? OR EXISTS(SELECT 1 FROM ik_d1_staging_originals o WHERE o.stage_id=s.value AND NOT EXISTS(SELECT 1 FROM ik_d1_staging_originals f WHERE f.stage_id=? AND f.original_key=o.original_key AND f.name=o.name AND f.sha256=o.sha256 AND f.size=o.size)))", ids, originals, final.id)),
|
|
64
|
+
guard(sql('EXISTS(SELECT 1 FROM ik_d1_staging WHERE id=? AND revision=?)', final.id, layout.length + identities + chunks + originals))]);
|
|
65
|
+
const revision = await head.store(final, layout.length + identities + chunks + originals);
|
|
66
|
+
await stagingAtomic(context, [...stagesFence(revisions), ...stageRevisionWrites(final, revision, true),
|
|
67
|
+
sql('DELETE FROM ik_d1_staging WHERE id IN (SELECT value FROM json_each(?))', ids), guard(sql('NOT EXISTS(SELECT 1 FROM ik_d1_staging WHERE id IN (SELECT value FROM json_each(?)))', ids)),
|
|
68
|
+
...controlUpdate(row, { phase: 'prepared', stages: '[]', hash_json: null, verified_revision: revision })]);
|
|
69
|
+
return progress(row, 'verifying');
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/** All privacy index rows of one step generation, in staged order. */
|
|
73
|
+
async function stageRows(context: D1ImportContext, stage: string): Promise<any[]> {
|
|
74
|
+
const rows: any[] = [];
|
|
75
|
+
for (let cursor = 0; ;) {
|
|
76
|
+
const { results } = await context.storage.database.prepare('SELECT rowid AS cursor,page_ordinal,ordinal,source_identity,target_identity FROM ik_d1_staging_index WHERE stage_id=? AND rowid>? ORDER BY rowid LIMIT 4096').bind(stage, cursor).all<any>();
|
|
77
|
+
if (!results.length) return rows;
|
|
78
|
+
rows.push(...results); cursor = results[results.length - 1].cursor;
|
|
79
|
+
}
|
|
80
|
+
}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import type { ImportPlan } from '../types.ts';
|
|
2
|
+
import type { D1ImportContext, D1Sql } from './types.ts';
|
|
3
|
+
import { guard, sql } from './database.ts';
|
|
4
|
+
import { exactRows } from './sets.ts';
|
|
5
|
+
|
|
6
|
+
/** Where a preparation spread over requests stands. `reviewable` carries the
|
|
7
|
+
* complete canonical fingerprint and the review counts and diagnostics. */
|
|
8
|
+
export interface ImportPreparationStep {
|
|
9
|
+
id: string;
|
|
10
|
+
state: 'preparing' | 'verifying' | 'reviewable' | 'published';
|
|
11
|
+
records: number;
|
|
12
|
+
total: number;
|
|
13
|
+
fingerprint?: string;
|
|
14
|
+
counts?: ImportPlan['counts'];
|
|
15
|
+
diagnostics?: ImportPlan['diagnostics'];
|
|
16
|
+
}
|
|
17
|
+
export interface StepStage { id: string; pages: number; records: number }
|
|
18
|
+
export type StepControl = Record<string, any> & { id: string; stages: string; phase: 'collecting' | 'hashing' | 'combining' | 'prepared'; revision: number };
|
|
19
|
+
|
|
20
|
+
export async function control(context: D1ImportContext, operation: string): Promise<StepControl | undefined> {
|
|
21
|
+
return (await context.storage.database.prepare('SELECT * FROM ik_d1_preparation_steps WHERE destination=? AND principal_id=? AND operation_key=?')
|
|
22
|
+
.bind(context.destination, context.authorization.principalId, operation).first<StepControl>()) ?? undefined;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/** Advance the control row exactly once from the state this step read. */
|
|
26
|
+
export function controlUpdate(row: StepControl, changes: Record<string, unknown>): D1Sql[] {
|
|
27
|
+
const keys = Object.keys(changes);
|
|
28
|
+
return [...exactRows('ik_d1_preparation_steps', Object.keys(row), [row]),
|
|
29
|
+
sql(`UPDATE ik_d1_preparation_steps SET ${keys.map(key => `${key}=?`).join(',')},revision=revision+1 WHERE id=? AND revision=?`, ...Object.values(changes), row.id, row.revision),
|
|
30
|
+
guard(sql('EXISTS(SELECT 1 FROM ik_d1_preparation_steps WHERE id=? AND revision=?)', row.id, row.revision + 1))];
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** Fence every generation the steps staged at the revision it was read at. */
|
|
34
|
+
export function stagesFence(stages: { id: string; revision: number }[]): D1Sql[] {
|
|
35
|
+
return exactRows('ik_d1_staging', ['id', 'revision', 'sealed_revision'], stages.map(stage => ({ id: stage.id, revision: stage.revision, sealed_revision: stage.revision })));
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export async function stageRevisions(context: D1ImportContext, stages: StepStage[]): Promise<{ id: string; revision: number }[]> {
|
|
39
|
+
const { results } = await context.storage.database.prepare('SELECT id,revision,sealed_revision,registered_revision FROM ik_d1_staging WHERE id IN (SELECT value FROM json_each(?))').bind(JSON.stringify(stages.map(stage => stage.id))).all<any>();
|
|
40
|
+
const rows = stages.map(stage => results.find(row => row.id === stage.id));
|
|
41
|
+
// A generation erased, cancelled or replaced since it was staged ends the
|
|
42
|
+
// preparation: cancel it and prepare the upload again.
|
|
43
|
+
if (rows.some(row => !row || row.sealed_revision !== row.revision || row.registered_revision === null)) throw new Error('IMPORT_PREPARATION_INTERRUPTED');
|
|
44
|
+
return rows.map(row => ({ id: row.id, revision: row.revision }));
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export const progress = (row: StepControl, state: ImportPreparationStep['state'], records = row.records): ImportPreparationStep => ({ id: row.id, state, records, total: row.total });
|
|
48
|
+
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
import type { ImportPlan } from '../types.ts';
|
|
2
|
+
import type { D1ImportContext } from './types.ts';
|
|
3
|
+
import type { UnverifiedStreamPages } from './stream-plan.ts';
|
|
4
|
+
import type { Sha256State } from '../provenance/resumable-sha256.ts';
|
|
5
|
+
import { createResumableCanonicalHash } from '../provenance/canonical-hash.ts';
|
|
6
|
+
import { canonicalHash } from '../provenance/canonical-stream.ts';
|
|
7
|
+
import { planIndex } from './erasure-index.ts';
|
|
8
|
+
import { assertStreamPage } from './stream-page-integrity.ts';
|
|
9
|
+
import { policyHash } from './undo-state.ts';
|
|
10
|
+
import { stagingAtomic } from './staging-transport.ts';
|
|
11
|
+
import { controlUpdate, progress, stageRevisions, stagesFence, type ImportPreparationStep, type StepControl, type StepStage } from './preparation-step-control.ts';
|
|
12
|
+
import { locate, mergedHeader, stageHeader, stageIndex, stagePages } from './preparation-step-pages.ts';
|
|
13
|
+
|
|
14
|
+
/** Pages one verifying step reads, checks and hashes (in each of two passes). */
|
|
15
|
+
export const STEP_PAGES = 125;
|
|
16
|
+
|
|
17
|
+
type Counts = ImportPlan['counts'];
|
|
18
|
+
type Writer = ReturnType<typeof createResumableCanonicalHash>;
|
|
19
|
+
/** The fingerprint's progress between requests. `frame` is the merged header
|
|
20
|
+
* without diagnostics: every field the document needs after its prefix. */
|
|
21
|
+
interface HashState { revisions: { id: string; revision: number }[]; pass: 'entries' | 'records'; page: number; first: boolean; sha: Sha256State; counts: Counts; frame: Record<string, any> }
|
|
22
|
+
|
|
23
|
+
const zero = (): Counts => ({ input: 0, unique: 0, duplicates: 0, mapped: 0, unmapped: 0, rejected: 0, effects: 0, byKind: Object.create(null) });
|
|
24
|
+
const documentOf = (header: Record<string, any>) => ({ ...header, entries: [], source: { ...header.source, records: [] } });
|
|
25
|
+
|
|
26
|
+
/** One verifying step: the same sorted keys, values and page order as
|
|
27
|
+
* planFingerprint and cold verification, written into a hash whose state is
|
|
28
|
+
* saved with the generations' revisions it read. */
|
|
29
|
+
export async function hashStep(context: D1ImportContext, row: StepControl): Promise<ImportPreparationStep> {
|
|
30
|
+
const stages: StepStage[] = JSON.parse(row.stages), pages = stages.reduce((n, stage) => n + stage.pages, 0);
|
|
31
|
+
const revisions = await stageRevisions(context, stages);
|
|
32
|
+
let state: HashState, writer: Writer;
|
|
33
|
+
if (!row.hash_json) {
|
|
34
|
+
const header = mergedHeader(await Promise.all(stages.map(stage => stageHeader(context, stage.id))));
|
|
35
|
+
if (header.version !== 1 || header.destination !== row.destination || header.namespace !== row.namespace || header.mappingVersion !== row.mapping_version || header.principalId !== row.principal_id || header.counts.unique !== row.total) throw new Error('PLAN_BINDING_MISMATCH');
|
|
36
|
+
writer = createResumableCanonicalHash();
|
|
37
|
+
prefix(writer, documentOf(header));
|
|
38
|
+
const { diagnostics: _, ...frame } = header;
|
|
39
|
+
state = { revisions, pass: 'entries', page: 0, first: true, sha: writer.checkpoint(), counts: zero(), frame };
|
|
40
|
+
} else {
|
|
41
|
+
state = JSON.parse(row.hash_json);
|
|
42
|
+
if (canonicalHash(state.revisions) !== canonicalHash(revisions)) throw new Error('IMPORT_PREPARATION_CHANGED');
|
|
43
|
+
writer = createResumableCanonicalHash(state.sha);
|
|
44
|
+
}
|
|
45
|
+
const headers = new Map<string, UnverifiedStreamPages['header']>();
|
|
46
|
+
try {
|
|
47
|
+
const end = Math.min(pages, state.page + STEP_PAGES);
|
|
48
|
+
while (state.page < end) {
|
|
49
|
+
const { stage, ordinal, first } = locate(stages, state.page), to = Math.min(stage.pages, end - first);
|
|
50
|
+
const read = await stagePages(context, stage.id, ordinal, to);
|
|
51
|
+
const index = state.pass === 'entries' ? await stageIndex(context, stage.id, ordinal, to) : [];
|
|
52
|
+
if (!headers.has(stage.id) && state.pass === 'entries') headers.set(stage.id, await stageHeader(context, stage.id));
|
|
53
|
+
let cursor = 0;
|
|
54
|
+
for (const { page, plan } of read) {
|
|
55
|
+
if (state.pass === 'entries') cursor = checkPage(page, plan, headers.get(stage.id)!, index, cursor, row.reversal_policy, state.counts);
|
|
56
|
+
for (const entry of plan.entries) { if (!state.first) writer.token(','); state.first = false; writer.value(state.pass === 'entries' ? entry : entry.source); }
|
|
57
|
+
}
|
|
58
|
+
if (state.pass === 'entries' && cursor !== index.length) throw new Error('IMPORT_STAGE_CONTENT_CHANGED');
|
|
59
|
+
state.page += to - ordinal;
|
|
60
|
+
}
|
|
61
|
+
let fingerprint: string | undefined;
|
|
62
|
+
if (state.page === pages && state.pass === 'entries') {
|
|
63
|
+
if (canonicalHash(state.counts) !== canonicalHash(state.frame.counts)) throw new Error('IMPORT_STREAM_COUNTS_CHANGED');
|
|
64
|
+
middle(writer, documentOf(state.frame)); state.pass = 'records'; state.page = 0; state.first = true;
|
|
65
|
+
} else if (state.page === pages) {
|
|
66
|
+
suffix(writer, documentOf(state.frame)); fingerprint = writer.digest();
|
|
67
|
+
}
|
|
68
|
+
const changes = fingerprint ? { phase: 'combining', fingerprint, hash_json: JSON.stringify({ revisions: state.revisions, frame: state.frame }) } : { hash_json: JSON.stringify({ ...state, sha: writer.checkpoint() }) };
|
|
69
|
+
await stagingAtomic(context, [...stagesFence(revisions), ...controlUpdate(row, changes)]);
|
|
70
|
+
return progress(row, 'verifying');
|
|
71
|
+
} finally { writer.clear(); }
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/** The checks cold verification makes on each page: its scoped projection of
|
|
75
|
+
* the header, normalized custody only, and its exact privacy index, index and
|
|
76
|
+
* policy hashes. Returns the next unread index row. */
|
|
77
|
+
function checkPage(page: any, plan: ImportPlan, header: UnverifiedStreamPages['header'], index: any[], cursor: number, reversal: string, counts: Counts): number {
|
|
78
|
+
assertStreamPage({ header }, plan);
|
|
79
|
+
if (plan.entries.some(entry => entry.effects.some(effect => effect.attachment)) || plan.source.artifacts.some(artifact => artifact.original?.status === 'available')) throw new Error('IMPORT_STREAM_NORMALIZED_SOURCE_REQUIRED');
|
|
80
|
+
for (const key of ['input', 'unique', 'duplicates', 'mapped', 'unmapped', 'rejected', 'effects'] as const) counts[key] += plan.counts[key];
|
|
81
|
+
for (const [kind, count] of Object.entries(plan.counts.byKind)) counts.byKind[kind] = (counts.byKind[kind] ?? 0) + count;
|
|
82
|
+
const identities = planIndex(plan, page.job_id).map(item => [page.ordinal, item.ordinal, item.source_identity, item.target_identity]);
|
|
83
|
+
if (identities.length !== page.identities || canonicalHash(identities) !== page.index_hash || policyHash(page.job_id, plan.fingerprint, reversal as never) !== page.policy_hash) throw new Error('IMPORT_STAGE_CONTENT_CHANGED');
|
|
84
|
+
for (const expected of identities) {
|
|
85
|
+
const actual = index[cursor++];
|
|
86
|
+
if (!actual || canonicalHash(expected) !== canonicalHash([actual.page_ordinal, actual.ordinal, actual.source_identity, actual.target_identity])) throw new Error('IMPORT_STAGE_CONTENT_CHANGED');
|
|
87
|
+
}
|
|
88
|
+
return cursor;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/** `{` and every key before `entries`, then `"entries":[`. */
|
|
92
|
+
function prefix(writer: Writer, document: Record<string, any>): void {
|
|
93
|
+
writer.token('{'); let first = true;
|
|
94
|
+
for (const key of Object.keys(document).sort()) {
|
|
95
|
+
if (!first) writer.token(','); first = false;
|
|
96
|
+
writer.value(key); writer.token(':');
|
|
97
|
+
if (key === 'entries') { writer.token('['); return; }
|
|
98
|
+
writer.value(document[key]);
|
|
99
|
+
}
|
|
100
|
+
throw new Error('IMPORT_STREAM_LAYOUT_CHANGED');
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/** `]`, the keys between `entries` and `source`, then `"source":{` with its keys
|
|
104
|
+
* before `records`, then `"records":[`. */
|
|
105
|
+
function middle(writer: Writer, document: Record<string, any>): void {
|
|
106
|
+
writer.token(']');
|
|
107
|
+
const keys = Object.keys(document).sort();
|
|
108
|
+
for (const key of keys.slice(keys.indexOf('entries') + 1)) {
|
|
109
|
+
writer.token(','); writer.value(key); writer.token(':');
|
|
110
|
+
if (key !== 'source') { writer.value(document[key]); continue; }
|
|
111
|
+
writer.token('{'); let first = true;
|
|
112
|
+
for (const sourceKey of Object.keys(document.source).sort()) {
|
|
113
|
+
if (!first) writer.token(','); first = false;
|
|
114
|
+
writer.value(sourceKey); writer.token(':');
|
|
115
|
+
if (sourceKey === 'records') { writer.token('['); return; }
|
|
116
|
+
writer.value(document.source[sourceKey]);
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
throw new Error('IMPORT_STREAM_LAYOUT_CHANGED');
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/** `]`, the rest of `source`, and every key after it. */
|
|
123
|
+
function suffix(writer: Writer, document: Record<string, any>): void {
|
|
124
|
+
writer.token(']');
|
|
125
|
+
const sourceKeys = Object.keys(document.source).sort();
|
|
126
|
+
for (const key of sourceKeys.slice(sourceKeys.indexOf('records') + 1)) { writer.token(','); writer.value(key); writer.token(':'); writer.value(document.source[key]); }
|
|
127
|
+
writer.token('}');
|
|
128
|
+
const keys = Object.keys(document).sort();
|
|
129
|
+
for (const key of keys.slice(keys.indexOf('source') + 1)) { writer.token(','); writer.value(key); writer.token(':'); writer.value(document[key]); }
|
|
130
|
+
writer.token('}');
|
|
131
|
+
}
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
import { Buffer } from 'node:buffer';
|
|
2
|
+
import { createHash } from 'node:crypto';
|
|
3
|
+
import type { ImportPlan } from '../types.ts';
|
|
4
|
+
import type { D1ImportContext } from './types.ts';
|
|
5
|
+
import type { UnverifiedStreamPages } from './stream-plan.ts';
|
|
6
|
+
import { canonicalHash } from '../provenance/canonical-stream.ts';
|
|
7
|
+
import { sha256 } from '../provenance/identity.ts';
|
|
8
|
+
import { storedPagePlan } from './preparation-reader.ts';
|
|
9
|
+
import type { StepStage } from './preparation-step-control.ts';
|
|
10
|
+
|
|
11
|
+
type Header = UnverifiedStreamPages['header'];
|
|
12
|
+
|
|
13
|
+
/** A step generation's stored header, checked against its preparation receipt. */
|
|
14
|
+
export async function stageHeader(context: D1ImportContext, stage: string): Promise<Header> {
|
|
15
|
+
const db = context.storage.database;
|
|
16
|
+
const receipt = await db.prepare('SELECT header_hash,header_size,header_chunks FROM ik_d1_preparation_receipts WHERE stage_id=?').bind(stage).first<any>();
|
|
17
|
+
if (!receipt) throw new Error('IMPORT_PREPARATION_INTERRUPTED');
|
|
18
|
+
const { results } = await db.prepare('SELECT ordinal,body,body_hash,body_size FROM ik_d1_staging_headers WHERE stage_id=? ORDER BY ordinal').bind(stage).all<any>();
|
|
19
|
+
const hash = createHash('sha256'); let bytes = 0;
|
|
20
|
+
results.forEach((row, ordinal) => {
|
|
21
|
+
if (row.ordinal !== ordinal || Buffer.byteLength(row.body) !== row.body_size || sha256(row.body) !== row.body_hash) throw new Error('IMPORT_PREPARATION_CHANGED');
|
|
22
|
+
hash.update(row.body); bytes += row.body_size;
|
|
23
|
+
});
|
|
24
|
+
if (results.length !== receipt.header_chunks || bytes !== receipt.header_size || hash.digest('hex') !== receipt.header_hash) throw new Error('IMPORT_PREPARATION_CHANGED');
|
|
25
|
+
const header = JSON.parse(results.map(row => row.body).join(''));
|
|
26
|
+
if (canonicalHash(header) !== receipt.header_hash) throw new Error('IMPORT_PREPARATION_CHANGED');
|
|
27
|
+
return header;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/** The header one collecting pass over the whole source would have built: the
|
|
31
|
+
* first slice's, with every later slice's counts added and its own diagnostics
|
|
32
|
+
* appended (each slice repeats the shared artifact diagnostics first). */
|
|
33
|
+
export function mergedHeader(headers: Header[]): Header {
|
|
34
|
+
const [first, ...rest] = headers, merged: Header = structuredClone(first);
|
|
35
|
+
merged.counts.byKind = Object.assign(Object.create(null), merged.counts.byKind);
|
|
36
|
+
const shared = first.source.artifacts.filter(artifact => artifact.original?.status === 'unavailable').length;
|
|
37
|
+
for (const header of rest) {
|
|
38
|
+
for (const key of ['input', 'unique', 'duplicates', 'mapped', 'unmapped', 'rejected', 'effects'] as const) merged.counts[key] += header.counts[key];
|
|
39
|
+
for (const [kind, count] of Object.entries(header.counts.byKind)) merged.counts.byKind[kind] = (merged.counts.byKind[kind] ?? 0) + count;
|
|
40
|
+
merged.diagnostics.push(...header.diagnostics.slice(shared));
|
|
41
|
+
const { counts: _a, diagnostics: _b, ...same } = header, { counts: _c, diagnostics: _d, ...expected } = first;
|
|
42
|
+
if (canonicalHash(same) !== canonicalHash(expected)) throw new Error('PLAN_BINDING_MISMATCH');
|
|
43
|
+
}
|
|
44
|
+
return merged;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** Which step generation holds global page `page`, and its ordinal there. */
|
|
48
|
+
export function locate(stages: StepStage[], page: number): { stage: StepStage; ordinal: number; first: number } {
|
|
49
|
+
let first = 0;
|
|
50
|
+
for (const stage of stages) { if (page < first + stage.pages) return { stage, ordinal: page - first, first }; first += stage.pages; }
|
|
51
|
+
throw new Error('IMPORT_STREAM_LAYOUT_CHANGED');
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/** Pages [from, to) of one step generation, each checked against its manifest. */
|
|
55
|
+
export async function stagePages(context: D1ImportContext, stage: string, from: number, to: number): Promise<{ page: any; plan: ImportPlan }[]> {
|
|
56
|
+
const db = context.storage.database;
|
|
57
|
+
const pages = (await db.prepare('SELECT * FROM ik_d1_staging_pages WHERE stage_id=? AND ordinal>=? AND ordinal<? ORDER BY ordinal').bind(stage, from, to).all<any>()).results;
|
|
58
|
+
const chunks = (await db.prepare('SELECT page_ordinal,ordinal,body,body_hash,body_size FROM ik_d1_staging_chunks WHERE stage_id=? AND page_ordinal>=? AND page_ordinal<? ORDER BY page_ordinal,ordinal').bind(stage, from, to).all<any>()).results;
|
|
59
|
+
if (pages.length !== to - from || pages.some((page, index) => page.ordinal !== from + index)) throw new Error('IMPORT_STREAM_LAYOUT_CHANGED');
|
|
60
|
+
let next = 0;
|
|
61
|
+
const result = pages.map(page => {
|
|
62
|
+
const parts: string[] = []; let bytes = 0;
|
|
63
|
+
for (let ordinal = 0; ordinal < page.chunks; ordinal++) {
|
|
64
|
+
const chunk = chunks[next++];
|
|
65
|
+
if (!chunk || chunk.page_ordinal !== page.ordinal || chunk.ordinal !== ordinal || typeof chunk.body !== 'string' || Buffer.byteLength(chunk.body) !== chunk.body_size || sha256(chunk.body) !== chunk.body_hash) throw new Error('IMPORT_STAGE_CONTENT_CHANGED');
|
|
66
|
+
parts.push(chunk.body); bytes += chunk.body_size;
|
|
67
|
+
}
|
|
68
|
+
return { page, plan: storedPagePlan(page, parts, bytes, context.storage.targets.planCodec) };
|
|
69
|
+
});
|
|
70
|
+
if (next !== chunks.length) throw new Error('IMPORT_STAGE_CONTENT_CHANGED');
|
|
71
|
+
return result;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/** The privacy index rows of pages [from, to), in the order they were staged. */
|
|
75
|
+
export async function stageIndex(context: D1ImportContext, stage: string, from: number, to: number): Promise<any[]> {
|
|
76
|
+
return (await context.storage.database.prepare('SELECT page_ordinal,ordinal,source_identity,target_identity FROM ik_d1_staging_index WHERE stage_id=? AND page_ordinal>=? AND page_ordinal<? ORDER BY rowid').bind(stage, from, to).all<any>()).results;
|
|
77
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/** Additive control rows for preparations spread over several requests. A row
|
|
2
|
+
* names the private generations its steps staged and the resumable fingerprint
|
|
3
|
+
* state; it holds no source value. A stream the steps published keeps the
|
|
4
|
+
* source digest they began with, so only that upload replays it. Apply after
|
|
5
|
+
* d1PreparationSchema. */
|
|
6
|
+
export const d1PreparationStepsSchema = [
|
|
7
|
+
"CREATE TABLE IF NOT EXISTS ik_d1_preparation_steps (id TEXT PRIMARY KEY, destination TEXT NOT NULL, namespace TEXT NOT NULL, mapping_version TEXT NOT NULL, principal_id TEXT NOT NULL, operation_key TEXT NOT NULL, reversal_policy TEXT NOT NULL, codec_id TEXT, source_digest TEXT NOT NULL, total INTEGER NOT NULL, records INTEGER NOT NULL DEFAULT 0, stages TEXT NOT NULL DEFAULT '[]', phase TEXT NOT NULL CHECK(phase IN ('collecting','hashing','combining','prepared')), hash_json TEXT, fingerprint TEXT, verified_revision INTEGER, revision INTEGER NOT NULL DEFAULT 0, UNIQUE(destination,principal_id,operation_key))",
|
|
8
|
+
"CREATE TRIGGER IF NOT EXISTS ik_d1_preparation_steps_binding BEFORE UPDATE OF id,destination,namespace,mapping_version,principal_id,operation_key,reversal_policy,codec_id,source_digest,total ON ik_d1_preparation_steps BEGIN SELECT RAISE(ABORT,'IMPORT_PREPARATION_BINDING_CHANGED'); END",
|
|
9
|
+
"CREATE TRIGGER IF NOT EXISTS ik_d1_preparation_steps_revision BEFORE UPDATE ON ik_d1_preparation_steps WHEN NEW.revision<>OLD.revision+1 BEGIN SELECT RAISE(ABORT,'IMPORT_PREPARATION_CHANGED'); END",
|
|
10
|
+
'CREATE TABLE IF NOT EXISTS ik_d1_stream_sources (stream_id TEXT PRIMARY KEY REFERENCES ik_d1_streams(id) ON DELETE CASCADE, destination TEXT NOT NULL, source_digest TEXT NOT NULL)',
|
|
11
|
+
];
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto';
|
|
2
|
+
import type { ImportMapping, SourceBundle } from '../types.ts';
|
|
3
|
+
import type { D1ImportContext } from './types.ts';
|
|
4
|
+
import { requestContext, reserveReturn } from './budget.ts';
|
|
5
|
+
import { authorize, guard, sql } from './database.ts';
|
|
6
|
+
import { canonicalJson, requireText, sha256 } from '../provenance/identity.ts';
|
|
7
|
+
import { collectStreamPages } from './stream-plan.ts';
|
|
8
|
+
import { stageStreamPages } from './stream-stage.ts';
|
|
9
|
+
import { preparationHeader } from './preparation-header.ts';
|
|
10
|
+
import { stagingAtomic, type StageBinding } from './staging-transport.ts';
|
|
11
|
+
import { discardStage } from './staging-lifecycle.ts';
|
|
12
|
+
import { exactRows } from './sets.ts';
|
|
13
|
+
import { policy } from './undo-state.ts';
|
|
14
|
+
import { hashStep } from './preparation-step-hash.ts';
|
|
15
|
+
import { combineStep } from './preparation-step-combine.ts';
|
|
16
|
+
import { preparationState } from './preparation-state.ts';
|
|
17
|
+
import { preparationHeaderRead } from './preparation-reader.ts';
|
|
18
|
+
import { control, controlUpdate, progress, stageRevisions, stagesFence, type ImportPreparationStep, type StepControl, type StepStage } from './preparation-step-control.ts';
|
|
19
|
+
export type { ImportPreparationStep } from './preparation-step-control.ts';
|
|
20
|
+
|
|
21
|
+
/** Source records one collecting step maps, normalizes and stages privately. */
|
|
22
|
+
export const STEP_RECORDS = 5000;
|
|
23
|
+
|
|
24
|
+
/** Normalized imports too large for one request's CPU. Each call does one bounded
|
|
25
|
+
* step and returns where the preparation stands; call again (in a new request)
|
|
26
|
+
* with the same source, mapping and operationKey until it is `reviewable`, show
|
|
27
|
+
* that review, then publish with `publishD1ImportPreparation(id)` and the
|
|
28
|
+
* acknowledged fingerprint as `reviewedFingerprint`.
|
|
29
|
+
*
|
|
30
|
+
* Collecting steps map and stage STEP_RECORDS records each as private, sealed
|
|
31
|
+
* generations under the usual privacy admission. Verifying steps then compute
|
|
32
|
+
* the complete canonical fingerprint from that stored custody with a resumable
|
|
33
|
+
* hash whose state stays in D1, fenced by every generation's revision, so a
|
|
34
|
+
* changed, missing or reordered page changes the fingerprint or fails the step.
|
|
35
|
+
* The last step combines the generations into one sealed preparation in SQL. */
|
|
36
|
+
export async function prepareD1ImportSteps(source: SourceBundle, mapping: ImportMapping, input: D1ImportContext): Promise<ImportPreparationStep> {
|
|
37
|
+
const context = requestContext(input); await authorize(context, 'create');
|
|
38
|
+
const operation = context.operationKey; requireText(operation, 'operationKey');
|
|
39
|
+
const db = context.storage.database;
|
|
40
|
+
const digest = sourceDigest(source);
|
|
41
|
+
const published = await db.prepare('SELECT s.id,d.source_digest FROM ik_d1_streams s LEFT JOIN ik_d1_stream_sources d ON d.stream_id=s.id WHERE s.destination=? AND s.principal_id=? AND s.operation_key=?').bind(context.destination, context.authorization.principalId, operation).first<{ id: string; source_digest: string | null }>();
|
|
42
|
+
// Only the upload these steps published replays; a different one (or an
|
|
43
|
+
// import published in one request, which kept no digest) is refused.
|
|
44
|
+
if (published && published.source_digest !== digest) throw new Error('OPERATION_KEY_CONFLICT');
|
|
45
|
+
if (published) return { id: published.id, state: 'published', records: source.records.length, total: source.records.length };
|
|
46
|
+
let row = await control(context, operation!);
|
|
47
|
+
if (!row) {
|
|
48
|
+
await stagingAtomic(context, [
|
|
49
|
+
guard(sql('NOT EXISTS(SELECT 1 FROM ik_d1_staging WHERE destination=? AND principal_id=? AND operation_key=?)', context.destination, context.authorization.principalId, operation)),
|
|
50
|
+
guard(sql('NOT EXISTS(SELECT 1 FROM ik_d1_streams WHERE destination=? AND principal_id=? AND operation_key=?)', context.destination, context.authorization.principalId, operation)),
|
|
51
|
+
sql('INSERT INTO ik_d1_preparation_steps(id,destination,namespace,mapping_version,principal_id,operation_key,reversal_policy,codec_id,source_digest,total,phase) VALUES(?,?,?,?,?,?,?,?,?,?,?)',
|
|
52
|
+
randomUUID(), context.destination, context.namespace, context.mappingVersion, context.authorization.principalId, operation, policy(context), context.storage.targets.planCodec?.id ?? null, digest, source.records.length, 'collecting')]);
|
|
53
|
+
row = (await control(context, operation!))!;
|
|
54
|
+
}
|
|
55
|
+
if (row.namespace !== context.namespace || row.mapping_version !== context.mappingVersion || row.reversal_policy !== policy(context)) throw new Error('OPERATION_KEY_CONFLICT');
|
|
56
|
+
if (row.codec_id !== (context.storage.targets.planCodec?.id ?? null)) throw new Error('IMPORT_PLAN_CODEC_REQUIRED');
|
|
57
|
+
if (row.phase === 'collecting') {
|
|
58
|
+
if (row.source_digest !== digest) throw new Error('OPERATION_KEY_CONFLICT');
|
|
59
|
+
return collectStep(source, mapping, context, row);
|
|
60
|
+
}
|
|
61
|
+
if (row.phase === 'hashing') return hashStep(context, row);
|
|
62
|
+
if (row.phase === 'combining') return combineStep(context, row);
|
|
63
|
+
const state = await preparationState(row.id, context, 'create');
|
|
64
|
+
const header = await preparationHeaderRead(state);
|
|
65
|
+
return { id: row.id, state: 'reviewable', records: row.total, total: row.total, fingerprint: state.stage.fingerprint, counts: header.counts, diagnostics: header.diagnostics };
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/** The records count, artifacts and cursor policy the preparation began with:
|
|
69
|
+
* a different upload under the same operation key is refused, not merged. */
|
|
70
|
+
function sourceDigest(source: SourceBundle): string {
|
|
71
|
+
return sha256(canonicalJson({ artifacts: source.artifacts, diagnostics: source.diagnostics, cursor: source.paginationCursorPolicy ?? null, records: source.records.length }));
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
async function collectStep(source: SourceBundle, mapping: ImportMapping, context: D1ImportContext, row: StepControl): Promise<ImportPreparationStep> {
|
|
75
|
+
const stages: StepStage[] = JSON.parse(row.stages);
|
|
76
|
+
await stageRevisions(context, stages);
|
|
77
|
+
const slice = source.records.slice(row.records, row.records + STEP_RECORDS);
|
|
78
|
+
const { stream } = await collectStreamPages({ ...source, records: slice }, mapping, context);
|
|
79
|
+
// Each step's generation has an internal operation key and a fingerprint no
|
|
80
|
+
// review produced, so it can never be published on its own.
|
|
81
|
+
const step = `${row.id}:${stages.length}`, stepContext = { ...context, operationKey: step };
|
|
82
|
+
// A request that died after sealing this step's generation, before recording
|
|
83
|
+
// it, left it unrecorded under the step's key: discard it (fenced by the
|
|
84
|
+
// control row this step read) and stage the slice again.
|
|
85
|
+
if (await context.storage.database.prepare('SELECT 1 FROM ik_d1_staging WHERE destination=? AND principal_id=? AND operation_key=?').bind(context.destination, context.authorization.principalId, step).first()) {
|
|
86
|
+
await stagingAtomic(context, [...exactRows('ik_d1_preparation_steps', Object.keys(row), [row]),
|
|
87
|
+
sql('DELETE FROM ik_d1_staging WHERE destination=? AND principal_id=? AND operation_key=?', context.destination, context.authorization.principalId, step)]);
|
|
88
|
+
}
|
|
89
|
+
const header = preparationHeader(stream.header, stepContext);
|
|
90
|
+
let stage: StageBinding | undefined;
|
|
91
|
+
try {
|
|
92
|
+
const staged = await stageStreamPages(stream, sha256(step), reserveReturn(stepContext, header.reserve + 8), binding => { stage = binding; }, header.store);
|
|
93
|
+
const records = row.records + slice.length, next = [...stages, { id: staged.stage.id, pages: stream.pages.length, records: slice.length }];
|
|
94
|
+
await stagingAtomic(context, [...stagesFence([{ id: staged.stage.id, revision: staged.revision }]),
|
|
95
|
+
...controlUpdate(row, { records, stages: JSON.stringify(next), phase: records >= row.total ? 'hashing' : 'collecting' })]);
|
|
96
|
+
return progress(row, records >= row.total ? 'verifying' : 'preparing', records);
|
|
97
|
+
} catch (error) {
|
|
98
|
+
if (stage) await discardStage(stage, context);
|
|
99
|
+
throw error;
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
import type { SourceBundle, ImportMapping } from '../types.ts';
|
|
2
|
+
import type { D1ImportContext } from './types.ts';
|
|
3
|
+
import { requestContext, reserveReturn } from './budget.ts';
|
|
4
|
+
import { authorize, guard, sql } from './database.ts';
|
|
5
|
+
import { requireText } from '../provenance/identity.ts';
|
|
6
|
+
import { collectStreamPages, streamView } from './stream-plan.ts';
|
|
7
|
+
import { reviewedSourceRecords } from './reviewed-source.ts';
|
|
8
|
+
import { planFingerprint } from '../plan/index.ts';
|
|
9
|
+
import { stageStreamPages } from './stream-stage.ts';
|
|
10
|
+
import { preparationHeader } from './preparation-header.ts';
|
|
11
|
+
import { preparationState, preparationRead, preparationStatus, readD1ImportPreparation } from './preparation-state.ts';
|
|
12
|
+
import { preparationHeaderRead, preparationPages } from './preparation-reader.ts';
|
|
13
|
+
import { verifyPreparationFingerprint } from './preparation-fingerprint.ts';
|
|
14
|
+
import { bindingPredicate, stageClear, stagingAtomic, type StageBinding } from './staging-transport.ts';
|
|
15
|
+
import { discardStage, publishedStage } from './staging-lifecycle.ts';
|
|
16
|
+
import { stagePublication, publishedStagePredicates } from './staging-publication.ts';
|
|
17
|
+
import { publishedStream } from './published-stream.ts';
|
|
18
|
+
import { publicationReturnQueries } from './staging-control.ts';
|
|
19
|
+
import { exactRows } from './sets.ts';
|
|
20
|
+
|
|
21
|
+
export { readD1ImportPreparation } from './preparation-state.ts';
|
|
22
|
+
export type { ImportPreparation } from './preparation-state.ts';
|
|
23
|
+
|
|
24
|
+
/** Explicitly unverified custody after acknowledgment. The expected fingerprint
|
|
25
|
+
* supplies no evidence that these pages were reviewed; cold verification must
|
|
26
|
+
* recompute it before the first public job or native effect can exist. */
|
|
27
|
+
export async function prepareD1ImportStream(source:SourceBundle,mapping:ImportMapping,input:D1ImportContext){
|
|
28
|
+
const context=requestContext(input);await authorize(context,'create');
|
|
29
|
+
const expected=context.reviewedFingerprint;
|
|
30
|
+
if(!expected||!/^[a-f0-9]{64}$/.test(expected))throw new Error('REVIEW_FINGERPRINT_MISMATCH');
|
|
31
|
+
const operation=context.operationKey??expected;requireText(operation,'operationKey');
|
|
32
|
+
const {stream,sourceHashes,pageSize}=await collectStreamPages(source,mapping,context);
|
|
33
|
+
const prior=await publishedStage(context,operation,expected);
|
|
34
|
+
const pending=await context.storage.database.prepare('SELECT id,fingerprint,revision,sealed_revision FROM ik_d1_staging WHERE destination=? AND principal_id=? AND operation_key=?').bind(context.destination,context.authorization.principalId,operation).first();
|
|
35
|
+
if(pending&&pending.fingerprint!==expected)throw new Error('OPERATION_KEY_CONFLICT');
|
|
36
|
+
const receipt=pending?await context.storage.database.prepare('SELECT stage_id FROM ik_d1_preparation_receipts WHERE stage_id=?').bind(pending.id).first():undefined;
|
|
37
|
+
const ready=!!receipt&&pending!.revision===pending!.sealed_revision;
|
|
38
|
+
if(prior||ready){
|
|
39
|
+
// Replay still verifies the submitted values. It cannot use the supplied
|
|
40
|
+
// digest to substitute a different mapping, timestamp or source.
|
|
41
|
+
const review=streamView({...stream,fingerprint:expected},context.storage.targets.planCodec);
|
|
42
|
+
review.source.records=reviewedSourceRecords(source.records,sourceHashes,pageSize,stream.header.counts.unique);
|
|
43
|
+
if(planFingerprint(review)!==expected)throw new Error('REVIEW_FINGERPRINT_MISMATCH');
|
|
44
|
+
if(prior)return prior;
|
|
45
|
+
return readD1ImportPreparation(pending!.id,context);
|
|
46
|
+
}
|
|
47
|
+
const header=preparationHeader(stream.header,context);
|
|
48
|
+
let stage:StageBinding|undefined;
|
|
49
|
+
try{
|
|
50
|
+
const prepared=await stageStreamPages(stream,expected,reserveReturn(context,header.reserve),binding=>{stage=binding;},header.store);
|
|
51
|
+
await preparationState(prepared.stage.id,context);
|
|
52
|
+
return preparationStatus(prepared.stage);
|
|
53
|
+
}catch(error){
|
|
54
|
+
if(stage)await discardStage(stage,context);
|
|
55
|
+
// This attempt has not verified its submitted stream. A concurrent public
|
|
56
|
+
// winner cannot make those different submitted values a successful replay.
|
|
57
|
+
// A new request takes the complete fingerprint-checked replay path above.
|
|
58
|
+
throw error;
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/** Cold continuation needs only stored safe custody and current native ports. */
|
|
63
|
+
export async function publishD1ImportPreparation(id:string,input:D1ImportContext){
|
|
64
|
+
const context=requestContext(input);await authorize(context,'create');
|
|
65
|
+
const state=await preparationState(id,context,'create');
|
|
66
|
+
const {stage,revision}=state;
|
|
67
|
+
try {
|
|
68
|
+
const header=await preparationHeaderRead(state),pages=await preparationPages(state);
|
|
69
|
+
// A preparation made in steps was verified step by step from this exact
|
|
70
|
+
// custody; its control row records the revision that verification covers.
|
|
71
|
+
const steps=await context.storage.database.prepare('SELECT * FROM ik_d1_preparation_steps WHERE id=?').bind(id).first<any>();
|
|
72
|
+
const verified=steps?.phase==='prepared'&&steps.verified_revision===revision&&steps.fingerprint===stage.fingerprint&&steps.destination===stage.destination&&steps.principal_id===stage.principal_id&&steps.operation_key===stage.operation_key;
|
|
73
|
+
if(!verified)await verifyPreparationFingerprint(state,header,pages);
|
|
74
|
+
await preparationRead(state,sql('SELECT 1 AS verified'));
|
|
75
|
+
// Receipt/schema triggers make deletion part of the same revision fences as
|
|
76
|
+
// every other private value. Existing final publication owns exact cleanup.
|
|
77
|
+
await stagingAtomic(context,[guard(bindingPredicate(stage)),guard(stageClear(stage)),
|
|
78
|
+
...steps?[...exactRows('ik_d1_preparation_steps',Object.keys(steps),[steps]),sql('DELETE FROM ik_d1_preparation_steps WHERE id=?',id)]:[],
|
|
79
|
+
...stagePublication(stage,revision),...steps?[sql('INSERT INTO ik_d1_stream_sources(stream_id,destination,source_digest) VALUES(?,?,?)',id,stage.destination,steps.source_digest)]:[]],publicationReturnQueries(context,stage),[sql('DELETE FROM ik_d1_publication_fences WHERE stream_id=?',id)],[...publishedStagePredicates(stage,pages),sql('NOT EXISTS(SELECT 1 FROM ik_d1_preparation_receipts WHERE stage_id=?) AND NOT EXISTS(SELECT 1 FROM ik_d1_staging_headers WHERE stage_id=?)',id,id)]);
|
|
80
|
+
return await publishedStream(context,id,stage.operation_key,stage.fingerprint);
|
|
81
|
+
} catch(error) {
|
|
82
|
+
await discardStage(stage,context);
|
|
83
|
+
const winner=await publishedStage(context,stage.operation_key,stage.fingerprint);
|
|
84
|
+
if(winner)return winner;
|
|
85
|
+
throw error;
|
|
86
|
+
}
|
|
87
|
+
}
|