@runeditrun/rer 0.2.77 → 0.2.79
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -0
- package/cli/export.mjs +1 -1
- package/cli/portability-migrations.mjs +35 -14
- package/cli/portability-worker.mjs +2 -2
- package/cli/portability.mjs +2 -2
- package/cli/server-proxy.mjs +30 -8
- package/cli/upstream-agent.mjs +14 -0
- package/global/CHANGELOG.md +13 -0
- package/global/README.md +32 -0
- package/modules/demo-host/native/visitor.mjs +1 -1
- package/modules/deploy-kit/owner-cli.mjs +15 -5
- package/modules/deploy-kit/provider.mjs +9 -4
- package/modules/import-kit/src/d1/README.md +1322 -0
- package/modules/import-kit/src/d1/apply.ts +115 -0
- package/modules/import-kit/src/d1/budget.ts +89 -0
- package/modules/import-kit/src/d1/compressed-plan.ts +63 -0
- package/modules/import-kit/src/d1/creation.ts +95 -0
- package/modules/import-kit/src/d1/custody-fence.ts +27 -0
- package/modules/import-kit/src/d1/custody-state-schema.ts +67 -0
- package/modules/import-kit/src/d1/database.ts +163 -0
- package/modules/import-kit/src/d1/entry-locator.ts +61 -0
- package/modules/import-kit/src/d1/erased-artifacts.ts +38 -0
- package/modules/import-kit/src/d1/erased-custody.ts +36 -0
- package/modules/import-kit/src/d1/erased-plan.ts +47 -0
- package/modules/import-kit/src/d1/erased-reversal.ts +19 -0
- package/modules/import-kit/src/d1/erasure-bounded.ts +22 -0
- package/modules/import-kit/src/d1/erasure-claims.ts +27 -0
- package/modules/import-kit/src/d1/erasure-compiler.ts +67 -0
- package/modules/import-kit/src/d1/erasure-disposition.ts +16 -0
- package/modules/import-kit/src/d1/erasure-fragments.ts +86 -0
- package/modules/import-kit/src/d1/erasure-index.ts +53 -0
- package/modules/import-kit/src/d1/erasure-pages.ts +62 -0
- package/modules/import-kit/src/d1/erasure-progress-schema.ts +8 -0
- package/modules/import-kit/src/d1/erasure-progress.ts +61 -0
- package/modules/import-kit/src/d1/erasure-related.ts +124 -0
- package/modules/import-kit/src/d1/erasure-report.ts +31 -0
- package/modules/import-kit/src/d1/erasure-schema.ts +24 -0
- package/modules/import-kit/src/d1/erasure-selection.ts +77 -0
- package/modules/import-kit/src/d1/erasure-snapshot.ts +15 -0
- package/modules/import-kit/src/d1/erasure-stage.ts +146 -0
- package/modules/import-kit/src/d1/erasure-step.ts +12 -0
- package/modules/import-kit/src/d1/erasure-superseded.ts +98 -0
- package/modules/import-kit/src/d1/erasure-values.ts +83 -0
- package/modules/import-kit/src/d1/erasure-writes.ts +46 -0
- package/modules/import-kit/src/d1/erasure.ts +170 -0
- package/modules/import-kit/src/d1/evolutions/0001-preserved-undo.ts +13 -0
- package/modules/import-kit/src/d1/expected-state.ts +91 -0
- package/modules/import-kit/src/d1/field-redaction.ts +86 -0
- package/modules/import-kit/src/d1/guards.ts +37 -0
- package/modules/import-kit/src/d1/history.ts +71 -0
- package/modules/import-kit/src/d1/index.ts +88 -0
- package/modules/import-kit/src/d1/integrity.ts +127 -0
- package/modules/import-kit/src/d1/inventory-snapshot.ts +84 -0
- package/modules/import-kit/src/d1/jobs.ts +97 -0
- package/modules/import-kit/src/d1/native-set.ts +48 -0
- package/modules/import-kit/src/d1/native-staging-admission.ts +19 -0
- package/modules/import-kit/src/d1/native-staging-context.ts +44 -0
- package/modules/import-kit/src/d1/native-staging-lifecycle.ts +33 -0
- package/modules/import-kit/src/d1/native-staging-offset.ts +74 -0
- package/modules/import-kit/src/d1/native-staging-publication.ts +18 -0
- package/modules/import-kit/src/d1/native-staging-rows.ts +72 -0
- package/modules/import-kit/src/d1/native-staging-schema.ts +28 -0
- package/modules/import-kit/src/d1/native-staging-state.ts +22 -0
- package/modules/import-kit/src/d1/native-staging.ts +64 -0
- package/modules/import-kit/src/d1/original-copies.ts +53 -0
- package/modules/import-kit/src/d1/original-storage.ts +56 -0
- package/modules/import-kit/src/d1/originals.ts +74 -0
- package/modules/import-kit/src/d1/plan-dictionary.ts +80 -0
- package/modules/import-kit/src/d1/plans.ts +121 -0
- package/modules/import-kit/src/d1/preparation-fingerprint.ts +57 -0
- package/modules/import-kit/src/d1/preparation-header.ts +33 -0
- package/modules/import-kit/src/d1/preparation-reader.ts +98 -0
- package/modules/import-kit/src/d1/preparation-schema.ts +16 -0
- package/modules/import-kit/src/d1/preparation-state.ts +56 -0
- package/modules/import-kit/src/d1/preparation-step-combine.ts +80 -0
- package/modules/import-kit/src/d1/preparation-step-control.ts +48 -0
- package/modules/import-kit/src/d1/preparation-step-hash.ts +131 -0
- package/modules/import-kit/src/d1/preparation-step-pages.ts +77 -0
- package/modules/import-kit/src/d1/preparation-steps-schema.ts +11 -0
- package/modules/import-kit/src/d1/preparation-steps.ts +102 -0
- package/modules/import-kit/src/d1/preparations.ts +87 -0
- package/modules/import-kit/src/d1/preserved-preview.ts +89 -0
- package/modules/import-kit/src/d1/preserved-reverse.ts +91 -0
- package/modules/import-kit/src/d1/provenance-set.ts +87 -0
- package/modules/import-kit/src/d1/publication-fence.ts +22 -0
- package/modules/import-kit/src/d1/publication-receipt-schema.ts +15 -0
- package/modules/import-kit/src/d1/published-stream.ts +11 -0
- package/modules/import-kit/src/d1/read-fence.ts +64 -0
- package/modules/import-kit/src/d1/read-results.ts +16 -0
- package/modules/import-kit/src/d1/redacted-custody.ts +76 -0
- package/modules/import-kit/src/d1/redaction-step.ts +125 -0
- package/modules/import-kit/src/d1/referenced-plans.ts +136 -0
- package/modules/import-kit/src/d1/retained-values.ts +35 -0
- package/modules/import-kit/src/d1/retirement-bounded.ts +71 -0
- package/modules/import-kit/src/d1/retirement-door.ts +105 -0
- package/modules/import-kit/src/d1/retirement-schema.ts +39 -0
- package/modules/import-kit/src/d1/retirement-scope.ts +94 -0
- package/modules/import-kit/src/d1/retirement-steps.ts +76 -0
- package/modules/import-kit/src/d1/retirement-tally.ts +74 -0
- package/modules/import-kit/src/d1/retirement.ts +136 -0
- package/modules/import-kit/src/d1/reversal-conversion-record.ts +19 -0
- package/modules/import-kit/src/d1/reversal-conversion-schema.ts +11 -0
- package/modules/import-kit/src/d1/reversal-conversion.ts +40 -0
- package/modules/import-kit/src/d1/reversal-fence-schema.ts +14 -0
- package/modules/import-kit/src/d1/reversal-fence.ts +106 -0
- package/modules/import-kit/src/d1/reversal-journal.ts +24 -0
- package/modules/import-kit/src/d1/reverse.ts +103 -0
- package/modules/import-kit/src/d1/reviewed-source.ts +23 -0
- package/modules/import-kit/src/d1/sets.ts +78 -0
- package/modules/import-kit/src/d1/staged-creation.ts +201 -0
- package/modules/import-kit/src/d1/staging-admission.ts +45 -0
- package/modules/import-kit/src/d1/staging-control.ts +43 -0
- package/modules/import-kit/src/d1/staging-erasure.ts +28 -0
- package/modules/import-kit/src/d1/staging-lifecycle.ts +49 -0
- package/modules/import-kit/src/d1/staging-plan.ts +15 -0
- package/modules/import-kit/src/d1/staging-privacy.ts +22 -0
- package/modules/import-kit/src/d1/staging-publication.ts +79 -0
- package/modules/import-kit/src/d1/staging-schema.ts +37 -0
- package/modules/import-kit/src/d1/staging-statements.ts +8 -0
- package/modules/import-kit/src/d1/staging-transport.ts +70 -0
- package/modules/import-kit/src/d1/stream-boundary.ts +11 -0
- package/modules/import-kit/src/d1/stream-create.ts +55 -0
- package/modules/import-kit/src/d1/stream-identity-admission.ts +37 -0
- package/modules/import-kit/src/d1/stream-input.ts +18 -0
- package/modules/import-kit/src/d1/stream-layout.ts +43 -0
- package/modules/import-kit/src/d1/stream-origin.ts +44 -0
- package/modules/import-kit/src/d1/stream-page-integrity.ts +18 -0
- package/modules/import-kit/src/d1/stream-partition.ts +50 -0
- package/modules/import-kit/src/d1/stream-plan.ts +133 -0
- package/modules/import-kit/src/d1/stream-receipt.ts +45 -0
- package/modules/import-kit/src/d1/stream-revision-schema.ts +35 -0
- package/modules/import-kit/src/d1/stream-snapshot.ts +26 -0
- package/modules/import-kit/src/d1/stream-stage.ts +79 -0
- package/modules/import-kit/src/d1/stream-state.ts +25 -0
- package/modules/import-kit/src/d1/stream-working-set.ts +30 -0
- package/modules/import-kit/src/d1/streams.ts +39 -0
- package/modules/import-kit/src/d1/target-guards.ts +9 -0
- package/modules/import-kit/src/d1/target-identities.ts +17 -0
- package/modules/import-kit/src/d1/transport.ts +10 -0
- package/modules/import-kit/src/d1/types.ts +162 -0
- package/modules/import-kit/src/d1/undo-observations.ts +39 -0
- package/modules/import-kit/src/d1/undo-report.ts +22 -0
- package/modules/import-kit/src/d1/undo-snapshot.ts +87 -0
- package/modules/import-kit/src/d1/undo-state.ts +84 -0
- package/modules/import-kit/src/d1/undo-types.ts +110 -0
- package/modules/import-kit/src/journal/authority.ts +12 -0
- package/modules/import-kit/src/plan/budget.ts +60 -0
- package/modules/import-kit/src/plan/index.ts +123 -0
- package/modules/import-kit/src/plan/size.ts +122 -0
- package/modules/import-kit/src/provenance/canonical-hash.ts +130 -0
- package/modules/import-kit/src/provenance/canonical-stream.ts +85 -0
- package/modules/import-kit/src/provenance/credential-names.ts +131 -0
- package/modules/import-kit/src/provenance/credentials.ts +317 -0
- package/modules/import-kit/src/provenance/identity.ts +51 -0
- package/modules/import-kit/src/provenance/json-snapshot.ts +21 -0
- package/modules/import-kit/src/provenance/resumable-sha256.ts +75 -0
- package/modules/import-kit/src/provenance/validate-json.ts +19 -0
- package/modules/import-kit/src/storage/originals.ts +87 -0
- package/modules/import-kit/src/storage/provenance.ts +64 -0
- package/modules/import-kit/src/storage/schema.ts +42 -0
- package/modules/import-kit/src/types.ts +47 -0
- package/modules/portability/README.md +98 -2
- package/modules/portability/backfills.mjs +42 -15
- package/modules/portability/ledgers.mjs +7 -1
- package/modules/portability/transient-tables.mjs +40 -0
- package/package.json +3 -1
- package/scripts/publish-check/reviewed.py +3 -1
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import { liveStreamOrigin, needsStreamOrigin } from './stream-origin.ts';
|
|
2
|
+
import { publicationFenceSchema } from './publication-fence.ts';
|
|
3
|
+
|
|
4
|
+
/** Additive private custody transport. Apply after the erasure and preserved
|
|
5
|
+
* undo evolutions; no existing migration or public table is replaced. */
|
|
6
|
+
export const d1StagingSchema = [
|
|
7
|
+
...publicationFenceSchema,
|
|
8
|
+
'CREATE TABLE IF NOT EXISTS ik_d1_staging (id TEXT PRIMARY KEY, generation TEXT NOT NULL, destination TEXT NOT NULL, namespace TEXT NOT NULL, mapping_version TEXT NOT NULL, principal_id TEXT NOT NULL, fingerprint TEXT NOT NULL, operation_key TEXT NOT NULL, total INTEGER NOT NULL, pages INTEGER NOT NULL, layout_hash TEXT NOT NULL, reversal_policy TEXT NOT NULL, revision INTEGER NOT NULL DEFAULT 0, registered_revision INTEGER, sealed_revision INTEGER, UNIQUE(destination,principal_id,operation_key))',
|
|
9
|
+
'CREATE TABLE IF NOT EXISTS ik_d1_staging_pages (stage_id TEXT NOT NULL REFERENCES ik_d1_staging(id) ON DELETE CASCADE, ordinal INTEGER NOT NULL, job_id TEXT NOT NULL UNIQUE, total INTEGER NOT NULL, fingerprint TEXT NOT NULL, plan_json TEXT NOT NULL, body_hash TEXT NOT NULL, body_size INTEGER NOT NULL, chunks INTEGER NOT NULL, identities INTEGER NOT NULL, index_hash TEXT NOT NULL, policy_hash TEXT NOT NULL, inline_plan INTEGER NOT NULL CHECK(inline_plan IN (0,1)), PRIMARY KEY(stage_id,ordinal))',
|
|
10
|
+
'CREATE TABLE IF NOT EXISTS ik_d1_staging_chunks (stage_id TEXT NOT NULL REFERENCES ik_d1_staging(id) ON DELETE CASCADE, page_ordinal INTEGER NOT NULL, ordinal INTEGER NOT NULL, body TEXT NOT NULL, body_hash TEXT NOT NULL, body_size INTEGER NOT NULL, PRIMARY KEY(stage_id,page_ordinal,ordinal))',
|
|
11
|
+
'CREATE TABLE IF NOT EXISTS ik_d1_staging_index (stage_id TEXT NOT NULL REFERENCES ik_d1_staging(id) ON DELETE CASCADE, page_ordinal INTEGER NOT NULL, ordinal INTEGER NOT NULL, source_identity TEXT NOT NULL, target_identity TEXT NOT NULL, PRIMARY KEY(stage_id,page_ordinal,ordinal,target_identity))',
|
|
12
|
+
'CREATE INDEX IF NOT EXISTS ik_d1_staging_source ON ik_d1_staging_index(source_identity,stage_id)',
|
|
13
|
+
'CREATE INDEX IF NOT EXISTS ik_d1_staging_target ON ik_d1_staging_index(target_identity,stage_id)',
|
|
14
|
+
'CREATE TABLE IF NOT EXISTS ik_d1_staging_originals (stage_id TEXT NOT NULL REFERENCES ik_d1_staging(id) ON DELETE CASCADE, original_key TEXT NOT NULL, name TEXT NOT NULL, sha256 TEXT NOT NULL, size INTEGER NOT NULL, PRIMARY KEY(stage_id,original_key))',
|
|
15
|
+
"CREATE TRIGGER IF NOT EXISTS ik_d1_staging_binding BEFORE UPDATE OF id,generation,destination,namespace,mapping_version,principal_id,fingerprint,operation_key,total,pages,layout_hash,reversal_policy ON ik_d1_staging BEGIN SELECT RAISE(ABORT,'IMPORT_STAGE_BINDING_CHANGED'); END",
|
|
16
|
+
...['pages', 'index', 'chunks', 'originals'].flatMap(table => ['INSERT', 'UPDATE', 'DELETE'].map(event => {
|
|
17
|
+
const refs = event === 'INSERT' ? 'id=NEW.stage_id' : event === 'DELETE' ? 'id=OLD.stage_id' : 'id=OLD.stage_id OR id=NEW.stage_id';
|
|
18
|
+
return `CREATE TRIGGER IF NOT EXISTS ik_d1_staging_${table}_${event.toLowerCase()} AFTER ${event} ON ik_d1_staging_${table} BEGIN UPDATE ik_d1_staging SET revision=revision+1,sealed_revision=NULL${['pages','index'].includes(table) ? ',registered_revision=NULL' : ''} WHERE ${refs}; END`;
|
|
19
|
+
})),
|
|
20
|
+
// Invalidating the complete privacy index also discards private values.
|
|
21
|
+
// An interrupted generation can never hide payload by losing its index.
|
|
22
|
+
...['INSERT', 'UPDATE', 'DELETE'].map(event => {
|
|
23
|
+
const refs = event === 'INSERT' ? 'stage_id=NEW.stage_id' : event === 'DELETE' ? 'stage_id=OLD.stage_id' : 'stage_id=OLD.stage_id OR stage_id=NEW.stage_id';
|
|
24
|
+
return `CREATE TRIGGER IF NOT EXISTS ik_d1_staging_index_custody_${event.toLowerCase()} AFTER ${event} ON ik_d1_staging_index WHEN EXISTS(SELECT 1 FROM ik_d1_staging_chunks WHERE ${refs}) OR EXISTS(SELECT 1 FROM ik_d1_staging_originals WHERE ${refs}) BEGIN DELETE FROM ik_d1_staging_chunks WHERE ${refs}; DELETE FROM ik_d1_staging_originals WHERE ${refs}; END`;
|
|
25
|
+
}),
|
|
26
|
+
...['chunks', 'originals'].map(table => `CREATE TRIGGER IF NOT EXISTS ik_d1_staging_${table}_registered BEFORE INSERT ON ik_d1_staging_${table} WHEN NOT EXISTS(SELECT 1 FROM ik_d1_staging WHERE id=NEW.stage_id AND registered_revision IS NOT NULL) BEGIN SELECT RAISE(ABORT,'IMPORT_STAGE_INDEX_REQUIRED'); END`),
|
|
27
|
+
];
|
|
28
|
+
|
|
29
|
+
/** Earlier streams shared version-three custody with ordinary jobs. Convert
|
|
30
|
+
* only actual page membership; durable origins reject known orphan custody.
|
|
31
|
+
* A legacy job whose structural evidence was already lost cannot be classified
|
|
32
|
+
* from its ID, operation key, payload or nullable reversal-policy binding. */
|
|
33
|
+
export const d1LegacyStreamReceiptUpgrade = [
|
|
34
|
+
`INSERT INTO ik_d1_assertions(ok) SELECT CASE WHEN NOT EXISTS(SELECT 1 FROM ik_jobs legacy WHERE ${needsStreamOrigin('legacy.id')} AND NOT ${liveStreamOrigin('legacy.id')}) THEN 1 ELSE 0 END`,
|
|
35
|
+
'UPDATE ik_original_sets SET version=4 WHERE version=3 AND job_id IN (SELECT job_id FROM ik_d1_stream_pages)',
|
|
36
|
+
'DELETE FROM ik_d1_assertions',
|
|
37
|
+
];
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { ImportAction } from '../types.ts';
|
|
2
|
+
import type { D1ImportContext, D1Sql } from './types.ts';
|
|
3
|
+
import { atomicStatements, finalAuthority, finalGuard } from './database.ts';
|
|
4
|
+
/** Release and assertion cleanup are mutations. All final predicates and current
|
|
5
|
+
* authority follow them, with no subsequent write in the transaction. */
|
|
6
|
+
export function stagingStatements(context: D1ImportContext, action: ImportAction, statements: D1Sql[], release: D1Sql[] = [], final: D1Sql[] = []): D1Sql[] {
|
|
7
|
+
return [...atomicStatements({...context,maxBatchBytes:undefined},action,statements),...release,...final.map(finalGuard),finalAuthority(context,action)];
|
|
8
|
+
}
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import { assertStagingIdentitiesClear } from './staging-privacy.ts';
|
|
2
|
+
import { stagingStatements } from './staging-statements.ts';
|
|
3
|
+
import { Buffer } from 'node:buffer';
|
|
4
|
+
import type { D1ImportContext, D1Sql } from './types.ts';
|
|
5
|
+
import { guard, prepare, sql } from './database.ts';
|
|
6
|
+
import { D1RequestLimitError, remaining } from './budget.ts';
|
|
7
|
+
|
|
8
|
+
export const STAGING_BATCH_BYTES = 2 * 1024 * 1024;
|
|
9
|
+
export const STAGING_ROWS_BYTES = 768 * 1024;
|
|
10
|
+
export const STAGING_CHUNK_CHARS = 64 * 1024;
|
|
11
|
+
import { batchBytes as stagingBatchBytes } from './transport.ts';
|
|
12
|
+
export { batchBytes as stagingBatchBytes } from './transport.ts';
|
|
13
|
+
export async function stagingAtomic(context: D1ImportContext, statements: D1Sql[], returnReads = 0, release: D1Sql[] = [], final: D1Sql[] = []): Promise<unknown[]> {
|
|
14
|
+
const packed = stagingStatements(context, 'create', statements, release, final);
|
|
15
|
+
if (packed.length + returnReads > remaining(context)) throw new D1RequestLimitError();
|
|
16
|
+
if (stagingBatchBytes(packed) > STAGING_BATCH_BYTES) throw new Error('IMPORT_STAGE_TRANSPORT_LIMIT');
|
|
17
|
+
return context.storage.database.batch(packed.map(s => prepare(context.storage.database, s)));
|
|
18
|
+
}
|
|
19
|
+
export function* stagingChunks(body: string): Generator<string> {
|
|
20
|
+
for (let start = 0; start < body.length;) {
|
|
21
|
+
let end = Math.min(start + STAGING_CHUNK_CHARS, body.length);
|
|
22
|
+
if (end < body.length && body.charCodeAt(end - 1) >= 0xd800 && body.charCodeAt(end - 1) <= 0xdbff) end--;
|
|
23
|
+
yield body.slice(start, end); start = end;
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
export interface StageBinding {
|
|
27
|
+
id: string; generation: string; destination: string; namespace: string; mapping_version: string;
|
|
28
|
+
principal_id: string; fingerprint: string; operation_key: string; total: number; pages: number; layout_hash: string; reversal_policy: 'strict' | 'preserve-conflicts';
|
|
29
|
+
}
|
|
30
|
+
export const bindingPredicate = (stage: StageBinding) => sql(`EXISTS(SELECT 1 FROM ik_d1_staging WHERE ${Object.keys(stage).map(key => `${key} IS ?`).join(' AND ')})`, ...Object.values(stage));
|
|
31
|
+
export const stageClear = (stage: StageBinding) => sql('NOT EXISTS(SELECT 1 FROM ik_d1_staging_index i WHERE i.stage_id=? AND (EXISTS(SELECT 1 FROM ik_d1_erased_sources s WHERE s.destination=? AND s.identity=i.source_identity) OR EXISTS(SELECT 1 FROM ik_d1_erased_targets t WHERE t.destination=? AND t.identity=i.target_identity)))', stage.id, stage.destination, stage.destination);
|
|
32
|
+
|
|
33
|
+
/** Buffers one table's bounded JSON rows, not all pages or SQL statements. */
|
|
34
|
+
export class StageWriter {
|
|
35
|
+
revision = 0;
|
|
36
|
+
private parts: string[] = [];
|
|
37
|
+
private bytes = 2;
|
|
38
|
+
private written = 0;
|
|
39
|
+
readonly context: D1ImportContext;
|
|
40
|
+
readonly stage: StageBinding;
|
|
41
|
+
readonly table: string;
|
|
42
|
+
readonly columns: string[];
|
|
43
|
+
constructor(context: D1ImportContext, stage: StageBinding, table: string, columns: string[]) {
|
|
44
|
+
this.context = context; this.stage = stage; this.table = table; this.columns = columns;
|
|
45
|
+
}
|
|
46
|
+
async append(row: unknown[]): Promise<void> {
|
|
47
|
+
const text = JSON.stringify(row), bytes = Buffer.byteLength(text) + 1;
|
|
48
|
+
if (bytes + 2 > STAGING_ROWS_BYTES) throw new Error('IMPORT_STAGE_TRANSPORT_LIMIT');
|
|
49
|
+
if (this.parts.length && this.bytes + bytes > STAGING_ROWS_BYTES) await this.flush();
|
|
50
|
+
this.parts.push(text); this.bytes += bytes;
|
|
51
|
+
}
|
|
52
|
+
async flush(): Promise<void> {
|
|
53
|
+
if (!this.parts.length) return;
|
|
54
|
+
const count = this.parts.length, body = `[${this.parts.join(',')}]`;
|
|
55
|
+
this.parts = []; this.bytes = 2;
|
|
56
|
+
if (this.table === 'index') await assertStagingIdentitiesClear(this.context, this.stage.destination, body, 'staging');
|
|
57
|
+
const writes = [guard(bindingPredicate(this.stage)), guard(sql('EXISTS(SELECT 1 FROM ik_d1_staging WHERE id=? AND revision=?)', this.stage.id, this.revision)), guard(stageClear(this.stage)),
|
|
58
|
+
sql(`INSERT INTO ik_d1_staging_${this.table}(stage_id,${this.columns.join(',')}) SELECT ?,${this.columns.map((_, index) => `json_extract(value,'$[${index}]')`).join(',')} FROM json_each(?)`, this.stage.id, body),
|
|
59
|
+
guard(bindingPredicate(this.stage)), guard(sql('EXISTS(SELECT 1 FROM ik_d1_staging WHERE id=? AND revision=?)', this.stage.id, this.revision + count)), guard(stageClear(this.stage)),
|
|
60
|
+
sql(`SELECT ${this.columns.join(',')} FROM ik_d1_staging_${this.table} WHERE stage_id=? ORDER BY rowid LIMIT ? OFFSET ?`, this.stage.id, count, this.written)];
|
|
61
|
+
const results = await stagingAtomic(this.context, writes);
|
|
62
|
+
// The read is part of the same guarded transaction, after all insertion
|
|
63
|
+
// triggers. Compare its exact values, never a newly observed revision.
|
|
64
|
+
const result = results[results.length - 4] as any;
|
|
65
|
+
const rows = Array.isArray(result) ? result : result?.results;
|
|
66
|
+
const expected = JSON.parse(body) as unknown[][];
|
|
67
|
+
if (!Array.isArray(rows) || rows.length !== count || rows.some((row, i) => this.columns.some((column, j) => row[column] !== expected[i][j]))) throw new Error('IMPORT_STAGE_CONTENT_CHANGED');
|
|
68
|
+
this.revision += count; this.written += count;
|
|
69
|
+
}
|
|
70
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { liveStreamOrigin, needsStreamOrigin } from './stream-origin.ts';
|
|
2
|
+
import type { ImportAction } from '../types.ts';
|
|
3
|
+
import { guard, sql } from './database.ts';
|
|
4
|
+
/** Ordinary child-job entry points obey the same sealed parent direction and
|
|
5
|
+
* page ordering as the stream API; an ID cannot bypass a parent's reversal. */
|
|
6
|
+
export function streamBoundary(id: string, action: ImportAction) {
|
|
7
|
+
const origin = guard(sql(`NOT EXISTS(SELECT 1 FROM ik_jobs child WHERE child.id=? AND ${needsStreamOrigin('child.id')} AND NOT ${liveStreamOrigin('child.id')})`,id));
|
|
8
|
+
if (!['apply','undo','rollback'].includes(action)) return [origin];
|
|
9
|
+
const forward=action==='apply';
|
|
10
|
+
return [origin,guard(sql(`NOT EXISTS(SELECT 1 FROM ik_d1_stream_pages p JOIN ik_d1_streams s ON s.id=p.stream_id WHERE p.job_id=? AND (s.state ${forward?'=':'<>'} 'undoing' OR EXISTS(SELECT 1 FROM ik_d1_stream_pages peer JOIN ik_jobs j ON j.id=peer.job_id WHERE peer.stream_id=p.stream_id AND peer.ordinal ${forward?'<':'>'} p.ordinal AND j.state ${forward?"<>'completed'":"NOT IN ('undone','rolled_back')"})))`,id))];
|
|
11
|
+
}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import { stageStreamPages } from './stream-stage.ts';
|
|
2
|
+
import { streamInputCheck } from './stream-input.ts';
|
|
3
|
+
import type { D1ImportContext } from './types.ts';
|
|
4
|
+
import { requestContext } from './budget.ts';
|
|
5
|
+
import { authorize, sql } from './database.ts';
|
|
6
|
+
import { cloneJson } from '../provenance/canonical-stream.ts';
|
|
7
|
+
import { requireText } from '../provenance/identity.ts';
|
|
8
|
+
import { streamFingerprintMatches, type ImportStreamPlan } from './stream-plan.ts';
|
|
9
|
+
import { publishedStream } from './published-stream.ts';
|
|
10
|
+
import { publicationReturnQueries } from './staging-control.ts';
|
|
11
|
+
import { stagingAtomic, type StageBinding } from './staging-transport.ts';
|
|
12
|
+
import { stagePublication, publishedStagePredicates } from './staging-publication.ts';
|
|
13
|
+
import { discardStage, publishedStage } from './staging-lifecycle.ts';
|
|
14
|
+
|
|
15
|
+
/** Publish all normalized pages and their exact layout atomically. The retained
|
|
16
|
+
* values are compressed safe plans, never original file bytes or callbacks. */
|
|
17
|
+
export async function createD1ImportStream(stream: ImportStreamPlan, input: D1ImportContext) {
|
|
18
|
+
const context = requestContext(input); await authorize(context, 'create');
|
|
19
|
+
const original = stream;
|
|
20
|
+
stream = { ...stream, header: cloneJson(stream.header), pages: [...stream.pages], lengths: [...stream.lengths] };
|
|
21
|
+
const h = stream.header;
|
|
22
|
+
if (!stream.pages.length || stream.pages.length !== stream.lengths.length || stream.lengths.some(length => !Number.isSafeInteger(length) || length < 0 || length === 0 && stream.pages.length !== 1)) throw new Error('IMPORT_STREAM_LAYOUT_CHANGED');
|
|
23
|
+
if (!context.reviewedFingerprint || context.reviewedFingerprint !== stream.fingerprint || !streamFingerprintMatches(stream,context.storage.targets.planCodec,original)) throw new Error('REVIEW_FINGERPRINT_MISMATCH');
|
|
24
|
+
if (stream.version !== 1 || h.destination !== context.destination || h.namespace !== context.namespace || h.mappingVersion !== context.mappingVersion || h.principalId !== context.authorization.principalId) throw new Error('PLAN_BINDING_MISMATCH');
|
|
25
|
+
for (const artifact of h.source.artifacts) {
|
|
26
|
+
requireText(artifact.name, 'artifact name');
|
|
27
|
+
if (!/^[a-f0-9]{64}$/.test(artifact.sha256) || !Number.isSafeInteger(artifact.size) || artifact.size < 0) throw new Error('INVALID_ARTIFACT');
|
|
28
|
+
if (artifact.original?.status === 'available') throw new Error('IMPORT_STREAM_NORMALIZED_SOURCE_REQUIRED');
|
|
29
|
+
}
|
|
30
|
+
const operation = context.operationKey ?? stream.fingerprint;
|
|
31
|
+
requireText(operation, 'operationKey');
|
|
32
|
+
const assertInput = streamInputCheck(original, stream, () => context.storage.targets.planCodec);
|
|
33
|
+
const prior = await publishedStage(context, operation, stream.fingerprint);
|
|
34
|
+
assertInput();
|
|
35
|
+
if (prior) return prior;
|
|
36
|
+
let stage: StageBinding | undefined;
|
|
37
|
+
try {
|
|
38
|
+
const staged = await stageStreamPages(stream, stream.fingerprint, context, binding => { stage = binding; }, undefined, assertInput);
|
|
39
|
+
assertInput();
|
|
40
|
+
const { pages, revision } = staged, binding = staged.stage;
|
|
41
|
+
await stagingAtomic(context, stagePublication(binding, revision), publicationReturnQueries(context, binding), [sql('DELETE FROM ik_d1_publication_fences WHERE stream_id=?', binding.id)], publishedStagePredicates(binding,pages));
|
|
42
|
+
const result = await publishedStream(context, binding.id, operation, stream.fingerprint);
|
|
43
|
+
assertInput();
|
|
44
|
+
return result;
|
|
45
|
+
} catch (error) {
|
|
46
|
+
// A successful concurrent publisher can be returned only after the ordinary
|
|
47
|
+
// exact binding, layout and current-authority checks. Cleanup names this
|
|
48
|
+
// attempt's generation, so it cannot remove the winner or a takeover.
|
|
49
|
+
if (stage) await discardStage(stage, context);
|
|
50
|
+
const winner = await publishedStage(context, operation, stream.fingerprint);
|
|
51
|
+
assertInput();
|
|
52
|
+
if (winner) return winner;
|
|
53
|
+
throw error;
|
|
54
|
+
}
|
|
55
|
+
}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import { Buffer } from 'node:buffer';
|
|
2
|
+
import type { D1ImportContext } from './types.ts';
|
|
3
|
+
import { D1RequestLimitError, remaining } from './budget.ts';
|
|
4
|
+
import { planIndex } from './erasure-index.ts';
|
|
5
|
+
import { expandCompressedPlan, type UnverifiedStreamPages } from './stream-plan.ts';
|
|
6
|
+
import { sql } from './database.ts';
|
|
7
|
+
import { assertStagingIdentitiesClear, stagingPrivacyClear } from './staging-privacy.ts';
|
|
8
|
+
import { STAGING_ROWS_BYTES } from './staging-transport.ts';
|
|
9
|
+
|
|
10
|
+
/** Establish complete identity admission before the first private write. Empty
|
|
11
|
+
* destination erasure sets prove it without another page decode. Otherwise
|
|
12
|
+
* retain only one decoded page and one bounded digest batch at a time. */
|
|
13
|
+
export async function admitStreamIdentities(stream: UnverifiedStreamPages, context: D1ImportContext, reservation: { executions: number; indexBatches: number }): Promise<void> {
|
|
14
|
+
const clear = await stagingPrivacyClear(context, sql('NOT EXISTS(SELECT 1 FROM ik_d1_erased_sources WHERE destination=?) AND NOT EXISTS(SELECT 1 FROM ik_d1_erased_targets WHERE destination=?)', context.destination, context.destination));
|
|
15
|
+
if (clear) return;
|
|
16
|
+
// The empty-set read has already consumed one of the reserved executions.
|
|
17
|
+
// Reserve all additional identity reads, all subsequent writes and cleanup.
|
|
18
|
+
if (reservation.executions - 1 + reservation.indexBatches > remaining(context)) throw new D1RequestLimitError();
|
|
19
|
+
let parts: string[] = [], bytes = 2, reads = 0;
|
|
20
|
+
const flush = async () => {
|
|
21
|
+
if (!parts.length) return;
|
|
22
|
+
if (++reads > reservation.indexBatches) throw new Error('IMPORT_STREAM_COUNTS_CHANGED');
|
|
23
|
+
const body = `[${parts.join(',')}]`;
|
|
24
|
+
parts = []; bytes = 2;
|
|
25
|
+
await assertStagingIdentitiesClear(context, context.destination, body, 'admission');
|
|
26
|
+
};
|
|
27
|
+
for (let page = 0; page < stream.pages.length; page++) {
|
|
28
|
+
const plan = expandCompressedPlan(stream.pages[page], context.storage.targets.planCodec);
|
|
29
|
+
for (const identity of planIndex(plan, '')) {
|
|
30
|
+
const text = JSON.stringify([page, identity.ordinal, identity.source_identity, identity.target_identity]), size = Buffer.byteLength(text) + 1;
|
|
31
|
+
if (!/^[a-f0-9]{64}$/.test(identity.source_identity) || identity.target_identity !== '' && !/^[a-f0-9]{64}$/.test(identity.target_identity) || size > 200) throw new Error('IMPORT_STREAM_PAGE_CHANGED');
|
|
32
|
+
if (parts.length && bytes + size > STAGING_ROWS_BYTES) await flush();
|
|
33
|
+
parts.push(text); bytes += size;
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
await flush();
|
|
37
|
+
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import { canonicalHash } from '../provenance/canonical-stream.ts';
|
|
2
|
+
import type { ImportStreamPlan } from './stream-plan.ts';
|
|
3
|
+
import type { D1PlanCodec } from './types.ts';
|
|
4
|
+
|
|
5
|
+
/** Compare the current caller values with the verified private snapshot. No
|
|
6
|
+
* mutable container, including a shallow-frozen one, grants reusable trust. */
|
|
7
|
+
export function streamInputCheck(original: ImportStreamPlan, snapshot: ImportStreamPlan, codec: () => D1PlanCodec | undefined): () => void {
|
|
8
|
+
const header = canonicalHash(snapshot.header), initialCodec = codec();
|
|
9
|
+
const codecId = initialCodec?.id, encode = initialCodec?.encode, decode = initialCodec?.decode;
|
|
10
|
+
return () => {
|
|
11
|
+
const current = codec();
|
|
12
|
+
if (original.version !== snapshot.version || original.fingerprint !== snapshot.fingerprint
|
|
13
|
+
|| canonicalHash(original.header) !== header
|
|
14
|
+
|| original.pages.length !== snapshot.pages.length || snapshot.pages.some((page, i) => page !== original.pages[i])
|
|
15
|
+
|| original.lengths.length !== snapshot.lengths.length || snapshot.lengths.some((length, i) => length !== original.lengths[i])
|
|
16
|
+
|| current?.id !== codecId || current?.encode !== encode || current?.decode !== decode) throw new Error('REVIEW_FINGERPRINT_MISMATCH');
|
|
17
|
+
};
|
|
18
|
+
}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import type { ImportAction, ImportJob } from '../types.ts';
|
|
2
|
+
import type { D1Database, D1ImportContext, D1Sql } from './types.ts';
|
|
3
|
+
import { guard, sql } from './database.ts';
|
|
4
|
+
import { needsStreamOrigin, streamOriginsMatch } from './stream-origin.ts';
|
|
5
|
+
import { readStreamReceipt } from './stream-receipt.ts';
|
|
6
|
+
|
|
7
|
+
// Retain only scalar SQL bindings. No page/job arrays, plan or authority result
|
|
8
|
+
// is cached. Every use still executes the predicate in its current transaction.
|
|
9
|
+
const layouts = new WeakMap<ImportJob, { predicate: D1Sql; reads: number }>();
|
|
10
|
+
const contexts = new WeakMap<D1ImportContext['authorization'], { database: D1Database; streamId: string; predicate: D1Sql }>();
|
|
11
|
+
export function rememberStreamContext(context: D1ImportContext, streamId: string, predicate: D1Sql): void {
|
|
12
|
+
contexts.set(context.authorization, { database: context.storage.database, streamId, predicate });
|
|
13
|
+
}
|
|
14
|
+
/** Current direction and immutable layout remain fixed across child progress. */
|
|
15
|
+
export function streamSnapshotPredicate(row: any, progress = false): D1Sql {
|
|
16
|
+
return sql(`EXISTS(SELECT 1 FROM ik_d1_streams s JOIN ik_d1_stream_revisions r ON r.stream_id=s.id WHERE s.id=? AND s.destination=? AND s.namespace=? AND s.mapping_version=? AND s.principal_id=? AND s.fingerprint=? AND s.operation_key=? AND s.total=? AND s.layout_hash=? AND s.state=? AND r.layout_revision=?${progress?' AND r.progress_revision=?':''} AND ${streamOriginsMatch('s')})`, row.id,row.destination,row.namespace,row.mapping_version,row.principal_id,row.fingerprint,row.operation_key,row.total,row.layout_hash,row.state,row.layout_revision,...progress?[row.progress_revision]:[]);
|
|
17
|
+
}
|
|
18
|
+
/** Every direct child entry validates the real complete parent layout before
|
|
19
|
+
* its boundary can authorize native work. A stream entry can reuse only its own
|
|
20
|
+
* already validated receipt on the same request database and parent identity. */
|
|
21
|
+
export async function captureJobLayout(job: ImportJob, row: any, context: D1ImportContext, action: ImportAction): Promise<void> {
|
|
22
|
+
if (row.stream_origin_required === 0) {
|
|
23
|
+
// A concurrent conversion of an ordinary job into a child must not bypass
|
|
24
|
+
// validation merely because this read found no structural stream origin.
|
|
25
|
+
layouts.set(job, { predicate: sql(`NOT EXISTS(SELECT 1 FROM ik_jobs child WHERE child.id=? AND ${needsStreamOrigin('child.id')})`, job.id), reads: 0 });
|
|
26
|
+
return;
|
|
27
|
+
}
|
|
28
|
+
if (row.stream_origin_required !== 1 || typeof row.origin_stream_id !== 'string' || row.origin_stream_id !== row.page_stream_id) throw new Error('IMPORT_ATOMIC_GUARD');
|
|
29
|
+
const shared = contexts.get(context.authorization);
|
|
30
|
+
if (shared?.database === context.storage.database && shared.streamId === row.origin_stream_id) {
|
|
31
|
+
layouts.set(job, { predicate: shared.predicate, reads: 0 });
|
|
32
|
+
return;
|
|
33
|
+
}
|
|
34
|
+
const receipt = await readStreamReceipt(context, row.origin_stream_id, action);
|
|
35
|
+
if (!receipt.jobs.some(child => child.id === job.id)) throw new Error('IMPORT_STREAM_LAYOUT_CHANGED');
|
|
36
|
+
layouts.set(job, { predicate: streamSnapshotPredicate(receipt.row), reads: receipt.queries });
|
|
37
|
+
}
|
|
38
|
+
export const jobLayoutReads = (job: ImportJob): number => layouts.get(job)?.reads ?? 0;
|
|
39
|
+
export const jobLayoutPredicate = (job: ImportJob): D1Sql | undefined => layouts.get(job)?.predicate;
|
|
40
|
+
export const jobLayoutGuards = (job: ImportJob): D1Sql[] => {
|
|
41
|
+
const predicate = jobLayoutPredicate(job);
|
|
42
|
+
return predicate ? [guard(predicate)] : [];
|
|
43
|
+
};
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import { d1StreamRevisionSchema, streamAnchorMatch } from './stream-revision-schema.ts';
|
|
2
|
+
/** A stream origin comes only from real page membership. It survives loss of
|
|
3
|
+
* that page or parent; ordinary legacy custody is never inferred from data. */
|
|
4
|
+
const parentFields = ['destination','namespace','mapping_version','principal_id','fingerprint','operation_key','total','layout_hash'];
|
|
5
|
+
const columns = ['job_id','stream_id','ordinal','page_total','job_operation_key',...parentFields.map(field=>'parent_'+field)];
|
|
6
|
+
const values = (p:string,s:string,j:string) => [`${p}.job_id`,`${s}.id`,`${p}.ordinal`,`${p}.total`,`${j}.operation_key`,...parentFields.map(field=>`${s}.${field}`)];
|
|
7
|
+
/** Nested membership checks share D1's expression-depth budget. Keep every
|
|
8
|
+
* exact comparison, but give the conjunction logarithmic rather than linear depth. */
|
|
9
|
+
function conjunction(terms:string[]):string {
|
|
10
|
+
if(terms.length===1)return terms[0];
|
|
11
|
+
const middle=Math.floor(terms.length/2);
|
|
12
|
+
return `(${conjunction(terms.slice(0,middle))} AND ${conjunction(terms.slice(middle))})`;
|
|
13
|
+
}
|
|
14
|
+
function exactOrigin(o:string,p:string,s:string,j:string):string {
|
|
15
|
+
return conjunction([...columns.map((column,index)=>`${o}.${column} IS ${values(p,s,j)[index]}`),
|
|
16
|
+
`${p}.stream_id IS ${s}.id`,`${j}.total IS ${p}.total`,...['destination','namespace','mapping_version','principal_id'].map(field=>`${j}.${field} IS ${s}.${field}`)]);
|
|
17
|
+
}
|
|
18
|
+
/** Both directions close the original membership set, including deleted peers. */
|
|
19
|
+
export function streamOriginsMatch(s:string):string {
|
|
20
|
+
return `EXISTS(SELECT 1 FROM ik_d1_stream_revisions sr WHERE sr.stream_id=${s}.id AND ${streamAnchorMatch(s,'sr')}) AND NOT EXISTS(SELECT 1 FROM ik_d1_stream_pages op WHERE op.stream_id=${s}.id AND NOT EXISTS(SELECT 1 FROM ik_d1_stream_origins oo JOIN ik_jobs oj ON oj.id=oo.job_id WHERE ${exactOrigin('oo','op',s,'oj')})) AND NOT EXISTS(SELECT 1 FROM ik_d1_stream_origins oo WHERE oo.stream_id=${s}.id AND NOT EXISTS(SELECT 1 FROM ik_d1_stream_pages op JOIN ik_jobs oj ON oj.id=op.job_id WHERE ${exactOrigin('oo','op',s,'oj')}))`;
|
|
21
|
+
}
|
|
22
|
+
export function liveStreamOrigin(job:string):string {
|
|
23
|
+
return `EXISTS(SELECT 1 FROM ik_d1_stream_origins o JOIN ik_d1_stream_pages p ON p.job_id=o.job_id JOIN ik_d1_streams s ON s.id=o.stream_id JOIN ik_jobs j ON j.id=o.job_id WHERE o.job_id=${job} AND ${exactOrigin('o','p','s','j')} AND s.state IN ('applying','undoing') AND ${streamOriginsMatch('s')})`;
|
|
24
|
+
}
|
|
25
|
+
export function needsStreamOrigin(job:string):string {
|
|
26
|
+
return `(EXISTS(SELECT 1 FROM ik_original_sets os WHERE os.job_id=${job} AND os.version=4) OR EXISTS(SELECT 1 FROM ik_d1_stream_origins oo WHERE oo.job_id=${job}) OR EXISTS(SELECT 1 FROM ik_d1_stream_pages op WHERE op.job_id=${job}))`;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
const capture = (where='') => `INSERT INTO ik_d1_stream_origins(${columns.join(',')}) SELECT ${values('p','s','j').join(',')} FROM ik_d1_stream_pages p JOIN ik_d1_streams s ON s.id=p.stream_id JOIN ik_jobs j ON j.id=p.job_id WHERE NOT EXISTS(SELECT 1 FROM ik_d1_stream_origins o WHERE o.job_id=p.job_id) AND j.total IS p.total AND ${['destination','namespace','mapping_version','principal_id'].map(field=>`j.${field} IS s.${field}`).join(' AND ')}${where}`;
|
|
30
|
+
/** Additive setup, after erasure tables and before legacy receipt conversion.
|
|
31
|
+
* The job-delete trigger also works when a SQLite caller disables foreign keys.
|
|
32
|
+
* Child fingerprints and progress can change through erasure; parent identity,
|
|
33
|
+
* layout and the original child operation binding cannot. */
|
|
34
|
+
export const d1StreamOriginSchema = [
|
|
35
|
+
`CREATE TABLE IF NOT EXISTS ik_d1_stream_origins (job_id TEXT PRIMARY KEY,stream_id TEXT NOT NULL,ordinal INTEGER NOT NULL,page_total INTEGER NOT NULL,job_operation_key TEXT NOT NULL,${parentFields.map(field=>`parent_${field} ${field==='total'?'INTEGER':'TEXT'} NOT NULL`).join(',')})`,
|
|
36
|
+
'CREATE INDEX IF NOT EXISTS ik_d1_stream_origin_parent ON ik_d1_stream_origins(stream_id,ordinal)',
|
|
37
|
+
`CREATE TRIGGER IF NOT EXISTS ik_d1_stream_origin_insert BEFORE INSERT ON ik_d1_stream_origins WHEN EXISTS(SELECT 1 FROM ik_d1_stream_origins WHERE job_id=NEW.job_id) OR NOT EXISTS(SELECT 1 FROM ik_d1_stream_pages p JOIN ik_d1_streams s ON s.id=p.stream_id JOIN ik_jobs j ON j.id=p.job_id WHERE ${exactOrigin('NEW','p','s','j')}) BEGIN SELECT RAISE(ABORT,'IMPORT_STREAM_ORIGIN_CHANGED'); END`,
|
|
38
|
+
"CREATE TRIGGER IF NOT EXISTS ik_d1_stream_origin_update BEFORE UPDATE ON ik_d1_stream_origins BEGIN SELECT RAISE(ABORT,'IMPORT_STREAM_ORIGIN_CHANGED'); END",
|
|
39
|
+
"CREATE TRIGGER IF NOT EXISTS ik_d1_stream_origin_delete BEFORE DELETE ON ik_d1_stream_origins WHEN EXISTS(SELECT 1 FROM ik_jobs WHERE id=OLD.job_id) BEGIN SELECT RAISE(ABORT,'IMPORT_STREAM_ORIGIN_CHANGED'); END",
|
|
40
|
+
'CREATE TRIGGER IF NOT EXISTS ik_d1_stream_origin_job_delete AFTER DELETE ON ik_jobs BEGIN DELETE FROM ik_d1_stream_origins WHERE job_id=OLD.id; END',
|
|
41
|
+
...['INSERT','UPDATE OF stream_id,ordinal,job_id,total'].map(event=>`CREATE TRIGGER IF NOT EXISTS ik_d1_stream_origin_page_${event==='INSERT'?'insert':'update'} AFTER ${event} ON ik_d1_stream_pages BEGIN ${capture(' AND p.job_id=NEW.job_id')}; END`),
|
|
42
|
+
capture(),
|
|
43
|
+
...d1StreamRevisionSchema,
|
|
44
|
+
];
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { ImportPlan } from '../types.ts';
|
|
2
|
+
import type { UnverifiedStreamPages } from './stream-plan.ts';
|
|
3
|
+
import { canonicalHash } from '../provenance/canonical-stream.ts';
|
|
4
|
+
import { canonicalJson } from '../provenance/identity.ts';
|
|
5
|
+
/** The complete reviewed view contains global metadata. Each retained page must
|
|
6
|
+
* be exactly its scoped projection, with no additional unreviewed source copy. */
|
|
7
|
+
export function assertStreamPage(stream: Pick<UnverifiedStreamPages, 'header'>, plan: ImportPlan) {
|
|
8
|
+
const h=stream.header;
|
|
9
|
+
const counts={input:plan.entries.length,unique:plan.entries.length,duplicates:0,mapped:0,unmapped:0,rejected:0,effects:0,byKind:Object.create(null) as Record<string,number>};
|
|
10
|
+
const identities=new Set(plan.entries.map(e=>canonicalJson([e.source.kind,e.source.sourceId])));
|
|
11
|
+
for(const e of plan.entries){counts.byKind[e.source.kind]=(counts.byKind[e.source.kind]??0)+1;counts.mapped+=Number(e.effects.length>0);counts.unmapped+=Number(e.unmapped.length>0);counts.rejected+=Number(!!e.rejection);counts.effects+=e.effects.length;}
|
|
12
|
+
const diagnostics=h.diagnostics.filter(d=>!d.kind||!d.sourceId||identities.has(canonicalJson([d.kind,d.sourceId])));
|
|
13
|
+
const same=(a:unknown,b:unknown)=>canonicalHash(a)===canonicalHash(b);
|
|
14
|
+
if (!same(Object.keys(plan).sort(),['counts','destination','diagnostics','entries','fingerprint','mappingVersion','namespace','principalId','source','version'])
|
|
15
|
+
|| !(['version','destination','namespace','mappingVersion','principalId'] as const).every(key=>plan[key]===h[key])
|
|
16
|
+
|| !same(counts,plan.counts) || !same(diagnostics,plan.diagnostics)
|
|
17
|
+
|| !same(plan.source,{...h.source,records:plan.entries.map(e=>e.source)})) throw new Error('IMPORT_STREAM_PAGE_CHANGED');
|
|
18
|
+
}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import { Buffer } from 'node:buffer';
|
|
2
|
+
import type { ImportPlan } from '../types.ts';
|
|
3
|
+
import { canonicalJson } from '../provenance/identity.ts';
|
|
4
|
+
import { planFingerprint } from '../plan/index.ts';
|
|
5
|
+
/** Retained literals and structural references, counted without allocating a
|
|
6
|
+
* second dictionary tree. Keys and container slots remain part of the bound. */
|
|
7
|
+
export function dictionaryWeight(value: unknown, dictionary = true, shared = new Set<string>()): number {
|
|
8
|
+
const weight = (value: unknown): number => {
|
|
9
|
+
if (typeof value === 'string') {
|
|
10
|
+
if (dictionary) { if (shared.has(value)) return 16; shared.add(value); }
|
|
11
|
+
return Buffer.byteLength(JSON.stringify(value));
|
|
12
|
+
}
|
|
13
|
+
if (Array.isArray(value)) return 2 + value.reduce((n, item) => n + 1 + weight(item), 0);
|
|
14
|
+
if (value && typeof value === 'object') return 2 + Object.entries(value).reduce((n, [key, item]) => n + Buffer.byteLength(JSON.stringify(key)) + 2 + weight(item), 0);
|
|
15
|
+
return Buffer.byteLength(JSON.stringify(value));
|
|
16
|
+
};
|
|
17
|
+
return weight(value);
|
|
18
|
+
}
|
|
19
|
+
/** Native units per stream child. Reversal and preserved-undo classification
|
|
20
|
+
* cost grows with a child's units, so many tiny effects split into more
|
|
21
|
+
* children instead of one child whose undo request outgrows the budget. */
|
|
22
|
+
export const STREAM_CHILD_UNITS = 100;
|
|
23
|
+
/** One unit per effect; an entry without effects still holds a decision. */
|
|
24
|
+
export const entryUnits = (entry: ImportPlan['entries'][number]): number => Math.max(1, entry.effects.length);
|
|
25
|
+
/** Preserve already normalized effects; partitioning never reruns the mapper.
|
|
26
|
+
* Large individual native entities stay atomic, while ordinary pages are byte
|
|
27
|
+
* bounded as well as unit bounded. */
|
|
28
|
+
export function* partitionPlan(plan: ImportPlan, dictionary = true, byteLimit = 512 * 1024, unitLimit = STREAM_CHILD_UNITS): Generator<ImportPlan> {
|
|
29
|
+
let start=0,size=0,units=0;
|
|
30
|
+
let shared = new Set<string>();
|
|
31
|
+
// Dictionary-backed custody retains each literal once regardless of its
|
|
32
|
+
// length. Counting expanded repetitions splits the dictionary over many
|
|
33
|
+
// pages and can make the complete compressed collection larger than memory.
|
|
34
|
+
// Non-dictionary custody still partitions by its expanded representation.
|
|
35
|
+
const part=(end:number):ImportPlan=>{
|
|
36
|
+
if(start===0&&end===plan.entries.length)return plan;
|
|
37
|
+
const entries=plan.entries.slice(start,end), identities=new Set(entries.map(e=>canonicalJson([e.source.kind,e.source.sourceId])));
|
|
38
|
+
const counts={input:entries.length,unique:entries.length,duplicates:0,mapped:0,unmapped:0,rejected:0,effects:0,byKind:Object.create(null) as Record<string,number>};
|
|
39
|
+
for(const entry of entries){counts.byKind[entry.source.kind]=(counts.byKind[entry.source.kind]??0)+1;counts.mapped+=Number(entry.effects.length>0);counts.unmapped+=Number(entry.unmapped.length>0);counts.rejected+=Number(!!entry.rejection);counts.effects+=entry.effects.length;}
|
|
40
|
+
const next={...plan,entries,source:{...plan.source,records:entries.map(e=>e.source)},counts,diagnostics:plan.diagnostics.filter(d=>!d.kind||!d.sourceId||identities.has(canonicalJson([d.kind,d.sourceId])))};
|
|
41
|
+
next.fingerprint=planFingerprint(next);return next;
|
|
42
|
+
};
|
|
43
|
+
for(let index=0;index<plan.entries.length;index++){
|
|
44
|
+
const entry=plan.entries[index];
|
|
45
|
+
let bytes=dictionaryWeight(entry,dictionary,shared)+dictionaryWeight(entry.source,dictionary,shared);
|
|
46
|
+
if(index>start&&(size+bytes>byteLimit||units+entryUnits(entry)>unitLimit)){yield part(index);start=index;size=0;units=0;shared=new Set();bytes=dictionaryWeight(entry,dictionary,shared)+dictionaryWeight(entry.source,dictionary,shared);}
|
|
47
|
+
size+=bytes;units+=entryUnits(entry);
|
|
48
|
+
}
|
|
49
|
+
yield part(plan.entries.length);
|
|
50
|
+
}
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
import { TargetIdentities } from './target-identities.ts';
|
|
2
|
+
import { reviewedSourceRecords } from './reviewed-source.ts';
|
|
3
|
+
import { compressedPlan, expandCompressedPlan } from './compressed-plan.ts';
|
|
4
|
+
export { compressedPlan, expandCompressedPlan } from './compressed-plan.ts';
|
|
5
|
+
import { partitionPlan } from './stream-partition.ts';
|
|
6
|
+
import { measurePlanWorkingSet, mergePlanPages } from './stream-working-set.ts';
|
|
7
|
+
import { Buffer } from 'node:buffer';
|
|
8
|
+
import type { ImportPlan, SourceBundle, ImportMapping } from '../types.ts';
|
|
9
|
+
import { buildStreamPagePlan, planFingerprint } from '../plan/index.ts';
|
|
10
|
+
import { canonicalHash, cloneJson } from '../provenance/canonical-stream.ts';
|
|
11
|
+
import { canonicalJson, sha256 } from '../provenance/identity.ts';
|
|
12
|
+
import type { D1ImportContext, D1PlanCodec } from './types.ts';
|
|
13
|
+
import { authorize } from './database.ts';
|
|
14
|
+
import { SNAPSHOT_BYTES } from './sets.ts';
|
|
15
|
+
import { requestContext } from './budget.ts';
|
|
16
|
+
|
|
17
|
+
export interface UnverifiedStreamPages {
|
|
18
|
+
version: 1;
|
|
19
|
+
/** Independently verified, compressed normalized pages; no mapper is retained. */
|
|
20
|
+
pages: string[];
|
|
21
|
+
lengths: number[];
|
|
22
|
+
header: Omit<ImportPlan, 'entries' | 'source' | 'fingerprint'> & { source: Omit<SourceBundle, 'records'> };
|
|
23
|
+
}
|
|
24
|
+
export interface ImportStreamPlan extends UnverifiedStreamPages { fingerprint: string }
|
|
25
|
+
const codecs = new WeakMap<ImportStreamPlan, D1PlanCodec>();
|
|
26
|
+
const reviewed = new WeakMap<ImportStreamPlan, { fingerprint: string; header: string; pages: string[]; lengths: number[]; codecId?: string; encode?: D1PlanCodec['encode']; decode?: D1PlanCodec['decode'] }>();
|
|
27
|
+
/** Reuse only an exact in-memory snapshot whose complete canonical fingerprint
|
|
28
|
+
* was already computed. A changed byte takes the ordinary verification path. */
|
|
29
|
+
export function streamFingerprintMatches(stream: ImportStreamPlan, codec?: D1PlanCodec, original = stream): boolean {
|
|
30
|
+
const prior = reviewed.get(original);
|
|
31
|
+
if (prior && prior.codecId === codec?.id && prior.encode === codec?.encode && prior.decode === codec?.decode
|
|
32
|
+
&& prior.fingerprint === stream.fingerprint && prior.header === canonicalHash(stream.header)
|
|
33
|
+
&& prior.pages.length === stream.pages.length && prior.pages.every((page, i) => page === stream.pages[i])
|
|
34
|
+
&& prior.lengths.length === stream.lengths.length && prior.lengths.every((length, i) => length === stream.lengths[i])) return true;
|
|
35
|
+
return planFingerprint(streamView(stream, codec)) === stream.fingerprint;
|
|
36
|
+
}
|
|
37
|
+
/** Array-compatible read view retaining only one normalized page at a time.
|
|
38
|
+
* The view is consumed internally for the original canonical fingerprint. */
|
|
39
|
+
export function streamView(stream: ImportStreamPlan, codec = codecs.get(stream)): ImportPlan {
|
|
40
|
+
const starts: number[] = []; let total = 0;
|
|
41
|
+
for (const size of stream.lengths) { starts.push(total); total += size; }
|
|
42
|
+
let current = -1, page: ImportPlan | undefined;
|
|
43
|
+
const at = (index: number) => {
|
|
44
|
+
let low = 0, high = starts.length;
|
|
45
|
+
while (low + 1 < high) { const mid = (low + high) >>> 1; if (starts[mid] <= index) low = mid; else high = mid; }
|
|
46
|
+
if (current !== low) { page = undefined; page = expandCompressedPlan(stream.pages[low],codec); current = low; }
|
|
47
|
+
return page!.entries[index - starts[low]];
|
|
48
|
+
};
|
|
49
|
+
const view = <T>(get: (index: number) => T): T[] => new Proxy(new Array<T>(total), {
|
|
50
|
+
get(target, key, receiver) { if (typeof key === 'string' && /^(0|[1-9][0-9]*)$/.test(key)) return Number(key) < total ? get(Number(key)) : undefined; return Reflect.get(target, key, receiver); },
|
|
51
|
+
has(target, key) { return typeof key === 'string' && /^(0|[1-9][0-9]*)$/.test(key) && Number(key) < total || Reflect.has(target, key); },
|
|
52
|
+
});
|
|
53
|
+
return { ...stream.header, source: { ...stream.header.source, records: view(index => at(index).source) }, entries: view(at), fingerprint: stream.fingerprint };
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/** Bounded normalized imports. Each input identity must be distinct; original
|
|
57
|
+
* bytes and unscoped source diagnostics use the existing general import API. */
|
|
58
|
+
export async function planD1ImportStream(source: SourceBundle, mapping: ImportMapping, input: D1ImportContext): Promise<ImportStreamPlan> {
|
|
59
|
+
input = requestContext(input);
|
|
60
|
+
const { stream: pages, sourceHashes, pageSize } = await collectStreamPages(source, mapping, input);
|
|
61
|
+
const stream: ImportStreamPlan = { ...pages, fingerprint: '' };
|
|
62
|
+
const codec = input.storage.targets.planCodec;
|
|
63
|
+
if (codec) codecs.set(stream,codec);
|
|
64
|
+
const initial = streamView(stream);
|
|
65
|
+
initial.source.records = reviewedSourceRecords(source.records, sourceHashes, pageSize, stream.header.counts.unique);
|
|
66
|
+
stream.fingerprint = planFingerprint(initial);
|
|
67
|
+
reviewed.set(stream, { fingerprint: stream.fingerprint, header: canonicalHash(stream.header), pages: [...stream.pages], lengths: [...stream.lengths], codecId: codec?.id, encode: codec?.encode, decode: codec?.decode });
|
|
68
|
+
await authorize(input, 'plan');
|
|
69
|
+
return stream;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/** Private preparation has no verified stream fingerprint. Only cold
|
|
73
|
+
* verification may turn these independently checked pages into a public job. */
|
|
74
|
+
export async function collectStreamPages(source: SourceBundle, mapping: ImportMapping, input: D1ImportContext): Promise<{ stream: UnverifiedStreamPages; sourceHashes: string[]; pageSize: number }> {
|
|
75
|
+
const context = requestContext(input); await authorize(context, 'plan');
|
|
76
|
+
if (source.diagnostics.length || source.artifacts.some(a => a.original?.status === 'available')) throw new Error('IMPORT_STREAM_NORMALIZED_SOURCE_REQUIRED');
|
|
77
|
+
const cursorPolicy = source.paginationCursorPolicy === undefined ? {} : { paginationCursorPolicy: source.paginationCursorPolicy };
|
|
78
|
+
const pages: string[] = [], lengths: number[] = [], sourceHashes: string[] = [], identities = new Set<string>(), targets = new TargetIdentities();
|
|
79
|
+
let header: ImportStreamPlan['header'] | undefined;
|
|
80
|
+
const pageSize = 100;
|
|
81
|
+
let pending: ImportPlan[] = [], pendingBytes = 0, pendingExpandedBytes = 0;
|
|
82
|
+
const flush = () => {
|
|
83
|
+
if (!pending.length) return;
|
|
84
|
+
const combined = mergePlanPages(pending);
|
|
85
|
+
for (const page of partitionPlan(combined, !!context.storage.targets.planCodec, 8 * 1024 * 1024)) { pages.push(compressedPlan(page,context.storage.targets.planCodec)); lengths.push(page.entries.length); }
|
|
86
|
+
pending = []; pendingBytes = 0; pendingExpandedBytes = 0;
|
|
87
|
+
};
|
|
88
|
+
for (let start = 0; start < source.records.length || !start; start += pageSize) {
|
|
89
|
+
const plan = buildStreamPagePlan({ ...cursorPolicy, artifacts: source.artifacts, diagnostics: [], records: source.records.slice(start, start + pageSize) }, mapping, context);
|
|
90
|
+
sourceHashes.push(canonicalHash(plan.source.records));
|
|
91
|
+
if (plan.counts.duplicates) throw new Error('IMPORT_STREAM_DUPLICATE_SOURCE');
|
|
92
|
+
const sharedDiagnostics = plan.source.artifacts.filter(a => a.original?.status === 'unavailable').length;
|
|
93
|
+
for (const entry of plan.entries) {
|
|
94
|
+
if (identities.has(entry.identity)) throw new Error('IMPORT_STREAM_DUPLICATE_SOURCE');
|
|
95
|
+
identities.add(entry.identity);
|
|
96
|
+
for (const effect of entry.effects) {
|
|
97
|
+
if (effect.attachment) throw new Error('IMPORT_STREAM_NORMALIZED_SOURCE_REQUIRED');
|
|
98
|
+
if (!targets.claim(plan.destination, effect.targetKind, effect.targetId)) throw new Error('DUPLICATE_TARGET');
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
if (!header) {
|
|
102
|
+
const { entries: _entries, fingerprint: _fingerprint, source: _source, ...metadata } = plan;
|
|
103
|
+
header = { ...cloneJson(metadata), source: { ...cursorPolicy, artifacts: cloneJson(plan.source.artifacts), diagnostics: [] } };
|
|
104
|
+
header.counts.byKind = Object.assign(Object.create(null),header.counts.byKind);
|
|
105
|
+
} else {
|
|
106
|
+
for (const key of ['input','unique','duplicates','mapped','unmapped','rejected','effects'] as const) header.counts[key] += plan.counts[key];
|
|
107
|
+
for (const [kind, count] of Object.entries(plan.counts.byKind)) header.counts.byKind[kind] = (header.counts.byKind[kind] ?? 0) + count;
|
|
108
|
+
header.diagnostics.push(...plan.diagnostics.slice(sharedDiagnostics));
|
|
109
|
+
}
|
|
110
|
+
const measurement=context.storage.targets.planCodec ? measurePlanWorkingSet(plan) : undefined;
|
|
111
|
+
if (measurement?.amplified) {
|
|
112
|
+
const {bytes,expandedBytes}=measurement;
|
|
113
|
+
if (pending.length && (pending.length === 2 || pendingBytes + bytes > 8 * 1024 * 1024 || pendingExpandedBytes + expandedBytes > 8 * 1024 * 1024)) flush();
|
|
114
|
+
pending.push(plan); pendingBytes += bytes; pendingExpandedBytes += expandedBytes;
|
|
115
|
+
if (pendingExpandedBytes >= 8 * 1024 * 1024) flush();
|
|
116
|
+
} else {
|
|
117
|
+
flush();
|
|
118
|
+
for (const page of partitionPlan(plan, !!context.storage.targets.planCodec)) { pages.push(compressedPlan(page,context.storage.targets.planCodec)); lengths.push(page.entries.length); }
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
flush();
|
|
122
|
+
identities.clear(); targets.clear();
|
|
123
|
+
return { stream: { version: 1, pages, lengths, header: header! }, sourceHashes, pageSize };
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/** The existing chunk manifest also fences compressed page bytes exactly. */
|
|
127
|
+
export function storedStreamPage(body: string, id: string, codec?: D1PlanCodec) {
|
|
128
|
+
const envelope=JSON.parse(body);
|
|
129
|
+
if ((envelope.expandedSize ?? envelope.size) <= SNAPSHOT_BYTES) return { body: envelope.d1Plan === 3 ? body : canonicalJson(expandCompressedPlan(body,codec)), chunks: [] as { job_id:string; ordinal:number; body:string }[] };
|
|
130
|
+
const chunks = [];
|
|
131
|
+
for (let offset=0; offset<body.length; offset+=SNAPSHOT_BYTES) chunks.push({job_id:id,ordinal:chunks.length,body:body.slice(offset,offset+SNAPSHOT_BYTES)});
|
|
132
|
+
return {body:canonicalJson({d1Plan:1,chunks:chunks.length,size:Buffer.byteLength(body),sha256:sha256(body)}),chunks};
|
|
133
|
+
}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { retiredError } from './retirement.ts';
|
|
2
|
+
import type { ImportAction } from '../types.ts';
|
|
3
|
+
import type { D1ImportContext } from './types.ts';
|
|
4
|
+
import { authority, guard, prepare, sql } from './database.ts';
|
|
5
|
+
import { packGuards, remaining, D1RequestLimitError } from './budget.ts';
|
|
6
|
+
import { streamOriginsMatch } from './stream-origin.ts';
|
|
7
|
+
import { batchBytes } from './transport.ts';
|
|
8
|
+
import { batchRows, checkBatchResults } from './read-results.ts';
|
|
9
|
+
import { streamJob } from './stream-snapshot.ts';
|
|
10
|
+
|
|
11
|
+
export type StreamCreationBinding = { operation: string; fingerprint: string };
|
|
12
|
+
/** The complete actual builder is also used to reserve the eventual return.
|
|
13
|
+
* Array size affects SQL outputs only, never predicate inputs or query count. */
|
|
14
|
+
export function streamReceipt(context: D1ImportContext, id: string, action: ImportAction, creation?: StreamCreationBinding) {
|
|
15
|
+
const authorization = () => packGuards([authority(context, action)], true);
|
|
16
|
+
const structural = () => packGuards([guard(sql(`EXISTS(SELECT 1 FROM ik_d1_streams s WHERE s.id=? AND s.destination=? AND s.namespace=? AND s.mapping_version=? AND s.principal_id=? AND s.state IN ('applying','undoing')${creation?' AND s.operation_key=? AND s.fingerprint=?':''} AND ${streamOriginsMatch('s')})`, id, context.destination, context.namespace, context.mappingVersion, context.authorization.principalId, ...creation?[creation.operation, creation.fingerprint]:[]))], true);
|
|
17
|
+
const beforeAuthority = authorization(), beforeLayout = structural();
|
|
18
|
+
const reads = [sql('SELECT s.*,r.layout_revision,r.progress_revision FROM ik_d1_streams s JOIN ik_d1_stream_revisions r ON r.stream_id=s.id WHERE s.id=?', id),
|
|
19
|
+
sql('SELECT * FROM ik_d1_stream_pages WHERE stream_id=? ORDER BY ordinal', id),
|
|
20
|
+
sql('SELECT j.id,j.destination,j.namespace,j.mapping_version,j.principal_id,j.operation_key,j.state,j.checkpoint,j.total,j.fence,j.error,j.reversal_policy,j.reversal_policy_hash FROM ik_jobs j JOIN ik_d1_stream_pages p ON p.job_id=j.id WHERE p.stream_id=? ORDER BY p.ordinal', id)];
|
|
21
|
+
const afterLayout = structural(), afterAuthority = authorization();
|
|
22
|
+
const statements = [...beforeAuthority, ...beforeLayout, ...reads, ...afterLayout, ...afterAuthority];
|
|
23
|
+
const limit = context.maxBatchBytes ?? 2 * 1024 * 1024;
|
|
24
|
+
if (!Number.isSafeInteger(limit) || limit < 1) throw new Error('INVALID_TRANSPORT_LIMIT');
|
|
25
|
+
if (batchBytes(statements) > Math.min(limit, 2 * 1024 * 1024)) throw new Error('IMPORT_D1_TRANSPORT_LIMIT');
|
|
26
|
+
return { statements, offset: beforeAuthority.length + beforeLayout.length,
|
|
27
|
+
authorityIndexes: [...beforeAuthority.keys(), ...afterAuthority.map((_, i) => statements.length - afterAuthority.length + i)],
|
|
28
|
+
layoutIndexes: [...beforeLayout.map((_, i) => beforeAuthority.length + i), ...afterLayout.map((_, i) => beforeAuthority.length + beforeLayout.length + reads.length + i)] };
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export async function readStreamReceipt(context: D1ImportContext, id: string, action: ImportAction, creation?: StreamCreationBinding) {
|
|
32
|
+
const receipt = streamReceipt(context, id, action, creation);
|
|
33
|
+
if (receipt.statements.length > remaining(context)) throw new D1RequestLimitError();
|
|
34
|
+
const result = await context.storage.database.batch(receipt.statements.map(statement => prepare(context.storage.database, statement)));
|
|
35
|
+
checkBatchResults(result, receipt.statements.length);
|
|
36
|
+
const valid = (index: number) => { const rows = batchRows(result[index]); return rows.length === 1 && rows[0].ok === 1; };
|
|
37
|
+
if (!receipt.authorityIndexes.every(valid)) throw new Error('IMPORT_ATOMIC_GUARD');
|
|
38
|
+
const parents = batchRows(result[receipt.offset]), pages = batchRows(result[receipt.offset + 1]), jobs = batchRows(result[receipt.offset + 2]);
|
|
39
|
+
if (parents.length > 1 || parents[0] && parents[0].id !== id) throw new Error('IMPORT_STREAM_LAYOUT_CHANGED');
|
|
40
|
+
if (!parents.length) { const retired = await retiredError(context.storage.database, id, 'stream'); if (retired) throw retired; }
|
|
41
|
+
const row = parents[0], job = streamJob(row, pages, jobs, context);
|
|
42
|
+
if (creation && (row.operation_key !== creation.operation || row.fingerprint !== creation.fingerprint)) throw new Error('OPERATION_KEY_CONFLICT');
|
|
43
|
+
if (!receipt.layoutIndexes.every(valid)) throw new Error('IMPORT_STREAM_LAYOUT_CHANGED');
|
|
44
|
+
return { row, pages, jobs, job, queries: receipt.statements.length };
|
|
45
|
+
}
|