@runeditrun/rer 0.2.77 → 0.2.79

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (167) hide show
  1. package/README.md +32 -0
  2. package/cli/export.mjs +1 -1
  3. package/cli/portability-migrations.mjs +35 -14
  4. package/cli/portability-worker.mjs +2 -2
  5. package/cli/portability.mjs +2 -2
  6. package/cli/server-proxy.mjs +30 -8
  7. package/cli/upstream-agent.mjs +14 -0
  8. package/global/CHANGELOG.md +13 -0
  9. package/global/README.md +32 -0
  10. package/modules/demo-host/native/visitor.mjs +1 -1
  11. package/modules/deploy-kit/owner-cli.mjs +15 -5
  12. package/modules/deploy-kit/provider.mjs +9 -4
  13. package/modules/import-kit/src/d1/README.md +1322 -0
  14. package/modules/import-kit/src/d1/apply.ts +115 -0
  15. package/modules/import-kit/src/d1/budget.ts +89 -0
  16. package/modules/import-kit/src/d1/compressed-plan.ts +63 -0
  17. package/modules/import-kit/src/d1/creation.ts +95 -0
  18. package/modules/import-kit/src/d1/custody-fence.ts +27 -0
  19. package/modules/import-kit/src/d1/custody-state-schema.ts +67 -0
  20. package/modules/import-kit/src/d1/database.ts +163 -0
  21. package/modules/import-kit/src/d1/entry-locator.ts +61 -0
  22. package/modules/import-kit/src/d1/erased-artifacts.ts +38 -0
  23. package/modules/import-kit/src/d1/erased-custody.ts +36 -0
  24. package/modules/import-kit/src/d1/erased-plan.ts +47 -0
  25. package/modules/import-kit/src/d1/erased-reversal.ts +19 -0
  26. package/modules/import-kit/src/d1/erasure-bounded.ts +22 -0
  27. package/modules/import-kit/src/d1/erasure-claims.ts +27 -0
  28. package/modules/import-kit/src/d1/erasure-compiler.ts +67 -0
  29. package/modules/import-kit/src/d1/erasure-disposition.ts +16 -0
  30. package/modules/import-kit/src/d1/erasure-fragments.ts +86 -0
  31. package/modules/import-kit/src/d1/erasure-index.ts +53 -0
  32. package/modules/import-kit/src/d1/erasure-pages.ts +62 -0
  33. package/modules/import-kit/src/d1/erasure-progress-schema.ts +8 -0
  34. package/modules/import-kit/src/d1/erasure-progress.ts +61 -0
  35. package/modules/import-kit/src/d1/erasure-related.ts +124 -0
  36. package/modules/import-kit/src/d1/erasure-report.ts +31 -0
  37. package/modules/import-kit/src/d1/erasure-schema.ts +24 -0
  38. package/modules/import-kit/src/d1/erasure-selection.ts +77 -0
  39. package/modules/import-kit/src/d1/erasure-snapshot.ts +15 -0
  40. package/modules/import-kit/src/d1/erasure-stage.ts +146 -0
  41. package/modules/import-kit/src/d1/erasure-step.ts +12 -0
  42. package/modules/import-kit/src/d1/erasure-superseded.ts +98 -0
  43. package/modules/import-kit/src/d1/erasure-values.ts +83 -0
  44. package/modules/import-kit/src/d1/erasure-writes.ts +46 -0
  45. package/modules/import-kit/src/d1/erasure.ts +170 -0
  46. package/modules/import-kit/src/d1/evolutions/0001-preserved-undo.ts +13 -0
  47. package/modules/import-kit/src/d1/expected-state.ts +91 -0
  48. package/modules/import-kit/src/d1/field-redaction.ts +86 -0
  49. package/modules/import-kit/src/d1/guards.ts +37 -0
  50. package/modules/import-kit/src/d1/history.ts +71 -0
  51. package/modules/import-kit/src/d1/index.ts +88 -0
  52. package/modules/import-kit/src/d1/integrity.ts +127 -0
  53. package/modules/import-kit/src/d1/inventory-snapshot.ts +84 -0
  54. package/modules/import-kit/src/d1/jobs.ts +97 -0
  55. package/modules/import-kit/src/d1/native-set.ts +48 -0
  56. package/modules/import-kit/src/d1/native-staging-admission.ts +19 -0
  57. package/modules/import-kit/src/d1/native-staging-context.ts +44 -0
  58. package/modules/import-kit/src/d1/native-staging-lifecycle.ts +33 -0
  59. package/modules/import-kit/src/d1/native-staging-offset.ts +74 -0
  60. package/modules/import-kit/src/d1/native-staging-publication.ts +18 -0
  61. package/modules/import-kit/src/d1/native-staging-rows.ts +72 -0
  62. package/modules/import-kit/src/d1/native-staging-schema.ts +28 -0
  63. package/modules/import-kit/src/d1/native-staging-state.ts +22 -0
  64. package/modules/import-kit/src/d1/native-staging.ts +64 -0
  65. package/modules/import-kit/src/d1/original-copies.ts +53 -0
  66. package/modules/import-kit/src/d1/original-storage.ts +56 -0
  67. package/modules/import-kit/src/d1/originals.ts +74 -0
  68. package/modules/import-kit/src/d1/plan-dictionary.ts +80 -0
  69. package/modules/import-kit/src/d1/plans.ts +121 -0
  70. package/modules/import-kit/src/d1/preparation-fingerprint.ts +57 -0
  71. package/modules/import-kit/src/d1/preparation-header.ts +33 -0
  72. package/modules/import-kit/src/d1/preparation-reader.ts +98 -0
  73. package/modules/import-kit/src/d1/preparation-schema.ts +16 -0
  74. package/modules/import-kit/src/d1/preparation-state.ts +56 -0
  75. package/modules/import-kit/src/d1/preparation-step-combine.ts +80 -0
  76. package/modules/import-kit/src/d1/preparation-step-control.ts +48 -0
  77. package/modules/import-kit/src/d1/preparation-step-hash.ts +131 -0
  78. package/modules/import-kit/src/d1/preparation-step-pages.ts +77 -0
  79. package/modules/import-kit/src/d1/preparation-steps-schema.ts +11 -0
  80. package/modules/import-kit/src/d1/preparation-steps.ts +102 -0
  81. package/modules/import-kit/src/d1/preparations.ts +87 -0
  82. package/modules/import-kit/src/d1/preserved-preview.ts +89 -0
  83. package/modules/import-kit/src/d1/preserved-reverse.ts +91 -0
  84. package/modules/import-kit/src/d1/provenance-set.ts +87 -0
  85. package/modules/import-kit/src/d1/publication-fence.ts +22 -0
  86. package/modules/import-kit/src/d1/publication-receipt-schema.ts +15 -0
  87. package/modules/import-kit/src/d1/published-stream.ts +11 -0
  88. package/modules/import-kit/src/d1/read-fence.ts +64 -0
  89. package/modules/import-kit/src/d1/read-results.ts +16 -0
  90. package/modules/import-kit/src/d1/redacted-custody.ts +76 -0
  91. package/modules/import-kit/src/d1/redaction-step.ts +125 -0
  92. package/modules/import-kit/src/d1/referenced-plans.ts +136 -0
  93. package/modules/import-kit/src/d1/retained-values.ts +35 -0
  94. package/modules/import-kit/src/d1/retirement-bounded.ts +71 -0
  95. package/modules/import-kit/src/d1/retirement-door.ts +105 -0
  96. package/modules/import-kit/src/d1/retirement-schema.ts +39 -0
  97. package/modules/import-kit/src/d1/retirement-scope.ts +94 -0
  98. package/modules/import-kit/src/d1/retirement-steps.ts +76 -0
  99. package/modules/import-kit/src/d1/retirement-tally.ts +74 -0
  100. package/modules/import-kit/src/d1/retirement.ts +136 -0
  101. package/modules/import-kit/src/d1/reversal-conversion-record.ts +19 -0
  102. package/modules/import-kit/src/d1/reversal-conversion-schema.ts +11 -0
  103. package/modules/import-kit/src/d1/reversal-conversion.ts +40 -0
  104. package/modules/import-kit/src/d1/reversal-fence-schema.ts +14 -0
  105. package/modules/import-kit/src/d1/reversal-fence.ts +106 -0
  106. package/modules/import-kit/src/d1/reversal-journal.ts +24 -0
  107. package/modules/import-kit/src/d1/reverse.ts +103 -0
  108. package/modules/import-kit/src/d1/reviewed-source.ts +23 -0
  109. package/modules/import-kit/src/d1/sets.ts +78 -0
  110. package/modules/import-kit/src/d1/staged-creation.ts +201 -0
  111. package/modules/import-kit/src/d1/staging-admission.ts +45 -0
  112. package/modules/import-kit/src/d1/staging-control.ts +43 -0
  113. package/modules/import-kit/src/d1/staging-erasure.ts +28 -0
  114. package/modules/import-kit/src/d1/staging-lifecycle.ts +49 -0
  115. package/modules/import-kit/src/d1/staging-plan.ts +15 -0
  116. package/modules/import-kit/src/d1/staging-privacy.ts +22 -0
  117. package/modules/import-kit/src/d1/staging-publication.ts +79 -0
  118. package/modules/import-kit/src/d1/staging-schema.ts +37 -0
  119. package/modules/import-kit/src/d1/staging-statements.ts +8 -0
  120. package/modules/import-kit/src/d1/staging-transport.ts +70 -0
  121. package/modules/import-kit/src/d1/stream-boundary.ts +11 -0
  122. package/modules/import-kit/src/d1/stream-create.ts +55 -0
  123. package/modules/import-kit/src/d1/stream-identity-admission.ts +37 -0
  124. package/modules/import-kit/src/d1/stream-input.ts +18 -0
  125. package/modules/import-kit/src/d1/stream-layout.ts +43 -0
  126. package/modules/import-kit/src/d1/stream-origin.ts +44 -0
  127. package/modules/import-kit/src/d1/stream-page-integrity.ts +18 -0
  128. package/modules/import-kit/src/d1/stream-partition.ts +50 -0
  129. package/modules/import-kit/src/d1/stream-plan.ts +133 -0
  130. package/modules/import-kit/src/d1/stream-receipt.ts +45 -0
  131. package/modules/import-kit/src/d1/stream-revision-schema.ts +35 -0
  132. package/modules/import-kit/src/d1/stream-snapshot.ts +26 -0
  133. package/modules/import-kit/src/d1/stream-stage.ts +79 -0
  134. package/modules/import-kit/src/d1/stream-state.ts +25 -0
  135. package/modules/import-kit/src/d1/stream-working-set.ts +30 -0
  136. package/modules/import-kit/src/d1/streams.ts +39 -0
  137. package/modules/import-kit/src/d1/target-guards.ts +9 -0
  138. package/modules/import-kit/src/d1/target-identities.ts +17 -0
  139. package/modules/import-kit/src/d1/transport.ts +10 -0
  140. package/modules/import-kit/src/d1/types.ts +162 -0
  141. package/modules/import-kit/src/d1/undo-observations.ts +39 -0
  142. package/modules/import-kit/src/d1/undo-report.ts +22 -0
  143. package/modules/import-kit/src/d1/undo-snapshot.ts +87 -0
  144. package/modules/import-kit/src/d1/undo-state.ts +84 -0
  145. package/modules/import-kit/src/d1/undo-types.ts +110 -0
  146. package/modules/import-kit/src/journal/authority.ts +12 -0
  147. package/modules/import-kit/src/plan/budget.ts +60 -0
  148. package/modules/import-kit/src/plan/index.ts +123 -0
  149. package/modules/import-kit/src/plan/size.ts +122 -0
  150. package/modules/import-kit/src/provenance/canonical-hash.ts +130 -0
  151. package/modules/import-kit/src/provenance/canonical-stream.ts +85 -0
  152. package/modules/import-kit/src/provenance/credential-names.ts +131 -0
  153. package/modules/import-kit/src/provenance/credentials.ts +317 -0
  154. package/modules/import-kit/src/provenance/identity.ts +51 -0
  155. package/modules/import-kit/src/provenance/json-snapshot.ts +21 -0
  156. package/modules/import-kit/src/provenance/resumable-sha256.ts +75 -0
  157. package/modules/import-kit/src/provenance/validate-json.ts +19 -0
  158. package/modules/import-kit/src/storage/originals.ts +87 -0
  159. package/modules/import-kit/src/storage/provenance.ts +64 -0
  160. package/modules/import-kit/src/storage/schema.ts +42 -0
  161. package/modules/import-kit/src/types.ts +47 -0
  162. package/modules/portability/README.md +98 -2
  163. package/modules/portability/backfills.mjs +42 -15
  164. package/modules/portability/ledgers.mjs +7 -1
  165. package/modules/portability/transient-tables.mjs +40 -0
  166. package/package.json +3 -1
  167. package/scripts/publish-check/reviewed.py +3 -1
@@ -0,0 +1,97 @@
1
+ import { captureJobLayout, jobLayoutGuards, jobLayoutReads } from './stream-layout.ts';
2
+ import { needsStreamOrigin } from './stream-origin.ts';
3
+ import { nativeStagingScope } from './native-staging-context.ts';
4
+ import { streamBoundary } from './stream-boundary.ts';
5
+ import { requestContext } from './budget.ts';
6
+ import { loadPlans, planBody, planGuards, planChunkReads, retainPlan, creationCodec, LARGE_PLAN_BYTES } from './plans.ts';
7
+ import { randomUUID } from 'node:crypto';
8
+ import type { ImportJob, ImportPlan, ImportMapping, SourceBundle, ImportAction } from '../types.ts';
9
+ import { buildImportPlan, planFingerprint } from '../plan/index.ts';
10
+ import { creationStatements, validateCreation } from './creation.ts';
11
+ import { planStagedCreation, runStagedCreation, stagedSize } from './staged-creation.ts';
12
+ import type { D1ImportContext, D1Sql } from './types.ts';
13
+ import { atomic, readAtomic, authorize, guard, sql } from './database.ts';
14
+ import { custodyGuards, inventoryGuards } from './integrity.ts';
15
+ import { guardErasedSources } from './erasure-claims.ts';
16
+ import { readOriginals } from './originals.ts';
17
+ import { retiredError } from './retirement.ts';
18
+ import { attachReversal, loadReversalState, reversalGuards, reversalState } from './undo-state.ts';
19
+
20
+ export function decode(row: any): ImportJob {
21
+ return { id: row.id, fingerprint: row.fingerprint, destination: row.destination, namespace: row.namespace, mappingVersion: row.mapping_version, principalId: row.principal_id, state: row.state, checkpoint: row.checkpoint, total: row.total, fence: row.fence, error: row.error };
22
+ }
23
+ const planReads = new WeakMap<ImportJob, number>();
24
+ export const storedReads = (job: ImportJob) => 1 + (planReads.get(job) ?? 0) + Number(reversalState(job).policy === 'preserve-conflicts') + jobLayoutReads(job);
25
+ export async function stored(id: string, context: D1ImportContext, action: ImportAction = 'read'): Promise<{ job: ImportJob; plan: ImportPlan }> {
26
+ const bounded = context.maxBatchBytes !== undefined;
27
+ const query = `SELECT j.*${bounded?',r.revision AS custody_revision':''},${needsStreamOrigin('j.id')} AS stream_origin_required,(SELECT stream_id FROM ik_d1_stream_origins WHERE job_id=j.id) AS origin_stream_id,(SELECT stream_id FROM ik_d1_stream_pages WHERE job_id=j.id) AS page_stream_id FROM ik_jobs j${bounded?' LEFT JOIN ik_d1_custody_revisions r ON r.job_id=j.id':''} WHERE j.id=?`;
28
+ const row = await context.storage.database.prepare(query).bind(id).first();
29
+ if (!row) throw await retiredError(context.storage.database, id, 'job') ?? new Error('IMPORT_JOB_NOT_FOUND');
30
+ // Unbounded guards run before their transaction's writes: a large plan is
31
+ // fenced by its custody revisions instead of resending it in every request.
32
+ const fenced = !bounded, job = decode(row), plan = (await loadPlans([row], context.storage.database, context.storage.targets.planCodec, undefined, fenced)).get(id)!;
33
+ planReads.set(job, planChunkReads(row.plan_json, fenced));
34
+ if (job.destination !== context.destination || job.namespace !== context.namespace || job.mappingVersion !== context.mappingVersion || job.principalId !== context.authorization.principalId) throw new Error('JOB_BINDING_MISMATCH');
35
+ if (context.reviewedFingerprint !== undefined && context.reviewedFingerprint !== job.fingerprint) throw new Error('REVIEW_FINGERPRINT_MISMATCH');
36
+ if (planFingerprint(plan) !== job.fingerprint || plan.fingerprint !== job.fingerprint || plan.destination !== job.destination || plan.namespace !== job.namespace || plan.mappingVersion !== job.mappingVersion || plan.principalId !== job.principalId || plan.entries.length !== job.total) throw new Error('STORED_PLAN_CHANGED');
37
+ job.erasedRecords = plan.entries.filter(entry => entry.erased).length;
38
+ attachReversal(job, await loadReversalState(row, plan, context));
39
+ await captureJobLayout(job, row, context, action);
40
+ return { job, plan };
41
+ }
42
+ export function boundary(job: ImportJob, plan: ImportPlan, action: ImportAction = 'read'): D1Sql[] {
43
+ return [...jobLayoutGuards(job), ...streamBoundary(job.id, action), ...planGuards(plan, job.id), ...reversalGuards(job.id, reversalState(job)), guard(sql('EXISTS(SELECT 1 FROM ik_jobs WHERE id=? AND fence=? AND checkpoint=? AND state=? AND fingerprint=? AND plan_json=? AND principal_id=? AND destination=? AND namespace=? AND mapping_version=? AND total=? AND error IS ?)', job.id, job.fence, job.checkpoint, job.state, job.fingerprint, planBody(plan), job.principalId, job.destination, job.namespace, job.mappingVersion, job.total, job.error))];
44
+ }
45
+ export async function planD1Import(source: SourceBundle, mapping: ImportMapping, context: D1ImportContext): Promise<ImportPlan> {
46
+ context = requestContext(context);
47
+ await authorize(context, 'plan');
48
+ const plan = buildImportPlan(source, mapping, context);
49
+ await authorize(context, 'plan');
50
+ return plan;
51
+ }
52
+ export async function createD1ImportJob(plan: ImportPlan, context: D1ImportContext): Promise<ImportJob> {
53
+ context = requestContext(context);
54
+ await authorize(context, 'create');
55
+ const operationKey = validateCreation(plan, context);
56
+ const revalidate = () => {
57
+ // The fingerprint check inside re-proves the scanned content is unchanged.
58
+ if (validateCreation(plan, context, false) !== operationKey) throw new Error('OPERATION_KEY_CONFLICT');
59
+ };
60
+ const db = context.storage.database;
61
+ const prior = await db.prepare('SELECT * FROM ik_jobs WHERE destination=? AND principal_id=? AND operation_key=?').bind(context.destination, context.authorization.principalId, operationKey).first();
62
+ // Lookup is an awaited boundary on both fresh creation and replay.
63
+ revalidate();
64
+ if (prior) {
65
+ const current = await stored(prior.id, { ...context, reviewedFingerprint: undefined }, 'create');
66
+ revalidate();
67
+ if (prior.fingerprint !== plan.fingerprint && current.plan.erasure?.reviewedFingerprint !== plan.fingerprint) throw new Error('OPERATION_KEY_CONFLICT');
68
+ await readOriginals(current.plan, current.job.id, context);
69
+ revalidate();
70
+ const inventory = await inventoryGuards(current.job, current.plan, context);
71
+ revalidate();
72
+ await atomic(context, 'create', [...boundary(current.job, current.plan), ...custodyGuards(current.plan, current.job.id), ...inventory]);
73
+ return current.job;
74
+ }
75
+ await guardErasedSources(plan, db);
76
+ revalidate();
77
+ const codec = creationCodec(plan, context.storage.targets.planCodec), unbounded = context.maxBatchBytes === undefined;
78
+ const returnReads = (body: string) => 1 + planChunkReads(body, unbounded) + Number(context.reversalPolicy === 'preserve-conflicts');
79
+ const id = randomUUID(), retained = retainPlan(plan, id, codec);
80
+ // A large plan is created in bounded steps; a retry resumes what was staged.
81
+ if (unbounded && stagedSize(retained.encoded) > LARGE_PLAN_BYTES) {
82
+ const creation = await planStagedCreation(plan, context, operationKey, retained.encoded, () => [randomUUID(), randomUUID(), id]);
83
+ revalidate();
84
+ return (await stored(await runStagedCreation(context, creation, returnReads(retained.body)), context, 'create')).job;
85
+ }
86
+ const creation = creationStatements(plan, { ...context, planCodec: codec }, id, operationKey, retained);
87
+ await atomic(context, 'create', creation, returnReads(retained.body));
88
+ return (await stored(id, context, 'create')).job;
89
+ }
90
+ export async function readD1ImportJob(id: string, context: D1ImportContext): Promise<ImportJob> {
91
+ context = requestContext(context);
92
+ await authorize(context, 'read');
93
+ const { job, plan } = await stored(id, context);
94
+ if(context.maxBatchBytes!==undefined)await readOriginals(plan,id,context);
95
+ await readAtomic(context, [...boundary(job, plan)], nativeStagingScope(job,plan));
96
+ return job;
97
+ }
@@ -0,0 +1,48 @@
1
+ import type { TargetRow } from '../types.ts';
2
+ import type { D1Sql, D1TargetAdapter } from './types.ts';
3
+ import { guard } from './database.ts';
4
+ import { packGuards } from './budget.ts';
5
+ import { batchBytes } from './transport.ts';
6
+ import { targetGuards } from './target-guards.ts';
7
+
8
+ /** One exact native-set guard and the target kinds it covers. */
9
+ export interface NativeGuard { guard: D1Sql; kinds: string[] }
10
+ export function nativeSetGuards(targets: D1TargetAdapter, rows: TargetRow[], destination: string): NativeGuard[] {
11
+ if (targets.matchesMany) {
12
+ const predicates = targets.matchesMany(rows, destination);
13
+ if (!predicates.length) throw new Error('IMPORT_NATIVE_SET_GUARD_REQUIRED');
14
+ const kinds = [...new Set(rows.map(row => row.targetKind))].sort();
15
+ return predicates.map(predicate => ({ guard: guard(predicate), kinds }));
16
+ }
17
+ return [{ guard: guard(targets.matchesCount!(rows.length, destination)), kinds: [] },
18
+ ...rows.flatMap(row => targetGuards(targets, row, row.destination, row.targetKind, row.targetId).map(statement => ({ guard: statement, kinds: [row.targetKind] })))];
19
+ }
20
+
21
+ const sizeCodes = new Set(['IMPORT_D1_TRANSPORT_LIMIT', 'IMPORT_D1_VALUE_LIMIT', 'IMPORT_D1_STATEMENT_LIMIT']);
22
+ /** A size limit of a native snapshot, naming the adapter's target kinds (never
23
+ * row content). `targetKinds` is empty when the native snapshot fits. */
24
+ export class ImportSizeLimitError extends Error {
25
+ readonly targetKinds: string[];
26
+ constructor(code: string, targetKinds: string[]) { super(code); this.name = 'ImportSizeLimitError'; this.targetKinds = targetKinds; }
27
+ }
28
+ function bytes(guards: D1Sql[]): number {
29
+ try { return batchBytes(packGuards(guards, true)); }
30
+ catch (error) { if (error instanceof Error && sizeCodes.has(error.message)) return Infinity; throw error; }
31
+ }
32
+ /** Names what made a persisted read exceed its limit: a native guard too large
33
+ * on its own, else the kind (or matchesMany set) with the largest guards, unless
34
+ * the custody fence outweighs the whole native snapshot. */
35
+ export function sizeLimit(error: unknown, native: NativeGuard[], custody: D1Sql[], limit: number): unknown {
36
+ if (!(error instanceof Error) || !sizeCodes.has(error.message)) return error;
37
+ const single = native.find(entry => bytes([entry.guard]) > limit);
38
+ if (single) return new ImportSizeLimitError(error.message, single.kinds);
39
+ const groups = new Map<string, { kinds: string[]; guards: D1Sql[] }>();
40
+ for (const entry of native) {
41
+ if (!entry.kinds.length) continue;
42
+ const key = JSON.stringify(entry.kinds), group = groups.get(key) ?? { kinds: entry.kinds, guards: [] };
43
+ group.guards.push(entry.guard); groups.set(key, group);
44
+ }
45
+ const [largest] = [...groups.values()].map(group => ({ kinds: group.kinds, size: bytes(group.guards) })).sort((a, b) => b.size - a.size);
46
+ const blamesCustody = !largest || bytes(custody) >= bytes(native.map(entry => entry.guard));
47
+ return new ImportSizeLimitError(error.message, blamesCustody ? [] : largest.kinds);
48
+ }
@@ -0,0 +1,19 @@
1
+ import { stagingStatements } from './staging-statements.ts';
2
+ import { Buffer } from 'node:buffer';
3
+ import type { D1ImportContext, D1Sql } from './types.ts';
4
+ import { batchBytes } from './transport.ts';
5
+ import { nativeStageRowsBytes, nativeRowStatements } from './native-staging-rows.ts';
6
+ import type { NativeStageBinding } from './native-staging-state.ts';
7
+ export function nativeStageCost(context:D1ImportContext,stage:NativeStageBinding,statements:D1Sql[],release:D1Sql[]=[],final:D1Sql[]=[]):number{
8
+ const packed=stagingStatements(context,stage.action,statements,release,final);
9
+ if(batchBytes(packed)>Math.min(context.maxBatchBytes??2*1024*1024,2*1024*1024))throw new Error('IMPORT_D1_TRANSPORT_LIMIT');
10
+ return packed.length;
11
+ }
12
+ /** Simulate the same greedy bounded batches before retaining a single byte. */
13
+ export function nativeRowsCost(context:D1ImportContext,stage:NativeStageBinding,baseline:D1Sql[],table:'index'|'values',columns:string[],rows:()=>Iterable<unknown[]>,revision:number):{queries:number;revision:number}{
14
+ const rowBytes=nativeStageRowsBytes(context);
15
+ let parts:string[]=[],bytes=2,written=0,queries=0;
16
+ const flush=()=>{if(!parts.length)return;queries+=nativeStageCost(context,stage,nativeRowStatements(stage,baseline,table,columns,'['+parts.join(',')+']',parts.length,written,revision));revision+=parts.length;written+=parts.length;parts=[];bytes=2;};
17
+ for(const row of rows()){const part=JSON.stringify(row),size=Buffer.byteLength(part)+1;if(size+2>rowBytes)throw new Error('IMPORT_D1_TRANSPORT_LIMIT');if(parts.length&&bytes+size>rowBytes)flush();parts.push(part);bytes+=size;}
18
+ flush();return {queries,revision};
19
+ }
@@ -0,0 +1,44 @@
1
+ import { jobLayoutGuards, jobLayoutPredicate } from './stream-layout.ts';
2
+ import { reversalState } from './undo-state.ts';
3
+ import type { ImportAction, ImportJob, ImportPlan, TargetRow } from '../types.ts';
4
+ import type { D1ImportContext, D1Sql } from './types.ts';
5
+ import { guard, sql } from './database.ts';
6
+ import { storedPlanSnapshot, planBody } from './plans.ts';
7
+ import { custodyGuards } from './integrity.ts';
8
+ import { streamBoundary } from './stream-boundary.ts';
9
+ export interface NativeStagingScope {
10
+ job: ImportJob; plan: ImportPlan; nativeGuards: D1Sql[]; postGuards: D1Sql[];
11
+ targets: { kind: string; id: string; present: boolean }[];
12
+
13
+ }
14
+ export const nativeStagingScope = (job: ImportJob, plan: ImportPlan): NativeStagingScope => ({job,plan,nativeGuards:[],postGuards:[],targets:[]});
15
+ export async function readStagedTarget(context: D1ImportContext, scope: NativeStagingScope, kind: string, id: string): Promise<TargetRow | undefined> {
16
+ const {database,targets}=context.storage;
17
+ if(context.maxBatchBytes===undefined || !targets.stagingRead){const row=await targets.read(database,context.destination,kind,id);scope.targets.push({kind,id,present:!!row});return row;}
18
+ const snapshot=await targets.stagingRead(database,context.destination,kind,id);
19
+ if(!snapshot.guards.length)throw new Error('IMPORT_NATIVE_STAGING_GUARD_REQUIRED');
20
+ scope.nativeGuards.push(...snapshot.guards.map(guard));
21
+ scope.targets.push({kind,id,present:!!snapshot.row});
22
+ return snapshot.row;
23
+ }
24
+ /** The revision was captured before loading any plan chunk. It fences the exact
25
+ * decoded, fingerprint-verified bytes, never a later revision read and rebound. */
26
+ export function nativeStagingBaseline(scope: NativeStagingScope, action: ImportAction): D1Sql[] {
27
+ const {job,plan}=scope, revision=storedPlanSnapshot(plan,job.id).revision;
28
+ if(revision===undefined||!Number.isSafeInteger(revision)||revision<0)throw new Error('IMPORT_NATIVE_STAGING_REVISION_REQUIRED');
29
+ return [...jobLayoutGuards(job), ...streamBoundary(job.id,action), ...custodyGuards(plan,job.id), ...nativeStagingPostBinding(scope),
30
+ guard(sql('EXISTS(SELECT 1 FROM ik_jobs j JOIN ik_d1_custody_revisions r ON r.job_id=j.id WHERE j.id=? AND j.fence=? AND j.checkpoint=? AND j.state=? AND j.fingerprint=? AND j.plan_json IS ? AND j.principal_id=? AND j.destination=? AND j.namespace=? AND j.mapping_version=? AND j.total=? AND r.revision=?)',job.id,job.fence,job.checkpoint,job.state,job.fingerprint,planBody(plan),job.principalId,job.destination,job.namespace,job.mappingVersion,job.total,revision)),...scope.nativeGuards];
31
+ }
32
+
33
+ /** Native progress may advance, but reviewed custody and owner binding cannot. */
34
+ export const nativeStagingPostBinding=(scope:NativeStagingScope):D1Sql[]=>[guard(nativeStagingPostPredicate(scope))];
35
+ export function nativeStagingPostPredicate(scope:NativeStagingScope):D1Sql{
36
+ const {job,plan}=scope, revision=storedPlanSnapshot(plan,job.id).revision, policy=reversalState(job);
37
+ const binding = sql('EXISTS(SELECT 1 FROM ik_jobs j JOIN ik_d1_custody_revisions r ON r.job_id=j.id WHERE j.id=? AND j.destination=? AND j.namespace=? AND j.mapping_version=? AND j.principal_id=? AND j.fingerprint=? AND j.plan_json IS ? AND j.total=? AND r.revision=? AND j.reversal_policy=? AND j.reversal_policy_hash IS ?)',job.id,job.destination,job.namespace,job.mappingVersion,job.principalId,job.fingerprint,planBody(plan),job.total,revision,policy.policy,policy.binding);
38
+ const layout = jobLayoutPredicate(job);
39
+ return layout ? sql(`(${binding.sql}) AND (${layout.sql})`, ...binding.params!, ...layout.params??[]) : binding;
40
+ }
41
+
42
+ export function expectedStagedTarget(scope:NativeStagingScope,kind:string,id:string,present:boolean):void{
43
+ scope.targets=scope.targets.map(target=>target.kind===kind&&target.id===id?{kind,id,present}:target);
44
+ }
@@ -0,0 +1,33 @@
1
+ import type { D1ImportContext, D1Sql } from './types.ts';
2
+ import { guard, sql, authorize } from './database.ts';
3
+ import { requestContext } from './budget.ts';
4
+ import { requireText } from '../provenance/identity.ts';
5
+ import { nativeStageAtomic, nativeStageGuards, type NativeStageBinding } from './native-staging-state.ts';
6
+ export function beginNativeStage(stage:NativeStageBinding,baseline:D1Sql[]):D1Sql[]{
7
+ return [...baseline,
8
+ sql('DELETE FROM ik_d1_native_staging WHERE destination=? AND namespace=? AND principal_id=? AND job_id=?',stage.destination,stage.namespace,stage.principal_id,stage.job_id),
9
+ sql(`INSERT INTO ik_d1_native_staging(${Object.keys(stage).join(',')}) VALUES(${Object.keys(stage).map(()=>'?').join(',')})`,...Object.values(stage)),
10
+ ...baseline,...nativeStageGuards(stage,0)];
11
+ }
12
+ export const discardNativeStage = (stage:NativeStageBinding):D1Sql[] => [sql('DELETE FROM ik_d1_native_staging WHERE id=? AND generation=? AND destination=? AND namespace=? AND principal_id=? AND job_id=?',stage.id,stage.generation,stage.destination,stage.namespace,stage.principal_id,stage.job_id)];
13
+ /** Current owner cancellation removes private values only, never public progress. */
14
+ export async function cancelD1NativeStaging(jobId:string,input:D1ImportContext):Promise<void>{
15
+ requireText(jobId,'jobId');const context=requestContext(input);await authorize(context,'create');
16
+ await nativeStageAtomic(context,'create',[guard(sql('EXISTS(SELECT 1 FROM ik_jobs WHERE id=? AND destination=? AND namespace=? AND principal_id=?)',jobId,context.destination,context.namespace,context.authorization.principalId)),sql('DELETE FROM ik_d1_native_staging WHERE job_id=? AND destination=? AND namespace=? AND principal_id=?',jobId,context.destination,context.namespace,context.authorization.principalId)]);
17
+ }
18
+
19
+ /** A privacy operator owns only the private erasure generation, not the
20
+ * importing operator's public job. Cancellation preserves all public custody. */
21
+ export async function cancelD1ErasureStaging(jobId:string,input:D1ImportContext):Promise<void>{
22
+ requireText(jobId,'jobId');const context=requestContext(input);await authorize(context,'erase');
23
+ const predicate=sql("NOT EXISTS(SELECT 1 FROM ik_d1_native_staging WHERE job_id=? AND destination=? AND namespace=? AND principal_id=? AND action='erase')",jobId,context.destination,context.namespace,context.authorization.principalId);
24
+ await nativeStageAtomic(context,'erase',[
25
+ guard(sql('EXISTS(SELECT 1 FROM ik_jobs WHERE id=? AND destination=?)',jobId,context.destination)),
26
+ sql("DELETE FROM ik_d1_native_staging WHERE job_id=? AND destination=? AND namespace=? AND principal_id=? AND action='erase'",jobId,context.destination,context.namespace,context.authorization.principalId),
27
+ ],0,[],[predicate]);
28
+ }
29
+
30
+ /** Reversal intent also cancels abandoned forward/private native work. */
31
+ export async function cancelNativeJobStaging(context:D1ImportContext,action:'undo'|'rollback',jobId:string,fence:number):Promise<void>{
32
+ await nativeStageAtomic(context,action,[guard(sql('EXISTS(SELECT 1 FROM ik_jobs WHERE id=? AND fence=? AND destination=? AND namespace=? AND principal_id=?)',jobId,fence,context.destination,context.namespace,context.authorization.principalId)),sql('DELETE FROM ik_d1_native_staging WHERE job_id=? AND destination=? AND namespace=? AND principal_id=?',jobId,context.destination,context.namespace,context.authorization.principalId)]);
33
+ }
@@ -0,0 +1,74 @@
1
+ import { Buffer } from 'node:buffer';
2
+ import { createHash } from 'node:crypto';
3
+ import type { D1ImportContext, D1Sql } from './types.ts';
4
+ import { guard, readAtomic, sql } from './database.ts';
5
+ import { guardPredicate } from './guards.ts';
6
+ import { nativeStageGuards, type NativeStageBinding } from './native-staging-state.ts';
7
+
8
+ export type NativeStageTable = 'index' | 'values';
9
+ export interface NativeStageProgress { written: number; revision: number }
10
+ export type NativeStageCheckpoint = (from: NativeStageProgress, to: NativeStageProgress) => D1Sql[];
11
+ declare const verifiedOffset: unique symbol;
12
+ /** Only a successful read fence can mint this capability; object casts are inert. */
13
+ export interface VerifiedNativeStageOffset { readonly [verifiedOffset]: true }
14
+ interface OffsetBinding extends NativeStageProgress {
15
+ context: D1ImportContext; database: D1ImportContext['storage']['database'];
16
+ stage: string; baseline: string; table: NativeStageTable;
17
+ }
18
+ const offsets = new WeakMap<VerifiedNativeStageOffset, OffsetBinding>();
19
+ const columns = {
20
+ index: Object.freeze(['ordinal', 'source_identity', 'target_identity']),
21
+ values: Object.freeze(['value_ordinal', 'ordinal', 'body', 'body_hash', 'body_size', 'value_chunks']),
22
+ };
23
+ const invalid = (): never => { throw new Error('IMPORT_NATIVE_STAGE_OFFSET_INVALID'); };
24
+ const integer = (n: number) => Number.isSafeInteger(n) && n >= 0;
25
+ function baselineIdentity(baseline: D1Sql[]): string {
26
+ const hash = createHash('sha256');
27
+ for (const statement of baseline) {
28
+ const scalar = guardPredicate(statement);
29
+ if (!scalar) invalid();
30
+ hash.update(JSON.stringify([scalar!.sql, (scalar!.params ?? []).map(value =>
31
+ value instanceof ArrayBuffer || ArrayBuffer.isView(value)
32
+ ? ['bytes', Buffer.from(value instanceof ArrayBuffer ? new Uint8Array(value) : new Uint8Array(value.buffer, value.byteOffset, value.byteLength)).toString('base64')]
33
+ : value)]) + '\n');
34
+ }
35
+ return hash.digest('hex');
36
+ }
37
+ export function validateNativeStageRows(context: D1ImportContext, stage: NativeStageBinding, table: NativeStageTable, selected?: string[]): void {
38
+ if ((table !== 'index' && table !== 'values') ||
39
+ context.destination !== stage.destination || context.namespace !== stage.namespace ||
40
+ context.mappingVersion !== stage.mapping_version || context.authorization.principalId !== stage.principal_id ||
41
+ ![stage.identities, stage.chunks_count, stage.values_count, stage.job_fence].every(integer)) invalid();
42
+ if (selected && (selected.length !== columns[table].length || selected.some((column, i) => column !== columns[table][i]))) invalid();
43
+ }
44
+ /** Revisions count every row mutation, so count equality proves an append-only
45
+ * prefix without fetching its bodies again. Registration separates the phases. */
46
+ export function nativeStageOffsetGuards(stage: NativeStageBinding, table: NativeStageTable, written: number, revision: number): D1Sql[] {
47
+ if ((table !== 'index' && table !== 'values') || !integer(written) || !integer(revision) ||
48
+ ![stage.identities, stage.chunks_count, stage.values_count].every(integer) ||
49
+ written > (table === 'index' ? stage.identities : stage.chunks_count) ||
50
+ revision !== written + (table === 'values' ? stage.identities : 0)) invalid();
51
+ const indexes = table === 'index' ? written : stage.identities;
52
+ const values = table === 'values' ? written : 0;
53
+ return [...nativeStageGuards(stage, revision), guard(sql(
54
+ `EXISTS(SELECT 1 FROM ik_d1_native_staging WHERE id=? AND sealed_revision IS NULL AND registered_revision ${table === 'index' ? 'IS NULL' : 'IS ?'}) AND (SELECT count(*) FROM ik_d1_native_staging_index WHERE stage_id=?)=? AND (SELECT count(*) FROM ik_d1_native_staging_values WHERE stage_id=?)=?`,
55
+ stage.id, ...(table === 'values' ? [stage.identities] : []), stage.id, indexes, stage.id, values))];
56
+ }
57
+ export async function verifyNativeStageOffset(context: D1ImportContext, stage: NativeStageBinding, baseline: D1Sql[], table: NativeStageTable, checkpoint: NativeStageProgress & { guards: D1Sql[] }): Promise<VerifiedNativeStageOffset> {
58
+ validateNativeStageRows(context, stage, table);
59
+ if (!checkpoint || !Array.isArray(checkpoint.guards) || !checkpoint.guards.length || checkpoint.guards.some(query => !guardPredicate(query))) invalid();
60
+ const binding: OffsetBinding = { context, database: context.storage.database, stage: JSON.stringify(stage), baseline: baselineIdentity(baseline), table, written: checkpoint.written, revision: checkpoint.revision };
61
+ const checks = [...baseline, ...nativeStageOffsetGuards(stage, table, binding.written, binding.revision), ...checkpoint.guards];
62
+ await readAtomic(context, checks, undefined, stage.action);
63
+ // Reject mutation of caller-owned binding objects across the awaited fence.
64
+ if (JSON.stringify(stage) !== binding.stage || baselineIdentity(baseline) !== binding.baseline || context.storage.database !== binding.database) invalid();
65
+ validateNativeStageRows(context, stage, table);
66
+ const token = Object.freeze({}) as VerifiedNativeStageOffset;
67
+ offsets.set(token, binding);
68
+ return token;
69
+ }
70
+ export function nativeStageOffsetProgress(token: VerifiedNativeStageOffset, context: D1ImportContext, stage: NativeStageBinding, baseline: D1Sql[], table: NativeStageTable): NativeStageProgress {
71
+ const binding = token && typeof token === 'object' ? offsets.get(token) : undefined;
72
+ if (!binding || binding.context !== context || binding.database !== context.storage.database || binding.stage !== JSON.stringify(stage) || binding.baseline !== baselineIdentity(baseline) || binding.table !== table) invalid();
73
+ return { written: binding!.written, revision: binding!.revision };
74
+ }
@@ -0,0 +1,18 @@
1
+ import { guardPredicate } from './guards.ts';
2
+ import type { D1ImportContext,D1Sql } from './types.ts';
3
+ import { guard,sql } from './database.ts';
4
+ import { streamBoundary } from './stream-boundary.ts';
5
+ import { nativeStageGuards,type NativeStageBinding } from './native-staging-state.ts';
6
+ import { discardNativeStage } from './native-staging-lifecycle.ts';
7
+
8
+ /** The final batch contains only compact references to already verified values. */
9
+ export function nativePublication(context:D1ImportContext,stage:NativeStageBinding,input:{baseline:D1Sql[];writes:D1Sql[];post:D1Sql[];jobs:string[];targets:{kind:string;id:string;present:boolean}[];binding:D1Sql[];final:D1Sql[];erasure?:boolean}){
10
+ if(!context.storage.targets.stagingPublication)throw new Error('IMPORT_NATIVE_STAGING_ADAPTER_REQUIRED');
11
+ const id=stage.id,sealed=stage.identities+stage.chunks_count,jobIds=JSON.stringify(input.jobs),protection=context.storage.targets.stagingPublication(id,context.destination,input.targets);
12
+ const publication=[...input.baseline,...nativeStageGuards(stage,sealed),guard(sql('EXISTS(SELECT 1 FROM ik_d1_native_staging WHERE id=? AND sealed_revision=revision AND registered_revision IS NOT NULL)',id)),...input.writes,
13
+ ...protection.protect,sql('INSERT INTO ik_d1_publication_fences(stream_id,job_id) SELECT ?,value FROM json_each(?)',id,jobIds),guard(sql('(SELECT count(*) FROM ik_d1_publication_fences WHERE stream_id=?)=json_array_length(?) AND NOT EXISTS(SELECT 1 FROM json_each(?) j WHERE NOT EXISTS(SELECT 1 FROM ik_d1_publication_fences f WHERE f.stream_id=? AND f.job_id=j.value))',id,jobIds,jobIds,id)),...streamBoundary(stage.job_id,stage.action),...input.binding,...input.post,...protection.seal,sql('UPDATE ik_d1_custody_revisions SET seal=? WHERE job_id IN (SELECT value FROM json_each(?))',id,jobIds),
14
+ ...nativeStageGuards(stage,sealed,!input.erasure),...discardNativeStage(stage),guard(sql('NOT EXISTS(SELECT 1 FROM ik_d1_native_staging WHERE id=?)',id))];
15
+ const release=[...protection.release,sql('DELETE FROM ik_d1_publication_fences WHERE stream_id=?',id)];
16
+ const final=[...streamBoundary(stage.job_id,stage.action).map(query=>guardPredicate(query)!),...input.final,...protection.final,sql('NOT EXISTS(SELECT 1 FROM json_each(?) j WHERE NOT EXISTS(SELECT 1 FROM ik_d1_custody_revisions r WHERE r.job_id=j.value AND r.seal=?))',jobIds,id),sql('NOT EXISTS(SELECT 1 FROM ik_d1_creation_plans WHERE job_id IN (SELECT value FROM json_each(?)))',jobIds)];
17
+ return {publication,release,final};
18
+ }
@@ -0,0 +1,72 @@
1
+ import { Buffer } from 'node:buffer';
2
+ import type { D1ImportContext, D1Sql } from './types.ts';
3
+ import { guard, sql } from './database.ts';
4
+ import { nativeStageAtomic, type NativeStageBinding } from './native-staging-state.ts';
5
+ import { nativeStageOffsetGuards, nativeStageOffsetProgress, validateNativeStageRows, type NativeStageCheckpoint, type VerifiedNativeStageOffset } from './native-staging-offset.ts';
6
+ export { verifyNativeStageOffset, type VerifiedNativeStageOffset, type NativeStageCheckpoint, type NativeStageProgress } from './native-staging-offset.ts';
7
+ export const NATIVE_STAGE_ROWS_BYTES = 512 * 1024;
8
+ /** Both insertion and atomic equality bind the body. Leave the rest of a
9
+ * smaller caller-selected transport limit for escaping, guards and journals. */
10
+ export const nativeStageRowsBytes = (context:D1ImportContext) => Math.min(NATIVE_STAGE_ROWS_BYTES, Math.floor(Math.min(context.maxBatchBytes??2*1024*1024,2*1024*1024)/4));
11
+ export interface NativeStageRowsOptions { offset?: VerifiedNativeStageOffset; checkpoint?: NativeStageCheckpoint }
12
+
13
+ /** Replay deterministic input from its beginning; only authenticated prior rows
14
+ * are skipped. Each new batch and its journal advance commit together. */
15
+ export class NativeStageRows {
16
+ revision = 0;
17
+ private committed = 0;
18
+ private replay = 0;
19
+ private skipped = 0;
20
+ private rows: string[] = [];
21
+ private bytes = 2;
22
+ private context: D1ImportContext;
23
+ private stage: NativeStageBinding;
24
+ private baseline: D1Sql[];
25
+ private table: 'index' | 'values';
26
+ private columns: string[];
27
+ private checkpoint?: NativeStageCheckpoint;
28
+ private rowBytes: number;
29
+ constructor(context: D1ImportContext, stage: NativeStageBinding, baseline: D1Sql[], table: 'index' | 'values', columns: string[], options: NativeStageRowsOptions = {}) {
30
+ validateNativeStageRows(context, stage, table, columns);
31
+ if (!options || typeof options !== 'object' || Array.isArray(options) || Object.keys(options).some(key => key !== 'offset' && key !== 'checkpoint') || options.checkpoint !== undefined && typeof options.checkpoint !== 'function') throw new Error('IMPORT_NATIVE_STAGE_OFFSET_INVALID');
32
+ this.context = context; this.stage = {...stage}; this.baseline = [...baseline]; this.table = table; this.columns = [...columns]; this.checkpoint = options.checkpoint;
33
+ this.rowBytes = nativeStageRowsBytes(context);
34
+ if (Object.hasOwn(options, 'offset')) {
35
+ const progress = nativeStageOffsetProgress(options.offset!, context, stage, baseline, table);
36
+ this.committed = this.replay = progress.written; this.revision = progress.revision;
37
+ }
38
+ }
39
+ get written(): number { return this.committed; }
40
+ async append(row: unknown[]): Promise<void> {
41
+ if (this.skipped < this.replay) { this.skipped++; return; }
42
+ const text = JSON.stringify(row), bytes = Buffer.byteLength(text) + 1;
43
+ if (bytes + 2 > this.rowBytes) throw new Error('IMPORT_D1_TRANSPORT_LIMIT');
44
+ if (this.rows.length && this.bytes + bytes > this.rowBytes) await this.flush();
45
+ this.rows.push(text); this.bytes += bytes;
46
+ }
47
+ async flush(): Promise<void> {
48
+ if (this.skipped < this.replay) throw new Error('IMPORT_NATIVE_STAGE_OFFSET_INPUT_INCOMPLETE');
49
+ if (!this.rows.length) return;
50
+ const body = '[' + this.rows.join(',') + ']', count = this.rows.length;
51
+ const writes = nativeRowStatements(this.stage, this.baseline, this.table, this.columns, body, count, this.committed, this.revision, this.checkpoint);
52
+ const result = await nativeStageAtomic(this.context, this.stage.action, writes), read = result[result.length - 4] as any;
53
+ const rows = Array.isArray(read) ? read : read?.results, expected = JSON.parse(body) as unknown[][];
54
+ if (!Array.isArray(rows) || rows.length !== count || rows.some((row, i) => this.columns.some((column, j) => row[column] !== expected[i][j]))) throw new Error('IMPORT_NATIVE_STAGE_CONTENT_CHANGED');
55
+ this.revision += count; this.committed += count;
56
+ this.rows = []; this.bytes = 2;
57
+ }
58
+ }
59
+
60
+ export function nativeRowStatements(stage: NativeStageBinding, baseline: D1Sql[], table: 'index' | 'values', columns: string[], body: string, count: number, written: number, revision: number, checkpoint?: NativeStageCheckpoint): D1Sql[] {
61
+ if (!Number.isSafeInteger(count) || count < 1) throw new Error('IMPORT_NATIVE_STAGE_OFFSET_INVALID');
62
+ const before = nativeStageOffsetGuards(stage, table, written, revision);
63
+ const after = nativeStageOffsetGuards(stage, table, written + count, revision + count);
64
+ const progress = checkpoint?.({written, revision}, {written: written + count, revision: revision + count}) ?? [];
65
+ return [...baseline, ...before,
66
+ sql(`INSERT INTO ik_d1_native_staging_${table}(stage_id,${columns.join(',')}) SELECT ?,${columns.map((_, i) => `json_extract(value,'$[${i}]')`).join(',')} FROM json_each(?)`, stage.id, body),
67
+ ...progress, ...baseline, ...after,
68
+ // Check inside the transaction, including changes caused by journal triggers.
69
+ // One extra body binding leaves room for guards under the 2 MiB ceiling.
70
+ guard(sql(`CASE WHEN NOT EXISTS(SELECT 1 FROM json_each(?) expected WHERE NOT EXISTS(SELECT 1 FROM ik_d1_native_staging_${table} actual WHERE actual.stage_id=? AND ${columns.map((column, i) => `actual.${column} IS json_extract(expected.value,'$[${i}]')`).join(' AND ')})) THEN 1 ELSE json_extract('{}','$IMPORT_NATIVE_STAGE_CONTENT_CHANGED') END`, body, stage.id)),
71
+ sql(`SELECT ${columns.join(',')} FROM ik_d1_native_staging_${table} WHERE stage_id=? ORDER BY rowid LIMIT ? OFFSET ?`, stage.id, count, written)];
72
+ }
@@ -0,0 +1,28 @@
1
+ import { publicationReceiptSchema } from './publication-receipt-schema.ts';
2
+ /** Additive, opt-in native transaction custody. No original byte APIs change. */
3
+ export const d1NativeStagingSchema = [
4
+ 'CREATE TABLE IF NOT EXISTS ik_d1_custody_revisions (job_id TEXT PRIMARY KEY REFERENCES ik_jobs(id) ON DELETE CASCADE,revision INTEGER NOT NULL DEFAULT 0,seal TEXT)',
5
+ 'INSERT OR IGNORE INTO ik_d1_custody_revisions(job_id) SELECT id FROM ik_jobs',
6
+ ...publicationReceiptSchema,
7
+ 'CREATE TRIGGER IF NOT EXISTS ik_d1_custody_job_insert AFTER INSERT ON ik_jobs BEGIN INSERT INTO ik_d1_custody_revisions(job_id) VALUES(NEW.id); END',
8
+ 'CREATE TRIGGER IF NOT EXISTS ik_d1_custody_job_update AFTER UPDATE OF plan_json ON ik_jobs BEGIN UPDATE ik_d1_custody_revisions SET revision=revision+1 WHERE job_id=OLD.id OR job_id=NEW.id; END',
9
+ ...['INSERT','UPDATE','DELETE'].map(event => {
10
+ const refs = event === 'INSERT' ? 'job_id=NEW.job_id' : event === 'DELETE' ? 'job_id=OLD.job_id' : 'job_id=OLD.job_id OR job_id=NEW.job_id';
11
+ return `CREATE TRIGGER IF NOT EXISTS ik_d1_custody_chunks_${event.toLowerCase()} AFTER ${event} ON ik_d1_plan_chunks BEGIN UPDATE ik_d1_custody_revisions SET revision=revision+1 WHERE ${refs}; END`;
12
+ }),
13
+ 'CREATE TABLE IF NOT EXISTS ik_d1_native_staging (id TEXT PRIMARY KEY,generation TEXT NOT NULL,destination TEXT NOT NULL,namespace TEXT NOT NULL,mapping_version TEXT NOT NULL,principal_id TEXT NOT NULL,job_id TEXT NOT NULL REFERENCES ik_jobs(id) ON DELETE CASCADE,fingerprint TEXT NOT NULL,job_fence INTEGER NOT NULL,action TEXT NOT NULL,values_count INTEGER NOT NULL,chunks_count INTEGER NOT NULL,identities INTEGER NOT NULL,revision INTEGER NOT NULL DEFAULT 0,registered_revision INTEGER,sealed_revision INTEGER,UNIQUE(destination,principal_id,job_id,action))',
14
+ 'CREATE TABLE IF NOT EXISTS ik_d1_native_staging_index (stage_id TEXT NOT NULL REFERENCES ik_d1_native_staging(id) ON DELETE CASCADE,ordinal INTEGER NOT NULL,source_identity TEXT NOT NULL,target_identity TEXT NOT NULL,PRIMARY KEY(stage_id,ordinal,target_identity))',
15
+ 'CREATE INDEX IF NOT EXISTS ik_d1_native_staging_source ON ik_d1_native_staging_index(source_identity,stage_id)',
16
+ 'CREATE INDEX IF NOT EXISTS ik_d1_native_staging_target ON ik_d1_native_staging_index(target_identity,stage_id)',
17
+ 'CREATE TABLE IF NOT EXISTS ik_d1_native_staging_values (stage_id TEXT NOT NULL REFERENCES ik_d1_native_staging(id) ON DELETE CASCADE,value_ordinal INTEGER NOT NULL,ordinal INTEGER NOT NULL,body TEXT NOT NULL,body_hash TEXT NOT NULL,body_size INTEGER NOT NULL,value_chunks INTEGER NOT NULL,PRIMARY KEY(stage_id,value_ordinal,ordinal))',
18
+ "CREATE TRIGGER IF NOT EXISTS ik_d1_native_staging_binding BEFORE UPDATE OF id,generation,destination,namespace,mapping_version,principal_id,job_id,fingerprint,job_fence,action,values_count,chunks_count,identities ON ik_d1_native_staging BEGIN SELECT RAISE(ABORT,'IMPORT_NATIVE_STAGE_BINDING_CHANGED'); END",
19
+ ...['index','values'].flatMap(table=>['INSERT','UPDATE','DELETE'].map(event=>{
20
+ const refs = event === 'INSERT' ? 'id=NEW.stage_id' : event === 'DELETE' ? 'id=OLD.stage_id' : 'id=OLD.stage_id OR id=NEW.stage_id';
21
+ return `CREATE TRIGGER IF NOT EXISTS ik_d1_native_staging_${table}_${event.toLowerCase()} AFTER ${event} ON ik_d1_native_staging_${table} BEGIN UPDATE ik_d1_native_staging SET revision=revision+1,sealed_revision=NULL${table==='index'?',registered_revision=NULL':''} WHERE ${refs}; END`;
22
+ })),
23
+ ...['INSERT','UPDATE','DELETE'].map(event=>{
24
+ const refs = event === 'INSERT' ? 'stage_id=NEW.stage_id' : event === 'DELETE' ? 'stage_id=OLD.stage_id' : 'stage_id=OLD.stage_id OR stage_id=NEW.stage_id';
25
+ return `CREATE TRIGGER IF NOT EXISTS ik_d1_native_staging_index_custody_${event.toLowerCase()} AFTER ${event} ON ik_d1_native_staging_index BEGIN DELETE FROM ik_d1_native_staging_values WHERE ${refs}; END`;
26
+ }),
27
+ "CREATE TRIGGER IF NOT EXISTS ik_d1_native_staging_values_registered BEFORE INSERT ON ik_d1_native_staging_values WHEN NOT EXISTS(SELECT 1 FROM ik_d1_native_staging WHERE id=NEW.stage_id AND registered_revision IS NOT NULL) BEGIN SELECT RAISE(ABORT,'IMPORT_NATIVE_STAGE_INDEX_REQUIRED'); END",
28
+ ];
@@ -0,0 +1,22 @@
1
+ import { stagingStatements } from './staging-statements.ts';
2
+ import type { D1ImportContext, D1Sql } from './types.ts';
3
+ import { guard, prepare, sql } from './database.ts';
4
+ import { batchBytes } from './transport.ts';
5
+ import { D1RequestLimitError, remaining } from './budget.ts';
6
+ import type { ImportAction } from '../types.ts';
7
+ export interface NativeStageBinding {
8
+ id: string; generation: string; destination: string; namespace: string;
9
+ mapping_version: string; principal_id: string; job_id: string; fingerprint: string;
10
+ job_fence: number; action: ImportAction; values_count: number; chunks_count: number; identities: number;
11
+ }
12
+ export const nativeStageBinding = (stage: NativeStageBinding) => sql(`EXISTS(SELECT 1 FROM ik_d1_native_staging WHERE ${Object.keys(stage).map(key=>`${key} IS ?`).join(' AND ')})`,...Object.values(stage));
13
+ export const nativeStageClear = (stage: NativeStageBinding) => sql('NOT EXISTS(SELECT 1 FROM ik_d1_native_staging_index i WHERE i.stage_id=? AND (EXISTS(SELECT 1 FROM ik_d1_erased_sources s WHERE s.destination=? AND s.identity=i.source_identity) OR EXISTS(SELECT 1 FROM ik_d1_erased_targets t WHERE t.destination=? AND t.identity=i.target_identity)))',stage.id,stage.destination,stage.destination);
14
+ export function nativeStageGuards(stage: NativeStageBinding, revision: number, clear=true): D1Sql[] {
15
+ return [guard(nativeStageBinding(stage)),guard(sql('EXISTS(SELECT 1 FROM ik_d1_native_staging WHERE id=? AND revision=?)',stage.id,revision)),...(clear?[guard(nativeStageClear(stage))]:[])];
16
+ }
17
+ export async function nativeStageAtomic(context: D1ImportContext, action: ImportAction, statements: D1Sql[], returnReads=0, release: D1Sql[]=[], final:D1Sql[]=[]): Promise<unknown[]> {
18
+ const packed=stagingStatements(context,action,statements,release,final);
19
+ if(packed.length+returnReads>remaining(context))throw new D1RequestLimitError();
20
+ if(batchBytes(packed)>Math.min(context.maxBatchBytes??2*1024*1024,2*1024*1024))throw new Error('IMPORT_D1_TRANSPORT_LIMIT');
21
+ return context.storage.database.batch(packed.map(query=>prepare(context.storage.database,query)));
22
+ }
@@ -0,0 +1,64 @@
1
+ import { guardPredicate } from './guards.ts';
2
+ import { nativePublication } from './native-staging-publication.ts';
3
+ import { randomUUID } from 'node:crypto';
4
+ import { Buffer } from 'node:buffer';
5
+ import type { ImportAction } from '../types.ts';
6
+ import type { D1ImportContext, D1Sql } from './types.ts';
7
+ import { guard, sql } from './database.ts';
8
+ import { D1RequestLimitError, remaining } from './budget.ts';
9
+ import { sha256 } from '../provenance/identity.ts';
10
+ import { planIndex,erasedIndexGuards } from './erasure-index.ts';
11
+ import { stagingChunks } from './staging-transport.ts';
12
+ import { replaceRetained } from './retained-values.ts';
13
+ import { nativeStagingBaseline, nativeStagingPostBinding, nativeStagingPostPredicate, type NativeStagingScope } from './native-staging-context.ts';
14
+ import { nativeStageAtomic, nativeStageGuards, type NativeStageBinding } from './native-staging-state.ts';
15
+ import { NativeStageRows } from './native-staging-rows.ts';
16
+ import { nativeRowsCost, nativeStageCost } from './native-staging-admission.ts';
17
+ import { beginNativeStage, discardNativeStage } from './native-staging-lifecycle.ts';
18
+ const indexColumns=['ordinal','source_identity','target_identity'];
19
+ const valueColumns=['value_ordinal','ordinal','body','body_hash','body_size','value_chunks'];
20
+ function* chunks(value:string):Generator<string>{if(!value.length)yield '';else yield* stagingChunks(value);}
21
+ /** Only explicitly described safe values enter custody. Authority bindings and
22
+ * arbitrary SQL parameters always remain in the final guarded transaction. */
23
+ export async function stageNativeTransaction(context:D1ImportContext,action:ImportAction,statements:D1Sql[],scope:NativeStagingScope,returnReads:number):Promise<unknown[]>{
24
+ if(!context.storage.targets.stagingPublication || scope.targets.length && !context.storage.targets.stagingRead)throw new Error('IMPORT_NATIVE_STAGING_ADAPTER_REQUIRED');
25
+ const baseline=nativeStagingBaseline(scope,action), id=randomUUID(), values:string[]=[], indexes=new Map<string,number>();
26
+ const reference=(value:string):D1Sql=>{
27
+ let ordinal=indexes.get(value);if(ordinal===undefined){ordinal=values.length;values.push(value);indexes.set(value,ordinal);}
28
+ return sql("(SELECT group_concat(body,'') FROM (SELECT body FROM ik_d1_native_staging_values WHERE stage_id=? AND value_ordinal=? ORDER BY ordinal))",id,ordinal);
29
+ };
30
+ const native=[...statements,...scope.postGuards].map(statement=>replaceRetained(statement,reference));
31
+ if(!values.length)throw new Error('IMPORT_NATIVE_STAGING_VALUES_REQUIRED');
32
+ // A related owner's exact plan guards can retain additional safe bytes. Its
33
+ // entire privacy membership joins the index before any values are written.
34
+ const plans=new Set([scope.plan,...[...statements,...scope.postGuards].flatMap(statement=>statement.retainedValues?.privacyPlan?[statement.retainedValues.privacyPlan]:[])]);
35
+ for(const plan of plans)if(plan.source.artifacts.some(artifact=>artifact.original?.status==='available')||plan.entries.some(entry=>entry.effects.some(effect=>effect.attachment!==undefined)))throw new Error('IMPORT_NATIVE_STAGE_ORIGINAL_BYTES_REFUSED');
36
+ const identities=new Map<string,{source_identity:string;target_identity:string}>();
37
+ for(const plan of plans)for(const row of planIndex(plan,''))if(!plan.entries[row.ordinal].erased)identities.set(row.source_identity+':'+row.target_identity,row);
38
+ const index=[...identities.values()].map((row,ordinal)=>[ordinal,row.source_identity,row.target_identity]);
39
+ const jobIds=[...new Set([scope.job.id,...[...statements,...scope.postGuards].flatMap(statement=>statement.retainedValues?.privacyJobId?[statement.retainedValues.privacyJobId]:[])])];
40
+ const sizes=values.map(value=>{let count=0;for(const _ of chunks(value))count++;return count;});
41
+ const stage:NativeStageBinding={id,generation:randomUUID(),destination:context.destination,namespace:context.namespace,mapping_version:context.mappingVersion,principal_id:context.authorization.principalId,job_id:scope.job.id,fingerprint:scope.plan.fingerprint,job_fence:scope.job.fence,action,values_count:values.length,chunks_count:sizes.reduce((a,b)=>a+b,0),identities:index.length};
42
+ const valueRows=function*(){for(const [valueOrdinal,value] of values.entries()){let ordinal=0;for(const body of chunks(value))yield [valueOrdinal,ordinal++,body,sha256(body),Buffer.byteLength(body),sizes[valueOrdinal]];}};
43
+ const registered=index.length, sealed=registered+stage.chunks_count;
44
+ const begin=beginNativeStage(stage,baseline), discard=discardNativeStage(stage);
45
+ const registration=[...baseline,...nativeStageGuards(stage,registered),guard(sql('(SELECT count(*) FROM ik_d1_native_staging_index WHERE stage_id=?)=?',id,stage.identities)),sql('UPDATE ik_d1_native_staging SET registered_revision=revision WHERE id=?',id),...baseline,...nativeStageGuards(stage,registered)];
46
+ const seal=[...baseline,...nativeStageGuards(stage,sealed),guard(sql('EXISTS(SELECT 1 FROM ik_d1_native_staging WHERE id=? AND registered_revision IS NOT NULL) AND (SELECT count(*) FROM ik_d1_native_staging_values WHERE stage_id=?)=? AND (SELECT count(DISTINCT value_ordinal) FROM ik_d1_native_staging_values WHERE stage_id=?)=? AND NOT EXISTS(SELECT 1 FROM ik_d1_native_staging_values WHERE stage_id=? GROUP BY value_ordinal HAVING value_ordinal<0 OR value_ordinal>=? OR min(ordinal)<>0 OR max(ordinal)<>count(*)-1 OR min(value_chunks)<>count(*) OR max(value_chunks)<>count(*))',id,id,stage.chunks_count,id,stage.values_count,id,stage.values_count)),sql('UPDATE ik_d1_native_staging SET sealed_revision=revision WHERE id=?',id),...baseline,...nativeStageGuards(stage,sealed)];
47
+ const after=native.splice(statements.length);
48
+ const {publication,release,final}=nativePublication(context,stage,{baseline,writes:native,post:after,jobs:jobIds,targets:scope.targets,binding:nativeStagingPostBinding(scope),final:[...[...plans].flatMap(plan=>erasedIndexGuards(plan).map(query=>guardPredicate(query)!)),sql("NOT EXISTS(SELECT 1 FROM json_each(?) i WHERE EXISTS(SELECT 1 FROM ik_d1_erased_sources s WHERE s.destination=? AND s.identity=json_extract(i.value,'$[1]')) OR EXISTS(SELECT 1 FROM ik_d1_erased_targets t WHERE t.destination=? AND t.identity=json_extract(i.value,'$[2]')))",JSON.stringify(index),stage.destination,stage.destination),nativeStagingPostPredicate(scope)]});
49
+ // Every exact SQL member, authority predicate, readback, cleanup and return
50
+ // read is admitted before begin can retain any normalized private bytes.
51
+ const indexCost=nativeRowsCost(context,stage,baseline,'index',indexColumns,()=>index,0);
52
+ const valueCost=nativeRowsCost(context,stage,baseline,'values',valueColumns,valueRows,registered);
53
+ const cost=indexCost.queries+valueCost.queries+[begin,registration,seal,discard].reduce((n,batch)=>n+nativeStageCost(context,stage,batch),0)+nativeStageCost(context,stage,publication,release,final)+returnReads;
54
+ if(cost>remaining(context))throw new D1RequestLimitError();
55
+ try{
56
+ await nativeStageAtomic(context,action,begin);
57
+ const identityRows=new NativeStageRows(context,stage,baseline,'index',indexColumns);for(const row of index)await identityRows.append(row);await identityRows.flush();
58
+ await nativeStageAtomic(context,action,registration);
59
+ const payload=new NativeStageRows(context,stage,baseline,'values',valueColumns);payload.revision=registered;for(const row of valueRows())await payload.append(row);await payload.flush();
60
+ await nativeStageAtomic(context,action,seal);
61
+ const results=await nativeStageAtomic(context,action,publication,returnReads,release,final);
62
+ return results;
63
+ }catch(error){await nativeStageAtomic(context,action,discard);throw error;}
64
+ }
@@ -0,0 +1,53 @@
1
+ import type { D1Sql } from './types.ts';
2
+ import type { StoredPlan } from './plans.ts';
3
+ import { guard, sql } from './database.ts';
4
+ import { jsonChunks, PACKED_JSON_BYTES } from './sets.ts';
5
+
6
+ /** A copy descriptor contains positions, never a second copy of the bytes. */
7
+ interface Copy {
8
+ job_id: string; kind: string; original_key: string; ordinal: number;
9
+ first: number; start: number; length: number; second: number; tail: number;
10
+ }
11
+ export function originalCopies(encoded: StoredPlan, chunks: any[]): Copy[] {
12
+ const body = encoded.chunks.map(c => c.body).join('');
13
+ const starts: number[] = []; let size = 0;
14
+ for (const chunk of encoded.chunks) { starts.push(size); size += chunk.body.length; }
15
+ const grouped = new Map<string, any[]>();
16
+ for (const chunk of chunks) {
17
+ const key = JSON.stringify([chunk.kind, chunk.original_key]);
18
+ const group = grouped.get(key) ?? []; group.push(chunk); grouped.set(key, group);
19
+ }
20
+ const copies: Copy[] = [];
21
+ for (const group of grouped.values()) {
22
+ // Valid canonical base64 needs no escaping in canonical JSON. Any exact
23
+ // occurrence is equivalent; no source identity is inferred from position.
24
+ const base64 = group.map(c => c.bytes).join('');
25
+ const found = body.indexOf(`"${base64}"`);
26
+ if (found < 0) throw new Error('ORIGINAL_CUSTODY_MISMATCH');
27
+ let position = found + 1, first = 0;
28
+ for (const chunk of group) {
29
+ while (first + 1 < starts.length && starts[first + 1] <= position) first++;
30
+ const local = position - starts[first], text = encoded.chunks[first].body;
31
+ const length = Math.min(chunk.bytes.length, text.length - local), tail = chunk.bytes.length - length;
32
+ // Plan chunks use UTF-8 bounds. SQLite substr counts Unicode code points,
33
+ // not JavaScript UTF-16 units. The copied base64 itself is ASCII.
34
+ const start = Array.from(text.slice(0, local)).length + 1;
35
+ if (tail && (!encoded.chunks[first + 1] || tail > encoded.chunks[first + 1].body.length)) throw new Error('ORIGINAL_CUSTODY_MISMATCH');
36
+ copies.push({ job_id: chunk.job_id, kind: chunk.kind, original_key: chunk.original_key, ordinal: chunk.ordinal, first, start, length, second: tail ? first + 1 : first, tail });
37
+ position += chunk.bytes.length;
38
+ }
39
+ }
40
+ return copies;
41
+ }
42
+ const value = (key: string) => `json_extract(e.value,'$.${key}')`;
43
+ const joins = `JOIN ik_d1_plan_chunks a ON a.job_id=${value('job_id')} AND a.ordinal=${value('first')} JOIN ik_d1_plan_chunks b ON b.job_id=${value('job_id')} AND b.ordinal=${value('second')}`;
44
+ const bytes = `substr(a.body,${value('start')},${value('length')}) || substr(b.body,1,${value('tail')})`;
45
+ export function retainCopies(copies: Copy[]): D1Sql[] {
46
+ return jsonChunks(copies, PACKED_JSON_BYTES).map(chunk => sql(`INSERT INTO ik_d1_original_chunks(job_id,kind,original_key,ordinal,bytes) SELECT ${['job_id','kind','original_key','ordinal'].map(value).join(',')},${bytes} FROM json_each(?) e ${joins}`, chunk));
47
+ }
48
+ /** The caller also guards the exact plan chunks in this same transaction.
49
+ * Equality with those immutable chunks proves original content without binding
50
+ * another complete base64 copy into the D1 request. Cardinality is separate. */
51
+ export function copyGuards(copies: Copy[]): D1Sql[] {
52
+ return jsonChunks(copies, PACKED_JSON_BYTES).map(chunk => guard(sql(`NOT EXISTS(SELECT 1 FROM json_each(?) e WHERE NOT EXISTS(SELECT 1 FROM ik_d1_original_chunks t ${joins} WHERE ${['job_id','kind','original_key','ordinal'].map(c => `t.${c} IS ${value(c)}`).join(' AND ')} AND t.bytes IS (${bytes})))`, chunk)));
53
+ }