@runeditrun/rer 0.2.77 → 0.2.79

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (167) hide show
  1. package/README.md +32 -0
  2. package/cli/export.mjs +1 -1
  3. package/cli/portability-migrations.mjs +35 -14
  4. package/cli/portability-worker.mjs +2 -2
  5. package/cli/portability.mjs +2 -2
  6. package/cli/server-proxy.mjs +30 -8
  7. package/cli/upstream-agent.mjs +14 -0
  8. package/global/CHANGELOG.md +13 -0
  9. package/global/README.md +32 -0
  10. package/modules/demo-host/native/visitor.mjs +1 -1
  11. package/modules/deploy-kit/owner-cli.mjs +15 -5
  12. package/modules/deploy-kit/provider.mjs +9 -4
  13. package/modules/import-kit/src/d1/README.md +1322 -0
  14. package/modules/import-kit/src/d1/apply.ts +115 -0
  15. package/modules/import-kit/src/d1/budget.ts +89 -0
  16. package/modules/import-kit/src/d1/compressed-plan.ts +63 -0
  17. package/modules/import-kit/src/d1/creation.ts +95 -0
  18. package/modules/import-kit/src/d1/custody-fence.ts +27 -0
  19. package/modules/import-kit/src/d1/custody-state-schema.ts +67 -0
  20. package/modules/import-kit/src/d1/database.ts +163 -0
  21. package/modules/import-kit/src/d1/entry-locator.ts +61 -0
  22. package/modules/import-kit/src/d1/erased-artifacts.ts +38 -0
  23. package/modules/import-kit/src/d1/erased-custody.ts +36 -0
  24. package/modules/import-kit/src/d1/erased-plan.ts +47 -0
  25. package/modules/import-kit/src/d1/erased-reversal.ts +19 -0
  26. package/modules/import-kit/src/d1/erasure-bounded.ts +22 -0
  27. package/modules/import-kit/src/d1/erasure-claims.ts +27 -0
  28. package/modules/import-kit/src/d1/erasure-compiler.ts +67 -0
  29. package/modules/import-kit/src/d1/erasure-disposition.ts +16 -0
  30. package/modules/import-kit/src/d1/erasure-fragments.ts +86 -0
  31. package/modules/import-kit/src/d1/erasure-index.ts +53 -0
  32. package/modules/import-kit/src/d1/erasure-pages.ts +62 -0
  33. package/modules/import-kit/src/d1/erasure-progress-schema.ts +8 -0
  34. package/modules/import-kit/src/d1/erasure-progress.ts +61 -0
  35. package/modules/import-kit/src/d1/erasure-related.ts +124 -0
  36. package/modules/import-kit/src/d1/erasure-report.ts +31 -0
  37. package/modules/import-kit/src/d1/erasure-schema.ts +24 -0
  38. package/modules/import-kit/src/d1/erasure-selection.ts +77 -0
  39. package/modules/import-kit/src/d1/erasure-snapshot.ts +15 -0
  40. package/modules/import-kit/src/d1/erasure-stage.ts +146 -0
  41. package/modules/import-kit/src/d1/erasure-step.ts +12 -0
  42. package/modules/import-kit/src/d1/erasure-superseded.ts +98 -0
  43. package/modules/import-kit/src/d1/erasure-values.ts +83 -0
  44. package/modules/import-kit/src/d1/erasure-writes.ts +46 -0
  45. package/modules/import-kit/src/d1/erasure.ts +170 -0
  46. package/modules/import-kit/src/d1/evolutions/0001-preserved-undo.ts +13 -0
  47. package/modules/import-kit/src/d1/expected-state.ts +91 -0
  48. package/modules/import-kit/src/d1/field-redaction.ts +86 -0
  49. package/modules/import-kit/src/d1/guards.ts +37 -0
  50. package/modules/import-kit/src/d1/history.ts +71 -0
  51. package/modules/import-kit/src/d1/index.ts +88 -0
  52. package/modules/import-kit/src/d1/integrity.ts +127 -0
  53. package/modules/import-kit/src/d1/inventory-snapshot.ts +84 -0
  54. package/modules/import-kit/src/d1/jobs.ts +97 -0
  55. package/modules/import-kit/src/d1/native-set.ts +48 -0
  56. package/modules/import-kit/src/d1/native-staging-admission.ts +19 -0
  57. package/modules/import-kit/src/d1/native-staging-context.ts +44 -0
  58. package/modules/import-kit/src/d1/native-staging-lifecycle.ts +33 -0
  59. package/modules/import-kit/src/d1/native-staging-offset.ts +74 -0
  60. package/modules/import-kit/src/d1/native-staging-publication.ts +18 -0
  61. package/modules/import-kit/src/d1/native-staging-rows.ts +72 -0
  62. package/modules/import-kit/src/d1/native-staging-schema.ts +28 -0
  63. package/modules/import-kit/src/d1/native-staging-state.ts +22 -0
  64. package/modules/import-kit/src/d1/native-staging.ts +64 -0
  65. package/modules/import-kit/src/d1/original-copies.ts +53 -0
  66. package/modules/import-kit/src/d1/original-storage.ts +56 -0
  67. package/modules/import-kit/src/d1/originals.ts +74 -0
  68. package/modules/import-kit/src/d1/plan-dictionary.ts +80 -0
  69. package/modules/import-kit/src/d1/plans.ts +121 -0
  70. package/modules/import-kit/src/d1/preparation-fingerprint.ts +57 -0
  71. package/modules/import-kit/src/d1/preparation-header.ts +33 -0
  72. package/modules/import-kit/src/d1/preparation-reader.ts +98 -0
  73. package/modules/import-kit/src/d1/preparation-schema.ts +16 -0
  74. package/modules/import-kit/src/d1/preparation-state.ts +56 -0
  75. package/modules/import-kit/src/d1/preparation-step-combine.ts +80 -0
  76. package/modules/import-kit/src/d1/preparation-step-control.ts +48 -0
  77. package/modules/import-kit/src/d1/preparation-step-hash.ts +131 -0
  78. package/modules/import-kit/src/d1/preparation-step-pages.ts +77 -0
  79. package/modules/import-kit/src/d1/preparation-steps-schema.ts +11 -0
  80. package/modules/import-kit/src/d1/preparation-steps.ts +102 -0
  81. package/modules/import-kit/src/d1/preparations.ts +87 -0
  82. package/modules/import-kit/src/d1/preserved-preview.ts +89 -0
  83. package/modules/import-kit/src/d1/preserved-reverse.ts +91 -0
  84. package/modules/import-kit/src/d1/provenance-set.ts +87 -0
  85. package/modules/import-kit/src/d1/publication-fence.ts +22 -0
  86. package/modules/import-kit/src/d1/publication-receipt-schema.ts +15 -0
  87. package/modules/import-kit/src/d1/published-stream.ts +11 -0
  88. package/modules/import-kit/src/d1/read-fence.ts +64 -0
  89. package/modules/import-kit/src/d1/read-results.ts +16 -0
  90. package/modules/import-kit/src/d1/redacted-custody.ts +76 -0
  91. package/modules/import-kit/src/d1/redaction-step.ts +125 -0
  92. package/modules/import-kit/src/d1/referenced-plans.ts +136 -0
  93. package/modules/import-kit/src/d1/retained-values.ts +35 -0
  94. package/modules/import-kit/src/d1/retirement-bounded.ts +71 -0
  95. package/modules/import-kit/src/d1/retirement-door.ts +105 -0
  96. package/modules/import-kit/src/d1/retirement-schema.ts +39 -0
  97. package/modules/import-kit/src/d1/retirement-scope.ts +94 -0
  98. package/modules/import-kit/src/d1/retirement-steps.ts +76 -0
  99. package/modules/import-kit/src/d1/retirement-tally.ts +74 -0
  100. package/modules/import-kit/src/d1/retirement.ts +136 -0
  101. package/modules/import-kit/src/d1/reversal-conversion-record.ts +19 -0
  102. package/modules/import-kit/src/d1/reversal-conversion-schema.ts +11 -0
  103. package/modules/import-kit/src/d1/reversal-conversion.ts +40 -0
  104. package/modules/import-kit/src/d1/reversal-fence-schema.ts +14 -0
  105. package/modules/import-kit/src/d1/reversal-fence.ts +106 -0
  106. package/modules/import-kit/src/d1/reversal-journal.ts +24 -0
  107. package/modules/import-kit/src/d1/reverse.ts +103 -0
  108. package/modules/import-kit/src/d1/reviewed-source.ts +23 -0
  109. package/modules/import-kit/src/d1/sets.ts +78 -0
  110. package/modules/import-kit/src/d1/staged-creation.ts +201 -0
  111. package/modules/import-kit/src/d1/staging-admission.ts +45 -0
  112. package/modules/import-kit/src/d1/staging-control.ts +43 -0
  113. package/modules/import-kit/src/d1/staging-erasure.ts +28 -0
  114. package/modules/import-kit/src/d1/staging-lifecycle.ts +49 -0
  115. package/modules/import-kit/src/d1/staging-plan.ts +15 -0
  116. package/modules/import-kit/src/d1/staging-privacy.ts +22 -0
  117. package/modules/import-kit/src/d1/staging-publication.ts +79 -0
  118. package/modules/import-kit/src/d1/staging-schema.ts +37 -0
  119. package/modules/import-kit/src/d1/staging-statements.ts +8 -0
  120. package/modules/import-kit/src/d1/staging-transport.ts +70 -0
  121. package/modules/import-kit/src/d1/stream-boundary.ts +11 -0
  122. package/modules/import-kit/src/d1/stream-create.ts +55 -0
  123. package/modules/import-kit/src/d1/stream-identity-admission.ts +37 -0
  124. package/modules/import-kit/src/d1/stream-input.ts +18 -0
  125. package/modules/import-kit/src/d1/stream-layout.ts +43 -0
  126. package/modules/import-kit/src/d1/stream-origin.ts +44 -0
  127. package/modules/import-kit/src/d1/stream-page-integrity.ts +18 -0
  128. package/modules/import-kit/src/d1/stream-partition.ts +50 -0
  129. package/modules/import-kit/src/d1/stream-plan.ts +133 -0
  130. package/modules/import-kit/src/d1/stream-receipt.ts +45 -0
  131. package/modules/import-kit/src/d1/stream-revision-schema.ts +35 -0
  132. package/modules/import-kit/src/d1/stream-snapshot.ts +26 -0
  133. package/modules/import-kit/src/d1/stream-stage.ts +79 -0
  134. package/modules/import-kit/src/d1/stream-state.ts +25 -0
  135. package/modules/import-kit/src/d1/stream-working-set.ts +30 -0
  136. package/modules/import-kit/src/d1/streams.ts +39 -0
  137. package/modules/import-kit/src/d1/target-guards.ts +9 -0
  138. package/modules/import-kit/src/d1/target-identities.ts +17 -0
  139. package/modules/import-kit/src/d1/transport.ts +10 -0
  140. package/modules/import-kit/src/d1/types.ts +162 -0
  141. package/modules/import-kit/src/d1/undo-observations.ts +39 -0
  142. package/modules/import-kit/src/d1/undo-report.ts +22 -0
  143. package/modules/import-kit/src/d1/undo-snapshot.ts +87 -0
  144. package/modules/import-kit/src/d1/undo-state.ts +84 -0
  145. package/modules/import-kit/src/d1/undo-types.ts +110 -0
  146. package/modules/import-kit/src/journal/authority.ts +12 -0
  147. package/modules/import-kit/src/plan/budget.ts +60 -0
  148. package/modules/import-kit/src/plan/index.ts +123 -0
  149. package/modules/import-kit/src/plan/size.ts +122 -0
  150. package/modules/import-kit/src/provenance/canonical-hash.ts +130 -0
  151. package/modules/import-kit/src/provenance/canonical-stream.ts +85 -0
  152. package/modules/import-kit/src/provenance/credential-names.ts +131 -0
  153. package/modules/import-kit/src/provenance/credentials.ts +317 -0
  154. package/modules/import-kit/src/provenance/identity.ts +51 -0
  155. package/modules/import-kit/src/provenance/json-snapshot.ts +21 -0
  156. package/modules/import-kit/src/provenance/resumable-sha256.ts +75 -0
  157. package/modules/import-kit/src/provenance/validate-json.ts +19 -0
  158. package/modules/import-kit/src/storage/originals.ts +87 -0
  159. package/modules/import-kit/src/storage/provenance.ts +64 -0
  160. package/modules/import-kit/src/storage/schema.ts +42 -0
  161. package/modules/import-kit/src/types.ts +47 -0
  162. package/modules/portability/README.md +98 -2
  163. package/modules/portability/backfills.mjs +42 -15
  164. package/modules/portability/ledgers.mjs +7 -1
  165. package/modules/portability/transient-tables.mjs +40 -0
  166. package/package.json +3 -1
  167. package/scripts/publish-check/reviewed.py +3 -1
@@ -0,0 +1,89 @@
1
+ import type { D1ImportContext, D1Sql, D1ProvenanceRow } from './types.ts';
2
+ import type { D1UndoEffect, D1PreservedUndoPreview } from './undo-types.ts';
3
+ import { canonicalJson, sha256 } from '../provenance/identity.ts';
4
+ import { requestContext } from './budget.ts';
5
+ import { readAtomic } from './database.ts';
6
+ import { stored, boundary } from './jobs.ts';
7
+ import { noSuccessors } from './history.ts';
8
+ import { custodyGuards, inventoryGuards, validatedProvenance } from './integrity.ts';
9
+ import { readOriginals } from './originals.ts';
10
+ import { reversalState, undoEvidence } from './undo-state.ts';
11
+ import { classifyUndo } from './undo-snapshot.ts';
12
+ import { nativeStagingScope } from './native-staging-context.ts';
13
+ import { validateReversalJournal } from './reversal-journal.ts';
14
+ import { exactRows } from './sets.ts';
15
+ import { sql, guard } from './database.ts';
16
+
17
+ export class D1ReversalPolicyConversionError extends Error {
18
+ readonly code: 'IMPORT_CONVERSION_UNAVAILABLE' | 'IMPORT_CONVERSION_REVIEW_CHANGED' | 'IMPORT_CONVERSION_ALREADY_CONVERTED';
19
+ readonly reason: string;
20
+ constructor(code: D1ReversalPolicyConversionError['code'], reason: string) {
21
+ super(`${code}: ${reason}`); this.name = 'D1ReversalPolicyConversionError';
22
+ this.code = code; this.reason = reason;
23
+ }
24
+ }
25
+ export const unavailable = (reason: string) => new D1ReversalPolicyConversionError('IMPORT_CONVERSION_UNAVAILABLE', reason);
26
+
27
+ /** The same preparation powers review and conversion. Its guards are reused
28
+ * inside the converting transaction, never replaced by a post-write read. */
29
+ export async function preparePreservedPreview(id: string, context: D1ImportContext) {
30
+ // Undo authority, including its approval predicates, with no assertion writes.
31
+ await readAtomic(context, [], undefined, 'undo');
32
+ const current = await stored(id, context, 'undo'), { job, plan } = current, state = reversalState(job);
33
+ if (state.conversionHash === undefined || !await context.storage.database.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='ik_d1_policy_conversions'").first()) throw unavailable('schema');
34
+ if (!['completed','undoing'].includes(job.state)) throw unavailable('state');
35
+ if (!context.storage.targets.preparePreservedUndo) throw unavailable('unfenceable');
36
+ const guards: D1Sql[] = [...boundary(job, plan, 'undo'), ...custodyGuards(plan, id)];
37
+ try { guards.push(await noSuccessors(id, context)); }
38
+ catch (error) { if (error instanceof Error && error.message === 'IMPORT_SUPERSEDED') throw unavailable('superseded'); throw error; }
39
+ await readOriginals(plan, id, context);
40
+ guards.push(...await inventoryGuards(job, plan, context));
41
+ if (state.policy === 'strict') {
42
+ const batches = (await context.storage.database.prepare('SELECT * FROM ik_batches WHERE job_id=? ORDER BY fence').bind(id).all()).results;
43
+ const applyFence = Math.max(0, ...batches.filter(row => row.action === 'apply').map(row => row.fence));
44
+ if (job.state === 'undoing') validateReversalJournal(job, batches.filter(row => row.action !== 'apply'), applyFence);
45
+ else if (job.fence !== applyFence) throw unavailable('journal');
46
+ guards.push(...exactRows('ik_batches', ['job_id','action','position','end_position','fence'], batches, sql('job_id=?', id)));
47
+ }
48
+ const rows = (await context.storage.database.prepare('SELECT * FROM ik_provenance WHERE job_id=? ORDER BY ordinal DESC,effect_ordinal DESC').bind(id).all()).results;
49
+ const effects: D1UndoEffect[] = [], previous: typeof state.dispositions = [], previousGuards: D1Sql[] = [];
50
+ for (const raw of rows) {
51
+ const p = await validatedProvenance(raw, context, guards) as D1ProvenanceRow;
52
+ // Retention of a replacement would release predecessor history. Conversion
53
+ // cannot authorize that, even if the adapter calls the row unchanged today.
54
+ if (!p.reversed && p.predecessorId !== undefined) throw unavailable('predecessor-history');
55
+ const effect = plan.entries[raw.ordinal].effects[raw.effect_ordinal];
56
+ const expected = { destination: p.destination, targetKind: p.targetKind, targetId: p.targetId, revision: p.appliedRevision, value: effect.value, ...(effect.attachment ? { attachment: effect.attachment } : {}) };
57
+ if (p.reversed) {
58
+ if (state.policy === 'strict' && job.state !== 'undoing') throw unavailable('superseded');
59
+ if (state.policy === 'strict') {
60
+ // A strict journal has no effect dispositions. A present replacement
61
+ // could instead be a successor's retained effect: never relabel that
62
+ // as an earlier successful strict reversal. Require its exact restored
63
+ // before-image (or absence), and fence it through publication.
64
+ const restored = await context.storage.targets.read(context.storage.database, context.destination, p.targetKind, p.targetId) ?? null;
65
+ if (canonicalJson(restored) !== canonicalJson(p.before ?? null)) throw unavailable('unfenceable-previous-reversal');
66
+ previousGuards.push(guard(context.storage.targets.matches(restored, context.destination, p.targetKind, p.targetId)));
67
+ previous.push({ jobId: id, effectId: p.id, targetKind: p.targetKind, targetId: p.targetId, entity: { table: p.targetKind, recordId: p.targetId }, disposition: 'reversed', reason: 'unchanged', expected: undoEvidence(expected, context), observed: undoEvidence(restored, context), protectedBy: [], evidence: { priorStrictReversal: true } });
68
+ }
69
+ } else effects.push({ provenance: p, expected, retained: state.dispositions.some(d => d.effectId === p.id && d.disposition === 'retained') });
70
+ }
71
+ let decision: ReturnType<typeof classifyUndo>;
72
+ try { decision = classifyUndo(effects, await context.storage.targets.preparePreservedUndo(context.storage.database, context.destination, structuredClone(effects)), context); }
73
+ catch (error) {
74
+ if (error instanceof Error && /^(IMPORT_UNDO_|IMPORT_NATIVE_SET_GUARD_REQUIRED|INVALID_UNDO_OWNERSHIP)/.test(error.message)) throw unavailable('unfenceable');
75
+ throw error;
76
+ }
77
+ const nativeGuards = [...previousGuards, ...decision.guards];
78
+ guards.push(...nativeGuards);
79
+ const dispositions = [...state.dispositions, ...previous, ...decision.ordered];
80
+ const reviewedFingerprint = sha256(canonicalJson({ jobId: id, fingerprint: job.fingerprint, state: job.state, fence: job.fence, policy: state.policy, binding: state.binding, dispositions, guards: nativeGuards.map(g => ({ sql: g.sql, params: g.params ?? [] })) }));
81
+ return { ...current, guards, nativeGuards, previous, reviewedFingerprint, dispositions, policy: state.policy };
82
+ }
83
+
84
+ export async function previewD1PreservedUndo(id: string, context: D1ImportContext): Promise<D1PreservedUndoPreview> {
85
+ context = requestContext({ ...context, reversalPolicy: undefined });
86
+ const prepared = await preparePreservedPreview(id, context);
87
+ await readAtomic(context, prepared.guards, nativeStagingScope(prepared.job, prepared.plan), 'undo');
88
+ return { job: prepared.job, policy: prepared.policy, reviewedFingerprint: prepared.reviewedFingerprint, dispositions: prepared.dispositions };
89
+ }
@@ -0,0 +1,91 @@
1
+ import { retainedSql } from './retained-values.ts';
2
+ import { changed, putRow, updateRows, deleteRows } from './expected-state.ts';
3
+ import { nativeStagingScope, readStagedTarget, expectedStagedTarget } from './native-staging-context.ts';
4
+ import { targetGuards } from './target-guards.ts';
5
+ import type { ImportJob, ImportPlan } from '../types.ts';
6
+ import type { D1ImportContext, D1Sql } from './types.ts';
7
+ import type { D1UndoDisposition, D1UndoEffect } from './undo-types.ts';
8
+ import { canonicalJson } from '../provenance/identity.ts';
9
+ import { D1RequestLimitError, remaining } from './budget.ts';
10
+ import { atomic, sql, guard, limits } from './database.ts';
11
+ import { stored, storedReads, boundary } from './jobs.ts';
12
+ import { custodyGuards, inventoryGuards, validatedProvenance } from './integrity.ts';
13
+ import { noSuccessors, releaseKeptPredecessors, restorePredecessor } from './history.ts';
14
+ import { readOriginals } from './originals.ts';
15
+ import { reversalState, reportHash, reversalGuards, undoEvidence } from './undo-state.ts';
16
+ import { classifyUndo } from './undo-snapshot.ts';
17
+ import { insertJsonRows } from './sets.ts';
18
+
19
+ /** Called after public authority, binding, reversal-mode and custody checks. */
20
+ export async function preserveReverse(current: { job: ImportJob; plan: ImportPlan }, context: D1ImportContext, action: 'undo' | 'rollback'): Promise<ImportJob> {
21
+ const { maxBatches, batchSize } = limits(context), id = current.job.id;
22
+ const reversing = action === 'undo' ? 'undoing' : 'rolling_back', terminal = action === 'undo' ? 'undone' : 'rolled_back';
23
+ const { database: db, targets } = context.storage;
24
+ if (!targets.preparePreservedUndo) throw new Error('IMPORT_PRESERVED_UNDO_ADAPTER_REQUIRED');
25
+ if (current.job.state !== reversing) {
26
+ await atomic(context, action, [...boundary(current.job, current.plan, action), await noSuccessors(id, context), ...custodyGuards(current.plan, id), ...await inventoryGuards(current.job, current.plan, context), changed(sql('UPDATE ik_jobs SET state=?,fence=fence+1 WHERE id=?', reversing, id),updateRows('ik_jobs',row=>row.id===id,row=>({...row,state:reversing,fence:current.job.fence+1})))], storedReads(current.job),nativeStagingScope(current.job,current.plan));
27
+ current = await stored(id, context, action);
28
+ }
29
+ let published: ImportJob | undefined;
30
+ try {
31
+ for (let count = 0; count < maxBatches; count++) {
32
+ const { job, plan } = current, state = reversalState(job);
33
+ if (job.state !== reversing) throw new Error('REVERSAL_MODE_CONFLICT');
34
+ await readOriginals(plan, id, context);
35
+ const scope=nativeStagingScope(job,plan);
36
+ const statements: D1Sql[] = [...boundary(job, plan, action), await noSuccessors(id, context), ...custodyGuards(plan, id), ...await inventoryGuards(job, plan, context)];
37
+ const { results: rows } = await db.prepare('SELECT * FROM ik_provenance WHERE job_id=? AND reversed=0 ORDER BY ordinal DESC,effect_ordinal DESC').bind(id).all();
38
+ const retained = new Set(state.dispositions.filter(d => d.disposition === 'retained').map(d => d.effectId));
39
+ const effects: D1UndoEffect[] = [];
40
+ for (const raw of rows) {
41
+ const p = await validatedProvenance(raw, context, statements), effect = plan.entries[raw.ordinal].effects[raw.effect_ordinal];
42
+ effects.push({ provenance: p, retained: retained.has(p.id), expected: { destination: p.destination, targetKind: p.targetKind, targetId: p.targetId, revision: p.appliedRevision, value: effect.value, ...(effect.attachment ? { attachment: effect.attachment } : {}) } });
43
+ }
44
+ // Effects a later import's kept effect released (releaseKeptPredecessors):
45
+ // reversed in provenance, never restorable, reported kept once.
46
+ const superseded: D1UndoDisposition[] = [];
47
+ for (const raw of (await db.prepare('SELECT * FROM ik_provenance p WHERE p.job_id=? AND p.reversed=1 AND NOT EXISTS(SELECT 1 FROM ik_d1_reversal_reports r WHERE r.job_id=p.job_id AND r.effect_id=p.id) ORDER BY ordinal DESC,effect_ordinal DESC').bind(id).all()).results) {
48
+ const p = await validatedProvenance(raw, context, statements), effect = plan.entries[raw.ordinal].effects[raw.effect_ordinal];
49
+ statements.push(guard(sql('NOT EXISTS(SELECT 1 FROM ik_d1_reversal_reports WHERE job_id=? AND effect_id=?)', id, p.id)));
50
+ superseded.push({ jobId: id, effectId: p.id, targetKind: p.targetKind, targetId: p.targetId, entity: { table: p.targetKind, recordId: p.targetId }, disposition: 'retained', reason: 'superseded',
51
+ expected: undoEvidence({ destination: p.destination, targetKind: p.targetKind, targetId: p.targetId, revision: p.appliedRevision, value: effect.value, ...(effect.attachment ? { attachment: effect.attachment } : {}) }, context),
52
+ observed: undoEvidence(null, context), protectedBy: [], evidence: null });
53
+ }
54
+ const decision = classifyUndo(effects, await targets.preparePreservedUndo(db, context.destination, structuredClone(effects)), context);
55
+ statements.push(...decision.guards);
56
+ if(context.maxBatchBytes!==undefined)for(const effect of effects)await readStagedTarget(context,scope,effect.provenance.targetKind,effect.provenance.targetId);
57
+ const selected: typeof decision.ordered = [], newRows: any[] = superseded.map((d, index) => ({ job_id: id, effect_id: d.effectId, position: state.cursor + index, record_json: canonicalJson(d) }));
58
+ for (const group of decision.groups) {
59
+ if (selected.length && selected.length + group.length > batchSize) break;
60
+ selected.push(...group);
61
+ if (selected.length >= batchSize) break;
62
+ }
63
+ for (const d of selected) {
64
+ const effect = effects.find(e => e.provenance.id === d.effectId)!, p = effect.provenance;
65
+ if (d.disposition === 'reversed') {
66
+ const restoration: D1Sql[] = [];
67
+ await restorePredecessor(p, context, statements, restoration);
68
+ const o = decision.observations.get(p.id)!;
69
+ statements.push(guard(o.guard ?? targets.matches(o.current, context.destination, p.targetKind, p.targetId)), ...(p.before ? targets.write(p.before) : targets.remove(context.destination, p.targetKind, p.targetId)), guard(targets.matches(p.before, context.destination, p.targetKind, p.targetId)), changed(retainedSql(['UPDATE ik_provenance SET reversed=1,record_json=', ' WHERE id=', ''], [canonicalJson({ ...p, reversed: true }), p.id], [0]),updateRows('ik_provenance',row=>row.id===p.id,row=>({...row,reversed:1,record_json:canonicalJson({...p,reversed:true})}))), ...restoration);
70
+ } else await releaseKeptPredecessors(p, context, statements);
71
+ expectedStagedTarget(scope,p.targetKind,p.targetId,d.disposition==='reversed'?!!p.before:!!decision.observations.get(p.id)!.current);
72
+ newRows.push({ job_id: id, effect_id: d.effectId, position: state.cursor + newRows.length, record_json: canonicalJson(d) });
73
+ }
74
+ if(context.maxBatchBytes!==undefined)for(const effect of effects){const p=effect.provenance,reverse=selected.some(d=>d.effectId===p.id&&d.disposition==='reversed'),row=reverse?p.before:decision.observations.get(p.id)!.current;scope.postGuards.push(...targetGuards(targets,row,context.destination,p.targetKind,p.targetId));}
75
+ statements.push(...insertJsonRows('ik_d1_reversal_reports', ['job_id','effect_id','position','record_json'], newRows));
76
+ const done = selected.length === decision.ordered.length, allRows = [...state.rows, ...newRows];
77
+ const next = { ...state, cursor: allRows.length, hash: reportHash(allRows), rows: allRows, dispositions: [...state.dispositions, ...superseded, ...selected] };
78
+ statements.push(changed(sql('DELETE FROM ik_sources WHERE job_id=? AND NOT EXISTS(SELECT 1 FROM ik_provenance WHERE ik_provenance.source_identity=ik_sources.identity AND reversed=0) AND NOT EXISTS(SELECT 1 FROM ik_d1_history WHERE ik_d1_history.source_identity=ik_sources.identity)', id),deleteRows('ik_sources',(row,expected)=>row.job_id===id&&![...expected.rows('ik_provenance').values()].some(p=>p.source_identity===row.identity&&!p.reversed)&&![...expected.rows('ik_d1_history').values()].some(p=>p.source_identity===row.identity))),
79
+ changed(sql('INSERT INTO ik_batches(job_id,action,position,end_position,fence) VALUES(?,?,?,?,?)', id, action, job.fence, job.fence + 1, job.fence + 1),putRow('ik_batches',{job_id:id,action,position:job.fence,end_position:job.fence+1,fence:job.fence+1})),
80
+ changed(sql('UPDATE ik_jobs SET state=?,fence=fence+1,error=NULL,reversal_cursor=?,reversal_hash=? WHERE id=?', done ? terminal : reversing, next.cursor, next.hash, id),updateRows('ik_jobs',row=>row.id===id,row=>({...row,state:done?terminal:reversing,fence:job.fence+1,error:null,reversal_cursor:next.cursor,reversal_hash:next.hash}))),
81
+ ...reversalGuards(id, next));
82
+ await atomic(context, action, statements, storedReads(job),scope);
83
+ current = await stored(id, context, action); published = current.job;
84
+ if (done || remaining(context) < 200) return published;
85
+ }
86
+ } catch (error) {
87
+ if (error instanceof D1RequestLimitError && published) return published;
88
+ throw error;
89
+ }
90
+ return current.job;
91
+ }
@@ -0,0 +1,87 @@
1
+ import type { InventorySnapshot } from './inventory-snapshot.ts';
2
+ import { planFingerprint } from '../plan/index.ts';
3
+ import { canonicalJson, sha256, attachmentHash } from '../provenance/identity.ts';
4
+ import { decodeProvenance, effectIdentity } from '../storage/provenance.ts';
5
+ import type { D1ImportContext, D1Sql, D1ProvenanceRow } from './types.ts';
6
+ import { guard, sql } from './database.ts';
7
+ import { selectKeys, exactRows, jobColumns, claimColumns, provenanceColumns, jsonChunks } from './sets.ts';
8
+ import { loadPlans, planGuards } from './plans.ts';
9
+ import { ownerHeads, loadReferencedPlans, isReferencedOnly, reportedJobs } from './referenced-plans.ts';
10
+ import { loadReversalState, reversalGuards,snapshotReversalGuards } from './undo-state.ts';
11
+ import { readOriginals } from './originals.ts';
12
+ import { custodyGuards } from './original-storage.ts';
13
+
14
+ /** One preflight closure, no process-wide cache. Every row used here is guarded
15
+ * with its complete stored fields before publication. Without `guards`, the
16
+ * caller fences `jobs` (every job whose rows were read) by custody revisions
17
+ * captured before the first read instead; validation is unchanged. */
18
+ export async function provenanceSet(raws: any[], context: D1ImportContext, guards: D1Sql[] | undefined, extraOwners: string[] = [], extraClaims: string[] = [], snapshot?: InventorySnapshot) {
19
+ const db = context.storage.database;
20
+ const read = (table: string, key: string, values: string[]) => snapshot ? Promise.resolve(snapshot.rows(table, key, values)) : selectKeys(db, table, key, values);
21
+ // A fenced read reads each owner's header and referenced entries, not its plan.
22
+ const referenced = !guards && !snapshot && context.maxBatchBytes === undefined;
23
+ const ownerRows = (ids: string[]) => referenced ? ownerHeads(db, ids) : read('ik_jobs', 'id', ids);
24
+ // Close preserved successors' history in both directions. Strict jobs retain
25
+ // their existing export contract; strict undo still uses restorePredecessor.
26
+ const initialOwners = await ownerRows([...extraOwners, ...raws.map(r => r.job_id)]);
27
+ const preservedOwners = new Set(initialOwners.filter(r => r.reversal_policy === 'preserve-conflicts').map(r => r.id));
28
+ const originalLength = raws.length, all = [...raws], known = new Map(raws.map(r => [r.id, r]));
29
+ let level = raws.filter(r => preservedOwners.has(r.job_id));
30
+ while (level.length) {
31
+ const ids = level.filter(r => !r.reversed).map(r => (decodeProvenance(r) as D1ProvenanceRow).predecessorId).filter((id): id is string => id !== undefined && !known.has(id));
32
+ const predecessors = await read('ik_d1_history', 'id', ids), next: any[] = [];
33
+ for (const prior of predecessors) { if (!known.has(prior.id)) { known.set(prior.id, prior); all.push(prior); next.push(prior); } }
34
+ for (const raw of level) {
35
+ const p = decodeProvenance(raw) as D1ProvenanceRow;
36
+ if (p.predecessorId === undefined || p.reversed) continue;
37
+ const prior = known.get(p.predecessorId);
38
+ if (p.predecessorId === p.id || !prior || prior.successor_id !== p.id || prior.reversed !== 0) throw new Error('IMPORT_HISTORY_MISMATCH');
39
+ }
40
+ level = next;
41
+ }
42
+ raws = all;
43
+ const owners = [...initialOwners, ...await ownerRows(raws.map(r => r.job_id).filter(id => !initialOwners.some(o => o.id === id)))];
44
+ const codec = context.storage.targets.planCodec, ownerMap = new Map(owners.map(r => [r.id, r]));
45
+ const plans = snapshot ? new Map(owners.map(owner => [owner.id, snapshot.plans.get(owner.id)!])) : referenced ? await loadReferencedPlans(owners, raws, db, codec) : await loadPlans(owners, db, codec, undefined, !!guards && context.maxBatchBytes === undefined);
46
+ // Preserved jobs with no reversal report yet: one existence read for all of them.
47
+ const reported = referenced ? await reportedJobs(db, owners.filter(o => o.reversal_policy === 'preserve-conflicts' && o.reversal_cursor === 0).map(o => o.id)) : undefined;
48
+ for (const owner of owners) {
49
+ const plan = plans.get(owner.id)!;
50
+ if ((!isReferencedOnly(plan) && planFingerprint(plan) !== owner.fingerprint) || plan.fingerprint !== owner.fingerprint || plan.version !== 1 || plan.destination !== owner.destination || plan.namespace !== owner.namespace || plan.mappingVersion !== owner.mapping_version || plan.principalId !== owner.principal_id || owner.total !== plan.entries.length || !Number.isSafeInteger(owner.checkpoint) || owner.checkpoint < 0 || owner.checkpoint > owner.total) throw new Error('PROVENANCE_JOB_MISMATCH');
51
+ const reversal=await loadReversalState(owner,plan,context,false,snapshot,reported&&owner.reversal_cursor===0&&!reported.has(owner.id)?[]:undefined);
52
+ guards?.push(...planGuards(plan,owner.id),...(snapshot?.tables.has('ik_d1_reversal_reports')?snapshotReversalGuards(owner,reversal):reversalGuards(owner.id,reversal)));
53
+ if(context.maxBatchBytes!==undefined){await readOriginals(plan,owner.id,context,snapshot);guards?.push(...custodyGuards(plan,owner.id));}
54
+ }
55
+ const claims = await read('ik_sources', 'identity', [...extraClaims, ...raws.filter(r => !r.reversed).map(r => r.source_identity)]);
56
+ const claimMap = new Map(claims.map(r => [r.identity, r]));
57
+ const successorIds = raws.filter(r => r.successor_id !== undefined).map(r => r.successor_id);
58
+ const activeSuccessors = await read('ik_provenance', 'id', successorIds);
59
+ const historicalSuccessors = await read('ik_d1_history', 'id', successorIds);
60
+ if (guards) {
61
+ guards.push(...exactRows('ik_jobs', jobColumns, owners), ...exactRows('ik_sources', claimColumns, claims));
62
+ // Each successor must exist in exactly one store, including at the boundary.
63
+ for (const chunk of jsonChunks([...new Set(successorIds)])) guards.push(guard(sql('NOT EXISTS(SELECT 1 FROM json_each(?) e WHERE (SELECT COUNT(*) FROM ik_provenance WHERE id=e.value)+(SELECT COUNT(*) FROM ik_d1_history WHERE id=e.value)<>1)', chunk)));
64
+ guards.push(...exactRows('ik_provenance', provenanceColumns, activeSuccessors), ...exactRows('ik_d1_history', [...provenanceColumns,'successor_id'], historicalSuccessors));
65
+ guards.push(...exactRows('ik_provenance', provenanceColumns, raws.filter(r => r.successor_id === undefined)), ...exactRows('ik_d1_history', [...provenanceColumns,'successor_id'], raws.filter(r => r.successor_id !== undefined)));
66
+ }
67
+ const successors = new Map<string, any[]>();
68
+ for (const row of [...activeSuccessors, ...historicalSuccessors]) successors.set(row.id, [...(successors.get(row.id) ?? []), row]);
69
+ const decoded = raws.map(raw => {
70
+ const p = decodeProvenance(raw), owner = ownerMap.get(p.jobId), plan = plans.get(p.jobId);
71
+ if (!owner || !plan) throw new Error('PROVENANCE_JOB_MISSING');
72
+ const entry = plan.entries[raw.ordinal], effect = entry?.effects[raw.effect_ordinal];
73
+ if (!Number.isSafeInteger(raw.ordinal) || !Number.isSafeInteger(raw.effect_ordinal) || !entry || !effect || raw.ordinal >= owner.checkpoint || p.id !== effectIdentity(p.jobId, entry.identity, raw.effect_ordinal) || p.sourceIdentity !== entry.identity || p.sourceKind !== entry.source.kind || p.sourceId !== entry.source.sourceId || p.sourceHash !== entry.source.payloadHash || p.artifactHash !== entry.source.artifactHash || canonicalJson(p.location) !== canonicalJson(entry.source.location) || p.destination !== plan.destination || p.targetKind !== effect.targetKind || p.targetId !== effect.targetId || p.valueHash !== sha256(canonicalJson(effect.value)) || p.attachmentHash !== attachmentHash(effect.attachment) || p.appliedRevision !== sha256(canonicalJson([p.jobId, entry.identity, raw.effect_ordinal, 'applied']))) throw new Error('PROVENANCE_ATTRIBUTION_MISMATCH');
74
+ if (p.before === undefined || (p.before ? p.before.destination !== p.destination || p.before.targetKind !== p.targetKind || p.before.targetId !== p.targetId || p.before.revision !== effect.expectedRevision : effect.expectedRevision !== undefined)) throw new Error('PROVENANCE_BEFORE_IMAGE_MISMATCH');
75
+ if (!p.reversed) {
76
+ const claim = claimMap.get(p.sourceIdentity);
77
+ if (!claim || claim.job_id !== p.jobId || claim.source_hash !== entry.source.payloadHash || claim.mapping_version !== plan.mappingVersion || claim.effects_hash !== sha256(canonicalJson(entry.effects))) throw new Error('PROVENANCE_SOURCE_CLAIM_MISMATCH');
78
+ }
79
+ if (raw.successor_id !== undefined) {
80
+ const found = successors.get(raw.successor_id) ?? [], next = found.length === 1 ? decodeProvenance(found[0]) as D1ProvenanceRow : undefined;
81
+ if (p.reversed || typeof raw.successor_id !== 'string' || !next || found[0].reversed !== 0 || next.reversed || next.id === p.id || next.predecessorId !== p.id || !next.before || next.destination !== p.destination || next.targetKind !== p.targetKind || next.targetId !== p.targetId || next.before.revision !== p.appliedRevision || sha256(canonicalJson(next.before.value)) !== p.valueHash || attachmentHash(next.before.attachment) !== p.attachmentHash) throw new Error('IMPORT_HISTORY_MISMATCH');
82
+ }
83
+ return p;
84
+ });
85
+ const jobs = new Set([...ownerMap.keys(), ...[...raws, ...activeSuccessors, ...historicalSuccessors, ...claims].map(row => row.job_id)]);
86
+ return { decoded: decoded.slice(0, originalLength), claims: claimMap, owners: ownerMap, plans, jobs };
87
+ }
@@ -0,0 +1,22 @@
1
+ /** Exact publication is immutable until private cleanup and the final current
2
+ * authority check finish. These guards carry only identities, never payloads. */
3
+ const guarded = [
4
+ ['ik_jobs', 'id', 'job_id'], ['ik_d1_streams', 'id', 'stream_id'],
5
+ ...['ik_d1_plan_chunks','ik_d1_plan_index','ik_originals','ik_d1_original_chunks','ik_original_sets','ik_d1_indexed_jobs','ik_d1_reversal_reports','ik_provenance','ik_sources','ik_d1_entries','ik_d1_history','ik_batches'].map(table => [table, 'job_id', 'job_id']),
6
+ ['ik_d1_stream_pages', 'stream_id', 'stream_id'],
7
+ ];
8
+ export const publicationFenceSchema = [
9
+ 'CREATE TABLE IF NOT EXISTS ik_d1_publication_fences (stream_id TEXT NOT NULL,job_id TEXT PRIMARY KEY)',
10
+ 'CREATE INDEX IF NOT EXISTS ik_d1_publication_fence_stream ON ik_d1_publication_fences(stream_id)',
11
+ ...guarded.flatMap(([table, column, fence]) => ['INSERT','UPDATE','DELETE'].map(event => {
12
+ const refs = event === 'INSERT' ? `${fence}=NEW.${column}` : event === 'DELETE' ? `${fence}=OLD.${column}` : `${fence}=OLD.${column} OR ${fence}=NEW.${column}`;
13
+ return `CREATE TRIGGER IF NOT EXISTS ik_d1_publication_${table}_${event.toLowerCase()} BEFORE ${event} ON ${table} WHEN EXISTS(SELECT 1 FROM ik_d1_publication_fences WHERE ${refs}) BEGIN SELECT RAISE(ABORT,'IMPORT_PUBLICATION_CHANGED'); END`;
14
+ })),
15
+ // A native publication uses its private generation as stream_id. Fence the
16
+ // real parent through the guarded child as well as through a stream receipt.
17
+ ...['UPDATE','DELETE'].map(event=>`CREATE TRIGGER IF NOT EXISTS ik_d1_publication_parent_${event.toLowerCase()} BEFORE ${event} ON ik_d1_streams WHEN EXISTS(SELECT 1 FROM ik_d1_stream_pages p JOIN ik_d1_publication_fences f ON f.job_id=p.job_id WHERE p.stream_id=OLD.id) BEGIN SELECT RAISE(ABORT,'IMPORT_PUBLICATION_CHANGED'); END`),
18
+ ...['INSERT','UPDATE','DELETE'].map(event=>{
19
+ const refs=event==='INSERT'?'job_id=NEW.job_id':event==='DELETE'?'job_id=OLD.job_id':'job_id=OLD.job_id OR job_id=NEW.job_id';
20
+ return `CREATE TRIGGER IF NOT EXISTS ik_d1_publication_child_page_${event.toLowerCase()} BEFORE ${event} ON ik_d1_stream_pages WHEN EXISTS(SELECT 1 FROM ik_d1_publication_fences WHERE ${refs}) BEGIN SELECT RAISE(ABORT,'IMPORT_PUBLICATION_CHANGED'); END`;
21
+ }),
22
+ ];
@@ -0,0 +1,15 @@
1
+ /** The known publication receipt survives private cleanup. Every public write
2
+ * invalidates it; a non-null receipt can be installed only under its fence. */
3
+ export const publicationReceiptSchema = [
4
+ "CREATE TRIGGER IF NOT EXISTS ik_d1_custody_receipt_revision AFTER UPDATE OF revision ON ik_d1_custody_revisions BEGIN UPDATE ik_d1_custody_revisions SET seal=NULL WHERE job_id=NEW.job_id; END",
5
+ ...['INSERT','UPDATE OF seal'].map(event=>`CREATE TRIGGER IF NOT EXISTS ik_d1_custody_receipt_${event.startsWith('INSERT')?'insert':'seal'} BEFORE ${event} ON ik_d1_custody_revisions WHEN NEW.seal IS NOT NULL AND NOT EXISTS(SELECT 1 FROM ik_d1_publication_fences WHERE job_id=NEW.job_id AND stream_id=NEW.seal) BEGIN SELECT RAISE(ABORT,'IMPORT_PUBLICATION_RECEIPT_CHANGED'); END`),
6
+ ...[['ik_jobs','id'],...['ik_d1_plan_chunks','ik_d1_plan_index','ik_originals','ik_d1_original_chunks','ik_original_sets','ik_d1_indexed_jobs','ik_d1_reversal_reports','ik_provenance','ik_sources','ik_d1_entries','ik_d1_history','ik_batches'].map(table=>[table,'job_id'])].flatMap(([table,column])=>['INSERT','UPDATE','DELETE'].map(event=>{
7
+ const refs=event==='INSERT'?`job_id=NEW.${column}`:event==='DELETE'?`job_id=OLD.${column}`:`job_id=OLD.${column} OR job_id=NEW.${column}`;
8
+ return `CREATE TRIGGER IF NOT EXISTS ik_d1_receipt_${table}_${event.toLowerCase()} AFTER ${event} ON ${table} BEGIN UPDATE ik_d1_custody_revisions SET seal=NULL WHERE ${refs}; END`;
9
+ })),
10
+ ...['INSERT','UPDATE','DELETE'].map(event=>{
11
+ const refs=event==='INSERT'?'job_id=NEW.job_id':event==='DELETE'?'job_id=OLD.job_id':'job_id=OLD.job_id OR job_id=NEW.job_id';
12
+ return `CREATE TRIGGER IF NOT EXISTS ik_d1_receipt_pages_${event.toLowerCase()} AFTER ${event} ON ik_d1_stream_pages BEGIN UPDATE ik_d1_custody_revisions SET seal=NULL WHERE ${refs}; END`;
13
+ }),
14
+ ...['UPDATE','DELETE'].map(event=>`CREATE TRIGGER IF NOT EXISTS ik_d1_receipt_stream_${event.toLowerCase()} BEFORE ${event} ON ik_d1_streams BEGIN UPDATE ik_d1_custody_revisions SET seal=NULL WHERE job_id IN (SELECT job_id FROM ik_d1_stream_pages WHERE stream_id=OLD.id); END`),
15
+ ];
@@ -0,0 +1,11 @@
1
+ import type { D1ImportContext } from './types.ts';
2
+ import { readStreamReceipt, streamReceipt } from './stream-receipt.ts';
3
+
4
+ /** Creation/replay and public status share one bounded, SELECT-only receipt.
5
+ * Creation has its own current authority and exact operation/fingerprint bind. */
6
+ export function publishedStreamReceipt(context: D1ImportContext, id: string, operation: string, fingerprint: string) {
7
+ return streamReceipt(context, id, 'create', { operation, fingerprint });
8
+ }
9
+ export async function publishedStream(context: D1ImportContext, id: string, operation: string, fingerprint: string) {
10
+ return (await readStreamReceipt(context, id, 'create', { operation, fingerprint })).job;
11
+ }
@@ -0,0 +1,64 @@
1
+ import type { ImportAction } from '../types.ts';
2
+ import type { D1ImportContext, D1Sql } from './types.ts';
3
+ import type { NativeStagingScope } from './native-staging-context.ts';
4
+ import { nativeStagingPostBinding } from './native-staging-context.ts';
5
+ import { authority, guard, prepare, sql } from './database.ts';
6
+ import { D1RequestLimitError, packGuards, remaining } from './budget.ts';
7
+ import { batchBytes } from './transport.ts';
8
+ import { batchRows, checkBatchResults } from './read-results.ts';
9
+ import { erasedIndexGuards } from './erasure-index.ts';
10
+ import { jsonChunks } from './sets.ts';
11
+
12
+ /** Check exact custody without writing private copies merely to read them.
13
+ * Every batch sees the same monotonic revisions, captured before any exact
14
+ * check. The final fence also retains the plan revision captured before decode.
15
+ * Native target reads have no such custody fence and remain one atomic batch. */
16
+ export async function readBoundedGuards(context: D1ImportContext, guards: D1Sql[], scope: NativeStagingScope, limit: number, action: ImportAction = 'read'): Promise<void> {
17
+ const ids = new Set([scope.job.id]);
18
+ for (const query of guards) {
19
+ if (query.retainedValues?.privacyJobId) ids.add(query.retainedValues.privacyJobId);
20
+ for (const snapshot of query.expectedSnapshots ?? []) {
21
+ for (const row of snapshot.rows) {
22
+ for (const key of ['job_id', 'owner_job_id']) if (typeof row[key] === 'string') ids.add(row[key]);
23
+ if (snapshot.table === 'ik_jobs' && typeof row.id === 'string') ids.add(row.id);
24
+ }
25
+ if (snapshot.selection?.key === 'job_id' || snapshot.table === 'ik_jobs' && snapshot.selection?.key === 'id') {
26
+ for (const id of snapshot.selection.values) ids.add(id);
27
+ }
28
+ }
29
+ }
30
+ const revisions: [string, number, number][] = [];
31
+ const grant = context.authorization.predicate(action, context.destination);
32
+ for (const body of jsonChunks([...ids])) {
33
+ const query = sql(`SELECT j.id,s.revision AS state_revision,r.revision AS custody_revision FROM ik_jobs j JOIN ik_d1_custody_states s ON s.job_id=j.id JOIN ik_d1_custody_revisions r ON r.job_id=j.id WHERE j.id IN (SELECT value FROM json_each(?)) AND (${grant.sql})`, body, ...grant.params ?? []);
34
+ if (batchBytes([query]) > limit) throw new Error('IMPORT_D1_TRANSPORT_LIMIT');
35
+ const rows = batchRows(await context.storage.database.prepare(query.sql).bind(...query.params!).all(), 'IMPORT_ATOMIC_GUARD');
36
+ for (const row of rows) {
37
+ if (!ids.has(row.id) || !Number.isSafeInteger(row.state_revision) || row.state_revision < 0 || !Number.isSafeInteger(row.custody_revision) || row.custody_revision < 0) throw new Error('IMPORT_ATOMIC_GUARD');
38
+ revisions.push([row.id, row.state_revision, row.custody_revision]);
39
+ }
40
+ }
41
+ if (revisions.length !== ids.size || new Set(revisions.map(row => row[0])).size !== ids.size) throw new Error('IMPORT_ATOMIC_GUARD');
42
+ const fences = jsonChunks(revisions).map(body => guard(sql("NOT EXISTS(SELECT 1 FROM json_each(?) e WHERE NOT EXISTS(SELECT 1 FROM ik_d1_custody_states s JOIN ik_d1_custody_revisions r ON r.job_id=s.job_id WHERE s.job_id=json_extract(e.value,'$[0]') AND s.revision=json_extract(e.value,'$[1]') AND r.revision=json_extract(e.value,'$[2]')))", body)));
43
+ const prefix = [authority(context, action), ...fences, ...nativeStagingPostBinding(scope)];
44
+ const batches: D1Sql[][] = []; let pending: D1Sql[] = [];
45
+ const packed = (members: D1Sql[]) => packGuards([...prefix, ...members], true);
46
+ for (const member of [...guards, ...erasedIndexGuards(scope.plan)]) {
47
+ if (batchBytes(packed([...pending, member])) > limit) {
48
+ if (!pending.length) throw new Error('IMPORT_D1_TRANSPORT_LIMIT');
49
+ batches.push(packed(pending)); pending = [];
50
+ if (batchBytes(packed([member])) > limit) throw new Error('IMPORT_D1_TRANSPORT_LIMIT');
51
+ }
52
+ pending.push(member);
53
+ }
54
+ batches.push(packed(pending));
55
+ if (batches.reduce((n, batch) => n + batch.length, 0) > remaining(context)) throw new D1RequestLimitError();
56
+ for (const batch of batches) {
57
+ const results = await context.storage.database.batch(batch.map(query => prepare(context.storage.database, query)));
58
+ checkBatchResults(results, batch.length, 'IMPORT_ATOMIC_GUARD');
59
+ for (const result of results) {
60
+ const rows = batchRows(result, 'IMPORT_ATOMIC_GUARD');
61
+ if (rows.length !== 1 || rows[0].ok !== 1) throw new Error('IMPORT_ATOMIC_GUARD');
62
+ }
63
+ }
64
+ }
@@ -0,0 +1,16 @@
1
+ /** Both supported port shapes must be successful, complete rowsets. An adapter
2
+ * cannot turn a failed or missing D1 envelope into an empty successful receipt. */
3
+ export function batchRows(result: unknown, error = 'IMPORT_D1_READ_RECEIPT_REQUIRED'): Record<string, any>[] {
4
+ let rows: unknown;
5
+ if (Array.isArray(result)) rows = result;
6
+ else if (result && typeof result === 'object' && 'results' in result) {
7
+ if (('success' in result && result.success !== true) || ('error' in result && result.error != null)) throw new Error(error);
8
+ rows = result.results;
9
+ }
10
+ if (!Array.isArray(rows) || rows.some(row => !row || typeof row !== 'object' || Array.isArray(row))) throw new Error(error);
11
+ return rows;
12
+ }
13
+ export function checkBatchResults(results: unknown, count: number, error = 'IMPORT_D1_READ_RECEIPT_REQUIRED'): asserts results is unknown[] {
14
+ if (!Array.isArray(results) || results.length !== count) throw new Error(error);
15
+ for (const result of results) batchRows(result, error);
16
+ }
@@ -0,0 +1,76 @@
1
+ import type { ImportPlan, TargetRow } from '../types.ts';
2
+ import type { D1FieldMatch, D1FieldRedaction, D1ImportContext, D1Sql } from './types.ts';
3
+ import type { InventorySnapshot } from './inventory-snapshot.ts';
4
+ import { canonicalJson, sha256 } from '../provenance/identity.ts';
5
+ import { inRange, redactFields, type EntryRange } from './field-redaction.ts';
6
+ import { withdrawnLocation } from './erased-artifacts.ts';
7
+ import { recordWrites } from './erasure-writes.ts';
8
+ import { targetGuards } from './target-guards.ts';
9
+ import { guard } from './database.ts';
10
+
11
+ /** Keep attribution and revision identities; change only the erased fields and
12
+ * their content bindings. A sanitized before-image is what undo will restore. */
13
+ export function redactedCustody(plan: ImportPlan, next: ImportPlan, id: string, snapshot: InventorySnapshot, fields: readonly D1FieldRedaction[], range?: EntryRange): D1Sql[] {
14
+ const writes: D1Sql[] = [];
15
+ for (const table of ['ik_provenance', 'ik_d1_history']) {
16
+ const rows: { id: string; body: string }[] = [];
17
+ for (const row of snapshot.rows(table, 'job_id', [id])) {
18
+ if (!inRange(range, row.ordinal)) continue;
19
+ const entry = next.entries[row.ordinal], effect = entry.effects[row.effect_ordinal];
20
+ if (!effect) continue; // Whole-record tombstones delete their provenance.
21
+ const record = JSON.parse(row.record_json);
22
+ const before = record.before ? { ...record.before, value: redactFields(record.before.value, fields) } : null;
23
+ const body = canonicalJson({ ...record, sourceHash: entry.source.payloadHash, valueHash: sha256(canonicalJson(effect.value)),
24
+ location: withdrawnLocation(plan, next, record.location, record.artifactHash), before });
25
+ if (body !== row.record_json) rows.push({ id: row.id, body });
26
+ }
27
+ writes.push(...recordWrites(table, rows));
28
+ }
29
+ return writes;
30
+ }
31
+
32
+ /** Sanitize the adapter's native set without claiming any unrelated edit as an
33
+ * imported value. Provenance retains the sanitized planned hash; other edits
34
+ * therefore continue to conflict on undo. */
35
+ export async function redactedNativeFields(kind: string, id: string, context: D1ImportContext, fields: readonly D1FieldRedaction[], counts: D1FieldMatch[], owned?: (row: TargetRow) => boolean): Promise<{ before: D1Sql[]; after: D1Sql[]; final: D1Sql[] }> {
36
+ const { database: db, targets } = context.storage, destination = context.destination;
37
+ if (owned) {
38
+ // One step of a resumable redaction rewrites only the rows its jobs own, each
39
+ // fenced by its exact snapshot; the final step proves no other row holds one.
40
+ const scope = targets.redaction ? await targets.redaction(db, destination, fields) : undefined;
41
+ const rows = (scope ? scope.rows : await targets.list(db, destination)).filter(owned);
42
+ const changed = rows.map(row => ({ row, next: { ...row, value: redactFields(row.value, fields, counts) } })).filter(({ row, next }) => next.value !== row.value);
43
+ const before = changed.map(({ row }) => row), after = changed.map(({ next }) => next);
44
+ if (scope) return { before: scope.matches(before).map(guard), after: [...scope.matches(before).map(guard), ...scope.write(after)], final: scope.matches(after).map(guard) };
45
+ const snapshot = (row: TargetRow) => targetGuards(targets, row, destination, row.targetKind, row.targetId);
46
+ return { before: before.flatMap(snapshot), after: changed.flatMap(({ row, next }) => [...snapshot(row), ...targets.write(next)]), final: after.flatMap(snapshot) };
47
+ }
48
+ if (targets.redaction) {
49
+ // Only rows holding a selected value are read, fenced and rewritten; one
50
+ // holding-set predicate proves no other row holds one before and after.
51
+ const scope = await targets.redaction(db, destination, fields);
52
+ const redacted = scope.rows.map(row => ({ ...row, value: redactFields(row.value, fields, counts) }));
53
+ if (redacted.some((row, index) => row.value === scope.rows[index].value)) throw new Error('IMPORT_REDACTION_SCOPE_MISMATCH');
54
+ const kept = (row: TargetRow) => row.targetKind !== kind || row.targetId !== id;
55
+ const rows = scope.rows.filter(kept), next = redacted.filter(kept);
56
+ return { before: [...scope.holding(scope.rows), ...scope.matches(scope.rows)].map(guard),
57
+ after: [...[...scope.holding(rows), ...scope.matches(rows)].map(guard), ...scope.write(next)],
58
+ final: [...scope.holding([]), ...scope.matches(next)].map(guard) };
59
+ }
60
+ const rows = await targets.list(db, destination);
61
+ const before = targets.matchesMany ? targets.matchesMany(rows, destination).map(guard)
62
+ : targets.matchesCount ? [guard(targets.matchesCount(rows.length, destination)), ...rows.flatMap(row => targetGuards(targets, row, destination, row.targetKind, row.targetId))] : [];
63
+ if (!before.length) throw new Error('IMPORT_NATIVE_SET_GUARD_REQUIRED');
64
+ const after: D1Sql[] = [], survivors: TargetRow[] = [];
65
+ for (const row of rows) {
66
+ const value = redactFields(row.value, fields, counts);
67
+ if (row.targetKind === kind && row.targetId === id) continue;
68
+ const next = value === row.value ? row : { ...row, value };
69
+ survivors.push(next);
70
+ if (next !== row) after.push(...targetGuards(targets, row, destination, row.targetKind, row.targetId), ...targets.write(next));
71
+ after.push(...targetGuards(targets, next, destination, row.targetKind, row.targetId));
72
+ }
73
+ const final = targets.matchesMany ? targets.matchesMany(survivors, destination).map(guard)
74
+ : [guard(targets.matchesCount!(survivors.length, destination)), ...survivors.flatMap(row => targetGuards(targets, row, destination, row.targetKind, row.targetId))];
75
+ return { before, after, final };
76
+ }
@@ -0,0 +1,125 @@
1
+ import { randomUUID } from 'node:crypto';
2
+ import type { D1FieldRedaction, D1ImportContext, D1RetirementProgress, D1Sql } from './types.ts';
3
+ import { packGuards, remaining, D1RequestLimitError } from './budget.ts';
4
+ import { authority, finalAuthority, guard, readonlyChecks, prepare, sql } from './database.ts';
5
+ import type { SourceRecord } from '../types.ts';
6
+ import { targetErasure } from './erasure.ts';
7
+ import { exactRows } from './sets.ts';
8
+ import { batchRows } from './read-results.ts';
9
+ import { batchBytes } from './transport.ts';
10
+ import { RPC_LIMIT } from '../../../d1-transport/index.mjs';
11
+ import type { EntryRange } from './field-redaction.ts';
12
+ import { addMatches, revisionsAfter, STEP_JOBS, writeProgress, type JobRevision, type RedactionPage, type RedactionProgress } from './retirement-steps.ts';
13
+ import { tallyJournal, tallyBefore, tallyStores, tallyWithdraw } from './retirement-tally.ts';
14
+
15
+ /** Statements one request may run (D1's per-invocation limit, as budget.ts counts it). */
16
+ const BUDGET = 1000;
17
+
18
+ /** A step that did not fit its request: the statements it read plus those its
19
+ * batch needed, or undefined when its reads alone ran out of the request. */
20
+ export interface Overflow { cost?: number }
21
+ /** The statements a batch needed that the request no longer had. Nothing was written. */
22
+ export const overflow = (needed: number): D1RequestLimitError => Object.assign(new D1RequestLimitError(), { needed });
23
+
24
+ /** A door lineage the steps sever as they read their jobs: copies `select`
25
+ * matches are bound to `target` (the erased target's digest), under the
26
+ * lineage's `fence`, so the final step reads none of the jobs they cleaned. */
27
+ export interface LineageSever { target: string; select: (record: SourceRecord) => boolean; fence: D1Sql[] }
28
+
29
+ /** Resumable redaction in bounded steps. A step takes as many pending jobs as fit
30
+ * one request (at least one, at most STEP_JOBS); a job that does not fit alone is
31
+ * redacted in pages of its entries. A step that does not fit writes nothing: the
32
+ * next is sized from what it would have cost and tried in the same request while
33
+ * that request still has room, otherwise in the next one, so every committed
34
+ * step stays inside the limit and the erasure reaches its final step. With
35
+ * `sever`, each step also severs the door lineage's copies in the jobs it reads. */
36
+ export async function redactionSteps(fields: readonly D1FieldRedaction[], context: D1ImportContext, fence: D1Sql[], progress: RedactionProgress, pending: JobRevision[], failed?: Overflow, sever?: LineageSever): Promise<D1RetirementProgress> {
37
+ let { span, page } = progress, unsized = failed;
38
+ for (;;) {
39
+ let job = page && pending.find(item => item.id === page!.job);
40
+ // A job changed since its last page starts again from its first entry; one no longer pending is done.
41
+ if (page && (!job || job.revision !== page.revision)) page = job ? { ...page, from: 0, revision: job.revision } : undefined;
42
+ if (unsized) {
43
+ const next = smaller(span, page, pending, unsized.cost);
44
+ ({ span, page } = next); job = page && pending.find(item => item.id === page!.job);
45
+ // Try it now only while this request has room for it; otherwise commit nothing and let the next request take it.
46
+ if (unsized.cost === undefined || unsized.cost * next.fraction > remaining(context)) {
47
+ return { phase: 'redacting', continuation: writeProgress({ ...progress, span, page }), jobsRedacted: 0, jobsPending: pending.length, fieldMatches: progress.matched };
48
+ }
49
+ }
50
+ const jobs = job ? [job] : pending.slice(0, span), range = page && job ? { from: page.from, to: Math.min(page.from + page.count, job.total) } : undefined;
51
+ const start = remaining(context);
52
+ let done: Awaited<ReturnType<typeof redactionStep>>;
53
+ try { done = await redactionStep(fields, context, fence, progress, jobs, range, sever); }
54
+ catch (error) {
55
+ if (!(error instanceof D1RequestLimitError)) throw error;
56
+ const needed = (error as { needed?: number }).needed;
57
+ unsized = { cost: needed === undefined ? undefined : start - remaining(context) + needed };
58
+ continue;
59
+ }
60
+ // A step that used under half a request lets the next one take twice as much.
61
+ const grow = (start - remaining(context)) * 2 <= BUDGET;
62
+ let next: RedactionPage | undefined, finished = jobs.length;
63
+ if (page && job && range) {
64
+ const revision = done.revisions.get(job.id) ?? null;
65
+ if (range.to < job.total) { next = { job: job.id, from: range.to, count: grow ? page.count * 2 : page.count, revision }; finished = 0; }
66
+ else { if (revision !== null) done.clean.set(job.id, revision); span = STEP_JOBS; }
67
+ } else if (grow) span = Math.min(STEP_JOBS, span * 2);
68
+ return { phase: 'redacting', continuation: writeProgress({ ...progress, clean: done.clean, matched: done.matched, span, page: next }),
69
+ jobsRedacted: done.redacted, jobsPending: pending.length - finished, fieldMatches: done.matched, ...done.receiptId ? { receiptId: done.receiptId } : {} };
70
+ }
71
+ }
72
+
73
+ /** The next, strictly smaller step after one cost `cost` statements: in proportion,
74
+ * with headroom, or half when its reads alone ran out. Fewer jobs first, then
75
+ * pages of one job's entries; `fraction` is its expected share of that cost. */
76
+ function smaller(span: number, page: RedactionPage | undefined, pending: JobRevision[], cost: number | undefined): { span: number; page?: RedactionPage; fraction: number } {
77
+ const size = (current: number) => Math.min(current - 1, cost === undefined ? Math.floor(current / 2) : Math.floor(current * BUDGET * 0.7 / cost));
78
+ if (page) {
79
+ const count = size(page.count);
80
+ if (count < 1) throw new D1RequestLimitError(); // One entry alone is more than a request.
81
+ return { span, page: { ...page, count }, fraction: count / page.count };
82
+ }
83
+ const jobs = Math.min(span, pending.length);
84
+ if (jobs > 1) { const fewer = Math.max(1, size(jobs)); return { span: fewer, fraction: fewer / jobs }; }
85
+ const [job] = pending, count = size(job.total);
86
+ if (count < 1) throw new D1RequestLimitError();
87
+ return { span: 1, page: { job: job.id, from: 0, count, revision: job.revision }, fraction: count / job.total };
88
+ }
89
+
90
+ /** One bounded step of a resumable redaction: read `jobs`' custody (with `range`,
91
+ * one job's page of entries), and in one batch with the restore checkpoint and a
92
+ * journal row rewrite the jobs holding a selected value (or, with `sever`, a
93
+ * lineage copy) together with the native rows they own. Jobs found free of both
94
+ * are fenced at the revision read before their plans. The caller's native
95
+ * statements wait for the final step, which also proves no other row holds one. */
96
+ async function redactionStep(fields: readonly D1FieldRedaction[], context: D1ImportContext, fence: D1Sql[], progress: RedactionProgress, jobs: JobRevision[], range?: EntryRange, sever?: LineageSever) {
97
+ const db = context.storage.database, destination = context.destination;
98
+ const erasure = await targetErasure(undefined, context, { fields, ...sever ? { source: sever.select } : {} }, new Set(),
99
+ { selection: { jobs: jobs.map(job => job.id), clean: new Map(), complete: false, ...range ? { range } : {} }, owned: true, ...sever ? { sever: sever.target } : {} });
100
+ const holding = new Set(erasure.jobs), free = jobs.filter(job => !holding.has(job.id) && job.revision !== null);
101
+ const clean = new Map(progress.clean), matched = addMatches(progress.matched, erasure.fieldMatches), revisions = new Map(jobs.map(job => [job.id, job.revision]));
102
+ // A page leaves its job pending until its last page.
103
+ if (!range) for (const job of free) clean.set(job.id, job.revision!);
104
+ let receiptId: string | undefined;
105
+ if (holding.size) {
106
+ receiptId = randomUUID();
107
+ const recordedAt = new Date().toISOString(), touched = { destination, jobs: [...holding].sort(), streams: [] as string[] };
108
+ touched.streams = (await db.prepare('SELECT DISTINCT stream_id AS id FROM ik_d1_stream_pages WHERE job_id IN (SELECT value FROM json_each(?))').bind(JSON.stringify(touched.jobs)).all<{ id: string }>()).results.map(row => row.id).sort();
109
+ const stores = await tallyStores(db);
110
+ const prefix = packGuards([authority(context, 'erase'), ...tallyBefore(stores, receiptId, touched), ...tallyWithdraw(stores, touched), ...erasure.before,
111
+ ...exactRows('ik_d1_custody_states', ['job_id', 'revision'], free.map(job => ({ job_id: job.id, revision: job.revision }))), ...sever?.fence.map(guard) ?? []]);
112
+ const suffix = [...packGuards([...erasure.after, ...tallyJournal(stores, receiptId, touched, { recordedAt, recordsErased: erasure.decisions.length, jobsRetired: 0, streamsRetired: 0 }, true),
113
+ authority(context, 'erase'), sql('DELETE FROM ik_d1_assertions')]), ...readonlyChecks(erasure.final), revisionsAfter(touched.jobs), finalAuthority(context, 'erase')];
114
+ const needed = prefix.length + fence.length + suffix.length, batch = [...prefix, ...fence, ...suffix];
115
+ if (needed > remaining(context)) throw overflow(needed);
116
+ // The batch must also fit one D1 RPC: a step too large in bytes is sized down like one too large in statements.
117
+ if (batchBytes(batch) > RPC_LIMIT) throw new D1RequestLimitError();
118
+ const results = await db.batch(batch.map(statement => prepare(db, statement)));
119
+ for (const row of batchRows(results[results.length - 2]) as { job_id: string; revision: number }[]) {
120
+ revisions.set(row.job_id, row.revision);
121
+ if (!range) clean.set(row.job_id, row.revision);
122
+ }
123
+ }
124
+ return { clean, matched, revisions, redacted: holding.size, receiptId };
125
+ }