@runeditrun/rer 0.2.77 → 0.2.79

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (167) hide show
  1. package/README.md +32 -0
  2. package/cli/export.mjs +1 -1
  3. package/cli/portability-migrations.mjs +35 -14
  4. package/cli/portability-worker.mjs +2 -2
  5. package/cli/portability.mjs +2 -2
  6. package/cli/server-proxy.mjs +30 -8
  7. package/cli/upstream-agent.mjs +14 -0
  8. package/global/CHANGELOG.md +13 -0
  9. package/global/README.md +32 -0
  10. package/modules/demo-host/native/visitor.mjs +1 -1
  11. package/modules/deploy-kit/owner-cli.mjs +15 -5
  12. package/modules/deploy-kit/provider.mjs +9 -4
  13. package/modules/import-kit/src/d1/README.md +1322 -0
  14. package/modules/import-kit/src/d1/apply.ts +115 -0
  15. package/modules/import-kit/src/d1/budget.ts +89 -0
  16. package/modules/import-kit/src/d1/compressed-plan.ts +63 -0
  17. package/modules/import-kit/src/d1/creation.ts +95 -0
  18. package/modules/import-kit/src/d1/custody-fence.ts +27 -0
  19. package/modules/import-kit/src/d1/custody-state-schema.ts +67 -0
  20. package/modules/import-kit/src/d1/database.ts +163 -0
  21. package/modules/import-kit/src/d1/entry-locator.ts +61 -0
  22. package/modules/import-kit/src/d1/erased-artifacts.ts +38 -0
  23. package/modules/import-kit/src/d1/erased-custody.ts +36 -0
  24. package/modules/import-kit/src/d1/erased-plan.ts +47 -0
  25. package/modules/import-kit/src/d1/erased-reversal.ts +19 -0
  26. package/modules/import-kit/src/d1/erasure-bounded.ts +22 -0
  27. package/modules/import-kit/src/d1/erasure-claims.ts +27 -0
  28. package/modules/import-kit/src/d1/erasure-compiler.ts +67 -0
  29. package/modules/import-kit/src/d1/erasure-disposition.ts +16 -0
  30. package/modules/import-kit/src/d1/erasure-fragments.ts +86 -0
  31. package/modules/import-kit/src/d1/erasure-index.ts +53 -0
  32. package/modules/import-kit/src/d1/erasure-pages.ts +62 -0
  33. package/modules/import-kit/src/d1/erasure-progress-schema.ts +8 -0
  34. package/modules/import-kit/src/d1/erasure-progress.ts +61 -0
  35. package/modules/import-kit/src/d1/erasure-related.ts +124 -0
  36. package/modules/import-kit/src/d1/erasure-report.ts +31 -0
  37. package/modules/import-kit/src/d1/erasure-schema.ts +24 -0
  38. package/modules/import-kit/src/d1/erasure-selection.ts +77 -0
  39. package/modules/import-kit/src/d1/erasure-snapshot.ts +15 -0
  40. package/modules/import-kit/src/d1/erasure-stage.ts +146 -0
  41. package/modules/import-kit/src/d1/erasure-step.ts +12 -0
  42. package/modules/import-kit/src/d1/erasure-superseded.ts +98 -0
  43. package/modules/import-kit/src/d1/erasure-values.ts +83 -0
  44. package/modules/import-kit/src/d1/erasure-writes.ts +46 -0
  45. package/modules/import-kit/src/d1/erasure.ts +170 -0
  46. package/modules/import-kit/src/d1/evolutions/0001-preserved-undo.ts +13 -0
  47. package/modules/import-kit/src/d1/expected-state.ts +91 -0
  48. package/modules/import-kit/src/d1/field-redaction.ts +86 -0
  49. package/modules/import-kit/src/d1/guards.ts +37 -0
  50. package/modules/import-kit/src/d1/history.ts +71 -0
  51. package/modules/import-kit/src/d1/index.ts +88 -0
  52. package/modules/import-kit/src/d1/integrity.ts +127 -0
  53. package/modules/import-kit/src/d1/inventory-snapshot.ts +84 -0
  54. package/modules/import-kit/src/d1/jobs.ts +97 -0
  55. package/modules/import-kit/src/d1/native-set.ts +48 -0
  56. package/modules/import-kit/src/d1/native-staging-admission.ts +19 -0
  57. package/modules/import-kit/src/d1/native-staging-context.ts +44 -0
  58. package/modules/import-kit/src/d1/native-staging-lifecycle.ts +33 -0
  59. package/modules/import-kit/src/d1/native-staging-offset.ts +74 -0
  60. package/modules/import-kit/src/d1/native-staging-publication.ts +18 -0
  61. package/modules/import-kit/src/d1/native-staging-rows.ts +72 -0
  62. package/modules/import-kit/src/d1/native-staging-schema.ts +28 -0
  63. package/modules/import-kit/src/d1/native-staging-state.ts +22 -0
  64. package/modules/import-kit/src/d1/native-staging.ts +64 -0
  65. package/modules/import-kit/src/d1/original-copies.ts +53 -0
  66. package/modules/import-kit/src/d1/original-storage.ts +56 -0
  67. package/modules/import-kit/src/d1/originals.ts +74 -0
  68. package/modules/import-kit/src/d1/plan-dictionary.ts +80 -0
  69. package/modules/import-kit/src/d1/plans.ts +121 -0
  70. package/modules/import-kit/src/d1/preparation-fingerprint.ts +57 -0
  71. package/modules/import-kit/src/d1/preparation-header.ts +33 -0
  72. package/modules/import-kit/src/d1/preparation-reader.ts +98 -0
  73. package/modules/import-kit/src/d1/preparation-schema.ts +16 -0
  74. package/modules/import-kit/src/d1/preparation-state.ts +56 -0
  75. package/modules/import-kit/src/d1/preparation-step-combine.ts +80 -0
  76. package/modules/import-kit/src/d1/preparation-step-control.ts +48 -0
  77. package/modules/import-kit/src/d1/preparation-step-hash.ts +131 -0
  78. package/modules/import-kit/src/d1/preparation-step-pages.ts +77 -0
  79. package/modules/import-kit/src/d1/preparation-steps-schema.ts +11 -0
  80. package/modules/import-kit/src/d1/preparation-steps.ts +102 -0
  81. package/modules/import-kit/src/d1/preparations.ts +87 -0
  82. package/modules/import-kit/src/d1/preserved-preview.ts +89 -0
  83. package/modules/import-kit/src/d1/preserved-reverse.ts +91 -0
  84. package/modules/import-kit/src/d1/provenance-set.ts +87 -0
  85. package/modules/import-kit/src/d1/publication-fence.ts +22 -0
  86. package/modules/import-kit/src/d1/publication-receipt-schema.ts +15 -0
  87. package/modules/import-kit/src/d1/published-stream.ts +11 -0
  88. package/modules/import-kit/src/d1/read-fence.ts +64 -0
  89. package/modules/import-kit/src/d1/read-results.ts +16 -0
  90. package/modules/import-kit/src/d1/redacted-custody.ts +76 -0
  91. package/modules/import-kit/src/d1/redaction-step.ts +125 -0
  92. package/modules/import-kit/src/d1/referenced-plans.ts +136 -0
  93. package/modules/import-kit/src/d1/retained-values.ts +35 -0
  94. package/modules/import-kit/src/d1/retirement-bounded.ts +71 -0
  95. package/modules/import-kit/src/d1/retirement-door.ts +105 -0
  96. package/modules/import-kit/src/d1/retirement-schema.ts +39 -0
  97. package/modules/import-kit/src/d1/retirement-scope.ts +94 -0
  98. package/modules/import-kit/src/d1/retirement-steps.ts +76 -0
  99. package/modules/import-kit/src/d1/retirement-tally.ts +74 -0
  100. package/modules/import-kit/src/d1/retirement.ts +136 -0
  101. package/modules/import-kit/src/d1/reversal-conversion-record.ts +19 -0
  102. package/modules/import-kit/src/d1/reversal-conversion-schema.ts +11 -0
  103. package/modules/import-kit/src/d1/reversal-conversion.ts +40 -0
  104. package/modules/import-kit/src/d1/reversal-fence-schema.ts +14 -0
  105. package/modules/import-kit/src/d1/reversal-fence.ts +106 -0
  106. package/modules/import-kit/src/d1/reversal-journal.ts +24 -0
  107. package/modules/import-kit/src/d1/reverse.ts +103 -0
  108. package/modules/import-kit/src/d1/reviewed-source.ts +23 -0
  109. package/modules/import-kit/src/d1/sets.ts +78 -0
  110. package/modules/import-kit/src/d1/staged-creation.ts +201 -0
  111. package/modules/import-kit/src/d1/staging-admission.ts +45 -0
  112. package/modules/import-kit/src/d1/staging-control.ts +43 -0
  113. package/modules/import-kit/src/d1/staging-erasure.ts +28 -0
  114. package/modules/import-kit/src/d1/staging-lifecycle.ts +49 -0
  115. package/modules/import-kit/src/d1/staging-plan.ts +15 -0
  116. package/modules/import-kit/src/d1/staging-privacy.ts +22 -0
  117. package/modules/import-kit/src/d1/staging-publication.ts +79 -0
  118. package/modules/import-kit/src/d1/staging-schema.ts +37 -0
  119. package/modules/import-kit/src/d1/staging-statements.ts +8 -0
  120. package/modules/import-kit/src/d1/staging-transport.ts +70 -0
  121. package/modules/import-kit/src/d1/stream-boundary.ts +11 -0
  122. package/modules/import-kit/src/d1/stream-create.ts +55 -0
  123. package/modules/import-kit/src/d1/stream-identity-admission.ts +37 -0
  124. package/modules/import-kit/src/d1/stream-input.ts +18 -0
  125. package/modules/import-kit/src/d1/stream-layout.ts +43 -0
  126. package/modules/import-kit/src/d1/stream-origin.ts +44 -0
  127. package/modules/import-kit/src/d1/stream-page-integrity.ts +18 -0
  128. package/modules/import-kit/src/d1/stream-partition.ts +50 -0
  129. package/modules/import-kit/src/d1/stream-plan.ts +133 -0
  130. package/modules/import-kit/src/d1/stream-receipt.ts +45 -0
  131. package/modules/import-kit/src/d1/stream-revision-schema.ts +35 -0
  132. package/modules/import-kit/src/d1/stream-snapshot.ts +26 -0
  133. package/modules/import-kit/src/d1/stream-stage.ts +79 -0
  134. package/modules/import-kit/src/d1/stream-state.ts +25 -0
  135. package/modules/import-kit/src/d1/stream-working-set.ts +30 -0
  136. package/modules/import-kit/src/d1/streams.ts +39 -0
  137. package/modules/import-kit/src/d1/target-guards.ts +9 -0
  138. package/modules/import-kit/src/d1/target-identities.ts +17 -0
  139. package/modules/import-kit/src/d1/transport.ts +10 -0
  140. package/modules/import-kit/src/d1/types.ts +162 -0
  141. package/modules/import-kit/src/d1/undo-observations.ts +39 -0
  142. package/modules/import-kit/src/d1/undo-report.ts +22 -0
  143. package/modules/import-kit/src/d1/undo-snapshot.ts +87 -0
  144. package/modules/import-kit/src/d1/undo-state.ts +84 -0
  145. package/modules/import-kit/src/d1/undo-types.ts +110 -0
  146. package/modules/import-kit/src/journal/authority.ts +12 -0
  147. package/modules/import-kit/src/plan/budget.ts +60 -0
  148. package/modules/import-kit/src/plan/index.ts +123 -0
  149. package/modules/import-kit/src/plan/size.ts +122 -0
  150. package/modules/import-kit/src/provenance/canonical-hash.ts +130 -0
  151. package/modules/import-kit/src/provenance/canonical-stream.ts +85 -0
  152. package/modules/import-kit/src/provenance/credential-names.ts +131 -0
  153. package/modules/import-kit/src/provenance/credentials.ts +317 -0
  154. package/modules/import-kit/src/provenance/identity.ts +51 -0
  155. package/modules/import-kit/src/provenance/json-snapshot.ts +21 -0
  156. package/modules/import-kit/src/provenance/resumable-sha256.ts +75 -0
  157. package/modules/import-kit/src/provenance/validate-json.ts +19 -0
  158. package/modules/import-kit/src/storage/originals.ts +87 -0
  159. package/modules/import-kit/src/storage/provenance.ts +64 -0
  160. package/modules/import-kit/src/storage/schema.ts +42 -0
  161. package/modules/import-kit/src/types.ts +47 -0
  162. package/modules/portability/README.md +98 -2
  163. package/modules/portability/backfills.mjs +42 -15
  164. package/modules/portability/ledgers.mjs +7 -1
  165. package/modules/portability/transient-tables.mjs +40 -0
  166. package/package.json +3 -1
  167. package/scripts/publish-check/reviewed.py +3 -1
@@ -0,0 +1,146 @@
1
+ import type { D1FieldMatch, D1ImportContext, D1Sql } from './types.ts';
2
+ import type { ErasureFragment } from './erasure-fragments.ts';
3
+ import { ErasureCompiler } from './erasure-compiler.ts';
4
+ import { authority, finalAuthority, guard, prepare, readAtomic, readonlyChecks, sql, withinBatchBytes } from './database.ts';
5
+ import { D1RequestLimitError, packGuards, remaining } from './budget.ts';
6
+ import { canonicalJson, sha256 } from '../provenance/identity.ts';
7
+ import { nativeStageAtomic, nativeStageGuards, type NativeStageBinding } from './native-staging-state.ts';
8
+ import { beginNativeStage, discardNativeStage } from './native-staging-lifecycle.ts';
9
+ import { nativeStageCost } from './native-staging-admission.ts';
10
+ import { NativeStageRows, nativeRowStatements, verifyNativeStageOffset } from './native-staging-rows.ts';
11
+ import { nativePublication } from './native-staging-publication.ts';
12
+ import { stagingStatements } from './staging-statements.ts';
13
+ import { guardPredicate } from './guards.ts';
14
+ import { erasureIndexColumns, erasureValueColumns, privacyRows, valueRows } from './erasure-values.ts';
15
+ import { completedErasure, erasureRequestContext, operationIdentity, pendingErasure, progressChange, progressGuard, progressInsert, readErasureProgress, type D1ErasureStep, type D1ErasureStepOptions, type ErasureProgress } from './erasure-progress.ts';
16
+
17
+ /** Advanced product-owned custody rewrites use the same runner as retirement.
18
+ * Fragments must describe authenticated normalized custody, in deterministic
19
+ * order. The binding identifies the product's immutable erasure intent. */
20
+ export interface D1ErasureStageSpec {
21
+ fragments: () => AsyncGenerator<ErasureFragment>;
22
+ snapshotGuards: D1Sql[];
23
+ targets: { kind: string; id: string; present: boolean }[];
24
+ final: D1Sql[];
25
+ fieldMatches?: D1FieldMatch[];
26
+ resultKeys?: symbol[];
27
+ direct?: boolean;
28
+ survivingJobs?: (compiler: ErasureCompiler) => string[];
29
+ publication?: (compiler: ErasureCompiler, receipt: { id: string; recordedAt: string }) => Promise<{ before: D1Sql[]; after: D1Sql[]; tail: D1Sql[] }>;
30
+ }
31
+ type Operation = { row?: ErasureProgress; requestHash: string; admissionOnly?: boolean };
32
+
33
+ export async function stageD1ImportErasure(input: D1ImportContext, options: D1ErasureStepOptions, binding: string,
34
+ build: (id: string, context: D1ImportContext) => Promise<D1ErasureStageSpec>): Promise<D1ErasureStep> {
35
+ const context = erasureRequestContext(input, options);
36
+ await readAtomic(context, [], undefined, 'erase');
37
+ const identity = operationIdentity(context, options, binding);
38
+ const row = await readErasureProgress(context, identity.id, identity.requestHash);
39
+ if (row?.phase === 'complete') return completedErasure(row);
40
+ return executeErasureStage(context, identity.id, await build(identity.id, context), { row, requestHash: identity.requestHash, admissionOnly: options.admissionOnly });
41
+ }
42
+
43
+ /** Without an operation this retains the original one-invocation contract.
44
+ * A journaled operation admits each step, not the sum of all future requests. */
45
+ export async function executeErasureStage(context: D1ImportContext, id: string, spec: D1ErasureStageSpec, operation?: Operation): Promise<D1ErasureStep> {
46
+ const db = context.storage.database, compiler = new ErasureCompiler(context, id, spec.fragments);
47
+ await compiler.inspect();
48
+ const recordedAt = new Date().toISOString(), createdAt = operation?.row?.created_at ?? recordedAt;
49
+ const own = await spec.publication?.(compiler, { id, recordedAt }) ?? { before: [], after: [], tail: [] };
50
+ const fresh: ErasureProgress = { id, destination: context.destination, request_hash: operation?.requestHash ?? '', generation: '', phase: 'index', index_rows: 0, value_rows: 0, revision: 0, records_erased: compiler.erased, field_matches_json: canonicalJson(spec.fieldMatches ?? []), created_at: createdAt };
51
+ const final = [...spec.final, ...compiler.erasedFinal];
52
+ const resultKeys = spec.resultKeys ?? [];
53
+ const resultsAt = (packed: D1Sql[]) => resultKeys.map(key => {
54
+ const index = packed.findIndex(row => row.resultKey === key);
55
+ if (index < 0) throw new Error('IMPORT_NATIVE_RESULT_MISSING');
56
+ return index;
57
+ });
58
+ const complete = (row: ErasureProgress) => progressChange(row, { phase: 'complete', records_erased: compiler.erased, field_matches_json: fresh.field_matches_json });
59
+ if (spec.direct !== false && compiler.direct && !operation?.row) {
60
+ const { before, writes, post } = compiler.direct;
61
+ const finished = { ...fresh, phase: 'complete' as const };
62
+ const protection = context.storage.targets.stagingPublication?.(id, context.destination, spec.targets) ?? { protect: [], seal: [], release: [], final: [] };
63
+ const snapshots = spec.snapshotGuards.map(row => guardPredicate(row) ? row : guard(row));
64
+ const packed = [...packGuards([authority(context, 'erase'), ...snapshots, ...own.before, ...before, ...writes, ...own.after,
65
+ ...protection.protect, ...post, ...protection.seal, ...protection.release, ...own.tail,
66
+ ...operation ? [progressInsert(finished)] : [], authority(context, 'erase'), sql('DELETE FROM ik_d1_assertions')]), ...readonlyChecks([...post, ...final.map(guard), ...protection.final.map(guard)]), finalAuthority(context, 'erase')];
67
+ if (packed.length <= remaining(context) && withinBatchBytes(context, packed)) {
68
+ const indexes = resultsAt(packed);
69
+ if (operation?.admissionOnly) return pendingErasure(fresh);
70
+ const result = await db.batch(packed.map(row => prepare(db, row)));
71
+ return completedErasure(finished, indexes.map(index => result[index]));
72
+ }
73
+ }
74
+ if (compiler.unsafeOriginals) throw new Error('IMPORT_NATIVE_STAGE_ORIGINAL_BYTES_REFUSED');
75
+ if (!context.storage.targets.stagingPublication || !spec.snapshotGuards.length) throw new Error('IMPORT_NATIVE_STAGING_ADAPTER_REQUIRED');
76
+ const jobs = [...compiler.jobs.values()], anchor = jobs[0];
77
+ if (!anchor) throw new Error('IMPORT_ERASURE_STAGE_ANCHOR_REQUIRED');
78
+ const baseline = [...compiler.baseline(), ...spec.snapshotGuards.map(row => guardPredicate(row) ? row : guard(row))];
79
+ const stage: NativeStageBinding = { id, generation: sha256(canonicalJson([compiler.fingerprints, baseline])), destination: context.destination, namespace: context.namespace, mapping_version: context.mappingVersion, principal_id: context.authorization.principalId, job_id: anchor.id, fingerprint: anchor.fingerprint, job_fence: anchor.fence, action: 'erase', values_count: compiler.values, chunks_count: compiler.payload.count, identities: compiler.indexes.count };
80
+ fresh.generation = stage.generation;
81
+ const current = operation?.row;
82
+ const stored = current ? await db.prepare('SELECT * FROM ik_d1_native_staging WHERE id=?').bind(id).first() : null;
83
+ const resumed = current && current.generation === stage.generation && stored && Object.keys(stage).every(key => stored[key] === stage[key as keyof NativeStageBinding]);
84
+ let progress = resumed ? current! : fresh, writes = 0;
85
+ const begin = [...operation ? current ? progressChange(current, { generation: fresh.generation, phase: 'index', index_rows: 0, value_rows: 0, revision: 0, records_erased: fresh.records_erased, field_matches_json: fresh.field_matches_json }) : [progressInsert(fresh)] : [], ...beginNativeStage(stage, baseline)];
86
+ const registered = stage.identities, sealed = registered + stage.chunks_count;
87
+ const registration = [...baseline, ...nativeStageGuards(stage, registered), guard(sql('(SELECT count(*) FROM ik_d1_native_staging_index WHERE stage_id=?)=?', id, registered)), sql('UPDATE ik_d1_native_staging SET registered_revision=revision WHERE id=?', id), ...baseline, ...nativeStageGuards(stage, registered)];
88
+ const seal = [...baseline, ...nativeStageGuards(stage, sealed), guard(sql('EXISTS(SELECT 1 FROM ik_d1_native_staging WHERE id=? AND registered_revision IS NOT NULL) AND (SELECT count(*) FROM ik_d1_native_staging_values WHERE stage_id=?)=? AND (SELECT count(DISTINCT value_ordinal) FROM ik_d1_native_staging_values WHERE stage_id=?)=? AND NOT EXISTS(SELECT 1 FROM ik_d1_native_staging_values WHERE stage_id=? GROUP BY value_ordinal HAVING min(ordinal)<>0 OR max(ordinal)<>count(*)-1 OR min(value_chunks)<>count(*) OR max(value_chunks)<>count(*))', id, id, stage.chunks_count, id, stage.values_count, id)), sql('UPDATE ik_d1_native_staging SET sealed_revision=revision WHERE id=?', id), ...baseline, ...nativeStageGuards(stage, sealed)];
89
+ const finished = { ...fresh, phase: 'sealed' as const, index_rows: registered, value_rows: stage.chunks_count, revision: sealed };
90
+ const publication = nativePublication(context, stage, { baseline, writes: [...own.before, ...compiler.compiled.before, ...compiler.compiled.writes, ...own.after], post: compiler.compiled.post, jobs: spec.survivingJobs?.(compiler) ?? jobs.map(row => row.id), targets: spec.targets, binding: [], final, erasure: true });
91
+ publication.release.push(...own.tail, ...operation ? complete(finished) : [], sql('DELETE FROM ik_d1_assertions'));
92
+ const packed = stagingStatements(context, 'erase', publication.publication, publication.release, publication.final), indexes = resultsAt(packed);
93
+ const publicationCost = nativeStageCost(context, stage, publication.publication, publication.release, publication.final);
94
+ const checkpoint = (table: 'index' | 'values') => (from: { written: number; revision: number }, to: { written: number; revision: number }) => {
95
+ const column = table === 'index' ? 'index_rows' : 'value_rows';
96
+ return operation ? progressChange({ ...progress, [column]: from.written, revision: from.revision }, { [column]: to.written, revision: to.revision }) : [];
97
+ };
98
+ const indexCost = compiler.indexes.cost(context, stage, baseline, 'index', 0, checkpoint('index'));
99
+ const valueCost = compiler.payload.cost(context, stage, baseline, 'values', registered, checkpoint('values'));
100
+ const registrationCost = nativeStageCost(context, stage, registration), sealCost = nativeStageCost(context, stage, seal);
101
+ if (!operation) {
102
+ const cost = 2 * compiler.readQueries + indexCost + valueCost + [begin, discardNativeStage(stage)].reduce((n, rows) => n + nativeStageCost(context, stage, rows), 0) + registrationCost + sealCost + publicationCost;
103
+ if (cost > remaining(context)) throw new D1RequestLimitError();
104
+ } else {
105
+ // At least one replay and useful flush must fit after inspection. Both
106
+ // registration and sealing are independently journaled atomic steps.
107
+ const flushCost = nativeStageCost(context, stage, nativeRowStatements(stage, baseline, 'values', erasureValueColumns, '[]', 1, 0, registered, checkpoint('values')));
108
+ if (Math.max(compiler.readQueries + nativeStageCost(context, stage, begin) + flushCost + 2, publicationCost, registrationCost + 2, sealCost + 2) > remaining(context)) throw new D1RequestLimitError();
109
+ if (operation.admissionOnly) return pendingErasure(progress);
110
+ }
111
+ const write = async (rows: D1Sql[], next: ErasureProgress, release: D1Sql[] = [], checks: D1Sql[] = []) => {
112
+ await nativeStageAtomic(context, 'erase', rows, 0, release, checks); progress = next; writes++;
113
+ };
114
+ try {
115
+ if (!resumed) await write(begin, fresh);
116
+ for (const table of ['index', 'values'] as const) {
117
+ if (table === 'index' ? progress.phase !== 'index' : progress.phase !== 'values') continue;
118
+ const column = table === 'index' ? 'index_rows' : 'value_rows';
119
+ const columns = table === 'index' ? erasureIndexColumns : erasureValueColumns;
120
+ const offset = operation ? await verifyNativeStageOffset(context, stage, baseline, table, { written: progress[column], revision: progress.revision, guards: [progressGuard(progress)] }) : undefined;
121
+ const writer = new NativeStageRows(context, stage, baseline, table, columns, { ...offset ? { offset } : {}, checkpoint: (from, to) => checkpoint(table)(from, to) });
122
+ if (!operation) writer.revision = table === 'index' ? 0 : registered;
123
+ try {
124
+ await compiler.replay(async (fragment, retained, start, indexStart) => {
125
+ for (const row of table === 'index' ? privacyRows(fragment, indexStart) : valueRows(retained, start)) await writer.append(row);
126
+ });
127
+ await writer.flush();
128
+ } finally {
129
+ if (writer.revision !== progress.revision) { progress = { ...progress, [column]: writer.written, revision: writer.revision }; writes++; }
130
+ }
131
+ const next = { ...progress, phase: table === 'index' ? 'values' as const : 'sealed' as const };
132
+ await write([...(table === 'index' ? registration : seal), ...operation ? progressChange(progress, { phase: next.phase }) : []], next);
133
+ }
134
+ const result = await nativeStageAtomic(context, 'erase', publication.publication, 0, publication.release, publication.final);
135
+ return completedErasure({ ...finished, phase: 'complete' }, indexes.map(index => result[index]));
136
+ } catch (error) {
137
+ if (operation) {
138
+ if (error instanceof D1RequestLimitError && writes) return pendingErasure(progress);
139
+ // A provider may have committed before losing its acknowledgement. Keep
140
+ // durable progress so a new invocation verifies the actual checkpoint.
141
+ throw error;
142
+ }
143
+ await nativeStageAtomic(context, 'erase', discardNativeStage(stage));
144
+ throw error;
145
+ }
146
+ }
@@ -0,0 +1,12 @@
1
+ import type { D1ImportContext, D1Sql } from './types.ts';
2
+ import { eraseBounded } from './erasure-bounded.ts';
3
+ import type { D1ErasureStep, D1ErasureStepOptions } from './erasure-progress.ts';
4
+ import { requireText } from '../provenance/identity.ts';
5
+
6
+ /** The target-only equivalent of retireD1ImportCustodyStep. Its operation key
7
+ * loads the journal's verified native-stage offset; arbitrary offsets are never
8
+ * accepted from a request. Use retirement when headers or restore fences apply. */
9
+ export function eraseD1ImportTargetStep(kind: string, id: string, nativeStatements: D1Sql[], context: D1ImportContext, options: D1ErasureStepOptions): Promise<D1ErasureStep> {
10
+ requireText(kind, 'target kind'); requireText(id, 'target id');
11
+ return eraseBounded(kind, id, nativeStatements, context, options);
12
+ }
@@ -0,0 +1,98 @@
1
+ import type { D1Database, D1Sql } from './types.ts';
2
+ import { guard, sql } from './database.ts';
3
+ import { jsonChunks } from './sets.ts';
4
+ import { jobLink } from './retirement-door.ts';
5
+ import type { JobRevision } from './retirement-steps.ts';
6
+
7
+ interface IndexRow { job_id: string; ordinal: number; source_identity: string; target_identity: string }
8
+ interface StateRow { store: 'h' | 'p'; id: string; job_id: string; ordinal: number; successor_id: string | null; reversed: number }
9
+
10
+ /** The erased target's own provenance in the jobs of its index: history rows
11
+ * (superseded by a later import), reversed rows and the live row, if any. */
12
+ const targetRows = `SELECT 'h' AS store,h.id,h.job_id,h.ordinal,h.successor_id,0 AS reversed FROM ik_d1_plan_index i JOIN ik_d1_history h ON h.job_id=i.job_id AND h.ordinal=i.ordinal WHERE i.target_identity=? AND h.destination=?
13
+ UNION ALL SELECT 'p',v.id,v.job_id,v.ordinal,NULL,v.reversed FROM ik_d1_plan_index i JOIN ik_provenance v ON v.job_id=i.job_id AND v.ordinal=i.ordinal WHERE i.target_identity=? AND v.destination=?`;
14
+
15
+ async function readTargetRows(db: D1Database, destination: string, target: string): Promise<StateRow[]> {
16
+ return (await db.prepare(targetRows).bind(target, destination, target, destination).all<StateRow>()).results;
17
+ }
18
+
19
+ /** Pending jobs with the erased target's provenance chain first, oldest first:
20
+ * jobs whose import of the target was reversed, then each superseded row in the
21
+ * order its successors replaced it, then the live row's job. A bounded step reads
22
+ * the jobs in this order, so it can erase each superseded row with all older ones. */
23
+ export async function chainFirst(db: D1Database, destination: string, target: string, pending: JobRevision[]): Promise<JobRevision[]> {
24
+ const rows = await readTargetRows(db, destination, target), order: string[] = [];
25
+ const history = rows.filter(row => row.store === 'h'), next = new Map(history.map(row => [row.id, row.successor_id!]));
26
+ const successors = new Set(history.map(row => row.successor_id)), byId = new Map(rows.map(row => [row.id, row]));
27
+ for (const row of rows) if (row.store === 'p' && row.reversed) order.push(row.job_id);
28
+ for (const head of history.filter(row => !successors.has(row.id))) {
29
+ for (let id: string | undefined = head.id, seen = 0; id !== undefined && byId.has(id) && seen <= rows.length; id = next.get(id), seen++) order.push(byId.get(id)!.job_id);
30
+ }
31
+ const rank = new Map<string, number>();
32
+ for (const [index, job] of order.entries()) if (!rank.has(job)) rank.set(job, index);
33
+ const last = order.length;
34
+ return pending.map((job, index) => ({ job, index })).sort((a, b) => (rank.get(a.job.id) ?? last) - (rank.get(b.job.id) ?? last) || a.index - b.index).map(({ job }) => job);
35
+ }
36
+
37
+ /** The erased target's own entries one bounded step may erase before the final
38
+ * step, bound to the target like the lineage copies the step severs: those whose
39
+ * import of the target was superseded (history) or reversed, in the door
40
+ * namespace that holds the target's live provenance. The live row, the native row
41
+ * it owns and every other own entry wait for the final step. A superseded row is
42
+ * erased only with every older row of its chain, and never under a newer row a
43
+ * preserve-conflicts job still walks back from, so no row left names a row that
44
+ * is gone; the live row keeps the door lineage readable for later steps. The
45
+ * guards fail the step's batch if that state changed after these reads. */
46
+ export async function supersededEntries(db: D1Database, destination: string, target: string, rows: IndexRow[]): Promise<{ identities: Set<string>; guards: D1Sql[] }> {
47
+ const none = { identities: new Set<string>(), guards: [] };
48
+ // A source also mapped to another target (or none) waits for the final step's shared-source check.
49
+ const mixed = new Set(rows.filter(row => row.target_identity !== target).map(row => row.source_identity));
50
+ const own = rows.filter(row => row.target_identity === target && !mixed.has(row.source_identity));
51
+ if (!own.length) return none;
52
+ const state = await readTargetRows(db, destination, target), live = state.filter(row => row.store === 'p' && !row.reversed);
53
+ if (live.length !== 1) return none;
54
+ const jobs = [...new Set([live[0].job_id, ...own.map(row => row.job_id)])], doors = new Map<string, Set<string>>();
55
+ for (const chunk of jsonChunks(jobs)) {
56
+ const { results } = await db.prepare(`SELECT DISTINCT j.id,p.installation_id,p.namespace FROM ik_jobs j JOIN ik_door_plans p ON ${jobLink} WHERE j.destination=? AND j.id IN (SELECT value FROM json_each(?))`)
57
+ .bind(destination, chunk).all<{ id: string; installation_id: string; namespace: string }>();
58
+ for (const row of results) doors.set(row.id, new Set([...doors.get(row.id) ?? [], JSON.stringify([row.installation_id, row.namespace])]));
59
+ }
60
+ const liveDoors = doors.get(live[0].job_id);
61
+ if (liveDoors?.size !== 1) return none;
62
+ const [door] = liveDoors, byEntry = new Map<string, StateRow[]>();
63
+ for (const row of state) byEntry.set(`${row.job_id}\u0000${row.ordinal}`, [...byEntry.get(`${row.job_id}\u0000${row.ordinal}`) ?? [], row]);
64
+ const bySource = new Map<string, StateRow[]>(), blocked = new Set<string>();
65
+ for (const row of own) {
66
+ const rows = byEntry.get(`${row.job_id}\u0000${row.ordinal}`) ?? [], doorsOf = doors.get(row.job_id);
67
+ // Never imported, still live, or outside the live row's door lineage: the final step erases it.
68
+ if (!rows.length || rows.some(item => item.store === 'p' && !item.reversed) || doorsOf?.size !== 1 || !doorsOf.has(door)) blocked.add(row.source_identity);
69
+ bySource.set(row.source_identity, [...bySource.get(row.source_identity) ?? [], ...rows]);
70
+ }
71
+ const accepted = new Set([...bySource.keys()].filter(identity => !blocked.has(identity)));
72
+ const history = [...accepted].flatMap(identity => bySource.get(identity)!.filter(row => row.store === 'h'));
73
+ const predecessors = new Map<string, string>(), policies = new Map<string, string>();
74
+ for (const chunk of jsonChunks(history.map(row => row.id))) {
75
+ for (const row of (await db.prepare('SELECT id,successor_id FROM ik_d1_history WHERE successor_id IN (SELECT value FROM json_each(?))').bind(chunk).all<{ id: string; successor_id: string }>()).results) predecessors.set(row.successor_id, row.id);
76
+ }
77
+ for (const chunk of jsonChunks(history.map(row => row.successor_id!))) {
78
+ const { results } = await db.prepare(`SELECT v.id,j.reversal_policy FROM ik_provenance v JOIN ik_jobs j ON j.id=v.job_id WHERE v.id IN (SELECT value FROM json_each(?))
79
+ UNION ALL SELECT h.id,j.reversal_policy FROM ik_d1_history h JOIN ik_jobs j ON j.id=h.job_id WHERE h.id IN (SELECT value FROM json_each(?))`).bind(chunk, chunk).all<{ id: string; reversal_policy: string }>();
80
+ for (const row of results) policies.set(row.id, row.reversal_policy);
81
+ }
82
+ for (let changed = true; changed;) {
83
+ changed = false;
84
+ const erased = new Set([...accepted].flatMap(identity => bySource.get(identity)!.filter(row => row.store === 'h').map(row => row.id)));
85
+ for (const identity of [...accepted]) {
86
+ const keep = bySource.get(identity)!.filter(row => row.store === 'h').every(row => {
87
+ const older = predecessors.get(row.id);
88
+ return (older === undefined || erased.has(older)) && (erased.has(row.successor_id!) || policies.get(row.successor_id!) === 'strict');
89
+ });
90
+ if (!keep) { accepted.delete(identity); changed = true; }
91
+ }
92
+ }
93
+ if (!accepted.size) return none;
94
+ const erased = [...accepted].flatMap(identity => bySource.get(identity)!.filter(row => row.store === 'h').map(row => row.id));
95
+ const guards = [guard(sql('EXISTS(SELECT 1 FROM ik_provenance WHERE id=? AND reversed=0)', live[0].id))];
96
+ if (erased.length) guards.push(guard(sql('NOT EXISTS(SELECT 1 FROM ik_d1_history WHERE successor_id IN (SELECT value FROM json_each(?)) AND id NOT IN (SELECT value FROM json_each(?)))', JSON.stringify(erased), JSON.stringify(erased))));
97
+ return { identities: accepted, guards };
98
+ }
@@ -0,0 +1,83 @@
1
+ import { guard, guardPredicate } from './guards.ts';
2
+ import { Buffer } from 'node:buffer';
3
+ import { createHash } from 'node:crypto';
4
+ import type { D1Sql,D1ImportContext } from './types.ts';
5
+ import type { ErasureFragment } from './erasure-fragments.ts';
6
+ import type { NativeStageBinding } from './native-staging-state.ts';
7
+ import { replaceRetained } from './retained-values.ts';
8
+ import { sql } from './database.ts';
9
+ import { planIndex } from './erasure-index.ts';
10
+ import { sha256 } from '../provenance/identity.ts';
11
+ import { stagingChunks } from './staging-transport.ts';
12
+ import { NATIVE_STAGE_ROWS_BYTES,nativeRowStatements } from './native-staging-rows.ts';
13
+ import { stagingStatements } from './staging-statements.ts';
14
+ import { rpcSize, rpcSizeBytes, wireSql, type RpcSize } from '../../../d1-transport/index.mjs';
15
+ import type { NativeStageCheckpoint } from './native-staging-rows.ts';
16
+
17
+ export const erasureIndexColumns=['ordinal','source_identity','target_identity'];
18
+ export const erasureValueColumns=['value_ordinal','ordinal','body','body_hash','body_size','value_chunks'];
19
+ export function* valueChunks(value:string){if(!value.length)yield '';else yield* stagingChunks(value);}
20
+ export function* valueRows(values:string[],start:number):Generator<unknown[]>{
21
+ for(const [i,value] of values.entries()){
22
+ let count=0;for(const _ of valueChunks(value))count++;
23
+ let ordinal=0;for(const body of valueChunks(value))yield [start+i,ordinal++,body,sha256(body),Buffer.byteLength(body),count];
24
+ }
25
+ }
26
+ export function* privacyRows(fragment:ErasureFragment,start:number):Generator<unknown[]>{
27
+ let ordinal=start;
28
+ for(const plan of fragment.plans){
29
+ for(const row of planIndex(plan,''))if(!plan.entries[row.ordinal].erased)yield [ordinal++,row.source_identity,row.target_identity];
30
+ }
31
+ }
32
+ /** Strip construction metadata immediately: it can reference a full plan or
33
+ * snapshot even when the prepared SQL already uses only a private value ID. */
34
+ export function bare(statement:D1Sql):D1Sql{
35
+ const scalar=guardPredicate(statement),source=scalar??statement;
36
+ const params=source.params?.map(value=>value instanceof ArrayBuffer?value.slice(0):ArrayBuffer.isView(value)?new Uint8Array(new Uint8Array(value.buffer,value.byteOffset,value.byteLength)):value);
37
+ // Reconstruct only an authenticated scalar. Neither the original statement
38
+ // nor its plan/snapshot/retained descriptors survive this compact copy.
39
+ const result=scalar?guard({sql:scalar.sql,params}):{sql:statement.sql,params};
40
+ if(statement.resultKey)result.resultKey=statement.resultKey;
41
+ return result;
42
+ }
43
+ const parameterIdentity=(value:any)=>value instanceof ArrayBuffer||ArrayBuffer.isView(value)?['bytes',sha256(value instanceof ArrayBuffer?new Uint8Array(value):new Uint8Array(value.buffer,value.byteOffset,value.byteLength))]:value;
44
+ export function prepareErasureFragment(fragment:ErasureFragment,id:string,start:number){
45
+ const values:string[]=[],ordinals=new Map<string,number>();
46
+ const reference=(value:string)=>{let ordinal=ordinals.get(value);if(ordinal===undefined){ordinal=start+values.length;values.push(value);ordinals.set(value,ordinal);}return sql("(SELECT group_concat(body,'') FROM (SELECT body FROM ik_d1_native_staging_values WHERE stage_id=? AND value_ordinal=? ORDER BY ordinal))",id,ordinal);};
47
+ const convert=(rows:D1Sql[])=>rows.map(row=>bare(replaceRetained(row,reference)));
48
+ const before=convert(fragment.before),writes=convert(fragment.writes),post=convert(fragment.post);
49
+ const hash=createHash('sha256');
50
+ for(const rows of [before,writes,post])for(const row of rows)hash.update(JSON.stringify([row.sql,(row.params??[]).map(parameterIdentity)])+'\n');
51
+ for(const value of values)hash.update(JSON.stringify([Buffer.byteLength(value),sha256(value)])+'\n');
52
+ return {before,writes,post,values,hash:hash.digest('hex')};
53
+ }
54
+
55
+ /** Retains numeric batch descriptors, never the values from prior fragments. */
56
+ export class ErasureRowsAdmission {
57
+ private parts:string[]=[];private bytes=2;private written=0;
58
+ private batches:{count:number;written:number;raw:RpcSize;encoded:RpcSize}[]=[];
59
+ private limit:number;
60
+ constructor(limit=NATIVE_STAGE_ROWS_BYTES){this.limit=limit;}
61
+ append(row:unknown[]){const text=JSON.stringify(row),bytes=Buffer.byteLength(text)+1;if(bytes+2>this.limit)throw new Error('IMPORT_D1_TRANSPORT_LIMIT');if(this.parts.length&&this.bytes+bytes>this.limit)this.flush();this.parts.push(text);this.bytes+=bytes;}
62
+ private flush(){
63
+ if(!this.parts.length)return;
64
+ const body='['+this.parts.join(',')+']',statement={sql:'SELECT value FROM json_each(?)',params:[body]},empty=rpcSize([{...statement,params:['']}]);
65
+ const delta=(size:RpcSize)=>({utf8:size.utf8-empty.utf8,units:size.units-empty.units,hasWide:size.hasWide});
66
+ this.batches.push({count:this.parts.length,written:this.written,raw:delta(rpcSize([statement])),encoded:delta(rpcSize([wireSql(statement)]))});
67
+ this.written+=this.parts.length;this.parts=[];this.bytes=2;
68
+ }
69
+ get count(){return this.written+this.parts.length;}
70
+ cost(context:D1ImportContext,stage:NativeStageBinding,baseline:D1Sql[],table:'index'|'values',revision:number,checkpoint?:NativeStageCheckpoint){
71
+ this.flush();let queries=0;
72
+ for(const batch of this.batches){
73
+ const members=stagingStatements(context,'erase',nativeRowStatements(stage,baseline,table,table==='index'?erasureIndexColumns:erasureValueColumns,'',batch.count,batch.written,revision,checkpoint));
74
+ const withBodies=(size:RpcSize,body:RpcSize)=>rpcSizeBytes({utf8:size.utf8+2*body.utf8,units:size.units+2*body.units,hasWide:size.hasWide||body.hasWide});
75
+ // Both insert and in-transaction equality carry this body. Keep only
76
+ // their numeric footprints; choose the same whole-call encoding as D1.
77
+ const bytes=Math.min(withBodies(rpcSize(members),batch.raw),withBodies(rpcSize(members.map(wireSql)),batch.encoded));
78
+ if(bytes>Math.min(context.maxBatchBytes??2*1024*1024,2*1024*1024))throw new Error('IMPORT_D1_TRANSPORT_LIMIT');
79
+ queries+=members.length;revision+=batch.count;
80
+ }
81
+ return queries;
82
+ }
83
+ }
@@ -0,0 +1,46 @@
1
+ import type { D1Sql } from './types.ts';
2
+ import { retainedTemplate as sql, retainedSql } from './retained-values.ts';
3
+ import { jsonChunks } from './sets.ts';
4
+ import { changed, updateRows, deleteRows } from './expected-state.ts';
5
+ /** Rewrite retained provenance or history bodies one packed chunk per statement,
6
+ * so a job-wide change (a withdrawn artifact's name) does not cost one per row. */
7
+ export function recordWrites(table: string, rows: { id: string; body: string }[]): D1Sql[] {
8
+ return jsonChunks(rows, 500000).map(chunk => {
9
+ const selected = new Map<string, string>(JSON.parse(chunk).map((row: { id: string; body: string }) => [row.id, row.body]));
10
+ // Joined by key, not a correlated lookup per row: cost stays linear in the chunk.
11
+ return changed(retainedSql([`UPDATE ${table} SET record_json=d.body FROM (SELECT json_extract(value,'$.id') AS id,json_extract(value,'$.body') AS body FROM json_each(`, `)) AS d WHERE ${table}.id=d.id`], [chunk], [0]),
12
+ updateRows(table, item => selected.has(item.id), item => ({ ...item, record_json: selected.get(item.id)! })));
13
+ });
14
+ }
15
+ export interface ErasedDecision { job_id: string; ordinal: number; identity: string; source_hash: string; effects_hash: string }
16
+ export interface ErasedJob { id: string; body: string; fingerprint: string; erasedRecords: number; policyHash: string | null }
17
+ /** Set writes keep repeated imports from spending one D1 statement per copy.
18
+ * Literal template boundaries make each normalized JSON binding retainable. */
19
+ export function erasureWrites(jobs: ErasedJob[], decisions: ErasedDecision[], redactions: ErasedDecision[] = []): D1Sql[] {
20
+ const result: D1Sql[] = [];
21
+ for (const chunk of jsonChunks(jobs, 500000)) {
22
+ const selected=new Map<string,ErasedJob>(JSON.parse(chunk).map((row:ErasedJob)=>[row.id,row]));
23
+ result.push(changed(sql`UPDATE ik_jobs SET plan_json=(SELECT json_extract(value,'$.body') FROM json_each(${chunk}) WHERE json_extract(value,'$.id')=ik_jobs.id),fingerprint=(SELECT json_extract(value,'$.fingerprint') FROM json_each(${chunk}) WHERE json_extract(value,'$.id')=ik_jobs.id) WHERE id IN (SELECT json_extract(value,'$.id') FROM json_each(${chunk}))`,updateRows('ik_jobs',row=>selected.has(row.id),row=>({...row,plan_json:selected.get(row.id)!.body,fingerprint:selected.get(row.id)!.fingerprint}))));
24
+ result.push(changed(sql`UPDATE ik_jobs SET reversal_policy_hash=(SELECT json_extract(value,'$.policyHash') FROM json_each(${chunk}) WHERE json_extract(value,'$.id')=ik_jobs.id) WHERE id IN (SELECT json_extract(value,'$.id') FROM json_each(${chunk}))`,updateRows('ik_jobs',row=>selected.has(row.id),row=>({...row,reversal_policy_hash:selected.get(row.id)!.policyHash}))));
25
+ result.push(changed(sql`UPDATE ik_d1_indexed_jobs SET fingerprint=(SELECT json_extract(value,'$.fingerprint') FROM json_each(${chunk}) WHERE json_extract(value,'$.id')=job_id) WHERE job_id IN (SELECT json_extract(value,'$.id') FROM json_each(${chunk}))`,updateRows('ik_d1_indexed_jobs',row=>selected.has(row.job_id),row=>({...row,fingerprint:selected.get(row.job_id)!.fingerprint}))));
26
+ result.push(changed(sql`UPDATE ik_d1_stream_pages SET erased_records=(SELECT json_extract(value,'$.erasedRecords') FROM json_each(${chunk}) WHERE json_extract(value,'$.id')=job_id) WHERE job_id IN (SELECT json_extract(value,'$.id') FROM json_each(${chunk}))`,updateRows('ik_d1_stream_pages',row=>selected.has(row.job_id),row=>({...row,erased_records:selected.get(row.job_id)!.erasedRecords}))));
27
+ }
28
+ for (const chunk of jsonChunks(decisions, 500000)) {
29
+ const selected:ErasedDecision[]=JSON.parse(chunk);
30
+ const match=(row:any)=>selected.some(decision=>decision.job_id===row.job_id&&decision.ordinal===row.ordinal);
31
+ for (const table of ['ik_provenance', 'ik_d1_history']) result.push(changed(retainedSql([`DELETE FROM ${table} WHERE EXISTS(SELECT 1 FROM json_each(`,") WHERE json_extract(value,'$.job_id')=job_id AND json_extract(value,'$.ordinal')=ordinal)"],[chunk],[0]),deleteRows(table,match)));
32
+ }
33
+ // Updates join each packed chunk by key instead of looking every row up in it:
34
+ // a correlated lookup costs rows x chunk, which took seconds at a few thousand.
35
+ for (const chunk of jsonChunks([...decisions, ...redactions], 500000)) {
36
+ const selected:ErasedDecision[]=JSON.parse(chunk),byEntry=new Map(selected.map(row=>[`${row.job_id} ${row.ordinal}`,row]));
37
+ // A source shared by several jobs keeps one claim, owned by one job; its
38
+ // entries can differ per job (a redacted entry keeps each job's own effects),
39
+ // so the claim takes the hashes of its own job's decision.
40
+ const bySource=new Map(selected.map(row=>[`${row.identity} ${row.job_id}`,row])),sources=JSON.stringify([...bySource.values()]);
41
+ result.push(changed(sql`UPDATE ik_sources SET source_hash=d.source_hash,effects_hash=d.effects_hash FROM (SELECT json_extract(value,'$.identity') AS identity,json_extract(value,'$.job_id') AS job_id,json_extract(value,'$.source_hash') AS source_hash,json_extract(value,'$.effects_hash') AS effects_hash FROM json_each(${sources})) AS d WHERE ik_sources.identity=d.identity AND ik_sources.job_id=d.job_id`,updateRows('ik_sources',row=>bySource.has(`${row.identity} ${row.job_id}`),row=>({...row,source_hash:bySource.get(`${row.identity} ${row.job_id}`)!.source_hash,effects_hash:bySource.get(`${row.identity} ${row.job_id}`)!.effects_hash}))));
42
+ result.push(changed(sql`UPDATE ik_d1_entries SET effects_hash=d.effects_hash FROM (SELECT json_extract(value,'$.job_id') AS job_id,json_extract(value,'$.ordinal') AS ordinal,json_extract(value,'$.effects_hash') AS effects_hash FROM json_each(${chunk})) AS d WHERE ik_d1_entries.job_id=d.job_id AND ik_d1_entries.ordinal=d.ordinal`,updateRows('ik_d1_entries',row=>byEntry.has(`${row.job_id} ${row.ordinal}`),row=>({...row,effects_hash:byEntry.get(`${row.job_id} ${row.ordinal}`)!.effects_hash}))));
43
+ result.push(changed(sql`DELETE FROM ik_sources WHERE identity IN (SELECT json_extract(value,'$.identity') FROM json_each(${chunk})) AND job_id IN (SELECT id FROM ik_jobs WHERE state IN ('undone','rolled_back')) AND NOT EXISTS(SELECT 1 FROM ik_provenance p WHERE p.source_identity=ik_sources.identity AND p.reversed=0) AND NOT EXISTS(SELECT 1 FROM ik_d1_history h WHERE h.source_identity=ik_sources.identity)`,deleteRows('ik_sources',(row,state)=>selected.some(decision=>decision.identity===row.identity)&&[...state.rows('ik_jobs').values()].some(job=>job.id===row.job_id&&['undone','rolled_back'].includes(job.state))&&![...state.rows('ik_provenance').values()].some(p=>p.source_identity===row.identity&&!p.reversed)&&![...state.rows('ik_d1_history').values()].some(p=>p.source_identity===row.identity))));
44
+ }
45
+ return result;
46
+ }
@@ -0,0 +1,170 @@
1
+ import { erasedCustody } from './erased-custody.ts';
2
+ import { targetGuards } from './target-guards.ts';
3
+ import type { D1FieldMatch, D1ImportContext, D1RelatedErasure, D1Sql } from './types.ts';
4
+ import { requestContext, packGuards, remaining, D1RequestLimitError } from './budget.ts';
5
+ import { readAtomic, authority, finalAuthority, readonlyChecks, guard, sql, prepare } from './database.ts';
6
+ import { decode } from './jobs.ts';
7
+ import { retainPlan } from './plans.ts';
8
+ import { inventoryGuards, custodyGuards } from './integrity.ts';
9
+ import { readOriginals } from './originals.ts';
10
+ import { exactRows, jsonChunks, selectedCounts, selectKeys } from './sets.ts';
11
+ import { canonicalJson, sha256, requireText } from '../provenance/identity.ts';
12
+ import { erasePlanRecords, type RelatedPlanErasure } from './erased-plan.ts';
13
+ import { fieldSelectionSnapshot, relatedSelection, type JobSelection } from './erasure-related.ts';
14
+ import { indexedDestinationGuard, planIndex, retainPlanIndex, targetIdentity, unindexedJobs } from './erasure-index.ts';
15
+ import { readInventorySnapshot } from './inventory-snapshot.ts';
16
+ import { erasureWrites, type ErasedDecision, type ErasedJob } from './erasure-writes.ts';
17
+ import { stagedSourceIdentities, stageErasure } from './staging-erasure.ts';
18
+ import { attachReversal, loadReversalState, reversalGuards, policyHash } from './undo-state.ts';
19
+ import { eraseReversalEvidence } from './erased-reversal.ts';
20
+ import type { ImportPlan } from '../types.ts';
21
+ import { changed, deleteRows } from './expected-state.ts';
22
+ import { eraseBounded } from './erasure-bounded.ts';
23
+ import { originalVersions } from './original-storage.ts';
24
+ import { fieldRedactions, inRange, redactPlanFields, unmatchedPath } from './field-redaction.ts';
25
+ import { redactedCustody, redactedNativeFields } from './redacted-custody.ts';
26
+ import { supersededEntries } from './erasure-superseded.ts';
27
+
28
+ /** Erase one normalized native entity and every retained copy of its source.
29
+ * nativeStatements execute in the same D1 batch as all kit custody changes.
30
+ * Current erase authority is destination-scoped, independently of job ownership.
31
+ * `related` also erases other retained records and originals the product
32
+ * recognizes as holding the same person's data; their effects are severed, so
33
+ * undo leaves those native records alone, and the product's nativeStatements
34
+ * must redact them. `related.fields` instead sanitizes peer values while
35
+ * preserving their effects, provenance and reversible before-images. */
36
+ export async function eraseD1ImportTarget(kind: string, id: string, nativeStatements: D1Sql[], context: D1ImportContext, related?: D1RelatedErasure): Promise<{ nativeResults: unknown[]; erasedRecords: number; fieldMatches: D1FieldMatch[] }> {
37
+ context = requestContext(context);
38
+ if (related) related = { ...related, fields: fieldRedactions(related.fields) };
39
+ if (context.originalErasurePolicy !== undefined && !['withdraw-affected-artifacts', 'withdraw-all-job-artifacts'].includes(context.originalErasurePolicy)) throw new Error('INVALID_ORIGINAL_ERASURE_POLICY');
40
+ requireText(kind, 'target kind'); requireText(id, 'target id');
41
+ await readAtomic(context, [], undefined, 'erase');
42
+ if(context.maxBatchBytes!==undefined){if(related)throw new Error('IMPORT_RELATED_ERASURE_UNBOUNDED_ONLY');return { ...await eraseBounded(kind,id,nativeStatements,context), fieldMatches: [] };}
43
+ const { before, after, final, decisions, fieldMatches } = await targetErasure({ kind, id }, context, related);
44
+ const db = context.storage.database;
45
+ const prefix = packGuards([authority(context, 'erase'), ...before]);
46
+ // Assertion cleanup can itself revoke authority through native SQL triggers.
47
+ // The last statement must abort the same batch without creating another row.
48
+ const suffix = [...packGuards([...after, authority(context, 'erase'), sql('DELETE FROM ik_d1_assertions')]), ...readonlyChecks(final), finalAuthority(context, 'erase')];
49
+ if (prefix.length + nativeStatements.length + suffix.length > remaining(context)) throw new D1RequestLimitError();
50
+ const results = await db.batch([...prefix, ...nativeStatements, ...suffix].map(statement => prepare(db, statement)));
51
+ return { nativeResults: results.slice(prefix.length, prefix.length + nativeStatements.length), erasedRecords: decisions.length, fieldMatches };
52
+ }
53
+
54
+ /** Restricts one bounded step of a resumable redaction: only `selection` jobs are
55
+ * read, and with `owned` only the native rows their active provenance owns are
56
+ * rewritten. `matched` carries match counts from earlier steps. `sever`, in a
57
+ * step with no target, is the erased target's digest: the related source copies
58
+ * the step finds are severed and bound to it, and so are the target's own records
59
+ * whose import of it was superseded or reversed (see erasure-superseded.ts); its
60
+ * other own records (indexed to it) wait for the final step and its native erasure. */
61
+ export interface ErasureStep { selection: JobSelection; owned?: boolean; matched?: D1FieldMatch[]; sever?: string }
62
+
63
+ /** The unbounded erasure of one target (or, with no target, only of the related
64
+ * fields) as guards before the native statements and writes after them. Jobs in
65
+ * `retiring` are left alone: the caller deletes their whole custody in the same
66
+ * batch, so their plans are not rewritten. */
67
+ export async function targetErasure(target: { kind: string; id: string } | undefined, context: D1ImportContext, related?: D1RelatedErasure, retiring: ReadonlySet<string> = new Set(), step?: ErasureStep): Promise<{ before: D1Sql[]; after: D1Sql[]; final: D1Sql[]; decisions: ErasedDecision[]; jobs: string[]; fieldMatches: D1FieldMatch[] }> {
68
+ const { database: db, targets } = context.storage;
69
+ const kind = target?.kind ?? '', id = target?.id ?? '';
70
+ if (await db.prepare(`${unindexedJobs} LIMIT 1`).bind(context.destination).first()) throw new Error('IMPORT_ERASURE_INDEX_UPGRADE_REQUIRED');
71
+ const digest = target ? targetIdentity(context.destination, kind, id) : undefined;
72
+ const indexed = digest ? await selectKeys(db, 'ik_d1_plan_index', 'target_identity', [digest]) : [];
73
+ // Copies earlier steps severed are tombstones bound to this target: a step's
74
+ // job selection, not this index, decides which jobs it reads.
75
+ const targetRows = step && digest ? await liveRows(db, context.destination, indexed) : indexed;
76
+ const scan = related ? await relatedSelection(context, related, step?.selection) : undefined;
77
+ let identities = new Set<string>([...targetRows.map(row => row.source_identity), ...digest ? await stagedSourceIdentities(digest, context) : [], ...scan?.identities ?? []]);
78
+ let sourceRows = await selectKeys(db, 'ik_d1_plan_index', 'source_identity', [...identities]);
79
+ let superseded: D1Sql[] = [];
80
+ if (!digest && step?.sever) {
81
+ const earlier = await supersededEntries(db, context.destination, step.sever, sourceRows);
82
+ const own = new Set(sourceRows.filter(row => row.target_identity === step.sever && !earlier.identities.has(row.source_identity)).map(row => row.source_identity));
83
+ identities = new Set([...identities].filter(identity => !own.has(identity)));
84
+ sourceRows = sourceRows.filter(row => !own.has(row.source_identity));
85
+ superseded = earlier.guards;
86
+ }
87
+ const bound = digest ?? step?.sever ?? '';
88
+ const severed: RelatedPlanErasure = { identities: new Set([...scan?.identities ?? []].filter(identity => identities.has(identity) && !targetRows.some(row => row.source_identity === identity))), target: bound };
89
+ const privateErasure = digest ? stageErasure(context.destination, digest, [...identities]) : { before: [], after: [] };
90
+ const ids = [...new Set([...sourceRows.map(row => row.job_id), ...scan?.originals.keys() ?? [], ...scan?.redactedJobs ?? []])];
91
+ const snapshot = await readInventorySnapshot(ids, db, targets.planCodec);
92
+ const before: D1Sql[] = [indexedDestinationGuard(context.destination), ...privateErasure.before, ...scan?.guards ?? [], ...superseded,
93
+ ...digest ? selectedCounts('ik_d1_plan_index', 'target_identity', [digest], indexed) : [],
94
+ ...selectedCounts('ik_d1_plan_index', 'source_identity', [...identities], sourceRows),
95
+ ...exactRows('ik_d1_plan_index', ['job_id','ordinal','source_identity','target_identity'], sourceRows)];
96
+ const after: D1Sql[] = [], final: D1Sql[] = [], changedJobs: ErasedJob[] = [], decisions: ErasedDecision[] = [], redactions: ErasedDecision[] = [];
97
+ const fields = related?.fields ?? [];
98
+ if (fields.length || related?.snapshot !== undefined) {
99
+ const range = step?.selection.range;
100
+ const owned = step?.owned ? new Set(snapshot.rows('ik_provenance', 'job_id', ids).filter(row => !row.reversed && inRange(range, row.ordinal)).map(row => `${row.target_kind}\u0000${row.target_id}`)) : undefined;
101
+ const native = await redactedNativeFields(kind, id, context, fields, scan!.fieldMatches, owned && (row => owned.has(`${row.targetKind}\u0000${row.targetId}`)));
102
+ if (related?.snapshot !== undefined && related.snapshot !== fieldSelectionSnapshot([...scan!.guards, ...native.before])) throw new Error('IMPORT_REDACTION_SNAPSHOT_CHANGED');
103
+ for (const earlier of step?.matched ?? []) { const count = scan!.fieldMatches.find(item => canonicalJson(item.path) === canonicalJson(earlier.path)); if (count) count.matches += earlier.matches; }
104
+ if (!step?.owned && unmatchedPath(fields, scan!.fieldMatches)) throw new Error('IMPORT_REDACTION_PATH_UNMATCHED');
105
+ before.push(...native.before); after.push(...native.after); final.push(...native.final);
106
+ }
107
+ const changedPlans = new Map<string, ImportPlan>();
108
+ const kept = ids.filter(job => !retiring.has(job));
109
+ for (const raw of snapshot.rows('ik_jobs', 'id', kept)) {
110
+ const plan = snapshot.plans.get(raw.id)!;
111
+ const originals = scan?.originals.get(raw.id);
112
+ if (plan.destination !== context.destination || (!originals && !scan?.redactedJobs.has(raw.id) && !plan.entries.some(entry => identities.has(entry.identity)))) throw new Error('IMPORT_ERASURE_INDEX_MISMATCH');
113
+ if (plan.entries.some(entry => identities.has(entry.identity) && !severed.identities.has(entry.identity) && entry.effects.some(effect => effect.targetKind !== kind || effect.targetId !== id))) throw new Error('IMPORT_ERASURE_SHARED_SOURCE');
114
+ const job = decode(raw);
115
+ const reversal = await loadReversalState(raw, plan, context, false);
116
+ attachReversal(job, reversal); before.push(...reversalGuards(job.id, reversal));
117
+ await readOriginals(plan, job.id, context, snapshot);
118
+ before.push(...custodyGuards(plan, job.id), ...await inventoryGuards(job, plan, context, snapshot));
119
+ let next = erasePlanRecords(plan, identities, context.originalErasurePolicy, { ...severed, originals });
120
+ if (fields.length) next = redactPlanFields(next, fields, context.originalErasurePolicy, step?.selection.range).plan;
121
+ const retained = retainPlan(next, job.id, next.source.artifacts.some(a => a.original?.status === 'available') ? undefined : targets.planCodec);
122
+ originalVersions.set(next,originalVersions.get(plan)!);
123
+ changedPlans.set(job.id, next);
124
+ after.push(...erasedCustody(plan, next, job.id, snapshot), ...retained.statements);
125
+ if (fields.length) after.push(...redactedCustody(plan, next, job.id, snapshot, fields, step?.selection.range));
126
+ // A severed record's index row moves from its native target (or none) to the erased target.
127
+ // Rewriting the index drops the job's indexed fingerprint (a schema trigger); retain it again.
128
+ if (canonicalJson(planIndex(next, job.id)) !== canonicalJson(planIndex(plan, job.id))) after.push(sql('DELETE FROM ik_d1_plan_index WHERE job_id=?', job.id), ...retainPlanIndex(next, job.id));
129
+ after.push(guard(sql('(SELECT count(*) FROM ik_d1_creation_plans WHERE job_id=? AND released=0)=? AND (SELECT count(*) FROM ik_d1_plan_chunks WHERE job_id=?)=?', job.id, retained.encoded.chunks.length, job.id, retained.encoded.chunks.length)));
130
+ changedJobs.push({ id: job.id, body: retained.body, fingerprint: next.fingerprint, erasedRecords: next.entries.filter(entry => entry.erased).length, policyHash: reversal.binding === null ? null : policyHash(job.id, next.fingerprint, reversal.policy) });
131
+ // Changing the fingerprint fences stale apply/undo evidence while preserving
132
+ // the apply journal's sequential fences for a partially imported file.
133
+ for (const [ordinal, entry] of next.entries.entries()) {
134
+ if (!entry.erased && (entry.source.payloadHash !== plan.entries[ordinal].source.payloadHash || canonicalJson(entry.effects) !== canonicalJson(plan.entries[ordinal].effects))) {
135
+ redactions.push({ job_id: job.id, ordinal, identity: entry.identity, source_hash: entry.source.payloadHash, effects_hash: sha256(canonicalJson(entry.effects)) });
136
+ }
137
+ if (!identities.has(entry.identity)) continue;
138
+ decisions.push({ job_id: job.id, ordinal, identity: entry.identity, source_hash: entry.source.payloadHash, effects_hash: sha256(canonicalJson(entry.effects)) });
139
+ }
140
+ }
141
+ const releaseCopies = jsonChunks(kept).flatMap(chunk => [sql('UPDATE ik_d1_creation_plans SET released=1 WHERE job_id IN (SELECT value FROM json_each(?))', chunk), sql('DELETE FROM ik_d1_creation_plans WHERE job_id IN (SELECT value FROM json_each(?))', chunk)]);
142
+ const removeChunks = [...releaseCopies, ...jsonChunks(kept).map(chunk => {const selected=new Set(JSON.parse(chunk));return changed(sql('DELETE FROM ik_d1_plan_chunks WHERE job_id IN (SELECT value FROM json_each(?))', chunk),deleteRows('ik_d1_plan_chunks',row=>selected.has(row.job_id)));})];
143
+ after.unshift(...removeChunks);
144
+ const reports = await eraseReversalEvidence(context, changedPlans);
145
+ before.push(...reports.before);
146
+ after.push(...releaseCopies, ...erasureWrites(changedJobs, decisions, redactions), ...reports.after, ...privateErasure.after);
147
+ if (fields.length || related?.snapshot !== undefined) for (const table of ['ik_d1_staging', 'ik_d1_native_staging', 'ik_d1_preparation_steps']) {
148
+ after.push(sql(`DELETE FROM ${table} WHERE destination=?`, context.destination), guard(sql(`NOT EXISTS(SELECT 1 FROM ${table} WHERE destination=?)`, context.destination)));
149
+ }
150
+ for (const [id, next] of changedPlans) after.push(...custodyGuards(next, id));
151
+ for (const chunk of jsonChunks([...identities])) after.push(sql('INSERT OR IGNORE INTO ik_d1_erased_sources(destination,identity) SELECT ?,value FROM json_each(?)', context.destination, chunk));
152
+ if (digest) {
153
+ after.push(sql('INSERT OR IGNORE INTO ik_d1_erased_targets(destination,identity) VALUES(?,?)', context.destination, digest));
154
+ const current = await targets.read(db, context.destination, kind, id);
155
+ before.push(...targetGuards(targets,current ?? null, context.destination, kind, id));
156
+ after.push(...targetGuards(targets,null, context.destination, kind, id));
157
+ } else if (bound && identities.size) after.push(sql('INSERT OR IGNORE INTO ik_d1_erased_targets(destination,identity) VALUES(?,?)', context.destination, bound));
158
+ return { before, after, final, decisions, jobs: ids, fieldMatches: scan?.fieldMatches ?? [] };
159
+ }
160
+
161
+ /** Index rows whose source is not an erased tombstone already. A tombstone's
162
+ * job is fenced like any other: by the step's selection or its clean revision. */
163
+ async function liveRows(db: D1ImportContext['storage']['database'], destination: string, rows: any[]): Promise<any[]> {
164
+ const erased = new Set<string>();
165
+ for (const chunk of jsonChunks([...new Set(rows.map(row => row.source_identity as string))])) {
166
+ const { results } = await db.prepare('SELECT identity FROM ik_d1_erased_sources WHERE destination=? AND identity IN (SELECT value FROM json_each(?))').bind(destination, chunk).all<{ identity: string }>();
167
+ for (const row of results) erased.add(row.identity);
168
+ }
169
+ return rows.filter(row => !erased.has(row.source_identity));
170
+ }
@@ -0,0 +1,13 @@
1
+ /** Additive evolution. Existing products apply these statements ONCE in their
2
+ * next numbered migration. Never edit an already applied product migration. */
3
+ export const preservedUndoEvolution = [
4
+ "ALTER TABLE ik_jobs ADD COLUMN reversal_policy TEXT NOT NULL DEFAULT 'strict' CHECK(reversal_policy IN ('strict','preserve-conflicts'))",
5
+ 'ALTER TABLE ik_jobs ADD COLUMN reversal_policy_hash TEXT',
6
+ 'ALTER TABLE ik_jobs ADD COLUMN reversal_cursor INTEGER NOT NULL DEFAULT 0 CHECK(reversal_cursor>=0)',
7
+ "ALTER TABLE ik_jobs ADD COLUMN reversal_hash TEXT NOT NULL DEFAULT '4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945'",
8
+ 'CREATE TABLE ik_d1_creation_plans (job_id TEXT NOT NULL REFERENCES ik_jobs(id), ordinal INTEGER NOT NULL, body TEXT NOT NULL, released INTEGER NOT NULL DEFAULT 0 CHECK(released IN (0,1)), PRIMARY KEY(job_id,ordinal))',
9
+ 'CREATE TRIGGER ik_d1_creation_plan_capture AFTER INSERT ON ik_d1_plan_chunks BEGIN INSERT INTO ik_d1_creation_plans(job_id,ordinal,body) VALUES(NEW.job_id,NEW.ordinal,NEW.body); END',
10
+ `CREATE TRIGGER ik_d1_creation_plan_immutable BEFORE UPDATE ON ik_d1_creation_plans WHEN NEW.job_id IS NOT OLD.job_id OR NEW.ordinal IS NOT OLD.ordinal OR NEW.body IS NOT OLD.body OR OLD.released<>0 OR NEW.released<>1 OR NOT EXISTS(SELECT 1 FROM ik_d1_plan_chunks p WHERE p.job_id=OLD.job_id AND p.ordinal=OLD.ordinal AND p.body IS OLD.body) BEGIN SELECT RAISE(ABORT,'IMPORT_CREATION_SNAPSHOT_CHANGED'); END`,
11
+ `CREATE TRIGGER ik_d1_creation_plan_release BEFORE DELETE ON ik_d1_creation_plans WHEN OLD.released<>1 BEGIN SELECT RAISE(ABORT,'IMPORT_CREATION_SNAPSHOT_CHANGED'); END`,
12
+ 'CREATE TABLE ik_d1_reversal_reports (job_id TEXT NOT NULL REFERENCES ik_jobs(id), effect_id TEXT NOT NULL, position INTEGER NOT NULL, record_json TEXT NOT NULL, PRIMARY KEY(job_id,effect_id), UNIQUE(job_id,position))',
13
+ ];