@runeditrun/rer 0.2.77 → 0.2.79

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (167) hide show
  1. package/README.md +32 -0
  2. package/cli/export.mjs +1 -1
  3. package/cli/portability-migrations.mjs +35 -14
  4. package/cli/portability-worker.mjs +2 -2
  5. package/cli/portability.mjs +2 -2
  6. package/cli/server-proxy.mjs +30 -8
  7. package/cli/upstream-agent.mjs +14 -0
  8. package/global/CHANGELOG.md +13 -0
  9. package/global/README.md +32 -0
  10. package/modules/demo-host/native/visitor.mjs +1 -1
  11. package/modules/deploy-kit/owner-cli.mjs +15 -5
  12. package/modules/deploy-kit/provider.mjs +9 -4
  13. package/modules/import-kit/src/d1/README.md +1322 -0
  14. package/modules/import-kit/src/d1/apply.ts +115 -0
  15. package/modules/import-kit/src/d1/budget.ts +89 -0
  16. package/modules/import-kit/src/d1/compressed-plan.ts +63 -0
  17. package/modules/import-kit/src/d1/creation.ts +95 -0
  18. package/modules/import-kit/src/d1/custody-fence.ts +27 -0
  19. package/modules/import-kit/src/d1/custody-state-schema.ts +67 -0
  20. package/modules/import-kit/src/d1/database.ts +163 -0
  21. package/modules/import-kit/src/d1/entry-locator.ts +61 -0
  22. package/modules/import-kit/src/d1/erased-artifacts.ts +38 -0
  23. package/modules/import-kit/src/d1/erased-custody.ts +36 -0
  24. package/modules/import-kit/src/d1/erased-plan.ts +47 -0
  25. package/modules/import-kit/src/d1/erased-reversal.ts +19 -0
  26. package/modules/import-kit/src/d1/erasure-bounded.ts +22 -0
  27. package/modules/import-kit/src/d1/erasure-claims.ts +27 -0
  28. package/modules/import-kit/src/d1/erasure-compiler.ts +67 -0
  29. package/modules/import-kit/src/d1/erasure-disposition.ts +16 -0
  30. package/modules/import-kit/src/d1/erasure-fragments.ts +86 -0
  31. package/modules/import-kit/src/d1/erasure-index.ts +53 -0
  32. package/modules/import-kit/src/d1/erasure-pages.ts +62 -0
  33. package/modules/import-kit/src/d1/erasure-progress-schema.ts +8 -0
  34. package/modules/import-kit/src/d1/erasure-progress.ts +61 -0
  35. package/modules/import-kit/src/d1/erasure-related.ts +124 -0
  36. package/modules/import-kit/src/d1/erasure-report.ts +31 -0
  37. package/modules/import-kit/src/d1/erasure-schema.ts +24 -0
  38. package/modules/import-kit/src/d1/erasure-selection.ts +77 -0
  39. package/modules/import-kit/src/d1/erasure-snapshot.ts +15 -0
  40. package/modules/import-kit/src/d1/erasure-stage.ts +146 -0
  41. package/modules/import-kit/src/d1/erasure-step.ts +12 -0
  42. package/modules/import-kit/src/d1/erasure-superseded.ts +98 -0
  43. package/modules/import-kit/src/d1/erasure-values.ts +83 -0
  44. package/modules/import-kit/src/d1/erasure-writes.ts +46 -0
  45. package/modules/import-kit/src/d1/erasure.ts +170 -0
  46. package/modules/import-kit/src/d1/evolutions/0001-preserved-undo.ts +13 -0
  47. package/modules/import-kit/src/d1/expected-state.ts +91 -0
  48. package/modules/import-kit/src/d1/field-redaction.ts +86 -0
  49. package/modules/import-kit/src/d1/guards.ts +37 -0
  50. package/modules/import-kit/src/d1/history.ts +71 -0
  51. package/modules/import-kit/src/d1/index.ts +88 -0
  52. package/modules/import-kit/src/d1/integrity.ts +127 -0
  53. package/modules/import-kit/src/d1/inventory-snapshot.ts +84 -0
  54. package/modules/import-kit/src/d1/jobs.ts +97 -0
  55. package/modules/import-kit/src/d1/native-set.ts +48 -0
  56. package/modules/import-kit/src/d1/native-staging-admission.ts +19 -0
  57. package/modules/import-kit/src/d1/native-staging-context.ts +44 -0
  58. package/modules/import-kit/src/d1/native-staging-lifecycle.ts +33 -0
  59. package/modules/import-kit/src/d1/native-staging-offset.ts +74 -0
  60. package/modules/import-kit/src/d1/native-staging-publication.ts +18 -0
  61. package/modules/import-kit/src/d1/native-staging-rows.ts +72 -0
  62. package/modules/import-kit/src/d1/native-staging-schema.ts +28 -0
  63. package/modules/import-kit/src/d1/native-staging-state.ts +22 -0
  64. package/modules/import-kit/src/d1/native-staging.ts +64 -0
  65. package/modules/import-kit/src/d1/original-copies.ts +53 -0
  66. package/modules/import-kit/src/d1/original-storage.ts +56 -0
  67. package/modules/import-kit/src/d1/originals.ts +74 -0
  68. package/modules/import-kit/src/d1/plan-dictionary.ts +80 -0
  69. package/modules/import-kit/src/d1/plans.ts +121 -0
  70. package/modules/import-kit/src/d1/preparation-fingerprint.ts +57 -0
  71. package/modules/import-kit/src/d1/preparation-header.ts +33 -0
  72. package/modules/import-kit/src/d1/preparation-reader.ts +98 -0
  73. package/modules/import-kit/src/d1/preparation-schema.ts +16 -0
  74. package/modules/import-kit/src/d1/preparation-state.ts +56 -0
  75. package/modules/import-kit/src/d1/preparation-step-combine.ts +80 -0
  76. package/modules/import-kit/src/d1/preparation-step-control.ts +48 -0
  77. package/modules/import-kit/src/d1/preparation-step-hash.ts +131 -0
  78. package/modules/import-kit/src/d1/preparation-step-pages.ts +77 -0
  79. package/modules/import-kit/src/d1/preparation-steps-schema.ts +11 -0
  80. package/modules/import-kit/src/d1/preparation-steps.ts +102 -0
  81. package/modules/import-kit/src/d1/preparations.ts +87 -0
  82. package/modules/import-kit/src/d1/preserved-preview.ts +89 -0
  83. package/modules/import-kit/src/d1/preserved-reverse.ts +91 -0
  84. package/modules/import-kit/src/d1/provenance-set.ts +87 -0
  85. package/modules/import-kit/src/d1/publication-fence.ts +22 -0
  86. package/modules/import-kit/src/d1/publication-receipt-schema.ts +15 -0
  87. package/modules/import-kit/src/d1/published-stream.ts +11 -0
  88. package/modules/import-kit/src/d1/read-fence.ts +64 -0
  89. package/modules/import-kit/src/d1/read-results.ts +16 -0
  90. package/modules/import-kit/src/d1/redacted-custody.ts +76 -0
  91. package/modules/import-kit/src/d1/redaction-step.ts +125 -0
  92. package/modules/import-kit/src/d1/referenced-plans.ts +136 -0
  93. package/modules/import-kit/src/d1/retained-values.ts +35 -0
  94. package/modules/import-kit/src/d1/retirement-bounded.ts +71 -0
  95. package/modules/import-kit/src/d1/retirement-door.ts +105 -0
  96. package/modules/import-kit/src/d1/retirement-schema.ts +39 -0
  97. package/modules/import-kit/src/d1/retirement-scope.ts +94 -0
  98. package/modules/import-kit/src/d1/retirement-steps.ts +76 -0
  99. package/modules/import-kit/src/d1/retirement-tally.ts +74 -0
  100. package/modules/import-kit/src/d1/retirement.ts +136 -0
  101. package/modules/import-kit/src/d1/reversal-conversion-record.ts +19 -0
  102. package/modules/import-kit/src/d1/reversal-conversion-schema.ts +11 -0
  103. package/modules/import-kit/src/d1/reversal-conversion.ts +40 -0
  104. package/modules/import-kit/src/d1/reversal-fence-schema.ts +14 -0
  105. package/modules/import-kit/src/d1/reversal-fence.ts +106 -0
  106. package/modules/import-kit/src/d1/reversal-journal.ts +24 -0
  107. package/modules/import-kit/src/d1/reverse.ts +103 -0
  108. package/modules/import-kit/src/d1/reviewed-source.ts +23 -0
  109. package/modules/import-kit/src/d1/sets.ts +78 -0
  110. package/modules/import-kit/src/d1/staged-creation.ts +201 -0
  111. package/modules/import-kit/src/d1/staging-admission.ts +45 -0
  112. package/modules/import-kit/src/d1/staging-control.ts +43 -0
  113. package/modules/import-kit/src/d1/staging-erasure.ts +28 -0
  114. package/modules/import-kit/src/d1/staging-lifecycle.ts +49 -0
  115. package/modules/import-kit/src/d1/staging-plan.ts +15 -0
  116. package/modules/import-kit/src/d1/staging-privacy.ts +22 -0
  117. package/modules/import-kit/src/d1/staging-publication.ts +79 -0
  118. package/modules/import-kit/src/d1/staging-schema.ts +37 -0
  119. package/modules/import-kit/src/d1/staging-statements.ts +8 -0
  120. package/modules/import-kit/src/d1/staging-transport.ts +70 -0
  121. package/modules/import-kit/src/d1/stream-boundary.ts +11 -0
  122. package/modules/import-kit/src/d1/stream-create.ts +55 -0
  123. package/modules/import-kit/src/d1/stream-identity-admission.ts +37 -0
  124. package/modules/import-kit/src/d1/stream-input.ts +18 -0
  125. package/modules/import-kit/src/d1/stream-layout.ts +43 -0
  126. package/modules/import-kit/src/d1/stream-origin.ts +44 -0
  127. package/modules/import-kit/src/d1/stream-page-integrity.ts +18 -0
  128. package/modules/import-kit/src/d1/stream-partition.ts +50 -0
  129. package/modules/import-kit/src/d1/stream-plan.ts +133 -0
  130. package/modules/import-kit/src/d1/stream-receipt.ts +45 -0
  131. package/modules/import-kit/src/d1/stream-revision-schema.ts +35 -0
  132. package/modules/import-kit/src/d1/stream-snapshot.ts +26 -0
  133. package/modules/import-kit/src/d1/stream-stage.ts +79 -0
  134. package/modules/import-kit/src/d1/stream-state.ts +25 -0
  135. package/modules/import-kit/src/d1/stream-working-set.ts +30 -0
  136. package/modules/import-kit/src/d1/streams.ts +39 -0
  137. package/modules/import-kit/src/d1/target-guards.ts +9 -0
  138. package/modules/import-kit/src/d1/target-identities.ts +17 -0
  139. package/modules/import-kit/src/d1/transport.ts +10 -0
  140. package/modules/import-kit/src/d1/types.ts +162 -0
  141. package/modules/import-kit/src/d1/undo-observations.ts +39 -0
  142. package/modules/import-kit/src/d1/undo-report.ts +22 -0
  143. package/modules/import-kit/src/d1/undo-snapshot.ts +87 -0
  144. package/modules/import-kit/src/d1/undo-state.ts +84 -0
  145. package/modules/import-kit/src/d1/undo-types.ts +110 -0
  146. package/modules/import-kit/src/journal/authority.ts +12 -0
  147. package/modules/import-kit/src/plan/budget.ts +60 -0
  148. package/modules/import-kit/src/plan/index.ts +123 -0
  149. package/modules/import-kit/src/plan/size.ts +122 -0
  150. package/modules/import-kit/src/provenance/canonical-hash.ts +130 -0
  151. package/modules/import-kit/src/provenance/canonical-stream.ts +85 -0
  152. package/modules/import-kit/src/provenance/credential-names.ts +131 -0
  153. package/modules/import-kit/src/provenance/credentials.ts +317 -0
  154. package/modules/import-kit/src/provenance/identity.ts +51 -0
  155. package/modules/import-kit/src/provenance/json-snapshot.ts +21 -0
  156. package/modules/import-kit/src/provenance/resumable-sha256.ts +75 -0
  157. package/modules/import-kit/src/provenance/validate-json.ts +19 -0
  158. package/modules/import-kit/src/storage/originals.ts +87 -0
  159. package/modules/import-kit/src/storage/provenance.ts +64 -0
  160. package/modules/import-kit/src/storage/schema.ts +42 -0
  161. package/modules/import-kit/src/types.ts +47 -0
  162. package/modules/portability/README.md +98 -2
  163. package/modules/portability/backfills.mjs +42 -15
  164. package/modules/portability/ledgers.mjs +7 -1
  165. package/modules/portability/transient-tables.mjs +40 -0
  166. package/package.json +3 -1
  167. package/scripts/publish-check/reviewed.py +3 -1
@@ -0,0 +1,74 @@
1
+ import type { D1Database, D1Sql } from './types.ts';
2
+ import { guard, sql } from './database.ts';
3
+ import { doorRetirementStore } from './retirement-door.ts';
4
+
5
+ // Bookkeeping that every custody change bumps, plus the erasure's own records.
6
+ const SKIPPED = new Set(['ik_d1_assertions', 'ik_d1_custody_states', 'ik_d1_custody_revisions', 'ik_d1_publication_fences',
7
+ 'ik_d1_erasure_tally', 'ik_d1_erasure_journal', 'ik_d1_erasure_progress', 'ik_d1_retirements', 'ik_d1_erased_sources', 'ik_d1_erased_targets', 'ik_targets',
8
+ 'ik_door_settings', // Installation opt-in only; no uploaded record content.
9
+ 'ik_door_revoked_keys', // Persistent key revocations must survive erasure.
10
+ 'ik_door_grants', // Grant nonce replay protection; no uploaded record content.
11
+ 'ik_door_plans', // Ids, hashes, fingerprint and job link of an import: its "via Business" history.
12
+ 'ik_door_approvals', // Who approved which fingerprint and when: the owner's approval record, not imported content.
13
+ 'ik_door_namespaces', // Head and pending plan ids per namespace: lineage pointers only.
14
+ 'ik_door_scopes', // Namespace-to-destination binding: keeps a namespace from moving to another destination.
15
+ 'ik_door_apply_failures', // Plan id and allowlisted reason code of a failed apply: audit fact, no record content.
16
+ 'ik_door_archives', // Archive id, tool, time, file name, hash, size and kind counts: custody receipt; its bytes are withdrawn.
17
+ ]);
18
+ // Door record content (uploaded chunks, upload headers, snapshot bodies, archive bytes) is
19
+ // scoped and withdrawn whole; see retirement-door.ts.
20
+ const COUNTERS = new Set(['revision', 'layout_revision', 'progress_revision', 'seal']);
21
+
22
+ export interface TallyScope { destination: string; jobs: string[]; streams: string[] }
23
+ export interface Store { name: string; image: string; where: string; params: (scope: TallyScope) => unknown[]; withdraw?: number }
24
+
25
+ /** Each custody store with the row image the journal counts and the rows an
26
+ * erasure of these jobs and streams can touch. */
27
+ export async function tallyStores(db: D1Database): Promise<Store[]> {
28
+ const { results } = await db.prepare("SELECT m.name AS store, p.name AS column, p.type AS type FROM sqlite_master m JOIN pragma_table_info(m.name) p WHERE m.type='table' AND m.name LIKE 'ik\\_%' ESCAPE '\\' ORDER BY m.name, p.cid").all<{ store: string; column: string; type: string }>();
29
+ const columns = new Map<string, { column: string; type: string }[]>(), installed = new Set(results.map(row => row.store));
30
+ for (const row of results) if (!SKIPPED.has(row.store)) columns.set(row.store, [...columns.get(row.store) ?? [], row]);
31
+ const jobs = (scope: TallyScope) => [JSON.stringify(scope.jobs)], streams = (scope: TallyScope) => [JSON.stringify(scope.streams)];
32
+ const destination = (scope: TallyScope) => [scope.destination];
33
+ return [...columns].map(([name, list]) => {
34
+ const names = list.map(item => item.column);
35
+ const image = `json_array(${list.filter(item => !COUNTERS.has(item.column)).map(item => /BLOB/i.test(item.type) ? `hex("${item.column}")` : `"${item.column}"`).join(',')})`;
36
+ const door = doorRetirementStore(name, installed);
37
+ if (door) return { name, image, ...door };
38
+ if (name === 'ik_jobs') return { name, image, where: 'id IN (SELECT value FROM json_each(?))', params: jobs };
39
+ if (name === 'ik_d1_streams') return { name, image, where: 'id IN (SELECT value FROM json_each(?))', params: streams };
40
+ if (name === 'ik_d1_stream_revisions') return { name, image, where: 'stream_id IN (SELECT value FROM json_each(?))', params: streams };
41
+ if (names.includes('stage_id')) {
42
+ const parent = name.startsWith('ik_d1_native_staging') ? 'ik_d1_native_staging' : 'ik_d1_staging';
43
+ return { name, image, where: `stage_id IN (SELECT id FROM ${parent} WHERE destination=?)`, params: destination };
44
+ }
45
+ if (names.includes('job_id')) return { name, image, where: 'job_id IN (SELECT value FROM json_each(?))', params: jobs };
46
+ if (names.includes('destination')) return { name, image, where: 'destination=?', params: destination };
47
+ throw new Error(`IMPORT_RETIREMENT_UNSCOPED_STORE:${name}`);
48
+ });
49
+ }
50
+
51
+ /** Images of every row in scope, taken first in the erasure batch. */
52
+ export function tallyBefore(stores: Store[], receipt: string, scope: TallyScope): D1Sql[] {
53
+ return stores.map(store => sql(`INSERT INTO ik_d1_erasure_tally(receipt_id,store,image) SELECT ?,?,${store.image} FROM ${store.name} WHERE ${store.where}`, receipt, store.name, ...store.params(scope)));
54
+ }
55
+
56
+ /** Opaque door content is withdrawn after all before-images are tallied and
57
+ * before kit job/header changes can sever its scope links, in the same batch. */
58
+ export function tallyWithdraw(stores: Store[], scope: TallyScope): D1Sql[] {
59
+ return stores.filter(store => store.withdraw !== undefined).sort((a, b) => a.withdraw! - b.withdraw!)
60
+ .map(store => sql(`DELETE FROM ${store.name} WHERE ${store.where}`, ...store.params(scope)));
61
+ }
62
+
63
+ /** Drops the images still present unchanged; the rest were deleted or rewritten.
64
+ * Unless the batch carries the restore checkpoint, none may remain. The journal
65
+ * row then counts them per store, and the tally is emptied. */
66
+ export function tallyJournal(stores: Store[], receipt: string, scope: TallyScope, entry: { recordedAt: string; recordsErased: number; jobsRetired: number; streamsRetired: number }, fenced: boolean): D1Sql[] {
67
+ return [
68
+ ...stores.map(store => sql(`DELETE FROM ik_d1_erasure_tally WHERE receipt_id=? AND store=? AND image IN (SELECT ${store.image} FROM ${store.name} WHERE ${store.where})`, receipt, store.name, ...store.params(scope))),
69
+ ...fenced ? [] : [guard(sql('NOT EXISTS(SELECT 1 FROM ik_d1_erasure_tally WHERE receipt_id=?)', receipt))],
70
+ sql("INSERT INTO ik_d1_erasure_journal(id,destination,recorded_at,records_erased,jobs_retired,streams_retired,stores_json) SELECT ?,?,?,?,?,?,(SELECT json_group_array(json_object('store',store,'rows',rows)) FROM (SELECT store, count(*) AS rows FROM ik_d1_erasure_tally WHERE receipt_id=? GROUP BY store ORDER BY store))",
71
+ receipt, scope.destination, entry.recordedAt, entry.recordsErased, entry.jobsRetired, entry.streamsRetired, receipt),
72
+ sql('DELETE FROM ik_d1_erasure_tally WHERE receipt_id=?', receipt),
73
+ ];
74
+ }
@@ -0,0 +1,136 @@
1
+ import { randomUUID } from 'node:crypto';
2
+ import type { D1CustodyRetirement, D1ErasureJournalEntry, D1ImportContext, D1ImportPerson, D1RetirementOptions, D1RetirementProgress, D1Sql } from './types.ts';
3
+ import { requestContext, packGuards, remaining, D1RequestLimitError } from './budget.ts';
4
+ import { authority, authorize, finalAuthority, guard, readonlyChecks, prepare, readAtomic, sql } from './database.ts';
5
+ import { requireText } from '../provenance/identity.ts';
6
+ import { targetErasure, type ErasureStep } from './erasure.ts';
7
+ import { destinationRevisions, lineageDigest, readProgress, type JobRevision, type RedactionProgress } from './retirement-steps.ts';
8
+ import { overflow, redactionSteps, type LineageSever } from './redaction-step.ts';
9
+ import { targetIdentity } from './erasure-index.ts';
10
+ import { headerAbsence, retirementScope, retirementWrites, scopeGuards } from './retirement-scope.ts';
11
+ import { tallyBefore, tallyJournal, tallyStores, tallyWithdraw, type Store } from './retirement-tally.ts';
12
+ import { fieldRedactions } from './field-redaction.ts';
13
+ import { doorCustodyInScope, doorLineage, type DoorLineage } from './retirement-door.ts';
14
+ import { retireD1ImportCustodyStep } from './retirement-bounded.ts';
15
+ import { chainFirst } from './erasure-superseded.ts';
16
+ import { batchBytes } from './transport.ts';
17
+ import { RPC_LIMIT } from '../../../d1-transport/index.mjs';
18
+ import type { D1ErasurePending, D1ErasureStepOptions } from './erasure-progress.ts';
19
+
20
+ /** Erase a person from import custody and journal it, in one D1 batch with the
21
+ * product's native statements.
22
+ *
23
+ * Their records (by exact native target) become zero-effect tombstones in every
24
+ * job, so undo and resume of everyone else's records keep working. A job or
25
+ * stream whose header holds one of the person's exact values cannot be renamed
26
+ * without breaking its source identities, so its whole custody is retired: no
27
+ * undo or resume, a plain retirement row naming the journal receipt, and the
28
+ * native rows it already wrote stay. Retirement closes over the streams, reused
29
+ * sources and preserved history that depend on a retired job. The journal row
30
+ * holds the receipt id, row counts per store and the time, never a value.
31
+ *
32
+ * `checkpoint` is the product's restore fence for rewritten rows (portability's
33
+ * redaction checkpoint). It commits in the batch whenever the batch rewrites or
34
+ * retires custody, so an archive older than the erasure is refused whole once
35
+ * the later journal is supplied; a batch without it proves no custody row changed. */
36
+ export function retireD1ImportCustody(person: D1ImportPerson, nativeStatements: D1Sql[], input: D1ImportContext, checkpoint: D1Sql[]): Promise<D1CustodyRetirement>;
37
+ export function retireD1ImportCustody(person: D1ImportPerson, nativeStatements: D1Sql[], input: D1ImportContext, checkpoint: D1Sql[], options: D1ErasureStepOptions): Promise<D1CustodyRetirement | D1ErasurePending>;
38
+ export function retireD1ImportCustody(person: D1ImportPerson, nativeStatements: D1Sql[], input: D1ImportContext, checkpoint: D1Sql[], options: D1RetirementOptions): Promise<D1CustodyRetirement | D1RetirementProgress>;
39
+ export async function retireD1ImportCustody(person: D1ImportPerson, nativeStatements: D1Sql[], input: D1ImportContext, checkpoint: D1Sql[], options: D1RetirementOptions | D1ErasureStepOptions = {}): Promise<D1CustodyRetirement | D1RetirementProgress | D1ErasurePending> {
40
+ if ('operationKey' in options) return retireD1ImportCustodyStep(person, nativeStatements, input, checkpoint, options);
41
+ person = { ...person, related: person.related ? { ...person.related, fields: fieldRedactions(person.related.fields) } : undefined };
42
+ const context = requestContext({ ...input, originalErasurePolicy: input.originalErasurePolicy ?? 'withdraw-affected-artifacts' });
43
+ if (context.maxBatchBytes !== undefined) throw new Error('IMPORT_RETIREMENT_OPERATION_KEY_REQUIRED');
44
+ const values = [...new Set(person.values ?? [])].sort();
45
+ for (const value of values) requireText(value, 'person value');
46
+ if (person.target) { requireText(person.target.kind, 'target kind'); requireText(person.target.id, 'target id'); }
47
+ const fields = person.related?.fields ?? [];
48
+ if (!person.target && !values.length && !fields.length) throw new Error('IMPORT_RETIREMENT_PERSON_REQUIRED');
49
+ // Without a native target only field redaction applies: whole-record erasure
50
+ // (`source`, `original`) binds what it severs to the erased target.
51
+ if (person.related && !person.target && (person.related.source || person.related.original || !fields.length)) throw new Error('IMPORT_RETIREMENT_TARGET_REQUIRED');
52
+ await readAtomic(context, [], undefined, 'erase');
53
+ const db = context.storage.database, destination = context.destination;
54
+ const scope = await retirementScope(db, destination, values), retiring = new Set(scope.jobs);
55
+ // Withdrawn door content is custody too: a values-only erasure that removes it is fenced.
56
+ let stores: Store[] | undefined;
57
+ const fenced = Boolean(person.target) || fields.length > 0 || scope.jobs.length + scope.streams.length + scope.stages.length + scope.anchors.length + scope.preparations.length > 0
58
+ || await doorCustodyInScope(db, destination, stores = await tallyStores(db));
59
+ if (fenced && !checkpoint.length) throw new Error('IMPORT_RETIREMENT_CHECKPOINT_REQUIRED');
60
+ const fence = fenced ? checkpoint : [];
61
+ // Only the checkpoint fences rows the tally does not count (another import's jobs, a seed's tables): unfenced caller SQL is refused.
62
+ if (nativeStatements.length && !fence.length) throw new Error('IMPORT_RETIREMENT_NATIVE_REQUIRES_CHECKPOINT');
63
+ let step: ErasureStep | undefined, progress: RedactionProgress | undefined, pending: JobRevision[] = [], lineage: DoorLineage | undefined, sever: LineageSever | undefined;
64
+ if (options.resumable && fields.length) {
65
+ progress = readProgress(options.continuation, destination, fields);
66
+ // Revisions before the lineage's plans: every job a step may read is then a known lineage member.
67
+ const jobs = await destinationRevisions(db, destination);
68
+ lineage = await doorLineage(person, db, destination, context.storage.targets.planCodec);
69
+ // Steps sever the lineage's copies in the jobs they read, so a job stays clean
70
+ // only under the lineage it was read under; a source joining it voids that.
71
+ const severing = lineageDigest(progress.nonce, lineage.sources), stale = new Set(severing === progress.lineage ? [] : lineage.jobs);
72
+ const clean = new Map(jobs.filter(job => job.revision !== null && !stale.has(job.id) && progress!.clean.get(job.id) === job.revision).map(job => [job.id, job.revision!]));
73
+ pending = jobs.filter(job => !clean.has(job.id));
74
+ progress = { ...progress, clean, lineage: severing };
75
+ if (lineage.select) {
76
+ sever = { target: targetIdentity(destination, person.target!.kind, person.target!.id), select: lineage.select, fence: lineage.fence };
77
+ // Steps read the target's superseded imports oldest first, so each erases them with every older one.
78
+ pending = await chainFirst(db, destination, sever.target, pending);
79
+ }
80
+ if (progress.page || pending.length > progress.span) {
81
+ if (person.related?.snapshot !== undefined) throw new Error('IMPORT_REDACTION_SNAPSHOT_UNBOUNDED_ONLY');
82
+ return redactionSteps(fields, context, fence, progress, pending, undefined, sever);
83
+ }
84
+ // The final step reads only pending jobs; a lineage job an earlier step cleaned holds no unsevered copy.
85
+ step = { selection: { jobs: pending.map(job => job.id), clean, complete: true }, matched: progress.matched };
86
+ }
87
+ const start = remaining(context);
88
+ let erasure: Awaited<ReturnType<typeof targetErasure>>, receiptId: string, prefix: D1Sql[], suffix: D1Sql[];
89
+ try {
90
+ lineage ??= await doorLineage(person, db, destination, context.storage.targets.planCodec);
91
+ erasure = person.target || fields.length ? await targetErasure(person.target, context, lineage.person.related, retiring, step) : { before: [], after: [], final: [], decisions: [], jobs: [], fieldMatches: [] };
92
+ receiptId = randomUUID(); const recordedAt = new Date().toISOString();
93
+ const touched = { destination, jobs: [...new Set([...erasure.jobs, ...scope.jobs])].sort(), streams: [] as string[] };
94
+ touched.streams = [...new Set([...scope.anchors, ...await pageStreams(db, touched.jobs)])].sort();
95
+ stores ??= await tallyStores(db);
96
+ const entry = { recordedAt, recordsErased: erasure.decisions.length, jobsRetired: scope.jobs.length, streamsRetired: scope.anchors.length };
97
+ prefix = packGuards([authority(context, 'erase'), ...tallyBefore(stores, receiptId, touched), ...tallyWithdraw(stores, touched), ...erasure.before, ...scopeGuards(scope), ...lineage.fence.map(guard)]);
98
+ suffix = [...packGuards([...erasure.after, ...retirementWrites(scope, destination, receiptId, recordedAt), ...headerAbsence(destination, values),
99
+ ...tallyJournal(stores, receiptId, touched, entry, fenced), authority(context, 'erase'), sql('DELETE FROM ik_d1_assertions')]), ...readonlyChecks(erasure.final), finalAuthority(context, 'erase')];
100
+ // One read after the batch returns its journal row.
101
+ const needed = prefix.length + nativeStatements.length + fence.length + suffix.length + 1;
102
+ if (needed > remaining(context)) throw overflow(needed);
103
+ // A final step too large for one D1 RPC goes on in steps too.
104
+ if (batchBytes([...prefix, ...nativeStatements, ...fence, ...suffix]) > RPC_LIMIT) throw new D1RequestLimitError();
105
+ } catch (error) {
106
+ // A redaction too large for one request goes on in steps; nothing was written.
107
+ if (!step || !pending.length || person.related?.snapshot !== undefined || !(error instanceof D1RequestLimitError)) throw error;
108
+ const needed = (error as { needed?: number }).needed;
109
+ return redactionSteps(fields, context, fence, progress!, pending, { cost: needed === undefined ? undefined : start - remaining(context) + needed }, sever);
110
+ }
111
+ const results = await db.batch([...prefix, ...nativeStatements, ...fence, ...suffix].map(statement => prepare(db, statement)));
112
+ const [journal] = await readJournal(db, destination, receiptId);
113
+ return { ...journal, phase: 'complete', fieldMatches: erasure.fieldMatches, retiredJobs: scope.jobs, retiredStreams: scope.anchors, nativeResults: results.slice(prefix.length, prefix.length + nativeStatements.length) };
114
+ }
115
+
116
+ const pageStreams = async (db: D1ImportContext['storage']['database'], jobs: string[]) =>
117
+ (await db.prepare('SELECT DISTINCT stream_id AS id FROM ik_d1_stream_pages WHERE job_id IN (SELECT value FROM json_each(?))').bind(JSON.stringify(jobs)).all<{ id: string }>()).results.map(row => row.id);
118
+
119
+ /** The destination's erasure journal, oldest first. Seeds show it, export it in
120
+ * archives and use it to explain a retired job or stream. */
121
+ export async function readD1ErasureJournal(input: D1ImportContext): Promise<D1ErasureJournalEntry[]> {
122
+ const context = requestContext(input);
123
+ await authorize(context, 'read');
124
+ return readJournal(context.storage.database, context.destination);
125
+ }
126
+
127
+ async function readJournal(db: D1ImportContext['storage']['database'], destination: string, receiptId?: string): Promise<D1ErasureJournalEntry[]> {
128
+ const { results } = await db.prepare(`SELECT * FROM ik_d1_erasure_journal WHERE destination=?${receiptId ? ' AND id=?' : ''} ORDER BY recorded_at, id`).bind(...[destination, receiptId].filter(value => value !== undefined)).all<any>();
129
+ return results.map(row => ({ receiptId: row.id, destination: row.destination, recordedAt: row.recorded_at, recordsErased: row.records_erased, jobsRetired: row.jobs_retired, streamsRetired: row.streams_retired, stores: JSON.parse(row.stores_json) }));
130
+ }
131
+
132
+ /** A job or stream that is gone because a person's erasure retired it. */
133
+ export async function retiredError(db: D1ImportContext['storage']['database'], id: string, kind: 'job' | 'stream'): Promise<Error | undefined> {
134
+ const row = await db.prepare('SELECT receipt_id FROM ik_d1_retirements WHERE id=? AND kind=?').bind(id, kind).first<{ receipt_id: string }>();
135
+ return row ? Object.assign(new Error(kind === 'job' ? 'IMPORT_JOB_RETIRED' : 'IMPORT_STREAM_RETIRED'), { receiptId: row.receipt_id }) : undefined;
136
+ }
@@ -0,0 +1,19 @@
1
+ import type { D1ImportContext, D1Sql } from './types.ts';
2
+ import type { D1ReversalPolicyConversion } from './undo-types.ts';
3
+ import { canonicalJson, sha256 } from '../provenance/identity.ts';
4
+ import { guard, sql } from './database.ts';
5
+
6
+ export async function loadConversion(raw: any, context: D1ImportContext): Promise<D1ReversalPolicyConversion | undefined> {
7
+ if (raw.reversal_conversion_hash == null) return undefined;
8
+ const row = await context.storage.database.prepare('SELECT record_json FROM ik_d1_policy_conversions WHERE job_id=?').bind(raw.id).first();
9
+ let record: D1ReversalPolicyConversion;
10
+ try { record = JSON.parse(row?.record_json); } catch { throw new Error('IMPORT_CONVERSION_INTEGRITY'); }
11
+ if (sha256(canonicalJson(record)) !== raw.reversal_conversion_hash || record.jobId !== raw.id || record.from !== 'strict' || record.to !== 'preserve-conflicts' || raw.reversal_policy !== record.to || !['completed','undoing'].includes(record.state) || !Number.isSafeInteger(record.fence) || record.fence < 0 || !Number.isSafeInteger(record.previouslyReversed) || record.previouslyReversed < 0 || record.policyHash !== sha256(canonicalJson([raw.id, record.jobFingerprint, record.to, 'reversal-policy-v1']))) throw new Error('IMPORT_CONVERSION_INTEGRITY');
12
+ return record;
13
+ }
14
+
15
+ export function conversionGuards(id: string, conversion?: D1ReversalPolicyConversion): D1Sql[] {
16
+ const body = conversion ? canonicalJson(conversion) : null;
17
+ return [guard(sql('EXISTS(SELECT 1 FROM ik_jobs WHERE id=? AND reversal_conversion_hash IS ?)', id, body === null ? null : sha256(body))),
18
+ ...(body === null ? [] : [guard(sql('EXISTS(SELECT 1 FROM ik_d1_policy_conversions WHERE job_id=? AND record_json=?)', id, body))])];
19
+ }
@@ -0,0 +1,11 @@
1
+ /** Additive migration after the existing D1 schemas. No personal values are
2
+ * retained: actor and reviewed content are represented by opaque digests. The
3
+ * journal survives whole-job privacy retirement without a foreign key. */
4
+ export const d1ReversalPolicyConversionSchema = [
5
+ 'ALTER TABLE ik_jobs ADD COLUMN reversal_conversion_hash TEXT',
6
+ 'CREATE TABLE IF NOT EXISTS ik_d1_policy_conversions (job_id TEXT PRIMARY KEY, record_json TEXT NOT NULL)',
7
+ "CREATE TRIGGER IF NOT EXISTS ik_d1_policy_conversion_insert BEFORE INSERT ON ik_d1_policy_conversions WHEN EXISTS(SELECT 1 FROM ik_d1_policy_conversions WHERE job_id=NEW.job_id OR rowid=NEW.rowid) BEGIN SELECT RAISE(ABORT,'IMPORT_POLICY_CONVERSION_APPEND_ONLY'); END",
8
+ ...['UPDATE','DELETE'].map(event => `CREATE TRIGGER IF NOT EXISTS ik_d1_policy_conversion_${event.toLowerCase()} BEFORE ${event} ON ik_d1_policy_conversions BEGIN SELECT RAISE(ABORT,'IMPORT_POLICY_CONVERSION_APPEND_ONLY'); END`),
9
+ 'CREATE TRIGGER IF NOT EXISTS ik_d1_policy_conversion_custody AFTER INSERT ON ik_d1_policy_conversions BEGIN UPDATE ik_d1_custody_states SET revision=revision+1 WHERE job_id=NEW.job_id; END',
10
+ "CREATE TRIGGER IF NOT EXISTS ik_d1_policy_conversion_binding BEFORE UPDATE ON ik_jobs WHEN OLD.reversal_conversion_hash IS NOT NULL AND (NEW.reversal_conversion_hash IS NOT OLD.reversal_conversion_hash OR NEW.reversal_policy<>'preserve-conflicts') BEGIN SELECT RAISE(ABORT,'IMPORT_POLICY_CONVERSION_APPEND_ONLY'); END",
11
+ ];
@@ -0,0 +1,40 @@
1
+ import type { D1ImportContext } from './types.ts';
2
+ import type { D1ReversalPolicyConversion } from './undo-types.ts';
3
+ import { canonicalJson, sha256 } from '../provenance/identity.ts';
4
+ import { requestContext } from './budget.ts';
5
+ import { atomic, preparedAtomic, withinBatchBytes, readonlyChecks, sql, readAtomic } from './database.ts';
6
+ import { stored, storedReads } from './jobs.ts';
7
+ import { changed, putRow, updateRows, ExpectedState } from './expected-state.ts';
8
+ import { insertJsonRows } from './sets.ts';
9
+ import { reversalState, policyHash, reportHash, reversalGuards } from './undo-state.ts';
10
+ import { conversionGuards } from './reversal-conversion-record.ts';
11
+ import { preparePreservedPreview, D1ReversalPolicyConversionError, unavailable } from './preserved-preview.ts';
12
+
13
+ export async function convertD1ReversalPolicy(id: string, context: D1ImportContext, options: { to: 'preserve-conflicts'; reviewedFingerprint: string }) {
14
+ context = requestContext({ ...context, reversalPolicy: undefined });
15
+ await readAtomic(context, [], undefined, 'undo');
16
+ const current = await stored(id, context, 'undo');
17
+ if (reversalState(current.job).policy !== 'strict') throw new D1ReversalPolicyConversionError('IMPORT_CONVERSION_ALREADY_CONVERTED', 'policy');
18
+ if (options?.to !== 'preserve-conflicts') throw unavailable('policy');
19
+ const prepared = await preparePreservedPreview(id, context), { job, previous } = prepared;
20
+ if (typeof options.reviewedFingerprint !== 'string' || options.reviewedFingerprint !== prepared.reviewedFingerprint) throw new D1ReversalPolicyConversionError('IMPORT_CONVERSION_REVIEW_CHANGED', 'fingerprint');
21
+ const record: D1ReversalPolicyConversion = { jobId: id, jobFingerprint: job.fingerprint, from: 'strict', to: 'preserve-conflicts', state: job.state as 'completed' | 'undoing', fence: job.fence, actorDigest: sha256(context.authorization.principalId), recordedAt: new Date().toISOString(), reviewedFingerprint: prepared.reviewedFingerprint, policyHash: policyHash(id, job.fingerprint, 'preserve-conflicts'), previouslyReversed: previous.length };
22
+ const body = canonicalJson(record), conversionHash = sha256(body);
23
+ const reports = previous.map((d, position) => ({ job_id: id, effect_id: d.effectId, position, record_json: canonicalJson(d) }));
24
+ const writes = [changed(sql('INSERT INTO ik_d1_policy_conversions(job_id,record_json) VALUES(?,?)', id, body), putRow('ik_d1_policy_conversions', {job_id:id,record_json:body})),
25
+ ...insertJsonRows('ik_d1_reversal_reports', ['job_id','effect_id','position','record_json'], reports),
26
+ changed(sql("UPDATE ik_jobs SET reversal_policy='preserve-conflicts',reversal_policy_hash=?,reversal_conversion_hash=?,reversal_cursor=?,reversal_hash=?,fence=fence+1 WHERE id=?", record.policyHash, conversionHash, reports.length, reportHash(reports), id), updateRows('ik_jobs', row => row.id===id, row => ({...row,reversal_policy:'preserve-conflicts',reversal_policy_hash:record.policyHash,reversal_conversion_hash:conversionHash,reversal_cursor:reports.length,reversal_hash:reportHash(reports),fence:job.fence+1}))),
27
+ ];
28
+ // Never stage conversion: refusal must leave every D1 byte unchanged. A
29
+ // conversion that cannot fit one atomic request is explicitly unavailable.
30
+ const statements = [...prepared.guards, ...writes, ...readonlyChecks([
31
+ ...new ExpectedState([...prepared.guards, ...writes]).guards(),
32
+ ...conversionGuards(id, record), ...prepared.nativeGuards,
33
+ ...reversalGuards(id, { policy: 'preserve-conflicts', binding: record.policyHash, cursor: reports.length, hash: reportHash(reports), rows: reports, dispositions: previous, conversion: record }),
34
+ ])];
35
+ if (!withinBatchBytes(context, preparedAtomic(context, 'undo', statements))) throw unavailable('transport');
36
+ // Preserved reports and the conversion record add two reads to the original
37
+ // strict return path. Admit them before committing, never fail after publish.
38
+ await atomic(context, 'undo', statements, storedReads(job) + 2);
39
+ return (await stored(id, { ...context, reversalPolicy: undefined }, 'undo')).job;
40
+ }
@@ -0,0 +1,14 @@
1
+ /** One row per reversing job: the custody revision its last verified reversal
2
+ * batch left behind. Every custody table of the job, its plan, originals and
3
+ * erasure tombstones advance ik_d1_custody_states, so an equal revision proves
4
+ * that nothing the complete inventory checked has changed since. Jobs that
5
+ * reuse another job's entries keep the complete per-batch inventory. */
6
+ export const d1ReversalFenceSchema = [
7
+ 'CREATE TABLE IF NOT EXISTS ik_d1_reversal_fences (job_id TEXT PRIMARY KEY REFERENCES ik_jobs(id) ON DELETE CASCADE,action TEXT NOT NULL,fence INTEGER NOT NULL,state_revision INTEGER NOT NULL,erased_records INTEGER NOT NULL,reused INTEGER NOT NULL)',
8
+ // A batch's next effects, and the claims it may release, are index lookups
9
+ // rather than scans of every job's provenance and claims.
10
+ 'CREATE INDEX IF NOT EXISTS ik_provenance_job ON ik_provenance(job_id,reversed,ordinal,effect_ordinal)',
11
+ 'CREATE INDEX IF NOT EXISTS ik_provenance_source ON ik_provenance(source_identity,reversed)',
12
+ 'CREATE INDEX IF NOT EXISTS ik_d1_history_source ON ik_d1_history(source_identity)',
13
+ 'CREATE INDEX IF NOT EXISTS ik_sources_job ON ik_sources(job_id)',
14
+ ];
@@ -0,0 +1,106 @@
1
+ import { retainedSql } from './retained-values.ts';
2
+ import { changed, putRow, updateRows, deleteRows } from './expected-state.ts';
3
+ import { targetGuards } from './target-guards.ts';
4
+ import { D1RequestLimitError, remaining } from './budget.ts';
5
+ import type { ImportJob } from '../types.ts';
6
+ import { canonicalJson, sha256, attachmentHash } from '../provenance/identity.ts';
7
+ import { decodeProvenance } from '../storage/provenance.ts';
8
+ import type { D1ImportContext, D1Sql, D1ProvenanceRow } from './types.ts';
9
+ import { atomic, atomicCount, guard, limits, sql } from './database.ts';
10
+ import { decode } from './jobs.ts';
11
+ import { restorePredecessor } from './history.ts';
12
+ import { attachReversal, EMPTY_REVERSAL_HASH, policy } from './undo-state.ts';
13
+ import { captureJobLayout, jobLayoutGuards } from './stream-layout.ts';
14
+ import { needsStreamOrigin } from './stream-origin.ts';
15
+ import { streamBoundary } from './stream-boundary.ts';
16
+
17
+ type Action = 'undo' | 'rollback';
18
+ const reversing = (action: Action) => action === 'undo' ? 'undoing' : 'rolling_back';
19
+ const terminal = (action: Action) => action === 'undo' ? 'undone' : 'rolled_back';
20
+
21
+ /** Appended after every other write of a verified strict reversal batch, so the
22
+ * captured revision includes this batch's own custody changes. Only the complete
23
+ * inventory path classifies entry reuse; a fenced batch carries it forward. */
24
+ export function recordReversalFence(id: string, action: Action, erasedRecords: number, verified: boolean): D1Sql[] {
25
+ const write = verified
26
+ ? sql('INSERT OR REPLACE INTO ik_d1_reversal_fences(job_id,action,fence,state_revision,erased_records,reused) SELECT j.id,?,j.fence,s.revision,?,EXISTS(SELECT 1 FROM ik_d1_entries e WHERE e.job_id=j.id AND e.owner_job_id<>j.id) FROM ik_jobs j JOIN ik_d1_custody_states s ON s.job_id=j.id WHERE j.id=?', action, erasedRecords, id)
27
+ : sql('UPDATE ik_d1_reversal_fences SET fence=(SELECT fence FROM ik_jobs WHERE id=?),state_revision=(SELECT revision FROM ik_d1_custody_states WHERE job_id=?) WHERE job_id=? AND action=?', id, id, id, action);
28
+ return [write, guard(sql('EXISTS(SELECT 1 FROM ik_d1_reversal_fences f JOIN ik_jobs j ON j.id=f.job_id JOIN ik_d1_custody_states s ON s.job_id=j.id WHERE f.job_id=? AND f.action=? AND f.fence=j.fence AND f.state_revision=s.revision AND f.erased_records=?)', id, action, erasedRecords))];
29
+ }
30
+
31
+ /** Releases claims no longer backing an unreversed effect: the whole job's on
32
+ * a verified or final batch (sources with zero effects release too), and only
33
+ * the batch's own sources on any other fenced batch. Reused sources belong to
34
+ * their owner. */
35
+ export function reversalProgress(job: ImportJob, action: Action, done: boolean, sources?: string[]): D1Sql[] {
36
+ const state = done ? terminal(action) : reversing(action), only = done ? undefined : sources;
37
+ const released = only ? sql("DELETE FROM ik_sources WHERE job_id=? AND identity IN (SELECT value FROM json_each(?)) AND NOT EXISTS(SELECT 1 FROM ik_provenance WHERE ik_provenance.source_identity=ik_sources.identity AND reversed=0) AND NOT EXISTS(SELECT 1 FROM ik_d1_history WHERE ik_d1_history.source_identity=ik_sources.identity)", job.id, JSON.stringify(only))
38
+ : sql('DELETE FROM ik_sources WHERE job_id=? AND NOT EXISTS(SELECT 1 FROM ik_provenance WHERE ik_provenance.source_identity=ik_sources.identity AND reversed=0) AND NOT EXISTS(SELECT 1 FROM ik_d1_history WHERE ik_d1_history.source_identity=ik_sources.identity)', job.id);
39
+ return [changed(released,deleteRows('ik_sources',(row,expected)=>row.job_id===job.id&&(!only||only.includes(row.identity))&&![...expected.rows('ik_provenance').values()].some(p=>p.source_identity===row.identity&&!p.reversed)&&![...expected.rows('ik_d1_history').values()].some(p=>p.source_identity===row.identity))), changed(sql('INSERT INTO ik_batches(job_id,action,position,end_position,fence) VALUES(?,?,?,?,?)', job.id, action, job.fence, job.fence + 1, job.fence + 1),putRow('ik_batches',{job_id:job.id,action,position:job.fence,end_position:job.fence+1,fence:job.fence+1})), changed(sql('UPDATE ik_jobs SET state=?,fence=fence+1,error=NULL WHERE id=?', state, job.id),updateRows('ik_jobs',row=>row.id===job.id,row=>({...row,state,fence:job.fence+1,error:null})))];
40
+ }
41
+
42
+ async function readFence(id: string, context: D1ImportContext): Promise<any> {
43
+ return context.storage.database.prepare(`SELECT j.*,s.revision AS state_revision,f.action AS fence_action,f.fence AS fence_fence,f.state_revision AS fence_revision,f.erased_records AS fence_erased,f.reused AS fence_reused,EXISTS(SELECT 1 FROM ik_d1_indexed_jobs i WHERE i.job_id=j.id AND i.fingerprint=j.fingerprint) AS indexed,${needsStreamOrigin('j.id')} AS stream_origin_required,(SELECT stream_id FROM ik_d1_stream_origins WHERE job_id=j.id) AS origin_stream_id,(SELECT stream_id FROM ik_d1_stream_pages WHERE job_id=j.id) AS page_stream_id FROM ik_jobs j JOIN ik_d1_custody_states s ON s.job_id=j.id LEFT JOIN ik_d1_reversal_fences f ON f.job_id=j.id WHERE j.id=?`).bind(id).first();
44
+ }
45
+ function current(row: any, action: Action, context: D1ImportContext): boolean {
46
+ return !!row && row.fence_action === action && row.fence_fence === row.fence && row.fence_revision === row.state_revision && row.state === reversing(action) && row.error === null && row.indexed === 1 && row.fence_reused === 0
47
+ && row.reversal_policy === 'strict' && (context.reversalPolicy === undefined || policy(context) === 'strict')
48
+ && row.destination === context.destination && row.namespace === context.namespace && row.mapping_version === context.mappingVersion && row.principal_id === context.authorization.principalId
49
+ && (context.reviewedFingerprint === undefined || context.reviewedFingerprint === row.fingerprint);
50
+ }
51
+ function fencedJob(row: any): ImportJob {
52
+ const job = decode(row); job.erasedRecords = row.fence_erased;
53
+ attachReversal(job, { policy: 'strict', binding: null, cursor: 0, hash: EMPTY_REVERSAL_HASH, rows: [], dispositions: [] });
54
+ return job;
55
+ }
56
+
57
+ /** Continue a strict, unbounded reversal whose custody is unchanged since its
58
+ * last verified batch. Each batch reads only its own provenance rows and their
59
+ * native targets; the custody revision replaces re-reading the whole job.
60
+ * Returns undefined when the fence is not current on entry: the caller then
61
+ * verifies the complete inventory as before. A fence that stops being current
62
+ * after this call's own batches ends the request with that published progress. */
63
+ export async function fencedReversal(id: string, action: Action, context: D1ImportContext, batches: number, published?: ImportJob): Promise<ImportJob | undefined> {
64
+ const { batchSize } = limits(context), { database: db, targets } = context.storage;
65
+ try {
66
+ for (let count = 0; count < batches; count++) {
67
+ const row = await readFence(id, context);
68
+ if (!current(row, action, context)) return count ? published : undefined;
69
+ const job = fencedJob(row);
70
+ await captureJobLayout(job, row, context, action);
71
+ const statements: D1Sql[] = [...jobLayoutGuards(job), ...streamBoundary(id, action),
72
+ guard(sql('NOT EXISTS(SELECT 1 FROM ik_d1_history WHERE job_id=?)', id)),
73
+ guard(sql('EXISTS(SELECT 1 FROM ik_jobs j JOIN ik_d1_custody_states s ON s.job_id=j.id JOIN ik_d1_reversal_fences f ON f.job_id=j.id WHERE j.id=? AND j.state=? AND j.fence=? AND j.fingerprint=? AND j.error IS NULL AND s.revision=? AND f.action=? AND f.fence=j.fence AND f.state_revision=s.revision)', id, job.state, job.fence, job.fingerprint, row.state_revision, action))];
74
+ const { results: rows } = await db.prepare('SELECT * FROM ik_provenance WHERE job_id=? AND reversed=0 ORDER BY ordinal DESC,effect_ordinal DESC LIMIT ?').bind(id, batchSize + 1).all();
75
+ const batch = rows.slice(0, batchSize), predecessors = new Map<string, any>();
76
+ for (const prior of (await db.prepare('SELECT * FROM ik_d1_history WHERE successor_id IN (SELECT value FROM json_each(?))').bind(JSON.stringify(batch.map(raw => raw.id))).all()).results) predecessors.set(prior.successor_id, prior);
77
+ // The request that publishes the first `count` rows, with its one return read.
78
+ const publication = (statements: D1Sql[], count: number) => [...statements, ...reversalProgress(job, action, count === rows.length, [...new Set(batch.slice(0, count).map(raw => raw.source_identity))]), ...recordReversalFence(id, action, job.erasedRecords!, false)];
79
+ let processed = 0, planned = atomicCount(context, action, publication(statements, 0), 1);
80
+ for (const raw of batch) {
81
+ // Leave room for this row's reads; the exact request is counted once built.
82
+ if (remaining(context) - planned < 70) { if (!processed) { if (published) return published; throw new D1RequestLimitError(); } break; }
83
+ // The verified inventory covered this exact row; its revision is unchanged.
84
+ const p = decodeProvenance(raw) as D1ProvenanceRow;
85
+ if (p.reversed || p.jobId !== id) throw new Error('PROVENANCE_STORAGE_MISMATCH');
86
+ const target = await targets.read(db, context.destination, p.targetKind, p.targetId);
87
+ if (!target || target.revision !== p.appliedRevision || sha256(canonicalJson(target.value)) !== p.valueHash || attachmentHash(target.attachment) !== p.attachmentHash) throw new Error('NATIVE_EDIT_CONFLICT');
88
+ const candidate = [...statements], restoration: D1Sql[] = [];
89
+ await restorePredecessor(p, context, candidate, restoration, predecessors.get(p.id) ?? null);
90
+ candidate.push(guard(retainedSql(['EXISTS(SELECT 1 FROM ik_provenance WHERE id=', ' AND record_json=', ' AND reversed=0)'], [raw.id, raw.record_json], [1])), ...targetGuards(targets, target, context.destination, p.targetKind, p.targetId), ...(p.before ? targets.write(p.before) : targets.remove(context.destination, p.targetKind, p.targetId)), ...targetGuards(targets, p.before, context.destination, p.targetKind, p.targetId), changed(retainedSql(['UPDATE ik_provenance SET reversed=1,record_json=', ' WHERE id=', ''], [canonicalJson({ ...p, reversed: true }), raw.id], [0]),updateRows('ik_provenance',row=>row.id===raw.id,row=>({...row,reversed:1,record_json:canonicalJson({...p,reversed:true})}))), ...restoration);
91
+ const spent = atomicCount(context, action, publication(candidate, processed + 1), 1);
92
+ if (spent > remaining(context)) { if (processed) break; if (published) return published; throw new D1RequestLimitError(); }
93
+ statements.splice(0, statements.length, ...candidate); processed++; planned = spent;
94
+ }
95
+ const done = processed === rows.length;
96
+ await atomic(context, action, publication(statements, processed), 1);
97
+ published = { ...job, state: done ? terminal(action) : reversing(action), fence: job.fence + 1, error: null };
98
+ attachReversal(published, { policy: 'strict', binding: null, cursor: 0, hash: EMPTY_REVERSAL_HASH, rows: [], dispositions: [] });
99
+ if (done || remaining(context) < 200) return published;
100
+ }
101
+ } catch (error) {
102
+ if (error instanceof D1RequestLimitError && published) return published;
103
+ throw error;
104
+ }
105
+ return published;
106
+ }
@@ -0,0 +1,24 @@
1
+ import type { ImportJob } from '../types.ts';
2
+ import type { D1ReversalPolicyConversion } from './undo-types.ts';
3
+
4
+ /** Conversion is its own durable fence increment, never an edit to an earlier
5
+ * apply/reversal batch. Account for that one gap at its recorded position. */
6
+ export function validateReversalJournal(job: ImportJob, batches: any[], applyFence: number, conversion?: D1ReversalPolicyConversion): void {
7
+ const action = ['undoing','undone'].includes(job.state) ? 'undo' : 'rollback';
8
+ let expectedFence = applyFence + 1;
9
+ if (conversion?.state === 'completed') {
10
+ if (conversion.fence !== applyFence) throw new Error('IMPORT_JOURNAL_MISMATCH');
11
+ expectedFence++;
12
+ }
13
+ let crossed = conversion?.state !== 'undoing';
14
+ const crossConversion = () => {
15
+ if (!crossed && conversion!.fence === expectedFence) { expectedFence++; crossed = true; }
16
+ };
17
+ for (const batch of batches) {
18
+ crossConversion();
19
+ if (batch.action !== action || batch.position !== expectedFence || batch.end_position !== expectedFence + 1 || batch.fence !== expectedFence + 1) throw new Error('IMPORT_JOURNAL_MISMATCH');
20
+ expectedFence++;
21
+ }
22
+ crossConversion();
23
+ if (!crossed || job.fence !== expectedFence) throw new Error('IMPORT_JOURNAL_MISMATCH');
24
+ }
@@ -0,0 +1,103 @@
1
+ import { retainedSql } from './retained-values.ts';
2
+ import { changed, updateRows } from './expected-state.ts';
3
+ import { cancelNativeJobStaging } from './native-staging-lifecycle.ts';
4
+ import { nativeStagingScope, readStagedTarget, expectedStagedTarget, type NativeStagingScope } from './native-staging-context.ts';
5
+ import { targetGuards } from './target-guards.ts';
6
+ import { D1RequestLimitError, requestContext, remaining } from './budget.ts';
7
+ import type { ImportJob } from '../types.ts';
8
+ import { canonicalJson, sha256, attachmentHash } from '../provenance/identity.ts';
9
+ import type { D1ImportContext, D1Sql } from './types.ts';
10
+ import { atomic, atomicCount, authorize, guard, limits, sql, fitsBatch } from './database.ts';
11
+ import { stored, boundary, storedReads } from './jobs.ts';
12
+ import { custodyGuards, validatedProvenance, inventoryGuards } from './integrity.ts';
13
+ import { noSuccessors, restorePredecessor } from './history.ts';
14
+ import { readOriginals } from './originals.ts';
15
+ import { reversalState } from './undo-state.ts';
16
+ import { preserveReverse } from './preserved-reverse.ts';
17
+ import { fencedReversal, recordReversalFence, reversalProgress } from './reversal-fence.ts';
18
+
19
+ export async function reverseD1Import(id: string, action: 'undo' | 'rollback', context: D1ImportContext): Promise<ImportJob> {
20
+ context = requestContext(context);
21
+ const { batchSize, maxBatches } = limits(context), state = action === 'undo' ? 'undoing' : 'rolling_back', final = action === 'undo' ? 'undone' : 'rolled_back';
22
+ await authorize(context, action);
23
+ // Unbounded strict reversal fences each later batch on the custody revision
24
+ // its last verified batch recorded, instead of re-reading the whole job.
25
+ const unbounded = context.maxBatchBytes === undefined;
26
+ const fenced = unbounded ? await fencedReversal(id, action, context, maxBatches) : undefined;
27
+ if (fenced) return fenced;
28
+ // Persist reversal intent before reading targets: a conflicting native edit may
29
+ // block this invocation, but a later request must never resume forward apply.
30
+ let current = await stored(id, context, action);
31
+ if(context.maxBatchBytes!==undefined)await cancelNativeJobStaging(context,action,id,current.job.fence);
32
+ if ((action === 'undo' && current.job.state === 'rolled_back') || (action === 'rollback' && current.job.state === 'undone')) throw new Error('REVERSAL_MODE_CONFLICT');
33
+ if (['undone', 'rolled_back'].includes(current.job.state)) {
34
+ await readOriginals(current.plan, id, context);
35
+ await atomic(context, action, [...boundary(current.job, current.plan, action), ...custodyGuards(current.plan, id), ...await inventoryGuards(current.job, current.plan, context)], 0, nativeStagingScope(current.job,current.plan));
36
+ return current.job;
37
+ }
38
+ if (current.job.state === (action === 'undo' ? 'rolling_back' : 'undoing')) throw new Error('REVERSAL_MODE_CONFLICT');
39
+ if (action === 'undo' && !['completed', 'undoing'].includes(current.job.state)) throw new Error('UNDO_REQUIRES_COMPLETED_IMPORT');
40
+ await readOriginals(current.plan, id, context);
41
+ if (reversalState(current.job).policy === 'preserve-conflicts') return preserveReverse(current, context, action);
42
+ if (current.job.state !== state) {
43
+ const intentScope=nativeStagingScope(current.job,current.plan);
44
+ intentScope.postGuards.push(guard(sql('EXISTS(SELECT 1 FROM ik_jobs WHERE id=? AND state=? AND fence=?)',id,state,current.job.fence+1)));
45
+ await atomic(context, action, [...boundary(current.job, current.plan, action), await noSuccessors(id, context), ...custodyGuards(current.plan, id), ...await inventoryGuards(current.job, current.plan, context), changed(sql('UPDATE ik_jobs SET state=?,fence=fence+1 WHERE id=?', state, id),updateRows('ik_jobs',row=>row.id===id,row=>({...row,state,fence:current.job.fence+1}))), ...unbounded ? recordReversalFence(id, action, current.job.erasedRecords!, true) : []], storedReads(current.job), intentScope);
46
+ const next = unbounded ? await fencedReversal(id, action, context, maxBatches) : undefined;
47
+ if (next) return next;
48
+ current = await stored(id, context, action);
49
+ }
50
+ let published: ImportJob | undefined;
51
+ try {
52
+ for (let count = 0; count < maxBatches; count++) {
53
+ const { job, plan } = current, { database: db, targets } = context.storage;
54
+ if (job.state === final) return job;
55
+ if (job.state !== state) throw new Error('REVERSAL_MODE_CONFLICT');
56
+ await readOriginals(plan, id, context);
57
+ const { results: rows } = await db.prepare('SELECT * FROM ik_provenance WHERE job_id=? AND reversed=0 ORDER BY ordinal DESC,effect_ordinal DESC LIMIT ?').bind(id, batchSize).all();
58
+ const statements: D1Sql[] = [...boundary(job, plan, action), await noSuccessors(id, context), ...custodyGuards(plan, id), ...await inventoryGuards(job, plan, context)];
59
+ const remainingRow = await db.prepare('SELECT COUNT(*) AS n FROM ik_provenance WHERE job_id=? AND reversed=0').bind(id).first();
60
+ const returnReads = storedReads(job), publication = (statements: D1Sql[], scope: NativeStagingScope, done: boolean) => ({
61
+ statements: [...statements, ...reversalProgress(job, action, done), ...unbounded ? recordReversalFence(id, action, job.erasedRecords!, true) : []],
62
+ scope: { ...scope, postGuards: [...scope.postGuards, guard(sql('EXISTS(SELECT 1 FROM ik_jobs WHERE id=? AND state=? AND fence=? AND error IS NULL)', id, done ? final : state, job.fence + 1))] },
63
+ }), cost = (request: ReturnType<typeof publication>) => atomicCount(context, action, request.statements, returnReads, request.scope);
64
+ let processed = 0, scope = nativeStagingScope(job, plan), planned = cost(publication(statements, scope, remainingRow!.n === 0));
65
+ for (const raw of rows) {
66
+ // Leave room for this row's reads; the exact request is counted once built.
67
+ if (remaining(context) - planned < 70) { if (!processed) { if (count > 0) return job; throw new D1RequestLimitError(); } break; }
68
+ const candidate = [...statements], nextScope = { ...scope, nativeGuards:[...scope.nativeGuards], postGuards:[...scope.postGuards], targets:[...scope.targets] };
69
+ const p = await validatedProvenance(raw, context, candidate);
70
+ const row = await readStagedTarget(context, nextScope, p.targetKind, p.targetId);
71
+ if (!row || row.revision !== p.appliedRevision || sha256(canonicalJson(row.value)) !== p.valueHash || attachmentHash(row.attachment) !== p.attachmentHash) throw new Error('NATIVE_EDIT_CONFLICT');
72
+ const restoration: D1Sql[] = [], post = targetGuards(targets,p.before,context.destination,p.targetKind,p.targetId);
73
+ await restorePredecessor(p, context, candidate, restoration);
74
+ candidate.push(guard(retainedSql(['EXISTS(SELECT 1 FROM ik_provenance WHERE id=', ' AND record_json=', ' AND reversed=0)'], [raw.id, raw.record_json], [1])), ...targetGuards(targets,row, context.destination, p.targetKind, p.targetId), ...(p.before ? targets.write(p.before) : targets.remove(context.destination, p.targetKind, p.targetId)), ...post, changed(retainedSql(['UPDATE ik_provenance SET reversed=1,record_json=', ' WHERE id=', ''], [canonicalJson({ ...p, reversed: true }), raw.id], [0]),updateRows('ik_provenance',row=>row.id===raw.id,row=>({...row,reversed:1,record_json:canonicalJson({...p,reversed:true})}))), ...restoration);
75
+ nextScope.postGuards.push(...post);
76
+ const next = publication(candidate, nextScope, remainingRow!.n === processed + 1), spent = cost(next);
77
+ if (spent > remaining(context)) { if (processed) break; if (count > 0) return job; throw new D1RequestLimitError(); }
78
+ const fits = fitsBatch(context, action, next.statements, next.scope);
79
+ if (!fits && processed) break;
80
+ statements.splice(0, statements.length, ...candidate);
81
+ expectedStagedTarget(nextScope,p.targetKind,p.targetId,!!p.before);
82
+ scope = nextScope; processed++; planned = spent;
83
+ if (!fits) break;
84
+ }
85
+ const done = remainingRow!.n === processed, request = publication(statements, scope, done);
86
+ await atomic(context, action, request.statements, returnReads, request.scope);
87
+ if (unbounded && !done) {
88
+ const next = await fencedReversal(id, action, context, maxBatches - count - 1, { ...job, state, fence: job.fence + 1, error: null });
89
+ if (next) return next;
90
+ }
91
+ current = await stored(id, context, action); published = current.job;
92
+ if (done || remaining(context) < 200) return published;
93
+ }
94
+ } catch (error) {
95
+ // Only actual reversal progress qualifies: intent alone still fails when
96
+ // no native batch fits, while preventing any later forward apply.
97
+ if (error instanceof D1RequestLimitError && published) return published;
98
+ throw error;
99
+ }
100
+ return current.job;
101
+ }
102
+ export const undoD1Import = (id: string, context: D1ImportContext) => reverseD1Import(id, 'undo', context);
103
+ export const rollbackD1Import = (id: string, context: D1ImportContext) => reverseD1Import(id, 'rollback', context);
@@ -0,0 +1,23 @@
1
+ import type { SourceRecord } from '../types.ts';
2
+ import { canonicalHash, cloneJson } from '../provenance/canonical-stream.ts';
3
+
4
+ /** During initial review, source records already remain available to the
5
+ * planner. Read a bounded fresh copy and fence it against the normalized page
6
+ * observed during mapping. Durable continuation uses only stored plans. */
7
+ export function reviewedSourceRecords(records: SourceRecord[], hashes: string[], pageSize: number, total: number): SourceRecord[] {
8
+ let current = -1, page: SourceRecord[] = [];
9
+ const at = (index: number): SourceRecord => {
10
+ const ordinal = Math.floor(index / pageSize);
11
+ if (current !== ordinal) {
12
+ const start = ordinal * pageSize;
13
+ const next = cloneJson(records.slice(start, Math.min(start + pageSize, total)));
14
+ if (records.length !== total || canonicalHash(next) !== hashes[ordinal]) throw new Error('IMPORT_STREAM_SOURCE_CHANGED');
15
+ page = next; current = ordinal;
16
+ }
17
+ return page[index % pageSize];
18
+ };
19
+ return new Proxy(new Array<SourceRecord>(total), {
20
+ get(target, key, receiver) { if (typeof key === 'string' && /^(0|[1-9][0-9]*)$/.test(key)) return Number(key) < total ? at(Number(key)) : undefined; return Reflect.get(target,key,receiver); },
21
+ has(target, key) { return typeof key === 'string' && /^(0|[1-9][0-9]*)$/.test(key) && Number(key) < total || Reflect.has(target,key); },
22
+ });
23
+ }