@runeditrun/rer 0.2.76 → 0.2.78

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (191) hide show
  1. package/README.md +32 -0
  2. package/cli/README.md +55 -0
  3. package/cli/contract.mjs +1 -1
  4. package/cli/deploy-command.mjs +4 -2
  5. package/cli/deploy-inputs.mjs +15 -3
  6. package/cli/deploy-installation-key.mjs +71 -0
  7. package/cli/deploy-secrets.mjs +2 -0
  8. package/cli/deploy.mjs +13 -2
  9. package/cli/deployment-record.mjs +2 -0
  10. package/cli/export.mjs +1 -1
  11. package/cli/main.mjs +1 -1
  12. package/cli/parse.mjs +3 -2
  13. package/cli/portability-migrations.mjs +35 -14
  14. package/cli/portability-worker.mjs +2 -2
  15. package/cli/portability.mjs +2 -2
  16. package/cli/runtime-front-door.mjs +1 -1
  17. package/global/CHANGELOG.md +14 -0
  18. package/global/README.md +32 -0
  19. package/modules/dependencies/README.md +71 -1
  20. package/modules/dependencies/auth.mjs +4 -4
  21. package/modules/dependencies/crypto.mjs +6 -1
  22. package/modules/dependencies/custody.mjs +16 -0
  23. package/modules/dependencies/index.d.mts +1 -1
  24. package/modules/dependencies/index.mjs +6 -5
  25. package/modules/dependencies/schema/keys-sealed.sql +4 -0
  26. package/modules/dependencies/sealing.mjs +55 -0
  27. package/modules/dependencies/store.mjs +62 -9
  28. package/modules/deploy-kit/README.md +1 -0
  29. package/modules/deploy-kit/owner-cli.mjs +19 -7
  30. package/modules/deploy-kit/owner-rekey.mjs +69 -0
  31. package/modules/deploy-kit/provider.mjs +9 -4
  32. package/modules/import-kit/src/d1/README.md +1322 -0
  33. package/modules/import-kit/src/d1/apply.ts +115 -0
  34. package/modules/import-kit/src/d1/budget.ts +89 -0
  35. package/modules/import-kit/src/d1/compressed-plan.ts +63 -0
  36. package/modules/import-kit/src/d1/creation.ts +95 -0
  37. package/modules/import-kit/src/d1/custody-fence.ts +27 -0
  38. package/modules/import-kit/src/d1/custody-state-schema.ts +67 -0
  39. package/modules/import-kit/src/d1/database.ts +163 -0
  40. package/modules/import-kit/src/d1/entry-locator.ts +61 -0
  41. package/modules/import-kit/src/d1/erased-artifacts.ts +38 -0
  42. package/modules/import-kit/src/d1/erased-custody.ts +36 -0
  43. package/modules/import-kit/src/d1/erased-plan.ts +47 -0
  44. package/modules/import-kit/src/d1/erased-reversal.ts +19 -0
  45. package/modules/import-kit/src/d1/erasure-bounded.ts +22 -0
  46. package/modules/import-kit/src/d1/erasure-claims.ts +27 -0
  47. package/modules/import-kit/src/d1/erasure-compiler.ts +67 -0
  48. package/modules/import-kit/src/d1/erasure-disposition.ts +16 -0
  49. package/modules/import-kit/src/d1/erasure-fragments.ts +86 -0
  50. package/modules/import-kit/src/d1/erasure-index.ts +53 -0
  51. package/modules/import-kit/src/d1/erasure-pages.ts +62 -0
  52. package/modules/import-kit/src/d1/erasure-progress-schema.ts +8 -0
  53. package/modules/import-kit/src/d1/erasure-progress.ts +61 -0
  54. package/modules/import-kit/src/d1/erasure-related.ts +124 -0
  55. package/modules/import-kit/src/d1/erasure-report.ts +31 -0
  56. package/modules/import-kit/src/d1/erasure-schema.ts +24 -0
  57. package/modules/import-kit/src/d1/erasure-selection.ts +77 -0
  58. package/modules/import-kit/src/d1/erasure-snapshot.ts +15 -0
  59. package/modules/import-kit/src/d1/erasure-stage.ts +146 -0
  60. package/modules/import-kit/src/d1/erasure-step.ts +12 -0
  61. package/modules/import-kit/src/d1/erasure-superseded.ts +98 -0
  62. package/modules/import-kit/src/d1/erasure-values.ts +83 -0
  63. package/modules/import-kit/src/d1/erasure-writes.ts +46 -0
  64. package/modules/import-kit/src/d1/erasure.ts +170 -0
  65. package/modules/import-kit/src/d1/evolutions/0001-preserved-undo.ts +13 -0
  66. package/modules/import-kit/src/d1/expected-state.ts +91 -0
  67. package/modules/import-kit/src/d1/field-redaction.ts +86 -0
  68. package/modules/import-kit/src/d1/guards.ts +37 -0
  69. package/modules/import-kit/src/d1/history.ts +71 -0
  70. package/modules/import-kit/src/d1/index.ts +88 -0
  71. package/modules/import-kit/src/d1/integrity.ts +127 -0
  72. package/modules/import-kit/src/d1/inventory-snapshot.ts +84 -0
  73. package/modules/import-kit/src/d1/jobs.ts +97 -0
  74. package/modules/import-kit/src/d1/native-set.ts +48 -0
  75. package/modules/import-kit/src/d1/native-staging-admission.ts +19 -0
  76. package/modules/import-kit/src/d1/native-staging-context.ts +44 -0
  77. package/modules/import-kit/src/d1/native-staging-lifecycle.ts +33 -0
  78. package/modules/import-kit/src/d1/native-staging-offset.ts +74 -0
  79. package/modules/import-kit/src/d1/native-staging-publication.ts +18 -0
  80. package/modules/import-kit/src/d1/native-staging-rows.ts +72 -0
  81. package/modules/import-kit/src/d1/native-staging-schema.ts +28 -0
  82. package/modules/import-kit/src/d1/native-staging-state.ts +22 -0
  83. package/modules/import-kit/src/d1/native-staging.ts +64 -0
  84. package/modules/import-kit/src/d1/original-copies.ts +53 -0
  85. package/modules/import-kit/src/d1/original-storage.ts +56 -0
  86. package/modules/import-kit/src/d1/originals.ts +74 -0
  87. package/modules/import-kit/src/d1/plan-dictionary.ts +80 -0
  88. package/modules/import-kit/src/d1/plans.ts +121 -0
  89. package/modules/import-kit/src/d1/preparation-fingerprint.ts +57 -0
  90. package/modules/import-kit/src/d1/preparation-header.ts +33 -0
  91. package/modules/import-kit/src/d1/preparation-reader.ts +98 -0
  92. package/modules/import-kit/src/d1/preparation-schema.ts +16 -0
  93. package/modules/import-kit/src/d1/preparation-state.ts +56 -0
  94. package/modules/import-kit/src/d1/preparation-step-combine.ts +80 -0
  95. package/modules/import-kit/src/d1/preparation-step-control.ts +48 -0
  96. package/modules/import-kit/src/d1/preparation-step-hash.ts +131 -0
  97. package/modules/import-kit/src/d1/preparation-step-pages.ts +77 -0
  98. package/modules/import-kit/src/d1/preparation-steps-schema.ts +11 -0
  99. package/modules/import-kit/src/d1/preparation-steps.ts +102 -0
  100. package/modules/import-kit/src/d1/preparations.ts +87 -0
  101. package/modules/import-kit/src/d1/preserved-preview.ts +89 -0
  102. package/modules/import-kit/src/d1/preserved-reverse.ts +91 -0
  103. package/modules/import-kit/src/d1/provenance-set.ts +87 -0
  104. package/modules/import-kit/src/d1/publication-fence.ts +22 -0
  105. package/modules/import-kit/src/d1/publication-receipt-schema.ts +15 -0
  106. package/modules/import-kit/src/d1/published-stream.ts +11 -0
  107. package/modules/import-kit/src/d1/read-fence.ts +64 -0
  108. package/modules/import-kit/src/d1/read-results.ts +16 -0
  109. package/modules/import-kit/src/d1/redacted-custody.ts +76 -0
  110. package/modules/import-kit/src/d1/redaction-step.ts +125 -0
  111. package/modules/import-kit/src/d1/referenced-plans.ts +136 -0
  112. package/modules/import-kit/src/d1/retained-values.ts +35 -0
  113. package/modules/import-kit/src/d1/retirement-bounded.ts +71 -0
  114. package/modules/import-kit/src/d1/retirement-door.ts +105 -0
  115. package/modules/import-kit/src/d1/retirement-schema.ts +39 -0
  116. package/modules/import-kit/src/d1/retirement-scope.ts +94 -0
  117. package/modules/import-kit/src/d1/retirement-steps.ts +76 -0
  118. package/modules/import-kit/src/d1/retirement-tally.ts +74 -0
  119. package/modules/import-kit/src/d1/retirement.ts +136 -0
  120. package/modules/import-kit/src/d1/reversal-conversion-record.ts +19 -0
  121. package/modules/import-kit/src/d1/reversal-conversion-schema.ts +11 -0
  122. package/modules/import-kit/src/d1/reversal-conversion.ts +40 -0
  123. package/modules/import-kit/src/d1/reversal-fence-schema.ts +14 -0
  124. package/modules/import-kit/src/d1/reversal-fence.ts +106 -0
  125. package/modules/import-kit/src/d1/reversal-journal.ts +24 -0
  126. package/modules/import-kit/src/d1/reverse.ts +103 -0
  127. package/modules/import-kit/src/d1/reviewed-source.ts +23 -0
  128. package/modules/import-kit/src/d1/sets.ts +78 -0
  129. package/modules/import-kit/src/d1/staged-creation.ts +201 -0
  130. package/modules/import-kit/src/d1/staging-admission.ts +45 -0
  131. package/modules/import-kit/src/d1/staging-control.ts +43 -0
  132. package/modules/import-kit/src/d1/staging-erasure.ts +28 -0
  133. package/modules/import-kit/src/d1/staging-lifecycle.ts +49 -0
  134. package/modules/import-kit/src/d1/staging-plan.ts +15 -0
  135. package/modules/import-kit/src/d1/staging-privacy.ts +22 -0
  136. package/modules/import-kit/src/d1/staging-publication.ts +79 -0
  137. package/modules/import-kit/src/d1/staging-schema.ts +37 -0
  138. package/modules/import-kit/src/d1/staging-statements.ts +8 -0
  139. package/modules/import-kit/src/d1/staging-transport.ts +70 -0
  140. package/modules/import-kit/src/d1/stream-boundary.ts +11 -0
  141. package/modules/import-kit/src/d1/stream-create.ts +55 -0
  142. package/modules/import-kit/src/d1/stream-identity-admission.ts +37 -0
  143. package/modules/import-kit/src/d1/stream-input.ts +18 -0
  144. package/modules/import-kit/src/d1/stream-layout.ts +43 -0
  145. package/modules/import-kit/src/d1/stream-origin.ts +44 -0
  146. package/modules/import-kit/src/d1/stream-page-integrity.ts +18 -0
  147. package/modules/import-kit/src/d1/stream-partition.ts +50 -0
  148. package/modules/import-kit/src/d1/stream-plan.ts +133 -0
  149. package/modules/import-kit/src/d1/stream-receipt.ts +45 -0
  150. package/modules/import-kit/src/d1/stream-revision-schema.ts +35 -0
  151. package/modules/import-kit/src/d1/stream-snapshot.ts +26 -0
  152. package/modules/import-kit/src/d1/stream-stage.ts +79 -0
  153. package/modules/import-kit/src/d1/stream-state.ts +25 -0
  154. package/modules/import-kit/src/d1/stream-working-set.ts +30 -0
  155. package/modules/import-kit/src/d1/streams.ts +39 -0
  156. package/modules/import-kit/src/d1/target-guards.ts +9 -0
  157. package/modules/import-kit/src/d1/target-identities.ts +17 -0
  158. package/modules/import-kit/src/d1/transport.ts +10 -0
  159. package/modules/import-kit/src/d1/types.ts +162 -0
  160. package/modules/import-kit/src/d1/undo-observations.ts +39 -0
  161. package/modules/import-kit/src/d1/undo-report.ts +22 -0
  162. package/modules/import-kit/src/d1/undo-snapshot.ts +87 -0
  163. package/modules/import-kit/src/d1/undo-state.ts +84 -0
  164. package/modules/import-kit/src/d1/undo-types.ts +110 -0
  165. package/modules/import-kit/src/journal/authority.ts +12 -0
  166. package/modules/import-kit/src/plan/budget.ts +60 -0
  167. package/modules/import-kit/src/plan/index.ts +123 -0
  168. package/modules/import-kit/src/plan/size.ts +122 -0
  169. package/modules/import-kit/src/provenance/canonical-hash.ts +130 -0
  170. package/modules/import-kit/src/provenance/canonical-stream.ts +85 -0
  171. package/modules/import-kit/src/provenance/credential-names.ts +131 -0
  172. package/modules/import-kit/src/provenance/credentials.ts +317 -0
  173. package/modules/import-kit/src/provenance/identity.ts +51 -0
  174. package/modules/import-kit/src/provenance/json-snapshot.ts +21 -0
  175. package/modules/import-kit/src/provenance/resumable-sha256.ts +75 -0
  176. package/modules/import-kit/src/provenance/validate-json.ts +19 -0
  177. package/modules/import-kit/src/storage/originals.ts +87 -0
  178. package/modules/import-kit/src/storage/provenance.ts +64 -0
  179. package/modules/import-kit/src/storage/schema.ts +42 -0
  180. package/modules/import-kit/src/types.ts +47 -0
  181. package/modules/operator-auth/docs/native-accounts.md +4 -1
  182. package/modules/operator-auth/identity/oidc/aes-gcm.mjs +21 -0
  183. package/modules/operator-auth/identity/oidc/crypto.mjs +5 -12
  184. package/modules/operator-auth/index.d.mts +1 -1
  185. package/modules/portability/README.md +98 -2
  186. package/modules/portability/backfills.mjs +42 -15
  187. package/modules/portability/ledgers.mjs +7 -1
  188. package/modules/portability/transient-tables.mjs +40 -0
  189. package/onboarding/local/environment.mjs +2 -2
  190. package/package.json +3 -1
  191. package/scripts/publish-check/reviewed.py +3 -1
@@ -0,0 +1,136 @@
1
+ import { Buffer } from 'node:buffer';
2
+ import type { ImportPlan } from '../types.ts';
3
+ import type { D1Database, D1PlanCodec } from './types.ts';
4
+ import { canonicalJson } from '../provenance/identity.ts';
5
+ import { jsonChunks } from './sets.ts';
6
+ import { loadPlans } from './plans.ts';
7
+ import { entryRun, validLocator, type EntryLocator } from './entry-locator.ts';
8
+
9
+ /** A fenced read (custody revisions captured before any custody read and
10
+ * compared in its final batch) checks each owner job against its stored plan
11
+ * header and fingerprint and reads only the plan entries its provenance
12
+ * references, parsed inside D1 from the inline plan or from the chunk run its
13
+ * entry locator names. Unreferenced entries never leave D1 and no SQL value
14
+ * grows past one run, so an owner job costs fixed statements and bytes however
15
+ * long a catch-up chain grows. A plan stored without a locator (compressed,
16
+ * staged or written before locators) and a preserved job with a reversal report
17
+ * still decode the whole plan. */
18
+ const referencedOnly = new WeakSet<ImportPlan>();
19
+ export const isReferencedOnly = (plan: ImportPlan): boolean => referencedOnly.has(plan);
20
+ // One page of referenced entries stays well inside one D1 result.
21
+ const PAGE_BYTES = 2 * 1024 * 1024;
22
+ // A run joined in SQL stays under D1's 2 MB value ceiling (chunks are at most 128 KiB).
23
+ const SQL_RUN_CHUNKS = 14;
24
+ const needsWholePlan = (owner: any) => owner.reversal_policy === 'preserve-conflicts' && owner.reversal_cursor !== 0;
25
+ const inline = "json_valid(plan_json) AND json_type(plan_json,'$.d1Plan') IS NULL";
26
+
27
+ /** The job fields the fenced read checks, an inline plan's header and entry
28
+ * count, and the stored chunk layout. plan_json stays only as an envelope, where
29
+ * it is not valid JSON, or where a preserved job's reversal report reads every entry. */
30
+ export async function ownerHeads(db: D1Database, ids: string[]): Promise<any[]> {
31
+ const rows: any[] = [];
32
+ if (!ids.length) return rows;
33
+ // Ordinary reads remain usable before an adopter installs the additive
34
+ // conversion migration. Select only fields the declared schema provides;
35
+ // converted owners still carry their audit binding through this fast path.
36
+ const columns = await db.prepare("SELECT name FROM pragma_table_info('ik_jobs')").all();
37
+ const conversionColumn = columns.results.some(column => column.name === 'reversal_conversion_hash') ? ',reversal_conversion_hash' : '';
38
+ for (const chunk of jsonChunks([...new Set(ids)])) rows.push(...(await db.prepare(`SELECT id,destination,namespace,mapping_version,principal_id,fingerprint,state,checkpoint,total,reversal_policy,reversal_policy_hash,reversal_cursor,reversal_hash${conversionColumn},
39
+ CASE WHEN ${inline} AND NOT (reversal_policy='preserve-conflicts' AND reversal_cursor<>0) THEN NULL ELSE plan_json END AS plan_json,
40
+ CASE WHEN ${inline} THEN json_extract(plan_json,'$.version','$.destination','$.namespace','$.mappingVersion','$.principalId','$.fingerprint') END AS header,
41
+ CASE WHEN ${inline} THEN json_array_length(plan_json,'$.entries') END AS entries,
42
+ (SELECT json_array(count(*),min(c.ordinal),max(c.ordinal)) FROM ik_d1_plan_chunks c WHERE c.job_id=ik_jobs.id) AS layout
43
+ FROM ik_jobs WHERE id IN (SELECT value FROM json_each(?))`).bind(chunk).all()).results);
44
+ return rows;
45
+ }
46
+ /** Which of these jobs hold any reversal report row. */
47
+ export async function reportedJobs(db: D1Database, ids: string[]): Promise<Set<string>> {
48
+ const found = new Set<string>();
49
+ for (const chunk of jsonChunks([...new Set(ids)])) for (const row of (await db.prepare('SELECT DISTINCT job_id FROM ik_d1_reversal_reports WHERE job_id IN (SELECT value FROM json_each(?))').bind(chunk).all<any>()).results) found.add(row.job_id);
50
+ return found;
51
+ }
52
+
53
+ type Run = [job: string, first: number, offset: number, last: number, end: number, base: number, positions: number[]];
54
+ /** Pages of whole result rows up to PAGE_BYTES (at least one row each), until
55
+ * `next` leaves nothing requested or a page comes back empty. */
56
+ async function pages(db: D1Database, query: string, binds: number, items: unknown[], next: (results: any[], requested: any[]) => any[]): Promise<void> {
57
+ for (const chunk of jsonChunks(items)) {
58
+ let requested: any[] = JSON.parse(chunk);
59
+ while (requested.length) {
60
+ const request = JSON.stringify(requested), { results } = await db.prepare(`SELECT * FROM (SELECT *,sum(octet_length(value)) OVER (ORDER BY page_key ROWS UNBOUNDED PRECEDING)-octet_length(value) AS page_before FROM (${query})) WHERE page_before<? ORDER BY page_key`).bind(...Array(binds).fill(request), PAGE_BYTES).all<any>();
61
+ if (!results.length) break;
62
+ requested = next(results, requested);
63
+ }
64
+ }
65
+ }
66
+ const entryOf = (row: { type: string; value: string }) => { if (row.type !== 'object') throw new Error('STORED_PLAN_CHANGED'); return JSON.parse(row.value); };
67
+ const field = (index: number) => `json_extract(r.value,'$[${index}]')`;
68
+ // The run's bytes, from its first chunk's offset to its last chunk's end offset, as one bracketed JSON array.
69
+ const [first, offset, last, end] = [1, 2, 3, 4].map(field);
70
+ const RUN = `'['||rtrim((SELECT group_concat(CAST(CASE WHEN c.ordinal=${first} AND c.ordinal=${last} THEN substr(CAST(c.body AS BLOB),${offset}+1,${end}-${offset}) WHEN c.ordinal=${first} THEN substr(CAST(c.body AS BLOB),${offset}+1) WHEN c.ordinal=${last} THEN substr(CAST(c.body AS BLOB),1,${end}) ELSE CAST(c.body AS BLOB) END AS TEXT),'' ORDER BY c.ordinal) FROM ik_d1_plan_chunks c WHERE c.job_id=${field(0)} AND c.ordinal BETWEEN ${first} AND ${last}),',')||']'`;
71
+ const INLINE_ENTRIES = `SELECT json_array(j.id,e.key) AS page_key,j.id AS job_id,e.key AS ordinal,e.type AS type,e.value AS value FROM ik_jobs j,json_each(j.plan_json,'$.entries') e WHERE j.id IN (SELECT json_extract(value,'$[0]') FROM json_each(?)) AND e.key IN (SELECT ${field(1)} FROM json_each(?) r WHERE ${field(0)}=j.id)`;
72
+ const RUN_ENTRIES = `SELECT json_array(r.key,e.key) AS page_key,r.key AS run,e.key AS position,e.type AS type,e.value AS value FROM json_each(?) r,json_each(CASE WHEN json_valid(${RUN}) THEN ${RUN} ELSE '[]' END) e WHERE e.key IN (SELECT value FROM json_each(r.value,'$[6]'))`;
73
+
74
+ /** A run too long to join in one SQL value (an entry larger than about 1.8 MB) is read as its chunks. */
75
+ async function readLongRun(db: D1Database, run: Run, entries: unknown[]): Promise<void> {
76
+ const [job, from, offset, to, end, base, positions] = run, parts: Buffer[] = [];
77
+ for (let ordinal = from; ordinal <= to; ordinal += 3) {
78
+ const { results } = await db.prepare('SELECT ordinal,body FROM ik_d1_plan_chunks WHERE job_id=? AND ordinal BETWEEN ? AND ? ORDER BY ordinal').bind(job, ordinal, Math.min(ordinal + 2, to)).all<any>();
79
+ for (const row of results) { if (row.ordinal !== from + parts.length || typeof row.body !== 'string') throw new Error('STORED_PLAN_CHANGED'); parts.push(Buffer.from(row.body)); }
80
+ }
81
+ if (parts.length !== to - from + 1) throw new Error('STORED_PLAN_CHANGED');
82
+ parts[parts.length - 1] = parts[parts.length - 1].subarray(0, end); parts[0] = parts[0].subarray(offset);
83
+ let list: unknown;
84
+ try { list = JSON.parse('[' + Buffer.concat(parts).toString('utf8').replace(/,$/, '') + ']'); } catch { throw new Error('STORED_PLAN_CHANGED'); }
85
+ if (!Array.isArray(list)) throw new Error('STORED_PLAN_CHANGED');
86
+ for (const position of positions) { const entry = list[position]; if (!entry || typeof entry !== 'object' || Array.isArray(entry)) throw new Error('STORED_PLAN_CHANGED'); entries[base + position] = entry; }
87
+ }
88
+
89
+ export async function loadReferencedPlans(owners: any[], raws: any[], db: D1Database, codec?: D1PlanCodec): Promise<Map<string, ImportPlan>> {
90
+ const plans = new Map<string, ImportPlan>(), locators = new Map<string, EntryLocator>(), full: any[] = [];
91
+ for (const owner of owners) {
92
+ const envelope = owner.plan_json === null ? null : JSON.parse(owner.plan_json), layout = JSON.parse(owner.layout);
93
+ if (envelope === null) {
94
+ if (typeof owner.header !== 'string' || !Number.isSafeInteger(owner.entries)) throw new Error('STORED_PLAN_CHANGED');
95
+ const [version, destination, namespace, mappingVersion, principalId, fingerprint] = JSON.parse(owner.header);
96
+ plans.set(owner.id, { version, destination, namespace, mappingVersion, principalId, fingerprint, entries: new Array(owner.entries) } as ImportPlan);
97
+ } else if (envelope?.d1Plan === 1 && envelope.entries !== undefined && !needsWholePlan(owner)) {
98
+ if (canonicalJson(envelope) !== owner.plan_json || canonicalJson(Object.keys(envelope).sort()) !== '["chunks","d1Plan","entries","sha256","size"]' || !Number.isSafeInteger(envelope.chunks) || envelope.chunks < 2
99
+ || canonicalJson(layout) !== canonicalJson([envelope.chunks, 0, envelope.chunks - 1]) || !validLocator(envelope.entries, envelope.chunks)) throw new Error('STORED_PLAN_CHANGED');
100
+ const [version, destination, namespace, mappingVersion, principalId, fingerprint] = envelope.entries.header;
101
+ locators.set(owner.id, envelope.entries);
102
+ plans.set(owner.id, { version, destination, namespace, mappingVersion, principalId, fingerprint, entries: new Array(envelope.entries.count) } as ImportPlan);
103
+ } else full.push(owner);
104
+ }
105
+ for (const plan of plans.values()) referencedOnly.add(plan);
106
+ const wanted = new Map<string, Set<number>>();
107
+ for (const raw of raws) if (plans.has(raw.job_id) && Number.isSafeInteger(raw.ordinal) && raw.ordinal >= 0 && raw.ordinal < plans.get(raw.job_id)!.entries.length) {
108
+ if (!wanted.has(raw.job_id)) wanted.set(raw.job_id, new Set());
109
+ wanted.get(raw.job_id)!.add(raw.ordinal);
110
+ }
111
+ // Inline plans: the referenced entries of each, by ordinal.
112
+ const pairs = [...wanted].filter(([id]) => !locators.has(id)).flatMap(([id, ordinals]) => [...ordinals].sort((a, b) => a - b).map(ordinal => [id, ordinal]));
113
+ await pages(db, INLINE_ENTRIES, 2, pairs, (results, requested) => {
114
+ for (const row of results) plans.get(row.job_id)!.entries[row.ordinal] = entryOf(row);
115
+ const read = new Set(results.map(row => canonicalJson([row.job_id, row.ordinal])));
116
+ return requested.filter(pair => !read.has(canonicalJson(pair)));
117
+ });
118
+ // Located plans: each run holding a referenced entry, parsed in D1, returning only those entries.
119
+ const runs = new Map<string, Run>();
120
+ for (const [id, ordinals] of wanted) {
121
+ const locator = locators.get(id);
122
+ if (locator) for (const ordinal of [...ordinals].sort((a, b) => a - b)) {
123
+ const [from, offset, to, end, base] = entryRun(locator, ordinal), key = canonicalJson([id, from]);
124
+ if (!runs.has(key)) runs.set(key, [id, from, offset, to, end, base, []]);
125
+ runs.get(key)![6].push(ordinal - base);
126
+ }
127
+ }
128
+ await pages(db, RUN_ENTRIES, 1, [...runs.values()].filter(run => run[3] - run[1] < SQL_RUN_CHUNKS), (results, requested: Run[]) => {
129
+ const read = new Set<string>();
130
+ for (const row of results) { const run = requested[row.run]; plans.get(run[0])!.entries[run[5] + row.position] = entryOf(row); read.add(canonicalJson([row.run, row.position])); }
131
+ return requested.map((run, index): Run => [...run.slice(0, 6), run[6].filter(position => !read.has(canonicalJson([index, position])))] as Run).filter(run => run[6].length);
132
+ });
133
+ for (const run of runs.values()) if (run[3] - run[1] >= SQL_RUN_CHUNKS) await readLongRun(db, run, plans.get(run[0])!.entries);
134
+ for (const [id, plan] of await loadPlans(full, db, codec)) plans.set(id, plan);
135
+ return plans;
136
+ }
@@ -0,0 +1,35 @@
1
+ import { guard, guardPredicate } from './guards.ts';
2
+ import type { D1Sql } from './types.ts';
3
+ /** A structural placeholder descriptor, supplied only for safe normalized values.
4
+ * SQL is assembled from literal parts; no SQL lexer or question-mark rewriting
5
+ * guesses whether text is a quoted literal, comment or numbered parameter. */
6
+ export function retainedSql(parts: string[], params: unknown[], retain: number[]): D1Sql {
7
+ if (parts.length !== params.length + 1 || retain.some(i=>!Number.isSafeInteger(i)||i<0||i>=params.length||typeof params[i]!=='string')) throw new Error('IMPORT_RETAINED_VALUE_DESCRIPTOR');
8
+ return { sql: parts.join('?'), params, retainedValues: { parts, indexes: retain } };
9
+ }
10
+ export function replaceRetained(statement: D1Sql, reference: (value: string) => D1Sql): D1Sql {
11
+ const scalar = guardPredicate(statement);
12
+ if (scalar) {
13
+ const replaced = guard(replaceRetained(scalar, reference));
14
+ replaced.expectedSnapshots = statement.expectedSnapshots;
15
+ replaced.expectedChanges = statement.expectedChanges;
16
+ replaced.finalChecks = statement.finalChecks;
17
+ replaced.resultKey = statement.resultKey;
18
+ return replaced;
19
+ }
20
+ const descriptor = statement.retainedValues;
21
+ if (!descriptor) return statement;
22
+ const { parts, indexes } = descriptor, params = statement.params ?? [];
23
+ if (parts.length!==params.length+1 || parts.join('?')!==statement.sql || indexes.some(i=>!Number.isSafeInteger(i)||i<0||i>=params.length||typeof params[i]!=='string')) throw new Error('IMPORT_RETAINED_VALUE_DESCRIPTOR');
24
+ const result: D1Sql = {...statement,sql:parts[0],params:[],retainedValues:undefined};
25
+ for(let i=0;i<params.length;i++){
26
+ const value=indexes.includes(i)?reference(params[i]):{sql:'?',params:[params[i]]};
27
+ result.sql+=value.sql+parts[i+1];result.params!.push(...value.params??[]);
28
+ }
29
+ return result;
30
+ }
31
+ /** Internal tagged SQL for already validated normalized JSON and identifiers.
32
+ * The template's literal boundaries, rather than a lexer, locate bindings. */
33
+ export function retainedTemplate(parts:TemplateStringsArray,...params:unknown[]):D1Sql{
34
+ return retainedSql([...parts],params,params.flatMap((value,index)=>typeof value==='string'?[index]:[]));
35
+ }
@@ -0,0 +1,71 @@
1
+ import type { D1CustodyRetirement, D1ImportContext, D1ImportPerson, D1Sql } from './types.ts';
2
+ import { canonicalJson, requireText } from '../provenance/identity.ts';
3
+ import { fieldRedactions } from './field-redaction.ts';
4
+ import { reserveReturn } from './budget.ts';
5
+ import { guardPredicate, copyStatement } from './guards.ts';
6
+ import { sql } from './database.ts';
7
+ import { prepareErasureSelection } from './erasure-selection.ts';
8
+ import { stageD1ImportErasure } from './erasure-stage.ts';
9
+ import { erasureRequestContext, erasureStatementBinding, type D1ErasurePending, type D1ErasureStepOptions } from './erasure-progress.ts';
10
+ import { headerAbsence, retirementScope, retirementWrites, scopeGuards } from './retirement-scope.ts';
11
+ import { tallyBefore, tallyJournal, tallyStores, tallyWithdraw } from './retirement-tally.ts';
12
+ import { doorCustodyInScope, doorLineage } from './retirement-door.ts';
13
+
14
+ /** Retire headers and erase records over bounded requests. Only the final
15
+ * publication changes public custody, native rows and the restore checkpoint.
16
+ * operationKey identifies that publication even when its acknowledgement is lost. */
17
+ export async function retireD1ImportCustodyStep(person: D1ImportPerson, nativeStatements: D1Sql[], input: D1ImportContext, checkpoint: D1Sql[], options: D1ErasureStepOptions): Promise<D1CustodyRetirement | D1ErasurePending> {
18
+ person = { ...person, values: [...new Set(person.values ?? [])].sort(), related: person.related ? { ...person.related, fields: fieldRedactions(person.related.fields) } : undefined };
19
+ for (const value of person.values!) requireText(value, 'person value');
20
+ if (person.target) { requireText(person.target.kind, 'target kind'); requireText(person.target.id, 'target id'); }
21
+ const fields = person.related?.fields ?? [];
22
+ if (!person.target && !person.values!.length && !fields.length) throw new Error('IMPORT_RETIREMENT_PERSON_REQUIRED');
23
+ if (person.related && !person.target && (person.related.source || person.related.original || !fields.length)) throw new Error('IMPORT_RETIREMENT_TARGET_REQUIRED');
24
+ const context = erasureRequestContext({ ...input, originalErasurePolicy: input.originalErasurePolicy ?? 'withdraw-affected-artifacts' }, options);
25
+ // Returning the completed receipt and retirement ids costs exactly two reads.
26
+ const nested = reserveReturn(context, 2);
27
+ const binding = canonicalJson(['retirement-v1', person.target ?? null, person.values, person.related ? { fields, snapshot: person.related.snapshot ?? null, source: person.related.source?.toString() ?? null, original: person.related.original?.toString() ?? null } : null, erasureStatementBinding(nativeStatements), erasureStatementBinding(checkpoint)]);
28
+ const keys = nativeStatements.map(() => Symbol('native retirement result'));
29
+ const native = nativeStatements.map((row, index) => Object.assign(copyStatement(row), { resultKey: keys[index] }));
30
+ const result = await stageD1ImportErasure(nested, options, binding, async (id, bounded) => {
31
+ const db = bounded.storage.database, destination = bounded.destination;
32
+ const scope = await retirementScope(db, destination, person.values!), retiring = new Set(scope.jobs);
33
+ // The stage is transport, not erased custody. Its private before-images
34
+ // would inflate the tally and retain values after a completed publication.
35
+ const stores = (await tallyStores(db)).filter(store => !/^ik_d1_native_staging/.test(store.name));
36
+ // Withdrawn door content is custody too: a values-only erasure that removes it is fenced.
37
+ const fenced = !!person.target || !!fields.length || Object.values(scope).some(ids => ids.length > 0) || await doorCustodyInScope(db, destination, stores);
38
+ if (fenced && !checkpoint.length) throw new Error('IMPORT_RETIREMENT_CHECKPOINT_REQUIRED');
39
+ if (native.length && !fenced) throw new Error('IMPORT_RETIREMENT_NATIVE_REQUIRES_CHECKPOINT');
40
+ // The binding stays the caller's intent; each step derives the door lineage it severs,
41
+ // and its publication fails if a catch-up joined that lineage after this read.
42
+ const lineage = await doorLineage(person, db, destination, bounded.storage.targets.planCodec);
43
+ const selection = await prepareErasureSelection(lineage.person, native, bounded, id, retiring);
44
+ const closure = scopeGuards(scope);
45
+ const absence = headerAbsence(destination, person.values!);
46
+ const retired = [sql('NOT EXISTS(SELECT 1 FROM ik_jobs WHERE id IN (SELECT value FROM json_each(?)))', JSON.stringify(scope.jobs)),
47
+ sql('NOT EXISTS(SELECT 1 FROM ik_d1_streams WHERE id IN (SELECT value FROM json_each(?)))', JSON.stringify(scope.streams))];
48
+ return { ...selection, resultKeys: keys, direct: !scope.jobs.length,
49
+ snapshotGuards: [...selection.snapshotGuards, ...closure],
50
+ survivingJobs: compiler => [...compiler.jobs.keys()].filter(job => !retiring.has(job)),
51
+ final: [...selection.final, ...absence.map(row => guardPredicate(row)!), ...retired, ...lineage.fence],
52
+ publication: async (compiler, receipt) => {
53
+ const jobs = [...compiler.jobs.keys()].sort();
54
+ const streams = [...new Set([...scope.anchors, ...(await db.prepare('SELECT DISTINCT stream_id AS id FROM ik_d1_stream_pages WHERE job_id IN (SELECT value FROM json_each(?)) ORDER BY id').bind(JSON.stringify(jobs)).all<{ id: string }>()).results.map(row => row.id)])].sort();
55
+ const touched = { destination, jobs, streams };
56
+ return {
57
+ before: [...closure, ...tallyBefore(stores, receipt.id, touched), ...tallyWithdraw(stores, touched)],
58
+ after: fenced ? checkpoint : [],
59
+ tail: [...retirementWrites(scope, destination, receipt.id, receipt.recordedAt), ...absence,
60
+ ...tallyJournal(stores, receipt.id, touched, { recordedAt: receipt.recordedAt, recordsErased: compiler.erased, jobsRetired: scope.jobs.length, streamsRetired: scope.anchors.length }, fenced)],
61
+ };
62
+ },
63
+ };
64
+ });
65
+ if (result.phase === 'pending') return result;
66
+ const db = context.storage.database;
67
+ const journal = await db.prepare('SELECT * FROM ik_d1_erasure_journal WHERE id=? AND destination=?').bind(result.operationId, context.destination).first<any>();
68
+ if (!journal) throw new Error('IMPORT_ERASURE_JOURNAL_MISSING');
69
+ const retired = (await db.prepare('SELECT id,kind FROM ik_d1_retirements WHERE receipt_id=? AND destination=? ORDER BY id').bind(result.operationId, context.destination).all<{ id: string; kind: string }>()).results;
70
+ return { phase: 'complete', receiptId: journal.id, destination: journal.destination, recordedAt: journal.recorded_at, recordsErased: journal.records_erased, jobsRetired: journal.jobs_retired, streamsRetired: journal.streams_retired, stores: JSON.parse(journal.stores_json), fieldMatches: result.fieldMatches, retiredJobs: retired.filter(row => row.kind === 'job').map(row => row.id), retiredStreams: retired.filter(row => row.kind === 'stream').map(row => row.id), nativeResults: result.nativeResults };
71
+ }
@@ -0,0 +1,105 @@
1
+ import type { D1Database, D1ImportPerson, D1PlanCodec, D1Sql } from './types.ts';
2
+ import { sql } from './database.ts';
3
+ import type { SourceRecord } from '../types.ts';
4
+ import type { Store, TallyScope } from './retirement-tally.ts';
5
+ import { loadPlans } from './plans.ts';
6
+ import { targetIdentity } from './erasure-index.ts';
7
+ import { canonicalJson } from '../provenance/identity.ts';
8
+
9
+ // Record content only, children before parents. Plans, approvals, namespaces,
10
+ // scopes and archive headers are bookkeeping (see retirement-tally.ts) and keep
11
+ // import history; an archive's bytes are content and go whole.
12
+ const ORDER = ['ik_door_chunks', 'ik_door_plan_chunks', 'ik_door_archive_chunks', 'ik_door_uploads'];
13
+
14
+ // Namespaces linked to a destination: its first-upload binding, or a plan
15
+ // applied by a kit job (old applied doors use their real job link, including
16
+ // the operation key used to recover an interrupted job-index update).
17
+ const linked = (mapped: boolean, destination: boolean) => `SELECT p.installation_id,p.namespace FROM ik_door_plans p JOIN ik_jobs j
18
+ ON j.id=p.job_id OR (p.job_id IS NULL AND j.operation_key='door:'||p.id) ${destination ? 'WHERE j.destination=?' : ''}
19
+ ${mapped ? `UNION SELECT installation_id,namespace FROM ik_door_scopes ${destination ? 'WHERE destination=?' : ''}` : ''}`;
20
+
21
+ /** A namespace is in scope when it is linked to the erased destination, or when
22
+ * it is linked to none: previous door code wrote uploads and plans without a
23
+ * binding, and no destination ever applied them, so any erasure withdraws them. */
24
+ function inScope(mapped: boolean) {
25
+ return `((installation_id,namespace) IN (${linked(mapped, true)}) OR (installation_id,namespace) NOT IN (${linked(mapped, false)}))`;
26
+ }
27
+
28
+ function where(name: string, mapped: boolean) {
29
+ const scoped = inScope(mapped);
30
+ return name === 'ik_door_chunks' ? `upload_id IN (SELECT id FROM ik_door_uploads WHERE ${scoped})`
31
+ : name === 'ik_door_plan_chunks' ? `plan_id IN (SELECT id FROM ik_door_plans WHERE ${scoped})`
32
+ : name === 'ik_door_archive_chunks' ? `archive_id IN (SELECT id FROM ik_door_archives WHERE ${scoped})`
33
+ : scoped;
34
+ }
35
+
36
+ const params = (mapped: boolean) => (scope: TallyScope) => mapped ? [scope.destination, scope.destination] : [scope.destination];
37
+
38
+ /** Withdraw a destination's opaque door bundles and snapshot bodies whole. They
39
+ * may contain rejected records or before-images that ordinary kit redaction
40
+ * cannot safely rewrite. `installed` lists every ik_ table in the database. */
41
+ export function doorRetirementStore(name: string, installed: ReadonlySet<string>) {
42
+ const withdraw = ORDER.indexOf(name);
43
+ if (withdraw < 0) return undefined;
44
+ const mapped = installed.has('ik_door_scopes');
45
+ return { where: where(name, mapped), withdraw, params: params(mapped) };
46
+ }
47
+
48
+ /** Whether an erasure of this destination withdraws door content, which makes
49
+ * it fenced: it then requires and commits the restore checkpoint. One read,
50
+ * and none where the door is not installed. */
51
+ export async function doorCustodyInScope(db: D1Database, destination: string, stores: readonly Store[]): Promise<boolean> {
52
+ const door = stores.filter(store => store.withdraw !== undefined);
53
+ if (!door.length) return false;
54
+ const scope = { destination, jobs: [], streams: [] };
55
+ const row = await db.prepare(`SELECT ${door.map(store => `EXISTS(SELECT 1 FROM ${store.name} WHERE ${store.where})`).join(' OR ')} AS found`)
56
+ .bind(...door.flatMap(store => store.params(scope))).first<{ found: number }>();
57
+ return row?.found === 1;
58
+ }
59
+
60
+ export const jobLink = "j.id=p.job_id OR (p.job_id IS NULL AND j.operation_key='door:'||p.id)";
61
+ /** What an erasure severs of a door import's lineage: the merged selector, the
62
+ * kit jobs that may hold a copy, and a fence: predicates that fail the erasure
63
+ * if a kit job joins one of those door namespaces after this read, with copies
64
+ * it never selected. `select` is the lineage's own selector, and `sources` names
65
+ * the severed source records by door namespace, kind and source id: a resumable
66
+ * erasure severs copies in steps, and a step's work stands only while they are
67
+ * unchanged. */
68
+ export interface DoorLineage { person: D1ImportPerson; jobs: string[]; fence: D1Sql[]; sources: string[]; select?: (record: SourceRecord) => boolean }
69
+ /** A door catch-up keeps each unchanged (or refused) record as a zero-effect
70
+ * entry in its own kit job, under a new kit namespace, with the record's payload
71
+ * and the bundle original. Erasing a target that a door import wrote severs those
72
+ * copies too: the same kind and source id in any kit job of the same door
73
+ * namespace is the same source record. Merged with the caller's own selector. */
74
+ export async function doorLineage(person: D1ImportPerson, db: D1Database, destination: string, codec?: D1PlanCodec): Promise<DoorLineage> {
75
+ const none: DoorLineage = { person, jobs: [], fence: [], sources: [] };
76
+ if (!person.target || !await db.prepare("SELECT 1 AS ok FROM sqlite_master WHERE type='table' AND name='ik_door_plans'").first()) return none;
77
+ // An erased entry (its source identity is a tombstone) holds nothing to follow,
78
+ // so copies an earlier step severed and bound to this target are not read again.
79
+ const { results } = await db.prepare(`SELECT DISTINCT j.*,i.ordinal AS entry_ordinal,p.installation_id AS door_installation,p.namespace AS door_namespace FROM ik_d1_plan_index i
80
+ JOIN ik_jobs j ON j.id=i.job_id JOIN ik_door_plans p ON ${jobLink} WHERE j.destination=? AND i.target_identity=?
81
+ AND NOT EXISTS(SELECT 1 FROM ik_d1_erased_sources s WHERE s.destination=j.destination AND s.identity=i.source_identity) ORDER BY j.id,i.ordinal`)
82
+ .bind(destination, targetIdentity(destination, person.target.kind, person.target.id)).all<any>();
83
+ if (!results.length) return none;
84
+ const plans = await loadPlans([...new Map(results.map(row => [row.id, row])).values()], db, codec);
85
+ const lineages = new Map<string, { id: string; namespace: string }[]>(), keys = new Set<string>(), sources = new Set<string>(), jobs = new Set<string>(), fence: D1Sql[] = [];
86
+ for (const row of results) {
87
+ const entry = plans.get(row.id)?.entries[row.entry_ordinal];
88
+ if (!entry || entry.erased) continue;
89
+ const door = canonicalJson([row.door_installation, row.door_namespace]);
90
+ if (!lineages.has(door)) {
91
+ const members = (await db.prepare(`SELECT DISTINCT j.id,j.namespace FROM ik_door_plans p JOIN ik_jobs j ON ${jobLink} WHERE p.installation_id=? AND p.namespace=? AND j.destination=? ORDER BY j.id`)
92
+ .bind(row.door_installation, row.door_namespace, destination).all<{ id: string; namespace: string }>()).results;
93
+ lineages.set(door, members);
94
+ for (const member of members) jobs.add(member.id);
95
+ fence.push(sql(`NOT EXISTS(SELECT 1 FROM ik_door_plans p JOIN ik_jobs j ON ${jobLink} WHERE p.installation_id=? AND p.namespace=? AND j.destination=? AND j.id NOT IN (SELECT value FROM json_each(?)))`,
96
+ row.door_installation, row.door_namespace, destination, JSON.stringify(members.map(member => member.id))));
97
+ }
98
+ sources.add(canonicalJson([row.door_installation, row.door_namespace, entry.source.kind, entry.source.sourceId]));
99
+ for (const member of lineages.get(door)!) keys.add(canonicalJson([member.namespace, entry.source.kind, entry.source.sourceId]));
100
+ }
101
+ if (!keys.size) return none;
102
+ const own = person.related?.source;
103
+ const lineage = (record: SourceRecord) => keys.has(canonicalJson([record.namespace, record.kind, record.sourceId]));
104
+ return { person: { ...person, related: { ...person.related, source: own ? record => own(record) || lineage(record) : lineage } }, jobs: [...jobs].sort(), fence, sources: [...sources].sort(), select: lineage };
105
+ }
@@ -0,0 +1,39 @@
1
+ import { streamParentFields } from './stream-revision-schema.ts';
2
+
3
+ const anchored = streamParentFields.filter(field => field !== 'namespace' && field !== 'operation_key');
4
+
5
+ /** SQLite's REPLACE removes a conflicting row without firing DELETE triggers, so
6
+ * an insert over an existing key is refused before conflict resolution. A store
7
+ * that applied an earlier d1RetirementSchema applies these as its next migration. */
8
+ export const d1RetirementAppendOnly = [
9
+ "CREATE TRIGGER IF NOT EXISTS ik_d1_erasure_journal_insert BEFORE INSERT ON ik_d1_erasure_journal WHEN EXISTS(SELECT 1 FROM ik_d1_erasure_journal WHERE id=NEW.id) BEGIN SELECT RAISE(ABORT,'IMPORT_ERASURE_JOURNAL_APPEND_ONLY'); END",
10
+ "CREATE TRIGGER IF NOT EXISTS ik_d1_retirement_insert BEFORE INSERT ON ik_d1_retirements WHEN EXISTS(SELECT 1 FROM ik_d1_retirements WHERE id=NEW.id AND kind=NEW.kind) BEGIN SELECT RAISE(ABORT,'IMPORT_RETIREMENT_CHANGED'); END",
11
+ ];
12
+
13
+ /** Additive upgrade for person erasure from import custody, applied after every
14
+ * other D1 schema. Existing applied migrations stay byte for byte.
15
+ *
16
+ * - `ik_d1_retirements`: jobs and streams whose whole custody was retired, with
17
+ * the journal receipt that did it. No header, namespace or operation key.
18
+ * Append-only like the journal.
19
+ * - `ik_d1_erasure_journal`: append-only (no update, delete or insert over an
20
+ * existing id), one row per erasure: receipt id, the stores touched with
21
+ * their row counts, and the time. It holds no erased value.
22
+ * - `ik_d1_erasure_tally`: transaction-local row images used to count the
23
+ * journal's rows; every erasure batch empties it before it commits.
24
+ * - The stream revision anchor keeps its monotonic counters and layout fields,
25
+ * and may drop its namespace and operation key (to '') once its parent stream
26
+ * is gone and the stream is recorded as retired. Nothing else may change. */
27
+ export const d1RetirementSchema = [
28
+ "CREATE TABLE IF NOT EXISTS ik_d1_retirements (id TEXT NOT NULL, kind TEXT NOT NULL CHECK(kind IN ('job','stream')), destination TEXT NOT NULL, receipt_id TEXT NOT NULL, retired_at TEXT NOT NULL, PRIMARY KEY(id, kind))",
29
+ "CREATE TABLE IF NOT EXISTS ik_d1_erasure_journal (id TEXT PRIMARY KEY, destination TEXT NOT NULL, recorded_at TEXT NOT NULL, records_erased INTEGER NOT NULL CHECK(records_erased>=0), jobs_retired INTEGER NOT NULL CHECK(jobs_retired>=0), streams_retired INTEGER NOT NULL CHECK(streams_retired>=0), stores_json TEXT NOT NULL CHECK(json_valid(stores_json)))",
30
+ 'CREATE INDEX IF NOT EXISTS ik_d1_erasure_journal_time ON ik_d1_erasure_journal(destination, recorded_at)',
31
+ ...d1RetirementAppendOnly,
32
+ "CREATE TRIGGER IF NOT EXISTS ik_d1_erasure_journal_update BEFORE UPDATE ON ik_d1_erasure_journal BEGIN SELECT RAISE(ABORT,'IMPORT_ERASURE_JOURNAL_APPEND_ONLY'); END",
33
+ "CREATE TRIGGER IF NOT EXISTS ik_d1_erasure_journal_delete BEFORE DELETE ON ik_d1_erasure_journal BEGIN SELECT RAISE(ABORT,'IMPORT_ERASURE_JOURNAL_APPEND_ONLY'); END",
34
+ "CREATE TRIGGER IF NOT EXISTS ik_d1_retirement_update BEFORE UPDATE ON ik_d1_retirements BEGIN SELECT RAISE(ABORT,'IMPORT_RETIREMENT_CHANGED'); END",
35
+ "CREATE TRIGGER IF NOT EXISTS ik_d1_retirement_delete BEFORE DELETE ON ik_d1_retirements BEGIN SELECT RAISE(ABORT,'IMPORT_RETIREMENT_CHANGED'); END",
36
+ 'CREATE TABLE IF NOT EXISTS ik_d1_erasure_tally (receipt_id TEXT NOT NULL, store TEXT NOT NULL, image TEXT NOT NULL)',
37
+ 'DROP TRIGGER IF EXISTS ik_d1_stream_revision_update',
38
+ `CREATE TRIGGER ik_d1_stream_revision_update BEFORE UPDATE ON ik_d1_stream_revisions WHEN NEW.stream_id IS NOT OLD.stream_id OR ${anchored.map(field => `NEW.${field} IS NOT OLD.${field}`).join(' OR ')} OR NEW.layout_revision<OLD.layout_revision OR NEW.progress_revision<=OLD.progress_revision OR ((NEW.namespace IS NOT OLD.namespace OR NEW.operation_key IS NOT OLD.operation_key) AND NOT (NEW.namespace='' AND NEW.operation_key='' AND NOT EXISTS(SELECT 1 FROM ik_d1_streams WHERE id=OLD.stream_id) AND EXISTS(SELECT 1 FROM ik_d1_retirements WHERE id=OLD.stream_id AND kind='stream'))) BEGIN SELECT RAISE(ABORT,'IMPORT_STREAM_REVISION_CHANGED'); END`,
39
+ ];
@@ -0,0 +1,94 @@
1
+ import type { D1Database, D1Sql } from './types.ts';
2
+ import { guard, sql } from './database.ts';
3
+ import { stepGenerations } from './staging-lifecycle.ts';
4
+
5
+ const among = 'IN (SELECT value FROM json_each(?))';
6
+ // Every header cell a product can fill from what a person typed or uploaded.
7
+ export const headerCells: { table: string; columns: string[]; destination: string }[] = [
8
+ { table: 'ik_jobs', columns: ['namespace', 'operation_key'], destination: 'destination' },
9
+ { table: 'ik_d1_streams', columns: ['namespace', 'operation_key'], destination: 'destination' },
10
+ { table: 'ik_d1_stream_revisions', columns: ['namespace', 'operation_key'], destination: 'destination' },
11
+ { table: 'ik_d1_stream_origins', columns: ['parent_namespace', 'parent_operation_key', 'job_operation_key'], destination: 'parent_destination' },
12
+ { table: 'ik_d1_staging', columns: ['namespace', 'operation_key'], destination: 'destination' },
13
+ { table: 'ik_d1_native_staging', columns: ['namespace'], destination: 'destination' },
14
+ { table: 'ik_d1_preparation_steps', columns: ['namespace', 'operation_key'], destination: 'destination' },
15
+ ];
16
+ const matching = (columns: string[]) => '(' + columns.map(column => `${column} ${among}`).join(' OR ') + ')';
17
+ const params = (columns: string[], values: string) => columns.map(() => values);
18
+
19
+ export interface RetirementScope { jobs: string[]; streams: string[]; stages: string[]; anchors: string[]; preparations: string[] }
20
+
21
+ /** Jobs, streams and private stages whose headers hold one of the exact values,
22
+ * closed over everything their deletion would break: whole streams, jobs whose
23
+ * entries reuse a retired job's sources, and preserved history in both directions. */
24
+ export async function retirementScope(db: D1Database, destination: string, values: string[]): Promise<RetirementScope> {
25
+ const jobs = new Set<string>(), streams = new Set<string>(), stages = new Set<string>(), anchors = new Set<string>(), preparations = new Set<string>();
26
+ const exact = JSON.stringify(values);
27
+ const read = async (query: string, ...bound: unknown[]) => (await db.prepare(query).bind(...bound).all<{ id: string }>()).results.map(row => row.id);
28
+ if (values.length) {
29
+ for (const id of await read(`SELECT id FROM ik_jobs WHERE destination=? AND ${matching(['namespace', 'operation_key'])}`, destination, exact, exact)) jobs.add(id);
30
+ for (const id of await read(`SELECT DISTINCT o.job_id AS id FROM ik_originals o JOIN ik_jobs j ON j.id=o.job_id WHERE j.destination=? AND o.name ${among}`, destination, exact)) jobs.add(id);
31
+ for (const id of await read(`SELECT id FROM ik_d1_streams WHERE destination=? AND ${matching(['namespace', 'operation_key'])}`, destination, exact, exact)) streams.add(id);
32
+ for (const id of await read(`SELECT stream_id AS id FROM ik_d1_stream_origins WHERE parent_destination=? AND ${matching(['parent_namespace', 'parent_operation_key', 'job_operation_key'])}`, destination, exact, exact, exact)) streams.add(id);
33
+ for (const id of await read(`SELECT stream_id AS id FROM ik_d1_stream_revisions WHERE destination=? AND ${matching(['namespace', 'operation_key'])}`, destination, exact, exact)) anchors.add(id);
34
+ for (const id of await read(`SELECT id FROM ik_d1_staging WHERE destination=? AND ${matching(['namespace', 'operation_key'])} UNION SELECT stage_id FROM ik_d1_staging_originals WHERE name ${among} AND stage_id IN (SELECT id FROM ik_d1_staging WHERE destination=?)`, destination, exact, exact, exact, destination)) stages.add(id);
35
+ // A preparation in steps goes whole: its control row and every generation under its internal keys.
36
+ for (const row of (await db.prepare(`SELECT id,destination,principal_id FROM ik_d1_preparation_steps WHERE destination=? AND ${matching(['namespace', 'operation_key'])}`).bind(destination, exact, exact).all<{ id: string; destination: string; principal_id: string }>()).results) {
37
+ preparations.add(row.id);
38
+ const generations = stepGenerations('SELECT id FROM ik_d1_staging', row);
39
+ for (const id of await read(generations.sql, ...generations.params!)) stages.add(id);
40
+ }
41
+ }
42
+ for (let size = -1; size !== jobs.size + streams.size;) {
43
+ size = jobs.size + streams.size;
44
+ const j = JSON.stringify([...jobs]), s = JSON.stringify([...streams]);
45
+ for (const id of await read(`SELECT job_id AS id FROM ik_d1_stream_pages WHERE stream_id ${among} UNION SELECT job_id FROM ik_d1_stream_origins WHERE stream_id ${among}`, s, s)) jobs.add(id);
46
+ for (const id of await read(`SELECT stream_id AS id FROM ik_d1_stream_pages WHERE job_id ${among} UNION SELECT stream_id FROM ik_d1_stream_origins WHERE job_id ${among}`, j, j)) streams.add(id);
47
+ for (const id of await read(`SELECT job_id AS id FROM ik_d1_entries WHERE owner_job_id ${among}`, j)) jobs.add(id);
48
+ for (const id of await read(`SELECT h.job_id AS id FROM ik_d1_history h JOIN ik_provenance p ON p.id=h.successor_id WHERE p.job_id ${among} UNION SELECT p.job_id FROM ik_d1_history h JOIN ik_provenance p ON p.id=h.successor_id WHERE h.job_id ${among}`, j, j)) jobs.add(id);
49
+ }
50
+ for (const id of streams) anchors.add(id);
51
+ return { jobs: [...jobs].sort(), streams: [...streams].sort(), stages: [...stages].sort(), anchors: [...anchors].sort(), preparations: [...preparations].sort() };
52
+ }
53
+
54
+ /** Commit-time proof that the closure read above is still the whole closure. */
55
+ export function scopeGuards(scope: RetirementScope): D1Sql[] {
56
+ const j = JSON.stringify(scope.jobs), s = JSON.stringify(scope.streams);
57
+ return [
58
+ guard(sql(`(SELECT count(*) FROM ik_jobs WHERE id ${among})=?`, j, scope.jobs.length)),
59
+ guard(sql(`NOT EXISTS(SELECT 1 FROM ik_d1_stream_pages WHERE (job_id ${among}) <> (stream_id ${among}))`, j, s)),
60
+ guard(sql(`NOT EXISTS(SELECT 1 FROM ik_d1_entries WHERE owner_job_id ${among} AND job_id NOT ${among})`, j, j)),
61
+ guard(sql(`NOT EXISTS(SELECT 1 FROM ik_d1_history h JOIN ik_provenance p ON p.id=h.successor_id WHERE (h.job_id ${among}) <> (p.job_id ${among}))`, j, j)),
62
+ ];
63
+ }
64
+
65
+ /** Deletes the whole custody of the retired jobs and streams, children first,
66
+ * and rewrites each retired stream's revision anchor to drop its header. */
67
+ export function retirementWrites(scope: RetirementScope, destination: string, receipt: string, retiredAt: string): D1Sql[] {
68
+ const j = JSON.stringify(scope.jobs), s = JSON.stringify(scope.streams);
69
+ const byJob = ['ik_d1_stream_pages', 'ik_d1_plan_index', 'ik_d1_indexed_jobs', 'ik_d1_entries', 'ik_d1_history', 'ik_provenance', 'ik_sources', 'ik_stages', 'ik_batches',
70
+ 'ik_d1_original_chunks', 'ik_originals', 'ik_original_sets', 'ik_d1_plan_chunks', 'ik_d1_reversal_reports', 'ik_d1_native_staging'];
71
+ return [
72
+ sql(`UPDATE ik_d1_creation_plans SET released=1 WHERE job_id ${among} AND released=0`, j),
73
+ sql(`DELETE FROM ik_d1_creation_plans WHERE job_id ${among}`, j),
74
+ ...byJob.map(table => sql(`DELETE FROM ${table} WHERE job_id ${among}`, j)),
75
+ sql(`DELETE FROM ik_jobs WHERE id ${among}`, j),
76
+ sql(`DELETE FROM ik_d1_streams WHERE id ${among}`, s),
77
+ sql(`DELETE FROM ik_d1_staging WHERE id ${among}`, JSON.stringify(scope.stages)),
78
+ sql(`DELETE FROM ik_d1_preparation_steps WHERE id ${among}`, JSON.stringify(scope.preparations)),
79
+ sql(`INSERT INTO ik_d1_retirements(id,kind,destination,receipt_id,retired_at) SELECT value,'job',?,?,? FROM json_each(?)`, destination, receipt, retiredAt, j),
80
+ sql(`INSERT INTO ik_d1_retirements(id,kind,destination,receipt_id,retired_at) SELECT value,'stream',?,?,? FROM json_each(?) WHERE NOT EXISTS(SELECT 1 FROM ik_d1_retirements WHERE id=value AND kind='stream')`, destination, receipt, retiredAt, JSON.stringify(scope.anchors)),
81
+ sql(`UPDATE ik_d1_stream_revisions SET namespace='',operation_key='',progress_revision=progress_revision+1 WHERE stream_id ${among} AND (namespace<>'' OR operation_key<>'')`, JSON.stringify(scope.anchors)),
82
+ ];
83
+ }
84
+
85
+ /** No header cell in this destination equals a named value after the batch. */
86
+ export function headerAbsence(destination: string, values: string[]): D1Sql[] {
87
+ if (!values.length) return [];
88
+ const exact = JSON.stringify(values);
89
+ return [
90
+ ...headerCells.map(({ table, columns, destination: column }) => guard(sql(`NOT EXISTS(SELECT 1 FROM ${table} WHERE ${column}=? AND ${matching(columns)})`, destination, ...params(columns, exact)))),
91
+ guard(sql(`NOT EXISTS(SELECT 1 FROM ik_originals o JOIN ik_jobs j ON j.id=o.job_id WHERE j.destination=? AND o.name ${among})`, destination, exact)),
92
+ guard(sql(`NOT EXISTS(SELECT 1 FROM ik_d1_staging_originals o JOIN ik_d1_staging g ON g.id=o.stage_id WHERE g.destination=? AND o.name ${among})`, destination, exact)),
93
+ ];
94
+ }
@@ -0,0 +1,76 @@
1
+ import { Buffer } from 'node:buffer';
2
+ import { randomUUID } from 'node:crypto';
3
+ import type { D1Database, D1FieldMatch, D1FieldRedaction, D1Sql } from './types.ts';
4
+ import { canonicalJson, sha256 } from '../provenance/identity.ts';
5
+ import { sql } from './database.ts';
6
+
7
+ /** The most jobs one resumable step reads, which bounds its CPU. A step takes
8
+ * fewer when they do not fit one request's statements (see redaction-step.ts). */
9
+ export const STEP_JOBS = 25;
10
+
11
+ /** A job too large for one step is redacted in pages of `count` entries; the
12
+ * next page starts at `from`. `revision` is the job's custody revision after
13
+ * the last page, so a job changed in between starts again from its first entry. */
14
+ export interface RedactionPage { job: string; from: number; count: number; revision: number | null }
15
+ /** Where a resumable redaction stands: the jobs already read and found free of
16
+ * the selected values and of unsevered door lineage copies (or rewritten), each
17
+ * at the custody revision it then had, and the matches counted so far. A changed
18
+ * revision makes a job pending again. `span` is how many jobs the next step
19
+ * reads, and `page` a job being redacted in pages. `lineage` is a salted digest
20
+ * of the door lineage those steps severed (see lineageDigest). It holds job ids,
21
+ * revisions, counts, sizes and salted digests of the selectors, never a selected
22
+ * value; it is bound to the destination and selectors it began with. */
23
+ export interface RedactionProgress { nonce: string; digest: string; clean: Map<string, number>; matched: D1FieldMatch[]; span: number; page?: RedactionPage; lineage?: string }
24
+
25
+ const digestOf = (nonce: string, destination: string, fields: readonly D1FieldRedaction[]) =>
26
+ sha256(canonicalJson([nonce, destination, fields.map(({ path, value, optional }) => [path, value, optional === true])]));
27
+
28
+ /** The door lineage a step severs, by its source records (door namespace, kind
29
+ * and source id), salted by the redaction's nonce. A job marked clean stays
30
+ * clean only under the same lineage: a source joining it makes copies in jobs
31
+ * an earlier step already read, so those are read again. */
32
+ export const lineageDigest = (nonce: string, sources: readonly string[]): string => sha256(canonicalJson([nonce, 'door-lineage', sources]));
33
+
34
+ export function readProgress(continuation: string | undefined, destination: string, fields: readonly D1FieldRedaction[]): RedactionProgress {
35
+ if (continuation === undefined) {
36
+ const nonce = randomUUID();
37
+ return { nonce, digest: digestOf(nonce, destination, fields), clean: new Map(), matched: [], span: STEP_JOBS };
38
+ }
39
+ let token: any;
40
+ try { token = JSON.parse(Buffer.from(continuation, 'base64url').toString('utf8')); } catch { throw new Error('IMPORT_REDACTION_CONTINUATION_INVALID'); }
41
+ if (token?.v !== 1 || typeof token.nonce !== 'string' || !Array.isArray(token.clean) || !Array.isArray(token.matched)
42
+ || !token.clean.every((item: unknown) => Array.isArray(item) && typeof item[0] === 'string' && Number.isSafeInteger(item[1]))
43
+ || token.span !== undefined && !(Number.isSafeInteger(token.span) && token.span >= 1 && token.span <= STEP_JOBS)
44
+ || token.page !== undefined && !(typeof token.page?.job === 'string' && [token.page.from, token.page.count].every(n => Number.isSafeInteger(n) && n >= 0) && token.page.count >= 1
45
+ && (token.page.revision === null || Number.isSafeInteger(token.page.revision)))
46
+ || token.lineage !== undefined && typeof token.lineage !== 'string') throw new Error('IMPORT_REDACTION_CONTINUATION_INVALID');
47
+ // A continuation from another erasure (other selectors or destination) must
48
+ // never let this one skip a job.
49
+ if (token.digest !== digestOf(token.nonce, destination, fields)) throw new Error('IMPORT_REDACTION_CONTINUATION_MISMATCH');
50
+ const { job, from, count, revision } = token.page ?? {};
51
+ return { nonce: token.nonce, digest: token.digest, clean: new Map(token.clean), matched: token.matched, span: token.span ?? STEP_JOBS, ...token.page ? { page: { job, from, count, revision } } : {}, ...token.lineage !== undefined ? { lineage: token.lineage } : {} };
52
+ }
53
+
54
+ export function writeProgress(progress: RedactionProgress): string {
55
+ return Buffer.from(JSON.stringify({ v: 1, nonce: progress.nonce, digest: progress.digest, clean: [...progress.clean], matched: progress.matched, span: progress.span, ...progress.page ? { page: progress.page } : {}, ...progress.lineage !== undefined ? { lineage: progress.lineage } : {} }), 'utf8').toString('base64url');
56
+ }
57
+
58
+ export function addMatches(total: D1FieldMatch[], step: readonly D1FieldMatch[]): D1FieldMatch[] {
59
+ const result = total.map(item => ({ path: [...item.path], matches: item.matches }));
60
+ for (const item of step) {
61
+ const prior = result.find(count => canonicalJson(count.path) === canonicalJson(item.path));
62
+ if (prior) prior.matches += item.matches; else result.push({ path: [...item.path], matches: item.matches });
63
+ }
64
+ return result;
65
+ }
66
+
67
+ /** A job of the destination with its custody revision and entry count. */
68
+ export interface JobRevision { id: string; revision: number | null; total: number }
69
+ /** Every job of the destination with its custody revision, read before any plan,
70
+ * so a change after the read always shows as a newer revision. */
71
+ export async function destinationRevisions(db: D1Database, destination: string): Promise<JobRevision[]> {
72
+ return (await db.prepare('SELECT j.id AS id,s.revision AS revision,j.total AS total FROM ik_jobs j LEFT JOIN ik_d1_custody_states s ON s.job_id=j.id WHERE j.destination=? ORDER BY j.id').bind(destination).all<JobRevision>()).results;
73
+ }
74
+
75
+ /** Read inside a step's batch, after its writes: the revisions it leaves. */
76
+ export const revisionsAfter = (jobs: string[]): D1Sql => sql('SELECT job_id,revision FROM ik_d1_custody_states WHERE job_id IN (SELECT value FROM json_each(?))', JSON.stringify(jobs));