@runeditrun/rer 0.2.77 → 0.2.79

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (167) hide show
  1. package/README.md +32 -0
  2. package/cli/export.mjs +1 -1
  3. package/cli/portability-migrations.mjs +35 -14
  4. package/cli/portability-worker.mjs +2 -2
  5. package/cli/portability.mjs +2 -2
  6. package/cli/server-proxy.mjs +30 -8
  7. package/cli/upstream-agent.mjs +14 -0
  8. package/global/CHANGELOG.md +13 -0
  9. package/global/README.md +32 -0
  10. package/modules/demo-host/native/visitor.mjs +1 -1
  11. package/modules/deploy-kit/owner-cli.mjs +15 -5
  12. package/modules/deploy-kit/provider.mjs +9 -4
  13. package/modules/import-kit/src/d1/README.md +1322 -0
  14. package/modules/import-kit/src/d1/apply.ts +115 -0
  15. package/modules/import-kit/src/d1/budget.ts +89 -0
  16. package/modules/import-kit/src/d1/compressed-plan.ts +63 -0
  17. package/modules/import-kit/src/d1/creation.ts +95 -0
  18. package/modules/import-kit/src/d1/custody-fence.ts +27 -0
  19. package/modules/import-kit/src/d1/custody-state-schema.ts +67 -0
  20. package/modules/import-kit/src/d1/database.ts +163 -0
  21. package/modules/import-kit/src/d1/entry-locator.ts +61 -0
  22. package/modules/import-kit/src/d1/erased-artifacts.ts +38 -0
  23. package/modules/import-kit/src/d1/erased-custody.ts +36 -0
  24. package/modules/import-kit/src/d1/erased-plan.ts +47 -0
  25. package/modules/import-kit/src/d1/erased-reversal.ts +19 -0
  26. package/modules/import-kit/src/d1/erasure-bounded.ts +22 -0
  27. package/modules/import-kit/src/d1/erasure-claims.ts +27 -0
  28. package/modules/import-kit/src/d1/erasure-compiler.ts +67 -0
  29. package/modules/import-kit/src/d1/erasure-disposition.ts +16 -0
  30. package/modules/import-kit/src/d1/erasure-fragments.ts +86 -0
  31. package/modules/import-kit/src/d1/erasure-index.ts +53 -0
  32. package/modules/import-kit/src/d1/erasure-pages.ts +62 -0
  33. package/modules/import-kit/src/d1/erasure-progress-schema.ts +8 -0
  34. package/modules/import-kit/src/d1/erasure-progress.ts +61 -0
  35. package/modules/import-kit/src/d1/erasure-related.ts +124 -0
  36. package/modules/import-kit/src/d1/erasure-report.ts +31 -0
  37. package/modules/import-kit/src/d1/erasure-schema.ts +24 -0
  38. package/modules/import-kit/src/d1/erasure-selection.ts +77 -0
  39. package/modules/import-kit/src/d1/erasure-snapshot.ts +15 -0
  40. package/modules/import-kit/src/d1/erasure-stage.ts +146 -0
  41. package/modules/import-kit/src/d1/erasure-step.ts +12 -0
  42. package/modules/import-kit/src/d1/erasure-superseded.ts +98 -0
  43. package/modules/import-kit/src/d1/erasure-values.ts +83 -0
  44. package/modules/import-kit/src/d1/erasure-writes.ts +46 -0
  45. package/modules/import-kit/src/d1/erasure.ts +170 -0
  46. package/modules/import-kit/src/d1/evolutions/0001-preserved-undo.ts +13 -0
  47. package/modules/import-kit/src/d1/expected-state.ts +91 -0
  48. package/modules/import-kit/src/d1/field-redaction.ts +86 -0
  49. package/modules/import-kit/src/d1/guards.ts +37 -0
  50. package/modules/import-kit/src/d1/history.ts +71 -0
  51. package/modules/import-kit/src/d1/index.ts +88 -0
  52. package/modules/import-kit/src/d1/integrity.ts +127 -0
  53. package/modules/import-kit/src/d1/inventory-snapshot.ts +84 -0
  54. package/modules/import-kit/src/d1/jobs.ts +97 -0
  55. package/modules/import-kit/src/d1/native-set.ts +48 -0
  56. package/modules/import-kit/src/d1/native-staging-admission.ts +19 -0
  57. package/modules/import-kit/src/d1/native-staging-context.ts +44 -0
  58. package/modules/import-kit/src/d1/native-staging-lifecycle.ts +33 -0
  59. package/modules/import-kit/src/d1/native-staging-offset.ts +74 -0
  60. package/modules/import-kit/src/d1/native-staging-publication.ts +18 -0
  61. package/modules/import-kit/src/d1/native-staging-rows.ts +72 -0
  62. package/modules/import-kit/src/d1/native-staging-schema.ts +28 -0
  63. package/modules/import-kit/src/d1/native-staging-state.ts +22 -0
  64. package/modules/import-kit/src/d1/native-staging.ts +64 -0
  65. package/modules/import-kit/src/d1/original-copies.ts +53 -0
  66. package/modules/import-kit/src/d1/original-storage.ts +56 -0
  67. package/modules/import-kit/src/d1/originals.ts +74 -0
  68. package/modules/import-kit/src/d1/plan-dictionary.ts +80 -0
  69. package/modules/import-kit/src/d1/plans.ts +121 -0
  70. package/modules/import-kit/src/d1/preparation-fingerprint.ts +57 -0
  71. package/modules/import-kit/src/d1/preparation-header.ts +33 -0
  72. package/modules/import-kit/src/d1/preparation-reader.ts +98 -0
  73. package/modules/import-kit/src/d1/preparation-schema.ts +16 -0
  74. package/modules/import-kit/src/d1/preparation-state.ts +56 -0
  75. package/modules/import-kit/src/d1/preparation-step-combine.ts +80 -0
  76. package/modules/import-kit/src/d1/preparation-step-control.ts +48 -0
  77. package/modules/import-kit/src/d1/preparation-step-hash.ts +131 -0
  78. package/modules/import-kit/src/d1/preparation-step-pages.ts +77 -0
  79. package/modules/import-kit/src/d1/preparation-steps-schema.ts +11 -0
  80. package/modules/import-kit/src/d1/preparation-steps.ts +102 -0
  81. package/modules/import-kit/src/d1/preparations.ts +87 -0
  82. package/modules/import-kit/src/d1/preserved-preview.ts +89 -0
  83. package/modules/import-kit/src/d1/preserved-reverse.ts +91 -0
  84. package/modules/import-kit/src/d1/provenance-set.ts +87 -0
  85. package/modules/import-kit/src/d1/publication-fence.ts +22 -0
  86. package/modules/import-kit/src/d1/publication-receipt-schema.ts +15 -0
  87. package/modules/import-kit/src/d1/published-stream.ts +11 -0
  88. package/modules/import-kit/src/d1/read-fence.ts +64 -0
  89. package/modules/import-kit/src/d1/read-results.ts +16 -0
  90. package/modules/import-kit/src/d1/redacted-custody.ts +76 -0
  91. package/modules/import-kit/src/d1/redaction-step.ts +125 -0
  92. package/modules/import-kit/src/d1/referenced-plans.ts +136 -0
  93. package/modules/import-kit/src/d1/retained-values.ts +35 -0
  94. package/modules/import-kit/src/d1/retirement-bounded.ts +71 -0
  95. package/modules/import-kit/src/d1/retirement-door.ts +105 -0
  96. package/modules/import-kit/src/d1/retirement-schema.ts +39 -0
  97. package/modules/import-kit/src/d1/retirement-scope.ts +94 -0
  98. package/modules/import-kit/src/d1/retirement-steps.ts +76 -0
  99. package/modules/import-kit/src/d1/retirement-tally.ts +74 -0
  100. package/modules/import-kit/src/d1/retirement.ts +136 -0
  101. package/modules/import-kit/src/d1/reversal-conversion-record.ts +19 -0
  102. package/modules/import-kit/src/d1/reversal-conversion-schema.ts +11 -0
  103. package/modules/import-kit/src/d1/reversal-conversion.ts +40 -0
  104. package/modules/import-kit/src/d1/reversal-fence-schema.ts +14 -0
  105. package/modules/import-kit/src/d1/reversal-fence.ts +106 -0
  106. package/modules/import-kit/src/d1/reversal-journal.ts +24 -0
  107. package/modules/import-kit/src/d1/reverse.ts +103 -0
  108. package/modules/import-kit/src/d1/reviewed-source.ts +23 -0
  109. package/modules/import-kit/src/d1/sets.ts +78 -0
  110. package/modules/import-kit/src/d1/staged-creation.ts +201 -0
  111. package/modules/import-kit/src/d1/staging-admission.ts +45 -0
  112. package/modules/import-kit/src/d1/staging-control.ts +43 -0
  113. package/modules/import-kit/src/d1/staging-erasure.ts +28 -0
  114. package/modules/import-kit/src/d1/staging-lifecycle.ts +49 -0
  115. package/modules/import-kit/src/d1/staging-plan.ts +15 -0
  116. package/modules/import-kit/src/d1/staging-privacy.ts +22 -0
  117. package/modules/import-kit/src/d1/staging-publication.ts +79 -0
  118. package/modules/import-kit/src/d1/staging-schema.ts +37 -0
  119. package/modules/import-kit/src/d1/staging-statements.ts +8 -0
  120. package/modules/import-kit/src/d1/staging-transport.ts +70 -0
  121. package/modules/import-kit/src/d1/stream-boundary.ts +11 -0
  122. package/modules/import-kit/src/d1/stream-create.ts +55 -0
  123. package/modules/import-kit/src/d1/stream-identity-admission.ts +37 -0
  124. package/modules/import-kit/src/d1/stream-input.ts +18 -0
  125. package/modules/import-kit/src/d1/stream-layout.ts +43 -0
  126. package/modules/import-kit/src/d1/stream-origin.ts +44 -0
  127. package/modules/import-kit/src/d1/stream-page-integrity.ts +18 -0
  128. package/modules/import-kit/src/d1/stream-partition.ts +50 -0
  129. package/modules/import-kit/src/d1/stream-plan.ts +133 -0
  130. package/modules/import-kit/src/d1/stream-receipt.ts +45 -0
  131. package/modules/import-kit/src/d1/stream-revision-schema.ts +35 -0
  132. package/modules/import-kit/src/d1/stream-snapshot.ts +26 -0
  133. package/modules/import-kit/src/d1/stream-stage.ts +79 -0
  134. package/modules/import-kit/src/d1/stream-state.ts +25 -0
  135. package/modules/import-kit/src/d1/stream-working-set.ts +30 -0
  136. package/modules/import-kit/src/d1/streams.ts +39 -0
  137. package/modules/import-kit/src/d1/target-guards.ts +9 -0
  138. package/modules/import-kit/src/d1/target-identities.ts +17 -0
  139. package/modules/import-kit/src/d1/transport.ts +10 -0
  140. package/modules/import-kit/src/d1/types.ts +162 -0
  141. package/modules/import-kit/src/d1/undo-observations.ts +39 -0
  142. package/modules/import-kit/src/d1/undo-report.ts +22 -0
  143. package/modules/import-kit/src/d1/undo-snapshot.ts +87 -0
  144. package/modules/import-kit/src/d1/undo-state.ts +84 -0
  145. package/modules/import-kit/src/d1/undo-types.ts +110 -0
  146. package/modules/import-kit/src/journal/authority.ts +12 -0
  147. package/modules/import-kit/src/plan/budget.ts +60 -0
  148. package/modules/import-kit/src/plan/index.ts +123 -0
  149. package/modules/import-kit/src/plan/size.ts +122 -0
  150. package/modules/import-kit/src/provenance/canonical-hash.ts +130 -0
  151. package/modules/import-kit/src/provenance/canonical-stream.ts +85 -0
  152. package/modules/import-kit/src/provenance/credential-names.ts +131 -0
  153. package/modules/import-kit/src/provenance/credentials.ts +317 -0
  154. package/modules/import-kit/src/provenance/identity.ts +51 -0
  155. package/modules/import-kit/src/provenance/json-snapshot.ts +21 -0
  156. package/modules/import-kit/src/provenance/resumable-sha256.ts +75 -0
  157. package/modules/import-kit/src/provenance/validate-json.ts +19 -0
  158. package/modules/import-kit/src/storage/originals.ts +87 -0
  159. package/modules/import-kit/src/storage/provenance.ts +64 -0
  160. package/modules/import-kit/src/storage/schema.ts +42 -0
  161. package/modules/import-kit/src/types.ts +47 -0
  162. package/modules/portability/README.md +98 -2
  163. package/modules/portability/backfills.mjs +42 -15
  164. package/modules/portability/ledgers.mjs +7 -1
  165. package/modules/portability/transient-tables.mjs +40 -0
  166. package/package.json +3 -1
  167. package/scripts/publish-check/reviewed.py +3 -1
@@ -0,0 +1,53 @@
1
+ import { guardErasedSources } from './erasure-claims.ts';
2
+ import type { ImportPlan } from '../types.ts';
3
+ import type { D1ImportContext, D1Sql } from './types.ts';
4
+ import { canonicalJson, sha256 } from '../provenance/identity.ts';
5
+ import { planFingerprint } from '../plan/index.ts';
6
+ import { requestContext } from './budget.ts';
7
+ import { atomic, authorize, guard, sql } from './database.ts';
8
+ import { exactRows, jobColumns, jsonChunks } from './sets.ts';
9
+ import { loadPlans, planGuards } from './plans.ts';
10
+
11
+ export const targetIdentity = (destination: string, kind: string, id: string): string => sha256(canonicalJson([destination, kind, id]));
12
+ export function planIndex(plan: ImportPlan, id: string) {
13
+ return plan.entries.flatMap((entry,ordinal)=>{
14
+ if(entry.erased && (entry.effects.length!==0 || typeof entry.erasedTargetIdentity!=='string' || !/^[a-f0-9]{64}$/.test(entry.erasedTargetIdentity)))throw new Error('IMPORT_ERASURE_BINDING_MISSING');
15
+ const targets=entry.erased?[entry.erasedTargetIdentity!]:[...new Set(entry.effects.map(effect=>targetIdentity(plan.destination,effect.targetKind,effect.targetId)))].concat(entry.effects.length?[]:['']);
16
+ return targets.map(target=>({job_id:id,ordinal,source_identity:entry.identity,target_identity:target}));
17
+ });
18
+ }
19
+
20
+ /** The erased target digest is part of the fingerprinted plan. It remains
21
+ * distinct from a source that never mapped an effect; both tombstones persist. */
22
+ export function erasedIndexGuards(plan:ImportPlan):D1Sql[]{
23
+ if(!plan.entries.some(entry=>entry.erased))return [];
24
+ return jsonChunks(planIndex(plan,'').filter(row=>plan.entries[row.ordinal].erased)).map(body=>guard(sql("NOT EXISTS(SELECT 1 FROM json_each(?) e WHERE NOT EXISTS(SELECT 1 FROM ik_d1_erased_sources s WHERE s.destination=? AND s.identity=json_extract(e.value,'$.source_identity')) OR NOT EXISTS(SELECT 1 FROM ik_d1_erased_targets t WHERE t.destination=? AND t.identity=json_extract(e.value,'$.target_identity')))",body,plan.destination,plan.destination)));
25
+ }
26
+ export function planIndexRows(plan: ImportPlan, id: string): D1Sql[] {
27
+ return jsonChunks(planIndex(plan, id), 1500000).map(chunk => sql("INSERT INTO ik_d1_plan_index(job_id,ordinal,source_identity,target_identity) SELECT json_extract(value,'$.job_id'),json_extract(value,'$.ordinal'),json_extract(value,'$.source_identity'),json_extract(value,'$.target_identity') FROM json_each(?)", chunk));
28
+ }
29
+ export function retainPlanIndex(plan: ImportPlan, id: string): D1Sql[] {
30
+ return [...planIndexRows(plan, id), sql('INSERT INTO ik_d1_indexed_jobs(job_id,fingerprint) VALUES(?,?)', id, plan.fingerprint)];
31
+ }
32
+ export const unindexedJobs = 'SELECT j.id FROM ik_jobs j LEFT JOIN ik_d1_indexed_jobs i ON i.job_id=j.id WHERE j.destination=? AND (i.job_id IS NULL OR i.fingerprint<>j.fingerprint)';
33
+
34
+ /** One bounded upgrade page. Run until complete after applying d1ErasureSchema.
35
+ * A fingerprint fence prevents an old page from indexing a concurrently changed
36
+ * plan. Index values are hashes only; no answers, source IDs or native IDs. */
37
+ export async function upgradeD1ImportErasureIndex(context: D1ImportContext, pageSize = 4): Promise<{ indexed: number; complete: boolean }> {
38
+ context = requestContext(context);
39
+ if (!Number.isSafeInteger(pageSize) || pageSize < 1 || pageSize > 100) throw new Error('INVALID_BATCH_LIMIT');
40
+ await authorize(context, 'erase');
41
+ const db = context.storage.database;
42
+ const { results: rows } = await db.prepare(`SELECT * FROM ik_jobs WHERE id IN (${unindexedJobs}) ORDER BY id LIMIT ?`).bind(context.destination, pageSize).all();
43
+ // Only one plan is materialized at a time, including old chunked plans.
44
+ for (const row of rows) {
45
+ const plan = (await loadPlans([row], db, context.storage.targets.planCodec)).get(row.id)!;
46
+ if (plan.fingerprint !== row.fingerprint || planFingerprint(plan) !== row.fingerprint || plan.destination !== row.destination || plan.entries.length !== row.total) throw new Error('STORED_PLAN_CHANGED');
47
+ await atomic(context, 'erase', [...await guardErasedSources(plan, db), ...exactRows('ik_jobs', jobColumns, [row]), ...planGuards(plan, row.id),
48
+ sql('DELETE FROM ik_d1_plan_index WHERE job_id=?', row.id), sql('DELETE FROM ik_d1_indexed_jobs WHERE job_id=?', row.id), ...retainPlanIndex(plan, row.id)]);
49
+ }
50
+ const pending = await db.prepare(`${unindexedJobs} LIMIT 1`).bind(context.destination).first();
51
+ return { indexed: rows.length, complete: !pending };
52
+ }
53
+ export const indexedDestinationGuard = (destination: string): D1Sql => guard(sql(`NOT EXISTS(${unindexedJobs})`, destination));
@@ -0,0 +1,62 @@
1
+ import type { D1Database,D1Sql } from './types.ts';
2
+ import { jsonChunks,jobColumns,provenanceColumns,claimColumns } from './sets.ts';
3
+
4
+ const PAGE_BYTES=2*1024*1024;
5
+ const UNKNOWN_BYTES=Number.MAX_SAFE_INTEGER;
6
+ // The unknown-size sentinel means an owner closure must occupy its own page.
7
+ // Saturate before addition so it remains an exact safe integer across SQL/JS.
8
+ const addBytes=(left:number,right:number):number=>left>=UNKNOWN_BYTES-right?UNKNOWN_BYTES:left+right;
9
+ const fields:[string,string[]][]=[
10
+ ['ik_provenance',provenanceColumns],['ik_d1_history',[...provenanceColumns,'successor_id']],
11
+ ['ik_sources',claimColumns],['ik_d1_entries',['job_id','ordinal','source_identity','owner_job_id','effects_hash']],
12
+ ['ik_batches',['job_id','action','position','end_position','fence']],
13
+ ['ik_originals',['job_id','kind','original_key','name','sha256','size','bytes']],
14
+ ['ik_original_sets',['job_id','version']],['ik_d1_original_chunks',['job_id','kind','original_key','ordinal','bytes']],
15
+ ['ik_d1_plan_chunks',['job_id','ordinal','body']],['ik_d1_creation_plans',['job_id','ordinal','body','released']],
16
+ ['ik_d1_plan_index',['job_id','ordinal','source_identity','target_identity']],['ik_d1_indexed_jobs',['job_id','fingerprint']],
17
+ ['ik_d1_reversal_reports',['job_id','effect_id','position','record_json']],['ik_d1_stream_pages',['stream_id','ordinal','job_id','total','erased_records']],
18
+ ];
19
+ const rowBytes=(alias:string,columns:string[])=>'768+6*('+columns.map(column=>`coalesce(length(CAST(${alias}.${column} AS BLOB)),0)`).join('+')+')';
20
+ const storedBytes=rowBytes('j',jobColumns)+'+'+fields.map(([table,columns])=>`(SELECT coalesce(sum(${rowBytes('r',columns)}),0) FROM ${table} r WHERE r.job_id=j.id)`).join('+');
21
+ // Chunk manifests deliberately occupy a page alone: compressed byte length is
22
+ // not an expanded working-set measurement. Inline envelopes declare the exact
23
+ // expanded size, which the ordinary decoder independently verifies on loading.
24
+ const expandedBytes="CASE json_extract(j.plan_json,'$.d1Plan') WHEN 1 THEN 9007199254740991 WHEN 3 THEN CASE WHEN typeof(json_extract(j.plan_json,'$.expandedSize'))='integer' AND json_extract(j.plan_json,'$.expandedSize')>=0 THEN json_extract(j.plan_json,'$.expandedSize') ELSE 9007199254740991 END WHEN 2 THEN CASE WHEN typeof(json_extract(j.plan_json,'$.size'))='integer' AND json_extract(j.plan_json,'$.size')>=0 THEN json_extract(j.plan_json,'$.size') ELSE 9007199254740991 END ELSE length(CAST(j.plan_json AS BLOB)) END";
25
+ const measuredBytes=`CASE WHEN (${expandedBytes})>=${UNKNOWN_BYTES}-min(${UNKNOWN_BYTES},(${storedBytes})) THEN ${UNKNOWN_BYTES} ELSE (${storedBytes})+(${expandedBytes}) END`;
26
+ const dependencies=`SELECT owner_job_id AS id FROM ik_d1_entries WHERE job_id=j.id
27
+ UNION SELECT h.job_id FROM (SELECT record_json FROM ik_provenance WHERE job_id=j.id UNION ALL SELECT record_json FROM ik_d1_history WHERE job_id=j.id) p JOIN ik_d1_history h ON h.id=json_extract(p.record_json,'$.predecessorId')
28
+ UNION SELECT p.job_id FROM ik_d1_history h JOIN ik_provenance p ON p.id=h.successor_id WHERE h.job_id=j.id
29
+ UNION SELECT p.job_id FROM ik_d1_history h JOIN ik_d1_history p ON p.id=h.successor_id WHERE h.job_id=j.id`;
30
+ interface Weight {bytes:number;dependencies:string[];revision:number}
31
+ export interface ErasurePage {ids:string[];fence:D1Sql;measuredBytes:number;readQueries?:number}
32
+
33
+ /** Group small independent or reused jobs with set reads. Only numeric sizes
34
+ * and ownership IDs span groups; no plan, provenance or original body is read
35
+ * here. Reused/history owners count once toward a group's complete closure. */
36
+ export async function measureErasurePages(ids:string[],db:D1Database):Promise<ErasurePage[]>{
37
+ const weights=new Map<string,Weight>();let frontier=[...new Set(ids)].sort();
38
+ while(frontier.length){
39
+ const next=new Set<string>();
40
+ for(const chunk of jsonChunks(frontier)){
41
+ const rows=(await db.prepare(`SELECT j.id,s.revision,${measuredBytes} AS bytes,(SELECT json_group_array(id) FROM (${dependencies})) AS dependencies FROM ik_jobs j JOIN ik_d1_custody_states s ON s.job_id=j.id WHERE j.id IN (SELECT value FROM json_each(?))`).bind(chunk).all()).results;
42
+ if(rows.length!==JSON.parse(chunk).length)throw new Error('IMPORT_JOB_NOT_FOUND');
43
+ for(const row of rows){if(!Number.isSafeInteger(Number(row.revision))||!Number.isSafeInteger(Number(row.bytes))||Number(row.bytes)<0)throw new Error('IMPORT_CUSTODY_STATE_REQUIRED');const related=[...new Set(JSON.parse(row.dependencies as string) as string[])].sort();weights.set(row.id as string,{bytes:Number(row.bytes),dependencies:related,revision:Number(row.revision)});for(const id of related)if(!weights.has(id))next.add(id);}
44
+ }
45
+ frontier=[...next].filter(id=>!weights.has(id)).sort();
46
+ }
47
+ const closure=(id:string)=>{const result=new Set<string>(),pending=[id];while(pending.length){const value=pending.pop()!;if(result.has(value))continue;result.add(value);pending.push(...weights.get(value)!.dependencies);}return result;};
48
+ const pages:ErasurePage[]=[];
49
+ const page=(ids:string[],owners:Set<string>,measuredBytes:number):ErasurePage=>{
50
+ const chunks=jsonChunks([...owners].sort().map(id=>[id,weights.get(id)!.revision]));
51
+ return {ids,measuredBytes,fence:{sql:chunks.map(()=>"NOT EXISTS(SELECT 1 FROM json_each(?) e WHERE NOT EXISTS(SELECT 1 FROM ik_d1_custody_states s WHERE s.job_id=json_extract(e.value,'$[0]') AND s.revision=json_extract(e.value,'$[1]')))").join(' AND '),params:chunks}};
52
+ };
53
+ let group:string[]=[],owners=new Set<string>(),bytes=0;
54
+ for(const id of ids){
55
+ const related=closure(id),extra=()=>[...related].filter(owner=>!owners.has(owner)).reduce((n,owner)=>addBytes(n,weights.get(owner)!.bytes),0);
56
+ if(group.length&&addBytes(bytes,extra())>PAGE_BYTES){pages.push(page(group,owners,bytes));group=[];owners=new Set();bytes=0;}
57
+ group.push(id);bytes=addBytes(bytes,extra());for(const owner of related)owners.add(owner);
58
+ if(bytes>=PAGE_BYTES){pages.push(page(group,owners,bytes));group=[];owners=new Set();bytes=0;}
59
+ }
60
+ if(group.length)pages.push(page(group,owners,bytes));
61
+ return pages;
62
+ }
@@ -0,0 +1,8 @@
1
+ /** Additive migration for resumable erasure. Progress contains opaque identities,
2
+ * digests and counts only; normalized private values remain in native staging. */
3
+ export const d1ErasureProgressSchema = [
4
+ "CREATE TABLE IF NOT EXISTS ik_d1_erasure_progress (id TEXT PRIMARY KEY,destination TEXT NOT NULL,request_hash TEXT NOT NULL,generation TEXT NOT NULL,phase TEXT NOT NULL CHECK(phase IN ('index','values','sealed','complete')),index_rows INTEGER NOT NULL CHECK(index_rows>=0),value_rows INTEGER NOT NULL CHECK(value_rows>=0),revision INTEGER NOT NULL CHECK(revision=index_rows+value_rows),records_erased INTEGER NOT NULL CHECK(records_erased>=0),field_matches_json TEXT NOT NULL CHECK(json_valid(field_matches_json)),created_at TEXT NOT NULL)",
5
+ "CREATE TRIGGER IF NOT EXISTS ik_d1_erasure_progress_insert BEFORE INSERT ON ik_d1_erasure_progress WHEN EXISTS(SELECT 1 FROM ik_d1_erasure_progress WHERE id=NEW.id) BEGIN SELECT RAISE(ABORT,'IMPORT_ERASURE_OPERATION_CHANGED'); END",
6
+ "CREATE TRIGGER IF NOT EXISTS ik_d1_erasure_progress_binding BEFORE UPDATE ON ik_d1_erasure_progress WHEN NEW.id IS NOT OLD.id OR NEW.destination IS NOT OLD.destination OR NEW.request_hash IS NOT OLD.request_hash OR NEW.created_at IS NOT OLD.created_at OR OLD.phase='complete' BEGIN SELECT RAISE(ABORT,'IMPORT_ERASURE_OPERATION_CHANGED'); END",
7
+ "CREATE TRIGGER IF NOT EXISTS ik_d1_erasure_progress_delete BEFORE DELETE ON ik_d1_erasure_progress WHEN OLD.phase='complete' BEGIN SELECT RAISE(ABORT,'IMPORT_ERASURE_OPERATION_CHANGED'); END",
8
+ ];
@@ -0,0 +1,61 @@
1
+ import type { D1FieldMatch, D1ImportContext, D1Sql } from './types.ts';
2
+ import { canonicalJson, requireText, sha256 } from '../provenance/identity.ts';
3
+ import { guard, sql } from './database.ts';
4
+ import { requestContext, limitStatements } from './budget.ts';
5
+
6
+ export interface D1ErasureStepOptions {
7
+ /** Stable product operation identity. Reuse it after a lost acknowledgement. */
8
+ operationKey: string;
9
+ /** Maximum executed SQL statements in this invocation, including reads. */
10
+ maxStatements?: number;
11
+ /** Compile and admit one future step without retaining or changing any data. */
12
+ admissionOnly?: boolean;
13
+ }
14
+ export interface D1ErasurePending {
15
+ phase: 'pending'; operationId: string; indexRows: number; valueRows: number;
16
+ }
17
+ export interface D1ErasureComplete {
18
+ phase: 'complete'; operationId: string; erasedRecords: number;
19
+ fieldMatches: D1FieldMatch[]; nativeResults: unknown[];
20
+ }
21
+ export type D1ErasureStep = D1ErasurePending | D1ErasureComplete;
22
+ export interface ErasureProgress {
23
+ id: string; destination: string; request_hash: string; generation: string;
24
+ phase: 'index' | 'values' | 'sealed' | 'complete'; index_rows: number;
25
+ value_rows: number; revision: number; records_erased: number;
26
+ field_matches_json: string; created_at: string;
27
+ }
28
+ export const erasureStatementBinding = (statements:D1Sql[]) => statements.map(row => [row.sql, (row.params??[]).map(value =>
29
+ value instanceof ArrayBuffer || ArrayBuffer.isView(value) ? ['bytes', sha256(value instanceof ArrayBuffer ? new Uint8Array(value) : new Uint8Array(value.buffer,value.byteOffset,value.byteLength))] : value)]);
30
+ export function erasureRequestContext(input: D1ImportContext, options: D1ErasureStepOptions): D1ImportContext {
31
+ if (!options || typeof options !== 'object' || Array.isArray(options) || Object.keys(options).some(key => !['operationKey', 'maxStatements', 'admissionOnly'].includes(key))) throw new Error('IMPORT_ERASURE_OPTIONS_INVALID');
32
+ const max = options.maxStatements ?? 1000;
33
+ if (!Number.isSafeInteger(max) || max < 1 || max > 1000) throw new Error('INVALID_ERASURE_STATEMENT_LIMIT');
34
+ const bytes = input.maxBatchBytes ?? 2 * 1024 * 1024;
35
+ if (!Number.isSafeInteger(bytes) || bytes < 1) throw new Error('INVALID_TRANSPORT_LIMIT');
36
+ const context = requestContext({ ...input, maxBatchBytes: Math.min(bytes, 2 * 1024 * 1024) });
37
+ return limitStatements(context, max);
38
+ }
39
+ export function operationIdentity(context: D1ImportContext, options: D1ErasureStepOptions, binding: string) {
40
+ requireText(options.operationKey, 'erasure operationKey'); requireText(binding, 'erasure binding');
41
+ return {
42
+ id: sha256(canonicalJson([context.destination, context.authorization.principalId, options.operationKey])),
43
+ requestHash: sha256(canonicalJson([context.destination, context.namespace, context.mappingVersion, context.authorization.principalId, context.originalErasurePolicy ?? null, binding])),
44
+ };
45
+ }
46
+ export async function readErasureProgress(context: D1ImportContext, id: string, requestHash: string): Promise<ErasureProgress | undefined> {
47
+ const row = await context.storage.database.prepare('SELECT * FROM ik_d1_erasure_progress WHERE id=?').bind(id).first<ErasureProgress>();
48
+ if (!row) return undefined;
49
+ if (row.destination !== context.destination || row.request_hash !== requestHash) throw new Error('IMPORT_ERASURE_OPERATION_CHANGED');
50
+ return row;
51
+ }
52
+ export const progressGuard = (row: ErasureProgress) => guard(sql('EXISTS(SELECT 1 FROM ik_d1_erasure_progress WHERE id=? AND request_hash=? AND generation=? AND phase=? AND index_rows=? AND value_rows=? AND revision=?)', row.id, row.request_hash, row.generation, row.phase, row.index_rows, row.value_rows, row.revision));
53
+ export function progressInsert(row: ErasureProgress): D1Sql {
54
+ return sql(`INSERT INTO ik_d1_erasure_progress(${Object.keys(row).join(',')}) VALUES(${Object.keys(row).map(() => '?').join(',')})`, ...Object.values(row));
55
+ }
56
+ export function progressChange(from: ErasureProgress, changes: Partial<ErasureProgress>): D1Sql[] {
57
+ const next = { ...from, ...changes }, keys = Object.keys(changes) as (keyof ErasureProgress)[];
58
+ return [progressGuard(from), sql(`UPDATE ik_d1_erasure_progress SET ${keys.map(key => `${key}=?`).join(',')} WHERE id=?`, ...keys.map(key => next[key]), from.id), progressGuard(next)];
59
+ }
60
+ export const pendingErasure = (row: ErasureProgress): D1ErasurePending => ({ phase: 'pending', operationId: row.id, indexRows: row.index_rows, valueRows: row.value_rows });
61
+ export const completedErasure = (row: ErasureProgress, nativeResults: unknown[] = []): D1ErasureComplete => ({ phase: 'complete', operationId: row.id, erasedRecords: row.records_erased, fieldMatches: JSON.parse(row.field_matches_json), nativeResults });
@@ -0,0 +1,124 @@
1
+ import type { D1FieldMatch, D1FieldRedaction, D1ImportContext, D1RelatedErasure, D1Sql } from './types.ts';
2
+ import { authorize, guard, sql } from './database.ts';
3
+ import { loadPlans } from './plans.ts';
4
+ import { readOriginals } from './originals.ts';
5
+ import { exactRows, jsonChunks } from './sets.ts';
6
+ import { fieldMatches, fieldRedactions, inRange, redactFields, type EntryRange } from './field-redaction.ts';
7
+ import { requestContext } from './budget.ts';
8
+ import { redactedNativeFields } from './redacted-custody.ts';
9
+ import { canonicalJson, sha256 } from '../provenance/identity.ts';
10
+
11
+ export interface RelatedSelection { identities: Set<string>; originals: Map<string, Set<string>>; guards: D1Sql[]; discoveryGuards?: D1Sql[]; redactedJobs: Set<string>; fieldMatches: D1FieldMatch[] }
12
+
13
+ /** Visit every retained plan and original in the destination once. The exact job
14
+ * set and fingerprints are fenced, so a job created or changed after this scan
15
+ * aborts the erasure instead of keeping an unscanned copy. */
16
+ /** Which jobs a bounded step reads. `clean` jobs were already read in an earlier
17
+ * step and are fenced by their unchanged custody revision instead; `complete`
18
+ * also proves the destination holds no other job. `range` limits a step over one
19
+ * job to a page of its entries: only they are counted and redacted. */
20
+ export interface JobSelection { jobs: string[]; clean: Map<string, number>; complete: boolean; range?: EntryRange }
21
+
22
+ export async function relatedSelection(context: D1ImportContext, related: D1RelatedErasure, selection?: JobSelection, ownStage?: string): Promise<RelatedSelection> {
23
+ const db = context.storage.database;
24
+ // Only compact headers span pages. Read the revision before any plan,
25
+ // original or before-image: it fences all copies observed by this scan.
26
+ const rows: { id: string; fingerprint: string; state_revision: number }[] = [];
27
+ for (const selected of selection ? jsonChunks(selection.jobs) : [undefined]) {
28
+ let cursor: string | undefined;
29
+ while (true) {
30
+ const scope = selected === undefined ? '' : ' AND j.id IN (SELECT value FROM json_each(?))';
31
+ const after = cursor === undefined ? '' : ' AND j.id>?';
32
+ const params = [context.destination, ...selected === undefined ? [] : [selected], ...cursor === undefined ? [] : [cursor]];
33
+ const { results } = await db.prepare(`SELECT j.id,j.fingerprint,s.revision AS state_revision FROM ik_jobs j LEFT JOIN ik_d1_custody_states s ON s.job_id=j.id WHERE j.destination=?${scope}${after} ORDER BY j.id LIMIT 32`).bind(...params).all<typeof rows[number]>();
34
+ for (const row of results) {
35
+ if (!Number.isSafeInteger(row.state_revision)) throw new Error('IMPORT_CUSTODY_STATE_REQUIRED');
36
+ rows.push(row);
37
+ }
38
+ if (results.length < 32) break;
39
+ cursor = results.at(-1)!.id;
40
+ }
41
+ }
42
+ rows.sort((a, b) => a.id < b.id ? -1 : a.id > b.id ? 1 : 0);
43
+ const identities = new Set<string>(), originals = new Map<string, Set<string>>(), redactedJobs = new Set<string>();
44
+ const fields = related.fields ?? [], counts = fieldMatches(fields);
45
+ for (const row of rows) {
46
+ const raw = await db.prepare('SELECT * FROM ik_jobs WHERE id=? AND destination=?').bind(row.id, context.destination).first();
47
+ if (!raw) throw new Error('IMPORT_JOB_NOT_FOUND');
48
+ if (raw.fingerprint !== row.fingerprint) throw new Error('IMPORT_CUSTODY_STATE_CHANGED');
49
+ const plan = (await loadPlans([raw], db, context.storage.targets.planCodec)).get(row.id)!;
50
+ if (related.source) for (const entry of plan.entries) if (!entry.erased && related.source(entry.source)) identities.add(entry.identity);
51
+ if (fields.length) {
52
+ for (const [ordinal, entry] of plan.entries.entries()) {
53
+ if (!inRange(selection?.range, ordinal)) continue;
54
+ if (redactFields(entry.source.payload, fields, counts) !== entry.source.payload) redactedJobs.add(row.id);
55
+ for (const effect of entry.effects) if (redactFields(effect.value, fields, counts) !== effect.value) redactedJobs.add(row.id);
56
+ }
57
+ }
58
+ if (fields.length || related.snapshot !== undefined) {
59
+ // Three near-limit records fit a bounded read. Release their decoded
60
+ // values immediately; the monotonic job revision carries the fence.
61
+ for (const table of ['ik_provenance', 'ik_d1_history']) {
62
+ let cursor: string | undefined;
63
+ while (true) {
64
+ const after = cursor === undefined ? '' : ' AND id>?';
65
+ const { results } = await db.prepare(`SELECT id,ordinal,record_json FROM ${table} WHERE job_id=?${after} ORDER BY id LIMIT 3`).bind(row.id, ...cursor === undefined ? [] : [cursor]).all();
66
+ for (const beforeImage of results) {
67
+ if (!inRange(selection?.range, beforeImage.ordinal)) continue;
68
+ const before = JSON.parse(beforeImage.record_json).before;
69
+ if (before && redactFields(before.value, fields, counts) !== before.value) redactedJobs.add(row.id);
70
+ }
71
+ if (results.length < 3) break;
72
+ cursor = results.at(-1)!.id;
73
+ }
74
+ }
75
+ }
76
+ if (!related.original) continue;
77
+ for (const original of await readOriginals(plan, row.id, context)) {
78
+ if (original.kind !== 'artifact' || original.original.status !== 'retained') continue;
79
+ if (!related.original(Buffer.from(original.original.base64, 'base64'), { name: original.name, sha256: original.sha256 })) continue;
80
+ if (!originals.has(row.id)) originals.set(row.id, new Set());
81
+ originals.get(row.id)!.add(original.sha256);
82
+ }
83
+ }
84
+ const jobs = rows.map(row => ({ id: row.id, fingerprint: row.fingerprint }));
85
+ const guards = selection ? exactRows('ik_jobs', ['id', 'fingerprint'], jobs) : exactRows('ik_jobs', ['id', 'fingerprint'], jobs, sql('destination=?', context.destination));
86
+ guards.push(...exactRows('ik_d1_custody_states', ['job_id', 'revision'], rows.map(row => ({ job_id: row.id, revision: row.state_revision }))));
87
+ if (selection?.complete) {
88
+ guards.push(guard(sql('(SELECT count(*) FROM ik_jobs WHERE destination=?)=?', context.destination, rows.length + selection.clean.size)));
89
+ guards.push(...exactRows('ik_d1_custody_states', ['job_id', 'revision'], [...selection.clean].map(([job_id, revision]) => ({ job_id, revision }))));
90
+ }
91
+ let discoveryGuards: D1Sql[] | undefined;
92
+ if (fields.length || related.snapshot !== undefined) {
93
+ // Incomplete private plans cannot be safely field-rewritten. Cancel every
94
+ // private preparation in this destination atomically, including other actors.
95
+ discoveryGuards = [...guards];
96
+ for (const table of ['ik_d1_staging', 'ik_d1_native_staging', 'ik_d1_preparation_steps']) {
97
+ const exclude = ownStage && table === 'ik_d1_native_staging' ? ' AND id<>?' : '';
98
+ const params = exclude ? [context.destination, ownStage] : [context.destination];
99
+ const { results } = await db.prepare(`SELECT id FROM ${table} WHERE destination=?${exclude} ORDER BY id`).bind(...params).all();
100
+ guards.push(...exactRows(table, ['id'], results, sql(`destination=?${exclude}`, ...params)));
101
+ // Discovery predates our preparation. Keep its public snapshot stable
102
+ // while publication permits precisely the stage this erasure creates.
103
+ discoveryGuards.push(...exactRows(table, ['id'], results, sql('destination=?', context.destination)));
104
+ }
105
+ }
106
+ return { identities, originals, guards, discoveryGuards, redactedJobs, fieldMatches: counts };
107
+ }
108
+
109
+ /** Read-only selector discovery for products with optional references. Erasure
110
+ * scans again and refuses an explicit path that no longer matches. */
111
+ export async function readD1ImportFieldMatches(fields: readonly D1FieldRedaction[], input: D1ImportContext): Promise<{ fieldMatches: D1FieldMatch[]; snapshot: string }> {
112
+ fields = fieldRedactions(fields);
113
+ const context = requestContext(input);
114
+ await authorize(context, 'read');
115
+ const selection = await relatedSelection(context, { fields, snapshot: '' });
116
+ const native = await redactedNativeFields('', '', context, fields, selection.fieldMatches);
117
+ await authorize(context, 'read');
118
+ return { fieldMatches: selection.fieldMatches, snapshot: fieldSelectionSnapshot([...selection.guards, ...native.before]) };
119
+ }
120
+
121
+ /** Bind discovery to exact custody/native predicates without returning their values. */
122
+ export function fieldSelectionSnapshot(guards: readonly D1Sql[]): string {
123
+ return sha256(canonicalJson(guards.map(({ sql, params }) => ({ sql, params: params ?? [] }))));
124
+ }
@@ -0,0 +1,31 @@
1
+ import type { InventorySnapshot } from './inventory-snapshot.ts';
2
+ import type { ImportPlan } from '../types.ts';
3
+ import type { D1ImportContext, D1Sql } from './types.ts';
4
+ import { canonicalJson } from '../provenance/identity.ts';
5
+ import { planFingerprint } from '../plan/index.ts';
6
+ import { effectIdentity } from '../storage/provenance.ts';
7
+ import { exactRows, insertJsonRows, jobColumns } from './sets.ts';
8
+ import { planGuards } from './plans.ts';
9
+ import { loadReversalState, reversalGuards, reportHash, policyHash } from './undo-state.ts';
10
+ import { erasedDisposition, redactDisposition } from './erasure-disposition.ts';
11
+ import { sql,guard } from './database.ts';
12
+ import { changed, deleteRows, updateRows } from './expected-state.ts';
13
+
14
+ /** One validated report page; callers release its values before reading another. */
15
+ export async function eraseReport(raw:any,plan:ImportPlan,next:ImportPlan,context:D1ImportContext,snapshot?:InventorySnapshot):Promise<{before:D1Sql[];after:D1Sql[]}>{
16
+ if(planFingerprint(plan)!==raw.fingerprint||plan.fingerprint!==raw.fingerprint||plan.destination!==raw.destination||plan.principalId!==raw.principal_id||plan.namespace!==raw.namespace||plan.mappingVersion!==raw.mapping_version||plan.entries.length!==raw.total)throw new Error('STORED_PLAN_CHANGED');
17
+ const state=await loadReversalState(raw,plan,context,false,snapshot);
18
+ let before=[...exactRows('ik_jobs',jobColumns,[raw]),...planGuards(plan,raw.id),...reversalGuards(raw.id,state)];
19
+ if(context.maxBatchBytes!==undefined){
20
+ if(!Number.isSafeInteger(raw.state_revision))throw new Error('IMPORT_CUSTODY_STATE_REQUIRED');
21
+ // This revision was read before the report. It covers all report/plan/job
22
+ // writes, so old cross-effect evidence never needs private transport.
23
+ const fence=guard(sql('EXISTS(SELECT 1 FROM ik_d1_custody_states WHERE job_id=? AND revision=?)',raw.id,raw.state_revision));
24
+ fence.expectedSnapshots=before.flatMap(row=>row.expectedSnapshots??[]);
25
+ before=[fence];
26
+ }
27
+ const erased=new Map(next.entries.filter(entry=>entry.erased).map(entry=>[effectIdentity(raw.id,entry.identity,0),entry.identity]));
28
+ const rows=state.rows.map((row,index)=>({job_id:raw.id,effect_id:row.effect_id,position:row.position,record_json:canonicalJson(erased.has(row.effect_id)?erasedDisposition(state.dispositions[index],erased.get(row.effect_id)!):redactDisposition(state.dispositions[index]))}));
29
+ const hash=reportHash(rows),binding=policyHash(raw.id,next.fingerprint,state.policy);
30
+ return {before,after:[changed(sql('DELETE FROM ik_d1_reversal_reports WHERE job_id=?',raw.id),deleteRows('ik_d1_reversal_reports',row=>row.job_id===raw.id)),...insertJsonRows('ik_d1_reversal_reports',['job_id','effect_id','position','record_json'],rows),changed(sql('UPDATE ik_jobs SET reversal_hash=?,reversal_policy_hash=? WHERE id=?',hash,binding,raw.id),updateRows('ik_jobs',row=>row.id===raw.id,row=>({...row,reversal_hash:hash,reversal_policy_hash:binding})))]};
31
+ }
@@ -0,0 +1,24 @@
1
+ /** Additive upgrade for existing D1 installations. Keep d1Schema unchanged so
2
+ * products can retain their original numbered migration byte for byte. */
3
+ export const d1ErasureSchema = [
4
+ 'CREATE TABLE IF NOT EXISTS ik_d1_streams (id TEXT PRIMARY KEY, destination TEXT NOT NULL, namespace TEXT NOT NULL, mapping_version TEXT NOT NULL, principal_id TEXT NOT NULL, fingerprint TEXT NOT NULL, operation_key TEXT NOT NULL, total INTEGER NOT NULL, layout_hash TEXT NOT NULL, state TEXT NOT NULL, UNIQUE(destination,principal_id,operation_key))',
5
+ 'CREATE TABLE IF NOT EXISTS ik_d1_stream_pages (stream_id TEXT NOT NULL REFERENCES ik_d1_streams(id), ordinal INTEGER NOT NULL, job_id TEXT NOT NULL UNIQUE REFERENCES ik_jobs(id), total INTEGER NOT NULL, erased_records INTEGER NOT NULL DEFAULT 0, PRIMARY KEY(stream_id,ordinal))',
6
+
7
+ // Page layout is sealed only after atomic creation. Progress metadata does not
8
+ // change membership; deleting or moving a page invalidates every child.
9
+ "CREATE TRIGGER IF NOT EXISTS ik_d1_stream_page_delete AFTER DELETE ON ik_d1_stream_pages BEGIN UPDATE ik_d1_streams SET state='invalid' WHERE id=OLD.stream_id; END",
10
+ "CREATE TRIGGER IF NOT EXISTS ik_d1_stream_page_update AFTER UPDATE OF stream_id,ordinal,job_id,total ON ik_d1_stream_pages BEGIN UPDATE ik_d1_streams SET state='invalid' WHERE id=OLD.stream_id OR id=NEW.stream_id; END",
11
+ "CREATE TRIGGER IF NOT EXISTS ik_d1_stream_page_insert AFTER INSERT ON ik_d1_stream_pages BEGIN UPDATE ik_d1_streams SET state='invalid' WHERE id=NEW.stream_id; END",
12
+
13
+ 'CREATE TABLE IF NOT EXISTS ik_d1_erased_sources (destination TEXT NOT NULL, identity TEXT NOT NULL, PRIMARY KEY(destination,identity))',
14
+ 'CREATE TABLE IF NOT EXISTS ik_d1_erased_targets (destination TEXT NOT NULL, identity TEXT NOT NULL, PRIMARY KEY(destination,identity))',
15
+ 'CREATE TABLE IF NOT EXISTS ik_d1_indexed_jobs (job_id TEXT PRIMARY KEY REFERENCES ik_jobs(id), fingerprint TEXT NOT NULL)',
16
+ 'CREATE TABLE IF NOT EXISTS ik_d1_plan_index (job_id TEXT NOT NULL REFERENCES ik_jobs(id), ordinal INTEGER NOT NULL, source_identity TEXT NOT NULL, target_identity TEXT NOT NULL, PRIMARY KEY(job_id,ordinal,target_identity))',
17
+ 'CREATE INDEX IF NOT EXISTS ik_d1_plan_index_source ON ik_d1_plan_index(source_identity,job_id)',
18
+ 'CREATE INDEX IF NOT EXISTS ik_d1_plan_index_target ON ik_d1_plan_index(target_identity,job_id)',
19
+ // Any derived-row mutation invalidates the completeness receipt. Rebuilding
20
+ // writes the receipt last, in the same guarded transaction as every row.
21
+ 'CREATE TRIGGER IF NOT EXISTS ik_d1_plan_index_insert AFTER INSERT ON ik_d1_plan_index BEGIN DELETE FROM ik_d1_indexed_jobs WHERE job_id=NEW.job_id; END',
22
+ 'CREATE TRIGGER IF NOT EXISTS ik_d1_plan_index_delete AFTER DELETE ON ik_d1_plan_index BEGIN DELETE FROM ik_d1_indexed_jobs WHERE job_id=OLD.job_id; END',
23
+ 'CREATE TRIGGER IF NOT EXISTS ik_d1_plan_index_update AFTER UPDATE ON ik_d1_plan_index BEGIN DELETE FROM ik_d1_indexed_jobs WHERE job_id=OLD.job_id OR job_id=NEW.job_id; END',
24
+ ];
@@ -0,0 +1,77 @@
1
+ import type { D1FieldMatch,D1ImportContext,D1ImportPerson,D1Sql } from './types.ts';
2
+ import { guard,sql } from './database.ts';
3
+ import { guardPredicate } from './guards.ts';
4
+ import { exactRows,jsonChunks,selectedCounts,selectKeys } from './sets.ts';
5
+ import { indexedDestinationGuard,targetIdentity,unindexedJobs } from './erasure-index.ts';
6
+ import { stageErasure,stagedSourceIdentities } from './staging-erasure.ts';
7
+ import { fieldSelectionSnapshot,relatedSelection } from './erasure-related.ts';
8
+ import { fieldRedactions,unmatchedPath } from './field-redaction.ts';
9
+ import { redactedNativeFields } from './redacted-custody.ts';
10
+ import { targetGuards } from './target-guards.ts';
11
+ import { erasureFragments,type ErasureFragment } from './erasure-fragments.ts';
12
+ import { measureErasurePages,type ErasurePage } from './erasure-pages.ts';
13
+
14
+ /** Select once before private preparation. Later passes reread only measured
15
+ * custody pages; native snapshots, selector counts and private-stage sets stay
16
+ * fixed while our own native stage is registered and populated. */
17
+ export async function prepareErasureSelection(person:D1ImportPerson,nativeStatements:D1Sql[],context:D1ImportContext,stageId:string,retiring:ReadonlySet<string>=new Set()):Promise<{
18
+ fragments:()=>AsyncGenerator<ErasureFragment>;snapshotGuards:D1Sql[];final:D1Sql[];
19
+ targets:{kind:string;id:string;present:boolean}[];fieldMatches:D1FieldMatch[];
20
+ }>{
21
+ const {database:db,targets}=context.storage,target=person.target,kind=target?.kind??'',targetId=target?.id??'';
22
+ const related=person.related?{...person.related,fields:fieldRedactions(person.related.fields)}:undefined;
23
+ if(await db.prepare(`${unindexedJobs} LIMIT 1`).bind(context.destination).first())throw new Error('IMPORT_ERASURE_INDEX_UPGRADE_REQUIRED');
24
+ const digest=target?targetIdentity(context.destination,kind,targetId):undefined;
25
+ const targetRows=digest?await selectKeys(db,'ik_d1_plan_index','target_identity',[digest]):[];
26
+ const scan=related?await relatedSelection(context,related,undefined,stageId):undefined;
27
+ const identities=new Set<string>([...targetRows.map(row=>row.source_identity),...digest?await stagedSourceIdentities(digest,context):[],...scan?.identities??[]]);
28
+ const direct=new Set<string>(targetRows.map(row=>row.source_identity));
29
+ const severed={identities:new Set([...scan?.identities??[]].filter(identity=>!direct.has(identity))),target:digest??''};
30
+ const sourceRows=await selectKeys(db,'ik_d1_plan_index','source_identity',[...identities]);
31
+ const ids=[...new Set<string>([...sourceRows.map(row=>row.job_id),...scan?.originals.keys()??[],...scan?.redactedJobs??[]])];
32
+ const reports=(await db.prepare("SELECT id FROM ik_jobs WHERE destination=? AND reversal_policy='preserve-conflicts' AND EXISTS(SELECT 1 FROM ik_d1_reversal_reports r WHERE r.job_id=ik_jobs.id) ORDER BY id").bind(context.destination).all()).results.map(row=>row.id as string);
33
+ const privateErasure=digest?stageErasure(context.destination,digest,[...identities],stageId):{before:[],after:[],final:[]};
34
+ const snapshot=target&&targets.stagingRead?await targets.stagingRead(db,context.destination,kind,targetId):undefined;
35
+ if(snapshot&&!snapshot.guards.length)throw new Error('IMPORT_NATIVE_STAGING_GUARD_REQUIRED');
36
+ const native=target?(snapshot?snapshot.row:await targets.read(db,context.destination,kind,targetId)):undefined;
37
+ const snapshotGuards=snapshot?snapshot.guards.map(guard):[];
38
+ const before=[indexedDestinationGuard(context.destination),...privateErasure.before,...scan?.guards??[],
39
+ ...digest?selectedCounts('ik_d1_plan_index','target_identity',[digest],targetRows):[],
40
+ ...selectedCounts('ik_d1_plan_index','source_identity',[...identities],sourceRows),
41
+ ...exactRows('ik_d1_plan_index',['job_id','ordinal','source_identity','target_identity'],sourceRows),
42
+ guard(sql("(SELECT count(*) FROM ik_jobs WHERE destination=? AND reversal_policy='preserve-conflicts' AND EXISTS(SELECT 1 FROM ik_d1_reversal_reports r WHERE r.job_id=ik_jobs.id))=?",context.destination,reports.length)),
43
+ ...target?targetGuards(targets,native??null,context.destination,kind,targetId):[]];
44
+ const writes=[...nativeStatements],post=target?targetGuards(targets,null,context.destination,kind,targetId):[],final=[...privateErasure.final];
45
+ const fields=related?.fields??[];
46
+ if(fields.length||related?.snapshot!==undefined){
47
+ const redaction=await redactedNativeFields(kind,targetId,context,fields,scan!.fieldMatches);
48
+ if(related?.snapshot!==undefined&&related.snapshot!==fieldSelectionSnapshot([...(scan!.discoveryGuards??scan!.guards),...redaction.before]))throw new Error('IMPORT_REDACTION_SNAPSHOT_CHANGED');
49
+ if(unmatchedPath(fields,scan!.fieldMatches))throw new Error('IMPORT_REDACTION_PATH_UNMATCHED');
50
+ before.push(...redaction.before);snapshotGuards.push(...redaction.before);
51
+ writes.push(...redaction.after);final.push(...redaction.final.map(query=>guardPredicate(query)??query));
52
+ }
53
+ const after=[...privateErasure.after];
54
+ if(fields.length||related?.snapshot!==undefined){
55
+ for(const table of ['ik_d1_staging','ik_d1_native_staging','ik_d1_preparation_steps']){
56
+ const exclude=table==='ik_d1_native_staging'?' AND id<>?':'',params=exclude?[context.destination,stageId]:[context.destination];
57
+ after.push(sql(`DELETE FROM ${table} WHERE destination=?${exclude}`,...params),guard(sql(`NOT EXISTS(SELECT 1 FROM ${table} WHERE destination=?${exclude})`,...params)));
58
+ final.push(sql(`NOT EXISTS(SELECT 1 FROM ${table} WHERE destination=?)`,context.destination));
59
+ }
60
+ }
61
+ for(const chunk of jsonChunks([...identities])){
62
+ after.push(sql('INSERT OR IGNORE INTO ik_d1_erased_sources(destination,identity) SELECT ?,value FROM json_each(?)',context.destination,chunk));
63
+ final.push(sql('NOT EXISTS(SELECT 1 FROM json_each(?) i WHERE NOT EXISTS(SELECT 1 FROM ik_d1_erased_sources s WHERE s.destination=? AND s.identity=i.value))',chunk,context.destination));
64
+ }
65
+ if(digest){
66
+ after.push(sql('INSERT OR IGNORE INTO ik_d1_erased_targets(destination,identity) VALUES(?,?)',context.destination,digest));
67
+ final.push(sql('EXISTS(SELECT 1 FROM ik_d1_erased_targets WHERE destination=? AND identity=?)',context.destination,digest));
68
+ }
69
+ let pages:ErasurePage[]|undefined;
70
+ const fragments=async function*():AsyncGenerator<ErasureFragment>{
71
+ yield {before,writes,post,plans:[],jobs:[],erased:0};
72
+ pages??=await measureErasurePages([...new Set([...ids,...reports,...retiring])],db);
73
+ yield* erasureFragments({ids,reports,identities,kind,targetId,severed,originals:scan?.originals,fields,redactedJobs:scan?.redactedJobs,retiring},context,pages);
74
+ yield {before:[],writes:after,post:[],plans:[],jobs:[],erased:0};
75
+ };
76
+ return {fragments,snapshotGuards,final,targets:target?[{kind,id:targetId,present:false}]:[],fieldMatches:scan?.fieldMatches??[]};
77
+ }
@@ -0,0 +1,15 @@
1
+ import type { D1Database,D1PlanCodec,D1Statement } from './types.ts';
2
+ import type { ErasurePage } from './erasure-pages.ts';
3
+ import { readInventorySnapshot } from './inventory-snapshot.ts';
4
+
5
+ /** Replays reuse the first measurement's owners, revisions and read ceiling.
6
+ * A changed closure cannot spend the reservation for publication or cleanup. */
7
+ export async function readErasureSnapshot(page:ErasurePage,database:D1Database,codec?:D1PlanCodec){
8
+ let reads=0;
9
+ const consume=()=>{if(page.readQueries!==undefined&&reads>=page.readQueries)throw new Error('IMPORT_CUSTODY_STATE_CHANGED');reads++;};
10
+ const wrap=(statement:D1Statement):D1Statement=>({bind(...values:unknown[]){return wrap(statement.bind(...values));},async first<T=Record<string,any>>(){consume();return statement.first<T>();},async all<T=Record<string,any>>(){consume();return statement.all<T>();}});
11
+ const measured:D1Database={prepare:query=>wrap(database.prepare(query)),async batch(){throw new Error('IMPORT_SNAPSHOT_WRITE_REFUSED');}};
12
+ const result=await readInventorySnapshot(page.ids,measured,codec,true,page.fence);
13
+ if(page.readQueries===undefined)page.readQueries=reads;
14
+ return result;
15
+ }