@runeditrun/rer 0.2.76 → 0.2.78
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -0
- package/cli/README.md +55 -0
- package/cli/contract.mjs +1 -1
- package/cli/deploy-command.mjs +4 -2
- package/cli/deploy-inputs.mjs +15 -3
- package/cli/deploy-installation-key.mjs +71 -0
- package/cli/deploy-secrets.mjs +2 -0
- package/cli/deploy.mjs +13 -2
- package/cli/deployment-record.mjs +2 -0
- package/cli/export.mjs +1 -1
- package/cli/main.mjs +1 -1
- package/cli/parse.mjs +3 -2
- package/cli/portability-migrations.mjs +35 -14
- package/cli/portability-worker.mjs +2 -2
- package/cli/portability.mjs +2 -2
- package/cli/runtime-front-door.mjs +1 -1
- package/global/CHANGELOG.md +14 -0
- package/global/README.md +32 -0
- package/modules/dependencies/README.md +71 -1
- package/modules/dependencies/auth.mjs +4 -4
- package/modules/dependencies/crypto.mjs +6 -1
- package/modules/dependencies/custody.mjs +16 -0
- package/modules/dependencies/index.d.mts +1 -1
- package/modules/dependencies/index.mjs +6 -5
- package/modules/dependencies/schema/keys-sealed.sql +4 -0
- package/modules/dependencies/sealing.mjs +55 -0
- package/modules/dependencies/store.mjs +62 -9
- package/modules/deploy-kit/README.md +1 -0
- package/modules/deploy-kit/owner-cli.mjs +19 -7
- package/modules/deploy-kit/owner-rekey.mjs +69 -0
- package/modules/deploy-kit/provider.mjs +9 -4
- package/modules/import-kit/src/d1/README.md +1322 -0
- package/modules/import-kit/src/d1/apply.ts +115 -0
- package/modules/import-kit/src/d1/budget.ts +89 -0
- package/modules/import-kit/src/d1/compressed-plan.ts +63 -0
- package/modules/import-kit/src/d1/creation.ts +95 -0
- package/modules/import-kit/src/d1/custody-fence.ts +27 -0
- package/modules/import-kit/src/d1/custody-state-schema.ts +67 -0
- package/modules/import-kit/src/d1/database.ts +163 -0
- package/modules/import-kit/src/d1/entry-locator.ts +61 -0
- package/modules/import-kit/src/d1/erased-artifacts.ts +38 -0
- package/modules/import-kit/src/d1/erased-custody.ts +36 -0
- package/modules/import-kit/src/d1/erased-plan.ts +47 -0
- package/modules/import-kit/src/d1/erased-reversal.ts +19 -0
- package/modules/import-kit/src/d1/erasure-bounded.ts +22 -0
- package/modules/import-kit/src/d1/erasure-claims.ts +27 -0
- package/modules/import-kit/src/d1/erasure-compiler.ts +67 -0
- package/modules/import-kit/src/d1/erasure-disposition.ts +16 -0
- package/modules/import-kit/src/d1/erasure-fragments.ts +86 -0
- package/modules/import-kit/src/d1/erasure-index.ts +53 -0
- package/modules/import-kit/src/d1/erasure-pages.ts +62 -0
- package/modules/import-kit/src/d1/erasure-progress-schema.ts +8 -0
- package/modules/import-kit/src/d1/erasure-progress.ts +61 -0
- package/modules/import-kit/src/d1/erasure-related.ts +124 -0
- package/modules/import-kit/src/d1/erasure-report.ts +31 -0
- package/modules/import-kit/src/d1/erasure-schema.ts +24 -0
- package/modules/import-kit/src/d1/erasure-selection.ts +77 -0
- package/modules/import-kit/src/d1/erasure-snapshot.ts +15 -0
- package/modules/import-kit/src/d1/erasure-stage.ts +146 -0
- package/modules/import-kit/src/d1/erasure-step.ts +12 -0
- package/modules/import-kit/src/d1/erasure-superseded.ts +98 -0
- package/modules/import-kit/src/d1/erasure-values.ts +83 -0
- package/modules/import-kit/src/d1/erasure-writes.ts +46 -0
- package/modules/import-kit/src/d1/erasure.ts +170 -0
- package/modules/import-kit/src/d1/evolutions/0001-preserved-undo.ts +13 -0
- package/modules/import-kit/src/d1/expected-state.ts +91 -0
- package/modules/import-kit/src/d1/field-redaction.ts +86 -0
- package/modules/import-kit/src/d1/guards.ts +37 -0
- package/modules/import-kit/src/d1/history.ts +71 -0
- package/modules/import-kit/src/d1/index.ts +88 -0
- package/modules/import-kit/src/d1/integrity.ts +127 -0
- package/modules/import-kit/src/d1/inventory-snapshot.ts +84 -0
- package/modules/import-kit/src/d1/jobs.ts +97 -0
- package/modules/import-kit/src/d1/native-set.ts +48 -0
- package/modules/import-kit/src/d1/native-staging-admission.ts +19 -0
- package/modules/import-kit/src/d1/native-staging-context.ts +44 -0
- package/modules/import-kit/src/d1/native-staging-lifecycle.ts +33 -0
- package/modules/import-kit/src/d1/native-staging-offset.ts +74 -0
- package/modules/import-kit/src/d1/native-staging-publication.ts +18 -0
- package/modules/import-kit/src/d1/native-staging-rows.ts +72 -0
- package/modules/import-kit/src/d1/native-staging-schema.ts +28 -0
- package/modules/import-kit/src/d1/native-staging-state.ts +22 -0
- package/modules/import-kit/src/d1/native-staging.ts +64 -0
- package/modules/import-kit/src/d1/original-copies.ts +53 -0
- package/modules/import-kit/src/d1/original-storage.ts +56 -0
- package/modules/import-kit/src/d1/originals.ts +74 -0
- package/modules/import-kit/src/d1/plan-dictionary.ts +80 -0
- package/modules/import-kit/src/d1/plans.ts +121 -0
- package/modules/import-kit/src/d1/preparation-fingerprint.ts +57 -0
- package/modules/import-kit/src/d1/preparation-header.ts +33 -0
- package/modules/import-kit/src/d1/preparation-reader.ts +98 -0
- package/modules/import-kit/src/d1/preparation-schema.ts +16 -0
- package/modules/import-kit/src/d1/preparation-state.ts +56 -0
- package/modules/import-kit/src/d1/preparation-step-combine.ts +80 -0
- package/modules/import-kit/src/d1/preparation-step-control.ts +48 -0
- package/modules/import-kit/src/d1/preparation-step-hash.ts +131 -0
- package/modules/import-kit/src/d1/preparation-step-pages.ts +77 -0
- package/modules/import-kit/src/d1/preparation-steps-schema.ts +11 -0
- package/modules/import-kit/src/d1/preparation-steps.ts +102 -0
- package/modules/import-kit/src/d1/preparations.ts +87 -0
- package/modules/import-kit/src/d1/preserved-preview.ts +89 -0
- package/modules/import-kit/src/d1/preserved-reverse.ts +91 -0
- package/modules/import-kit/src/d1/provenance-set.ts +87 -0
- package/modules/import-kit/src/d1/publication-fence.ts +22 -0
- package/modules/import-kit/src/d1/publication-receipt-schema.ts +15 -0
- package/modules/import-kit/src/d1/published-stream.ts +11 -0
- package/modules/import-kit/src/d1/read-fence.ts +64 -0
- package/modules/import-kit/src/d1/read-results.ts +16 -0
- package/modules/import-kit/src/d1/redacted-custody.ts +76 -0
- package/modules/import-kit/src/d1/redaction-step.ts +125 -0
- package/modules/import-kit/src/d1/referenced-plans.ts +136 -0
- package/modules/import-kit/src/d1/retained-values.ts +35 -0
- package/modules/import-kit/src/d1/retirement-bounded.ts +71 -0
- package/modules/import-kit/src/d1/retirement-door.ts +105 -0
- package/modules/import-kit/src/d1/retirement-schema.ts +39 -0
- package/modules/import-kit/src/d1/retirement-scope.ts +94 -0
- package/modules/import-kit/src/d1/retirement-steps.ts +76 -0
- package/modules/import-kit/src/d1/retirement-tally.ts +74 -0
- package/modules/import-kit/src/d1/retirement.ts +136 -0
- package/modules/import-kit/src/d1/reversal-conversion-record.ts +19 -0
- package/modules/import-kit/src/d1/reversal-conversion-schema.ts +11 -0
- package/modules/import-kit/src/d1/reversal-conversion.ts +40 -0
- package/modules/import-kit/src/d1/reversal-fence-schema.ts +14 -0
- package/modules/import-kit/src/d1/reversal-fence.ts +106 -0
- package/modules/import-kit/src/d1/reversal-journal.ts +24 -0
- package/modules/import-kit/src/d1/reverse.ts +103 -0
- package/modules/import-kit/src/d1/reviewed-source.ts +23 -0
- package/modules/import-kit/src/d1/sets.ts +78 -0
- package/modules/import-kit/src/d1/staged-creation.ts +201 -0
- package/modules/import-kit/src/d1/staging-admission.ts +45 -0
- package/modules/import-kit/src/d1/staging-control.ts +43 -0
- package/modules/import-kit/src/d1/staging-erasure.ts +28 -0
- package/modules/import-kit/src/d1/staging-lifecycle.ts +49 -0
- package/modules/import-kit/src/d1/staging-plan.ts +15 -0
- package/modules/import-kit/src/d1/staging-privacy.ts +22 -0
- package/modules/import-kit/src/d1/staging-publication.ts +79 -0
- package/modules/import-kit/src/d1/staging-schema.ts +37 -0
- package/modules/import-kit/src/d1/staging-statements.ts +8 -0
- package/modules/import-kit/src/d1/staging-transport.ts +70 -0
- package/modules/import-kit/src/d1/stream-boundary.ts +11 -0
- package/modules/import-kit/src/d1/stream-create.ts +55 -0
- package/modules/import-kit/src/d1/stream-identity-admission.ts +37 -0
- package/modules/import-kit/src/d1/stream-input.ts +18 -0
- package/modules/import-kit/src/d1/stream-layout.ts +43 -0
- package/modules/import-kit/src/d1/stream-origin.ts +44 -0
- package/modules/import-kit/src/d1/stream-page-integrity.ts +18 -0
- package/modules/import-kit/src/d1/stream-partition.ts +50 -0
- package/modules/import-kit/src/d1/stream-plan.ts +133 -0
- package/modules/import-kit/src/d1/stream-receipt.ts +45 -0
- package/modules/import-kit/src/d1/stream-revision-schema.ts +35 -0
- package/modules/import-kit/src/d1/stream-snapshot.ts +26 -0
- package/modules/import-kit/src/d1/stream-stage.ts +79 -0
- package/modules/import-kit/src/d1/stream-state.ts +25 -0
- package/modules/import-kit/src/d1/stream-working-set.ts +30 -0
- package/modules/import-kit/src/d1/streams.ts +39 -0
- package/modules/import-kit/src/d1/target-guards.ts +9 -0
- package/modules/import-kit/src/d1/target-identities.ts +17 -0
- package/modules/import-kit/src/d1/transport.ts +10 -0
- package/modules/import-kit/src/d1/types.ts +162 -0
- package/modules/import-kit/src/d1/undo-observations.ts +39 -0
- package/modules/import-kit/src/d1/undo-report.ts +22 -0
- package/modules/import-kit/src/d1/undo-snapshot.ts +87 -0
- package/modules/import-kit/src/d1/undo-state.ts +84 -0
- package/modules/import-kit/src/d1/undo-types.ts +110 -0
- package/modules/import-kit/src/journal/authority.ts +12 -0
- package/modules/import-kit/src/plan/budget.ts +60 -0
- package/modules/import-kit/src/plan/index.ts +123 -0
- package/modules/import-kit/src/plan/size.ts +122 -0
- package/modules/import-kit/src/provenance/canonical-hash.ts +130 -0
- package/modules/import-kit/src/provenance/canonical-stream.ts +85 -0
- package/modules/import-kit/src/provenance/credential-names.ts +131 -0
- package/modules/import-kit/src/provenance/credentials.ts +317 -0
- package/modules/import-kit/src/provenance/identity.ts +51 -0
- package/modules/import-kit/src/provenance/json-snapshot.ts +21 -0
- package/modules/import-kit/src/provenance/resumable-sha256.ts +75 -0
- package/modules/import-kit/src/provenance/validate-json.ts +19 -0
- package/modules/import-kit/src/storage/originals.ts +87 -0
- package/modules/import-kit/src/storage/provenance.ts +64 -0
- package/modules/import-kit/src/storage/schema.ts +42 -0
- package/modules/import-kit/src/types.ts +47 -0
- package/modules/operator-auth/docs/native-accounts.md +4 -1
- package/modules/operator-auth/identity/oidc/aes-gcm.mjs +21 -0
- package/modules/operator-auth/identity/oidc/crypto.mjs +5 -12
- package/modules/operator-auth/index.d.mts +1 -1
- package/modules/portability/README.md +98 -2
- package/modules/portability/backfills.mjs +42 -15
- package/modules/portability/ledgers.mjs +7 -1
- package/modules/portability/transient-tables.mjs +40 -0
- package/onboarding/local/environment.mjs +2 -2
- package/package.json +3 -1
- package/scripts/publish-check/reviewed.py +3 -1
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
import { copyStatement } from './guards.ts';
|
|
2
|
+
import type { D1Sql } from './types.ts';
|
|
3
|
+
import { guard } from './database.ts';
|
|
4
|
+
import { retainedSql } from './retained-values.ts';
|
|
5
|
+
import { exactRows, selectedCounts, jsonChunks } from './sets.ts';
|
|
6
|
+
|
|
7
|
+
type Row = Record<string, any>;
|
|
8
|
+
const keys: Record<string, string[]> = {
|
|
9
|
+
ik_jobs: ['id'], ik_provenance: ['id'], ik_sources: ['identity'],
|
|
10
|
+
ik_d1_history: ['id'], ik_d1_entries: ['job_id','ordinal'],
|
|
11
|
+
ik_batches: ['job_id','action','position'], ik_original_sets: ['job_id'],
|
|
12
|
+
ik_originals: ['job_id','kind','original_key'],
|
|
13
|
+
ik_d1_original_chunks: ['job_id','kind','original_key','ordinal'],
|
|
14
|
+
ik_d1_plan_chunks: ['job_id','ordinal'], ik_d1_reversal_reports: ['job_id','effect_id'],
|
|
15
|
+
ik_d1_indexed_jobs: ['job_id'], ik_d1_plan_index: ['job_id','ordinal','target_identity'],
|
|
16
|
+
ik_d1_stream_pages: ['stream_id','ordinal'], ik_d1_streams: ['id'],
|
|
17
|
+
ik_d1_policy_conversions: ['job_id'],
|
|
18
|
+
};
|
|
19
|
+
export interface ExpectedSnapshot { table: string; columns: string[]; rows: Row[]; selection?: { key: string; values: string[] } }
|
|
20
|
+
export interface ExpectedChange { table: string; apply(rows: Map<string, Row>, state: ExpectedState): void }
|
|
21
|
+
const identity = (table: string, row: Row) => JSON.stringify(keys[table].map(key => row[key]));
|
|
22
|
+
|
|
23
|
+
/** Transaction-local intended state. Snapshots come from the already validated
|
|
24
|
+
* read, and edits come from explicit maintained mutations, never observed SQL
|
|
25
|
+
* after a write. No SQL parsing or newly read revision can bless changed data. */
|
|
26
|
+
export class ExpectedState {
|
|
27
|
+
private tables = new Map<string, Map<string, Row>>();
|
|
28
|
+
private deleted = new Map<string, Map<string, Row>>();
|
|
29
|
+
private selections: { table: string; key: string; values: string[] }[] = [];
|
|
30
|
+
constructor(statements: D1Sql[]) {
|
|
31
|
+
for (const statement of statements) for (const snapshot of statement.expectedSnapshots ?? []) {
|
|
32
|
+
if (!keys[snapshot.table]) continue;
|
|
33
|
+
const rows = this.rows(snapshot.table);
|
|
34
|
+
for (const row of snapshot.rows) {
|
|
35
|
+
const id = identity(snapshot.table,row), prior = rows.get(id) ?? {};
|
|
36
|
+
// A later assertion may already describe the intended after-state.
|
|
37
|
+
// Keep the first complete preflight fields; mutations below own changes.
|
|
38
|
+
for (const column of snapshot.columns) if (!(column in prior)) prior[column] = row[column];
|
|
39
|
+
rows.set(id, prior);
|
|
40
|
+
}
|
|
41
|
+
if (snapshot.selection) this.selections.push({table:snapshot.table,...snapshot.selection});
|
|
42
|
+
}
|
|
43
|
+
for (const statement of statements) for (const change of typeof statement.expectedChanges==='function'?statement.expectedChanges():statement.expectedChanges??[]) change.apply(this.rows(change.table), this);
|
|
44
|
+
}
|
|
45
|
+
rows(table: string): Map<string, Row> {
|
|
46
|
+
let rows = this.tables.get(table); if (!rows) this.tables.set(table, rows = new Map()); return rows;
|
|
47
|
+
}
|
|
48
|
+
put(table: string, row: Row): void {
|
|
49
|
+
const id=identity(table,row);this.deleted.get(table)?.delete(id);
|
|
50
|
+
this.rows(table).set(id, {...row});
|
|
51
|
+
}
|
|
52
|
+
remove(table:string,id:string,row:Row):void{
|
|
53
|
+
let deleted=this.deleted.get(table);if(!deleted)this.deleted.set(table,deleted=new Map());
|
|
54
|
+
deleted.set(id,Object.fromEntries([...new Set([...keys[table],'job_id'])].filter(key=>key in row).map(key=>[key,row[key]])));
|
|
55
|
+
this.rows(table).delete(id);
|
|
56
|
+
}
|
|
57
|
+
guards(excludeJobs:ReadonlySet<string>=new Set()): D1Sql[] {
|
|
58
|
+
const result: D1Sql[] = [];
|
|
59
|
+
for (const [table, rows] of this.tables) {
|
|
60
|
+
// Related jobs can have different validated field projections. Compare
|
|
61
|
+
// every known field without inventing NULL values for unread columns.
|
|
62
|
+
const groups=new Map<string,{columns:string[];rows:Row[]}>();
|
|
63
|
+
for(const row of rows.values()){
|
|
64
|
+
if(excludeJobs.has(table==='ik_jobs'?row.id:row.job_id))continue;
|
|
65
|
+
const columns=Object.keys(row).sort(),key=JSON.stringify(columns);
|
|
66
|
+
let group=groups.get(key);if(!group)groups.set(key,group={columns,rows:[]});group.rows.push(row);
|
|
67
|
+
}
|
|
68
|
+
for(const {columns,rows:group} of groups.values())if(columns.length)result.push(...exactRows(table,columns,group,undefined,undefined,!['ik_originals','ik_d1_original_chunks'].includes(table)));
|
|
69
|
+
}
|
|
70
|
+
for(const [table,deleted] of this.deleted){
|
|
71
|
+
const rows=[...deleted.values()].filter(row=>!excludeJobs.has(table==='ik_jobs'?row.id:row.job_id));
|
|
72
|
+
for(const chunk of jsonChunks(rows))result.push(guard(retainedSql(['NOT EXISTS(SELECT 1 FROM json_each(',`) e JOIN ${table} t ON ${keys[table].map(key=>`t.${key} IS json_extract(e.value,'$.${key}')`).join(' AND ')})`],[chunk],[0])));
|
|
73
|
+
}
|
|
74
|
+
const seen=new Set<string>();
|
|
75
|
+
for(let selection of this.selections){
|
|
76
|
+
if(selection.key==='job_id'||selection.table==='ik_jobs'&&selection.key==='id')selection={...selection,values:selection.values.filter(id=>!excludeJobs.has(id))};
|
|
77
|
+
const stamp=JSON.stringify(selection);if(seen.has(stamp))continue;seen.add(stamp);
|
|
78
|
+
result.push(...selectedCounts(selection.table,selection.key,selection.values,[...this.rows(selection.table).values()]));
|
|
79
|
+
}
|
|
80
|
+
return result;
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
export function changed(query: D1Sql, ...changes: ExpectedChange[]): D1Sql { const result=copyStatement(query); result.expectedChanges=changes; return result; }
|
|
84
|
+
export function changedLazy(query:D1Sql,changes:()=>ExpectedChange[]):D1Sql{const result=copyStatement(query);result.expectedChanges=changes;return result;}
|
|
85
|
+
export function putRow(table: string, row: Row): ExpectedChange { return {table,apply(_rows,state){state.put(table,row)}}; }
|
|
86
|
+
export function updateRows(table: string, matches: (row: Row) => boolean, update: (row: Row) => Row): ExpectedChange {
|
|
87
|
+
return {table,apply(rows,state){for(const [id,row] of [...rows])if(matches(row)){state.remove(table,id,row);state.put(table,update({...row}));}}};
|
|
88
|
+
}
|
|
89
|
+
export function deleteRows(table: string, matches: (row: Row, state: ExpectedState) => boolean): ExpectedChange {
|
|
90
|
+
return {table,apply(rows,state){for(const [id,row] of rows)if(matches(row,state))state.remove(table,id,row);}};
|
|
91
|
+
}
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import type { Json, ImportPlan, OriginalErasurePolicy } from '../types.ts';
|
|
2
|
+
import type { D1FieldRedaction, D1FieldMatch } from './types.ts';
|
|
3
|
+
import { requireText, canonicalJson, sha256, sourceIdentity } from '../provenance/identity.ts';
|
|
4
|
+
import { planFingerprint } from '../plan/index.ts';
|
|
5
|
+
import { withdrawArtifacts } from './erased-artifacts.ts';
|
|
6
|
+
|
|
7
|
+
/** Snapshot the caller's selectors before any awaited reads. Never retain the
|
|
8
|
+
* selected private values in the plan or journal. */
|
|
9
|
+
export function fieldRedactions(fields: readonly D1FieldRedaction[] = []): D1FieldRedaction[] {
|
|
10
|
+
return fields.map(({ path, value, optional }) => {
|
|
11
|
+
requireText(value, 'redaction value');
|
|
12
|
+
if (!Array.isArray(path) || !path.length || path.length > 256 || path.some(segment =>
|
|
13
|
+
typeof segment !== 'string' && !(Number.isSafeInteger(segment) && segment >= 0))) throw new Error('IMPORT_INVALID_REDACTION_PATH');
|
|
14
|
+
if (optional !== undefined && typeof optional !== 'boolean') throw new Error('IMPORT_INVALID_REDACTION_PATH');
|
|
15
|
+
return { path: [...path], value, ...optional ? { optional } : {} };
|
|
16
|
+
});
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/** A path matched nowhere is a typo unless every selector on it is optional. */
|
|
20
|
+
export function unmatchedPath(fields: readonly D1FieldRedaction[], counts: readonly D1FieldMatch[]): boolean {
|
|
21
|
+
return counts.some(count => count.matches === 0 && fields.some(field => !field.optional && canonicalJson(field.path) === canonicalJson(count.path)));
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export function fieldMatches(fields: readonly D1FieldRedaction[]): D1FieldMatch[] {
|
|
25
|
+
return [...new Map(fields.map(field => [canonicalJson(field.path), { path: [...field.path], matches: 0 }])).values()];
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
const removed = Symbol('removed JSON field');
|
|
29
|
+
/** Copy only changed branches. Wildcards select arrays, never arbitrary object
|
|
30
|
+
* properties; exact leaves prevent substring replacement in unrelated text. */
|
|
31
|
+
export function redactFields(value: Json, fields: readonly D1FieldRedaction[], counts?: D1FieldMatch[]): Json {
|
|
32
|
+
const visit = (input: Json, selected: readonly D1FieldRedaction[], at: number): Json | typeof removed => {
|
|
33
|
+
const matched = selected.filter(field => field.path.length === at && input === field.value);
|
|
34
|
+
if (matched.length) {
|
|
35
|
+
for (const count of counts ?? []) if (matched.some(field => canonicalJson(field.path) === canonicalJson(count.path))) count.matches++;
|
|
36
|
+
return removed;
|
|
37
|
+
}
|
|
38
|
+
if (input === null || typeof input !== 'object') return input;
|
|
39
|
+
const result: Json[] | Record<string, Json> = Array.isArray(input) ? [] : {};
|
|
40
|
+
let changed = false;
|
|
41
|
+
for (const [key, item] of Object.entries(input)) {
|
|
42
|
+
const matching = selected.filter(field => field.path.length > at && (Array.isArray(input)
|
|
43
|
+
? field.path[at] === '*' || field.path[at] === Number(key) : field.path[at] === key));
|
|
44
|
+
const next = matching.length ? visit(item, matching, at + 1) : item;
|
|
45
|
+
changed ||= next !== item;
|
|
46
|
+
if (next === removed) continue;
|
|
47
|
+
if (Array.isArray(result)) result.push(next);
|
|
48
|
+
else Object.defineProperty(result, key, { value: next, enumerable: true, writable: true, configurable: true });
|
|
49
|
+
}
|
|
50
|
+
return changed ? result : input;
|
|
51
|
+
};
|
|
52
|
+
return visit(value, fields, 0) as Json;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/** Entries `from` (inclusive) to `to` (exclusive) of one job's plan: one page
|
|
56
|
+
* of a job too large to redact in one request. */
|
|
57
|
+
export interface EntryRange { from: number; to: number }
|
|
58
|
+
export const inRange = (range: EntryRange | undefined, ordinal: number): boolean => !range || ordinal >= range.from && ordinal < range.to;
|
|
59
|
+
|
|
60
|
+
/** With `range`, only those entries and the source records they collapse are
|
|
61
|
+
* redacted; the rest of the plan stays as it is until a later page. */
|
|
62
|
+
export function redactPlanFields(plan: ImportPlan, fields: readonly D1FieldRedaction[], policy?: OriginalErasurePolicy, range?: EntryRange): { plan: ImportPlan; identities: Set<string> } {
|
|
63
|
+
const identities = new Set<string>();
|
|
64
|
+
const paged = range && new Set(plan.entries.slice(range.from, range.to).map(entry => entry.identity));
|
|
65
|
+
const record = (source: ImportPlan['source']['records'][number]) => {
|
|
66
|
+
if (paged && !paged.has(sourceIdentity(plan.destination, plan.namespace, source.kind, source.sourceId))) return source;
|
|
67
|
+
const payload = redactFields(source.payload, fields);
|
|
68
|
+
return payload === source.payload ? source : { ...source, payload, payloadHash: sha256(canonicalJson(payload)) };
|
|
69
|
+
};
|
|
70
|
+
const entries = plan.entries.map((entry, ordinal) => {
|
|
71
|
+
if (!inRange(range, ordinal)) return entry;
|
|
72
|
+
const source = record(entry.source);
|
|
73
|
+
const effects = entry.effects.map(effect => {
|
|
74
|
+
const value = redactFields(effect.value, fields);
|
|
75
|
+
return value === effect.value ? effect : { ...effect, value };
|
|
76
|
+
});
|
|
77
|
+
if (source !== entry.source || effects.some((effect, index) => effect !== entry.effects[index])) identities.add(entry.identity);
|
|
78
|
+
return { ...entry, source, effects };
|
|
79
|
+
});
|
|
80
|
+
const next = { ...plan, entries, source: { ...plan.source, records: plan.source.records.map(record) },
|
|
81
|
+
erasure: plan.erasure ?? { reviewedFingerprint: plan.fingerprint } };
|
|
82
|
+
if (identities.size && policy === undefined && plan.source.artifacts.some(a => a.original?.status === 'available')) throw new Error('IMPORT_ERASURE_RETAINED_ORIGINAL');
|
|
83
|
+
withdrawArtifacts(plan, next, identities, policy);
|
|
84
|
+
next.fingerprint = planFingerprint(next);
|
|
85
|
+
return { plan: next, identities };
|
|
86
|
+
}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import type { D1Sql } from './types.ts';
|
|
2
|
+
|
|
3
|
+
// Scalar SQL and bindings cannot change after admission. Expected-state and
|
|
4
|
+
// retained-value annotations remain extensible on the maintained statement.
|
|
5
|
+
// Only this WeakMap conveys predicate provenance; arbitrary DML has none.
|
|
6
|
+
const predicates = new WeakMap<D1Sql, D1Sql>();
|
|
7
|
+
export function guard(predicate: D1Sql): D1Sql {
|
|
8
|
+
const retained = predicate.retainedValues;
|
|
9
|
+
const scalar: D1Sql = { sql: predicate.sql, params: [...(predicate.params ?? [])],
|
|
10
|
+
...(retained ? { retainedValues: { ...retained, parts: [...retained.parts], indexes: [...retained.indexes] } } : {}) };
|
|
11
|
+
Object.freeze(scalar.params);
|
|
12
|
+
if (scalar.retainedValues) {
|
|
13
|
+
Object.freeze(scalar.retainedValues.parts); Object.freeze(scalar.retainedValues.indexes);
|
|
14
|
+
Object.freeze(scalar.retainedValues);
|
|
15
|
+
}
|
|
16
|
+
Object.freeze(scalar);
|
|
17
|
+
const prefix = 'INSERT INTO ik_d1_assertions(ok) SELECT CASE WHEN (', suffix = ') THEN 1 ELSE 0 END';
|
|
18
|
+
const statement: D1Sql = { sql: prefix + scalar.sql + suffix, params: scalar.params };
|
|
19
|
+
Object.defineProperty(statement, 'sql', { writable: false });
|
|
20
|
+
Object.defineProperty(statement, 'params', { writable: false });
|
|
21
|
+
if (retained) {
|
|
22
|
+
const parts = [...retained.parts]; parts[0] = prefix + parts[0]; parts[parts.length - 1] += suffix;
|
|
23
|
+
statement.retainedValues = { ...retained, parts, indexes: [...retained.indexes] };
|
|
24
|
+
}
|
|
25
|
+
predicates.set(statement, scalar);
|
|
26
|
+
return statement;
|
|
27
|
+
}
|
|
28
|
+
export const guardPredicate = (statement: D1Sql): D1Sql | undefined => predicates.get(statement);
|
|
29
|
+
|
|
30
|
+
/** Copy construction annotations without discarding trusted scalar provenance.
|
|
31
|
+
* A copied arbitrary statement remains arbitrary; SQL text never grants trust. */
|
|
32
|
+
export function copyStatement(statement: D1Sql): D1Sql {
|
|
33
|
+
const scalar = guardPredicate(statement);
|
|
34
|
+
if (!scalar) return { ...statement };
|
|
35
|
+
const { sql: _sql, params: _params, ...metadata } = statement;
|
|
36
|
+
return Object.assign(guard(scalar), metadata);
|
|
37
|
+
}
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
import type { TargetRow } from '../types.ts';
|
|
2
|
+
import type { D1ImportContext, D1Sql, D1ProvenanceRow } from './types.ts';
|
|
3
|
+
import { canonicalJson, sha256, attachmentHash } from '../provenance/identity.ts';
|
|
4
|
+
import { guard, sql } from './database.ts';
|
|
5
|
+
import { validatedProvenance } from './integrity.ts';
|
|
6
|
+
import { changed, putRow, deleteRows, updateRows } from './expected-state.ts';
|
|
7
|
+
import { retainedSql } from './retained-values.ts';
|
|
8
|
+
|
|
9
|
+
export const provenanceColumns = 'id,job_id,ordinal,effect_ordinal,source_identity,destination,target_kind,target_id,reversed,record_json';
|
|
10
|
+
|
|
11
|
+
/** Transfer only a currently attributed exact native snapshot, explicitly reviewed
|
|
12
|
+
* through expectedRevision. Original attribution remains unchanged in history. */
|
|
13
|
+
export async function supersedeTarget(successor: D1ProvenanceRow, before: TargetRow | undefined, context: D1ImportContext, statements: D1Sql[]): Promise<void> {
|
|
14
|
+
const db = context.storage.database;
|
|
15
|
+
const prior = await db.prepare('SELECT * FROM ik_provenance WHERE destination=? AND target_kind=? AND target_id=? AND reversed=0').bind(successor.destination, successor.targetKind, successor.targetId).first();
|
|
16
|
+
if (!prior) {
|
|
17
|
+
statements.push(guard(sql('NOT EXISTS(SELECT 1 FROM ik_provenance WHERE destination=? AND target_kind=? AND target_id=? AND reversed=0)', successor.destination, successor.targetKind, successor.targetId)));
|
|
18
|
+
return;
|
|
19
|
+
}
|
|
20
|
+
const p = await validatedProvenance(prior, context, statements);
|
|
21
|
+
if (!before || before.revision !== p.appliedRevision || sha256(canonicalJson(before.value)) !== p.valueHash || attachmentHash(before.attachment) !== p.attachmentHash) throw new Error('NATIVE_EDIT_CONFLICT');
|
|
22
|
+
const owner = await db.prepare('SELECT state FROM ik_jobs WHERE id=?').bind(p.jobId).first();
|
|
23
|
+
if (owner?.state !== 'completed') throw new Error('TARGET_OWNER_BUSY');
|
|
24
|
+
successor.predecessorId = p.id;
|
|
25
|
+
statements.push(guard(sql("EXISTS(SELECT 1 FROM ik_jobs WHERE id=? AND state='completed')", p.jobId)), changed(sql(`INSERT INTO ik_d1_history(${provenanceColumns},successor_id) SELECT ${provenanceColumns},? FROM ik_provenance WHERE id=?`, successor.id, p.id),putRow('ik_d1_history',{...prior,successor_id:successor.id})), changed(sql('DELETE FROM ik_provenance WHERE id=?', p.id),deleteRows('ik_provenance',row=>row.id===p.id)));
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/** Undo cannot remove a historical layer underneath any newer import. Guarded
|
|
29
|
+
* again in each mutation batch so a racing successor cannot be overlooked. */
|
|
30
|
+
export async function noSuccessors(id: string, context: D1ImportContext): Promise<D1Sql> {
|
|
31
|
+
if (await context.storage.database.prepare('SELECT id FROM ik_d1_history WHERE job_id=? LIMIT 1').bind(id).first()) throw new Error('IMPORT_SUPERSEDED');
|
|
32
|
+
return guard(sql('NOT EXISTS(SELECT 1 FROM ik_d1_history WHERE job_id=?)', id));
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** `prior` may come from one read for a whole batch; undefined reads it here. */
|
|
36
|
+
export async function restorePredecessor(p: D1ProvenanceRow, context: D1ImportContext, before: D1Sql[], after: D1Sql[], prior?: any): Promise<void> {
|
|
37
|
+
if (prior === undefined) prior = await context.storage.database.prepare('SELECT * FROM ik_d1_history WHERE successor_id=?').bind(p.id).first();
|
|
38
|
+
if (!prior) {
|
|
39
|
+
if (p.predecessorId !== undefined) throw new Error('IMPORT_HISTORY_MISMATCH');
|
|
40
|
+
before.push(guard(sql('NOT EXISTS(SELECT 1 FROM ik_d1_history WHERE successor_id=?)', p.id)));
|
|
41
|
+
return;
|
|
42
|
+
}
|
|
43
|
+
const predecessor = await validatedProvenance(prior, context, before);
|
|
44
|
+
if (p.predecessorId !== predecessor.id) throw new Error('IMPORT_HISTORY_MISMATCH');
|
|
45
|
+
if (!p.before || p.before.destination !== predecessor.destination || p.before.targetKind !== predecessor.targetKind || p.before.targetId !== predecessor.targetId || p.before.revision !== predecessor.appliedRevision || sha256(canonicalJson(p.before.value)) !== predecessor.valueHash || attachmentHash(p.before.attachment) !== predecessor.attachmentHash) throw new Error('IMPORT_HISTORY_MISMATCH');
|
|
46
|
+
const {successor_id,...restored}=prior;
|
|
47
|
+
after.push(changed(sql(`INSERT INTO ik_provenance(${provenanceColumns}) SELECT ${provenanceColumns} FROM ik_d1_history WHERE id=? AND successor_id=?`, predecessor.id, p.id),putRow('ik_provenance',restored)), changed(sql('DELETE FROM ik_d1_history WHERE id=? AND successor_id=?', predecessor.id, p.id),deleteRows('ik_d1_history',row=>row.id===predecessor.id&&row.successor_id===p.id)));
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/** A preserving undo kept `p` for the owner, so the history beneath it can never
|
|
51
|
+
* be restored. In the same batch each predecessor goes back to its own job's
|
|
52
|
+
* provenance as reversed, its job's claim goes once nothing else backs it, and
|
|
53
|
+
* `p` stops naming it. That job's own undo then reports it kept ('superseded'),
|
|
54
|
+
* so a lineage with a catch-up undoes fully instead of stopping at IMPORT_SUPERSEDED. */
|
|
55
|
+
export async function releaseKeptPredecessors(p: D1ProvenanceRow, context: D1ImportContext, statements: D1Sql[]): Promise<void> {
|
|
56
|
+
if (p.predecessorId === undefined) return;
|
|
57
|
+
const db = context.storage.database;
|
|
58
|
+
for (let successor = p.id, prior = await db.prepare('SELECT * FROM ik_d1_history WHERE successor_id=?').bind(successor).first(); prior;
|
|
59
|
+
successor = prior.id, prior = await db.prepare('SELECT * FROM ik_d1_history WHERE successor_id=?').bind(successor).first()) {
|
|
60
|
+
const h = await validatedProvenance(prior, context, statements), record = canonicalJson({ ...h, reversed: true });
|
|
61
|
+
const { successor_id, ...row } = prior;
|
|
62
|
+
statements.push(
|
|
63
|
+
changed(retainedSql([`INSERT INTO ik_provenance(${provenanceColumns}) SELECT id,job_id,ordinal,effect_ordinal,source_identity,destination,target_kind,target_id,1,`, ' FROM ik_d1_history WHERE id=', ' AND successor_id=', ''], [record, h.id, successor], [0]), putRow('ik_provenance', { ...row, reversed: 1, record_json: record })),
|
|
64
|
+
changed(sql('DELETE FROM ik_d1_history WHERE id=? AND successor_id=?', h.id, successor), deleteRows('ik_d1_history', row => row.id === h.id)),
|
|
65
|
+
changed(sql('DELETE FROM ik_sources WHERE job_id=? AND identity=? AND NOT EXISTS(SELECT 1 FROM ik_provenance WHERE source_identity=? AND reversed=0) AND NOT EXISTS(SELECT 1 FROM ik_d1_history WHERE source_identity=?)', h.jobId, h.sourceIdentity, h.sourceIdentity, h.sourceIdentity),
|
|
66
|
+
deleteRows('ik_sources', (row, expected) => row.job_id === h.jobId && row.identity === h.sourceIdentity && ![...expected.rows('ik_provenance').values()].some(v => v.source_identity === row.identity && !v.reversed) && ![...expected.rows('ik_d1_history').values()].some(v => v.source_identity === row.identity))),
|
|
67
|
+
);
|
|
68
|
+
}
|
|
69
|
+
const { predecessorId: _released, ...kept } = p, record = canonicalJson(kept);
|
|
70
|
+
statements.push(changed(retainedSql(['UPDATE ik_provenance SET record_json=', ' WHERE id=', ' AND reversed=0'], [record, p.id], [0]), updateRows('ik_provenance', row => row.id === p.id, row => ({ ...row, record_json: record }))));
|
|
71
|
+
}
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
import { captureCustody } from './custody-fence.ts';
|
|
2
|
+
import { nativeSetGuards, sizeLimit } from './native-set.ts';
|
|
3
|
+
import { requestContext } from './budget.ts';
|
|
4
|
+
import { provenanceSet } from './provenance-set.ts';
|
|
5
|
+
/** Worker entry point. Requires nodejs_compat for shared crypto and byte helpers;
|
|
6
|
+
* deliberately does not import the synchronous node:sqlite entry point. */
|
|
7
|
+
export type * from './types.ts';
|
|
8
|
+
export type * from './undo-types.ts';
|
|
9
|
+
export type * from '../types.ts';
|
|
10
|
+
export { d1ErasureSchema } from './erasure-schema.ts';
|
|
11
|
+
export { d1StagingSchema, d1LegacyStreamReceiptUpgrade } from './staging-schema.ts';
|
|
12
|
+
export { d1PreparationSchema } from './preparation-schema.ts';
|
|
13
|
+
export { d1PreparationStepsSchema } from './preparation-steps-schema.ts';
|
|
14
|
+
export { prepareD1ImportSteps, type ImportPreparationStep } from './preparation-steps.ts';
|
|
15
|
+
export { prepareD1ImportStream, publishD1ImportPreparation, readD1ImportPreparation, type ImportPreparation } from './preparations.ts';
|
|
16
|
+
export { cancelD1ImportStaging } from './staging-lifecycle.ts';
|
|
17
|
+
export { requestContext as d1RequestContext } from './budget.ts';
|
|
18
|
+
export { upgradeD1ImportErasureIndex } from './erasure-index.ts';
|
|
19
|
+
export { d1Schema, initializeD1ImportStorage, preservedUndoEvolution } from './database.ts';
|
|
20
|
+
export { readD1ImportUndoReport } from './undo-report.ts';
|
|
21
|
+
export { previewD1PreservedUndo, D1ReversalPolicyConversionError } from './preserved-preview.ts';
|
|
22
|
+
export { convertD1ReversalPolicy } from './reversal-conversion.ts';
|
|
23
|
+
export { d1ReversalPolicyConversionSchema } from './reversal-conversion-schema.ts';
|
|
24
|
+
export { planD1Import, createD1ImportJob, readD1ImportJob } from './jobs.ts';
|
|
25
|
+
export { applyD1ImportBatches } from './apply.ts';
|
|
26
|
+
export { undoD1Import, rollbackD1Import } from './reverse.ts';
|
|
27
|
+
export { readD1ImportFieldMatches } from './erasure-related.ts';
|
|
28
|
+
export { eraseD1ImportTarget } from './erasure.ts';
|
|
29
|
+
export { eraseD1ImportTargetStep } from './erasure-step.ts';
|
|
30
|
+
export { retireD1ImportCustody, readD1ErasureJournal } from './retirement.ts';
|
|
31
|
+
export { retireD1ImportCustodyStep } from './retirement-bounded.ts';
|
|
32
|
+
export { stageD1ImportErasure, type D1ErasureStageSpec } from './erasure-stage.ts';
|
|
33
|
+
export type { D1ErasureStepOptions, D1ErasurePending, D1ErasureComplete, D1ErasureStep } from './erasure-progress.ts';
|
|
34
|
+
export { d1ErasureProgressSchema } from './erasure-progress-schema.ts';
|
|
35
|
+
export { NativeStageRows, verifyNativeStageOffset, type VerifiedNativeStageOffset, type NativeStageCheckpoint, type NativeStageProgress } from './native-staging-rows.ts';
|
|
36
|
+
export type { NativeStageBinding } from './native-staging-state.ts';
|
|
37
|
+
export type { ErasureFragment } from './erasure-fragments.ts';
|
|
38
|
+
export { d1RetirementSchema, d1RetirementAppendOnly } from './retirement-schema.ts';
|
|
39
|
+
export { readD1ImportOriginals } from './originals.ts';
|
|
40
|
+
export { canonicalJson, sha256 } from '../provenance/identity.ts';
|
|
41
|
+
export { PLAN_BYTES_LIMIT, PlanTooLargeError, planContentBytes } from '../plan/size.ts';
|
|
42
|
+
export { ImportSizeLimitError } from './native-set.ts';
|
|
43
|
+
import type { D1ImportContext } from './types.ts';
|
|
44
|
+
import type { PersistedImport } from '../types.ts';
|
|
45
|
+
import { authorize, readAtomic, guard, sql } from './database.ts';
|
|
46
|
+
import { readD1ImportJob } from './jobs.ts';
|
|
47
|
+
import { applyD1ImportBatches } from './apply.ts';
|
|
48
|
+
import { undoD1Import, rollbackD1Import } from './reverse.ts';
|
|
49
|
+
export async function resumeD1Import(id: string, context: D1ImportContext) {
|
|
50
|
+
context = requestContext(context);
|
|
51
|
+
// Reversal verifies its own authority and custody; reading the whole job
|
|
52
|
+
// first would make every reversal request cost the full job again.
|
|
53
|
+
const reversing = await context.storage.database.prepare("SELECT state FROM ik_jobs WHERE id=? AND state IN ('undoing','rolling_back')").bind(id).first();
|
|
54
|
+
const job = reversing ?? await readD1ImportJob(id, context);
|
|
55
|
+
if (job.state === 'undoing' || job.state === 'undone') return undoD1Import(id, context);
|
|
56
|
+
if (job.state === 'rolling_back' || job.state === 'rolled_back') return rollbackD1Import(id, context);
|
|
57
|
+
return applyD1ImportBatches(id, context);
|
|
58
|
+
}
|
|
59
|
+
export async function readD1PersistedImport(context: D1ImportContext): Promise<PersistedImport> {
|
|
60
|
+
context = requestContext(context);
|
|
61
|
+
await authorize(context, 'read');
|
|
62
|
+
const { targets, database } = context.storage;
|
|
63
|
+
if (!targets.matchesMany && !targets.matchesCount) throw new Error('IMPORT_NATIVE_SET_GUARD_REQUIRED');
|
|
64
|
+
// Captured before any custody read: the final read fences each job's revision
|
|
65
|
+
// instead of resending its plan, so catch-up chains cost one row per job.
|
|
66
|
+
const fence = await captureCustody(context);
|
|
67
|
+
const rows = await targets.list(database, context.destination);
|
|
68
|
+
const { results } = await context.storage.database.prepare('SELECT * FROM ik_provenance WHERE reversed=0 AND (destination=? OR job_id IN (SELECT id FROM ik_jobs WHERE destination=?))').bind(context.destination, context.destination).all();
|
|
69
|
+
const { decoded: provenance, jobs } = await provenanceSet(results, context, undefined);
|
|
70
|
+
const native = nativeSetGuards(targets, rows, context.destination);
|
|
71
|
+
const custody = [...fence(jobs), guard(sql('(? = (SELECT COUNT(*) FROM ik_provenance WHERE reversed=0 AND (destination=? OR job_id IN (SELECT id FROM ik_jobs WHERE destination=?))))', results.length, context.destination, context.destination))];
|
|
72
|
+
const limit = Math.min(context.maxBatchBytes ?? 2 * 1024 * 1024, 2 * 1024 * 1024);
|
|
73
|
+
try { await readAtomic({ ...context, maxBatchBytes: limit }, [...custody, ...native.map(entry => entry.guard)]); }
|
|
74
|
+
catch (error) { throw sizeLimit(error, native, custody, limit); }
|
|
75
|
+
return { rows, provenance };
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
export { planD1ImportStream, createD1ImportStream, readD1ImportStream, resumeD1ImportStream, undoD1ImportStream, type ImportStreamPlan } from './streams.ts';
|
|
79
|
+
export { preflightD1ImportJob } from './creation.ts';
|
|
80
|
+
export type { D1CreatePreflightContext, D1CreatePreflight } from './creation.ts';
|
|
81
|
+
|
|
82
|
+
export { d1NativeStagingSchema } from './native-staging-schema.ts';
|
|
83
|
+
export { retainedSql } from './retained-values.ts';
|
|
84
|
+
export { cancelD1NativeStaging, cancelD1ErasureStaging } from './native-staging-lifecycle.ts';
|
|
85
|
+
export { d1CustodyStateSchema, d1CustodyStateReplaceSchema } from './custody-state-schema.ts';
|
|
86
|
+
export { d1ReversalFenceSchema } from './reversal-fence-schema.ts';
|
|
87
|
+
|
|
88
|
+
export { d1StreamOriginSchema } from './stream-origin.ts';
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
import { guardErasedSources } from './erasure-claims.ts';
|
|
2
|
+
import type { InventorySnapshot } from './inventory-snapshot.ts';
|
|
3
|
+
import type { ImportJob, ImportPlan, PlanEntry, ProvenanceRow } from '../types.ts';
|
|
4
|
+
import { canonicalJson, sha256 } from '../provenance/identity.ts';
|
|
5
|
+
import type { D1ImportContext, D1Sql } from './types.ts';
|
|
6
|
+
import { guard, sql } from './database.ts';
|
|
7
|
+
import { claimColumns, exactRows, selectKeys, selectedCounts } from './sets.ts';
|
|
8
|
+
import { provenanceSet } from './provenance-set.ts';
|
|
9
|
+
import { reversalState } from './undo-state.ts';
|
|
10
|
+
import { validateReversalJournal } from './reversal-journal.ts';
|
|
11
|
+
export { custodyGuards } from './original-storage.ts';
|
|
12
|
+
// Only the current preflight: inventory already emits exact guards for every
|
|
13
|
+
// memoized row. Replaced before each publication and never shared by requests.
|
|
14
|
+
const inventories = new WeakMap<D1ImportContext, Map<string, { raw: string; decoded: ProvenanceRow }>>();
|
|
15
|
+
|
|
16
|
+
export const releaseInventory=(context:D1ImportContext):void=>{inventories.delete(context);};
|
|
17
|
+
|
|
18
|
+
export async function validatedProvenance(raw: any, context: D1ImportContext, guards: D1Sql[]): Promise<ProvenanceRow> {
|
|
19
|
+
const cached = inventories.get(context)?.get(raw.id);
|
|
20
|
+
if (cached?.raw === canonicalJson(raw)) return cached.decoded;
|
|
21
|
+
return (await provenanceSet([raw], context, guards)).decoded[0];
|
|
22
|
+
}
|
|
23
|
+
function validateReusedEntry(entry: PlanEntry, mappingVersion: string, owner: any, plan: ImportPlan | undefined, decision: any): void {
|
|
24
|
+
if (!owner || !plan) throw new Error('PROVENANCE_JOB_MISSING');
|
|
25
|
+
const index = plan.entries.findIndex(e => e.identity === entry.identity), original = plan.entries[index];
|
|
26
|
+
const effectsHash = sha256(canonicalJson(entry.effects));
|
|
27
|
+
if (plan.mappingVersion !== mappingVersion || !decision || decision.job_id !== owner.id || decision.ordinal !== index || decision.owner_job_id !== owner.id || decision.source_identity !== entry.identity || decision.effects_hash !== effectsHash || !original || index >= owner.checkpoint || original.source.payloadHash !== entry.source.payloadHash || sha256(canonicalJson(original.effects)) !== effectsHash) throw new Error('IMPORT_INVENTORY_MISMATCH');
|
|
28
|
+
}
|
|
29
|
+
/** A matching source claim is insufficient, especially when no native effects
|
|
30
|
+
* exist: validate and fence the actual owner's committed decision before reuse. */
|
|
31
|
+
export async function reusedSourceGuards(entry: PlanEntry, ownerId: string, context: D1ImportContext, guards: D1Sql[]): Promise<void> {
|
|
32
|
+
const state = await provenanceSet([], context, guards, [ownerId]);
|
|
33
|
+
const owner = state.owners.get(ownerId), plan = state.plans.get(ownerId);
|
|
34
|
+
const index = plan?.entries.findIndex(e => e.identity === entry.identity) ?? -1;
|
|
35
|
+
const decision = await context.storage.database.prepare('SELECT * FROM ik_d1_entries WHERE job_id=? AND ordinal=?').bind(ownerId, index).first();
|
|
36
|
+
validateReusedEntry(entry, context.mappingVersion, owner, plan, decision);
|
|
37
|
+
guards.push(...exactRows('ik_d1_entries', ['job_id','ordinal','source_identity','owner_job_id','effects_hash'], [decision]));
|
|
38
|
+
}
|
|
39
|
+
/** Complete inventory closure, validated once with bounded set reads. No cache
|
|
40
|
+
* survives this preflight; exact full-field snapshots fence the atomic batch. */
|
|
41
|
+
export async function inventoryGuards(job: ImportJob, plan: ImportPlan, context: D1ImportContext, snapshot?: InventorySnapshot): Promise<D1Sql[]> {
|
|
42
|
+
const db = context.storage.database, guards: D1Sql[] = snapshot ? [] : await guardErasedSources(plan, context.storage.database);
|
|
43
|
+
if (!Number.isSafeInteger(job.checkpoint) || job.checkpoint < 0 || job.checkpoint > job.total) throw new Error('CHECKPOINT_CONFLICT');
|
|
44
|
+
const read = (table: string, key: string, values: string[]) => snapshot ? Promise.resolve(snapshot.rows(table, key, values)) : selectKeys(db, table, key, values);
|
|
45
|
+
const entries = (await read('ik_d1_entries', 'job_id', [job.id])).sort((a, b) => a.ordinal - b.ordinal);
|
|
46
|
+
const provenance = await read('ik_provenance', 'job_id', [job.id]);
|
|
47
|
+
const history = await read('ik_d1_history', 'job_id', [job.id]);
|
|
48
|
+
if (entries.length !== job.checkpoint) throw new Error('IMPORT_INVENTORY_MISMATCH');
|
|
49
|
+
const reusedOwners = [...new Set(entries.filter(r => r.owner_job_id !== job.id).map(r => r.owner_job_id))];
|
|
50
|
+
const reusedActive = await read('ik_provenance', 'job_id', reusedOwners);
|
|
51
|
+
const reusedHistory = await read('ik_d1_history', 'job_id', reusedOwners);
|
|
52
|
+
const reusedEntries = await read('ik_d1_entries', 'job_id', reusedOwners);
|
|
53
|
+
const reusedClaims = await read('ik_sources', 'job_id', reusedOwners);
|
|
54
|
+
guards.push(...selectedCounts('ik_provenance', 'job_id', reusedOwners, reusedActive),
|
|
55
|
+
...selectedCounts('ik_d1_history', 'job_id', reusedOwners, reusedHistory),
|
|
56
|
+
...selectedCounts('ik_d1_entries', 'job_id', reusedOwners, reusedEntries),
|
|
57
|
+
...selectedCounts('ik_sources', 'job_id', reusedOwners, reusedClaims));
|
|
58
|
+
guards.push(...exactRows('ik_sources', claimColumns, reusedClaims));
|
|
59
|
+
guards.push(...exactRows('ik_d1_entries', ['job_id','ordinal','source_identity','owner_job_id','effects_hash'], reusedEntries));
|
|
60
|
+
const all = [...provenance, ...history, ...reusedActive, ...reusedHistory];
|
|
61
|
+
const state = await provenanceSet(all, context, guards, [job.id, ...reusedOwners], entries.map(r => r.source_identity), snapshot);
|
|
62
|
+
let expectedEffects = 0;
|
|
63
|
+
const expectedClaims = new Set([...provenance, ...history].filter(r => !r.reversed).map(r => r.source_identity));
|
|
64
|
+
for (const [ordinal, recorded] of entries.entries()) {
|
|
65
|
+
const entry = plan.entries[ordinal], effectsHash = sha256(canonicalJson(entry.effects));
|
|
66
|
+
if (recorded.ordinal !== ordinal || recorded.source_identity !== entry.identity || recorded.effects_hash !== effectsHash) throw new Error('IMPORT_INVENTORY_MISMATCH');
|
|
67
|
+
if (recorded.owner_job_id === job.id) {
|
|
68
|
+
expectedEffects += entry.effects.length;
|
|
69
|
+
if (!entry.effects.length && !['undoing', 'rolling_back', 'undone', 'rolled_back'].includes(job.state)) expectedClaims.add(entry.identity);
|
|
70
|
+
} else {
|
|
71
|
+
const owner = state.owners.get(recorded.owner_job_id), ownerPlan = state.plans.get(recorded.owner_job_id);
|
|
72
|
+
const index = ownerPlan?.entries.findIndex(e => e.identity === entry.identity);
|
|
73
|
+
const decision = reusedEntries.find(e => e.job_id === recorded.owner_job_id && e.ordinal === index);
|
|
74
|
+
validateReusedEntry(entry, plan.mappingVersion, owner, ownerPlan, decision);
|
|
75
|
+
// Non-reversed effects validate their claims in provenanceSet. A source
|
|
76
|
+
// with no effects needs the same check here on every later checkpoint.
|
|
77
|
+
// Reversal may legitimately release its claim while retaining decisions.
|
|
78
|
+
if (!entry.effects.length && !['undoing', 'rolling_back', 'undone', 'rolled_back'].includes(owner.state)) {
|
|
79
|
+
const claim = state.claims.get(entry.identity);
|
|
80
|
+
if (!claim || claim.job_id !== owner.id || claim.source_hash !== entry.source.payloadHash
|
|
81
|
+
|| claim.mapping_version !== ownerPlan!.mappingVersion || claim.effects_hash !== effectsHash) throw new Error('PROVENANCE_SOURCE_CLAIM_MISMATCH');
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
const reversal = reversalState(job);
|
|
86
|
+
const reportEffects = expectedEffects + reversal.dispositions.filter(d => d.redacted === 'erased').length;
|
|
87
|
+
if (reversal.cursor > reportEffects || (reversal.policy === 'preserve-conflicts' && ['undone','rolled_back'].includes(job.state) && reversal.cursor !== reportEffects)) throw new Error('IMPORT_REVERSAL_INTEGRITY');
|
|
88
|
+
if (new Set([...provenance, ...history].map(r => r.id)).size !== expectedEffects || provenance.length + history.length !== expectedEffects) throw new Error('IMPORT_INVENTORY_MISMATCH');
|
|
89
|
+
const ownedClaims = await read('ik_sources', 'job_id', [job.id]);
|
|
90
|
+
// A source-only claim may still be present immediately after reversal intent;
|
|
91
|
+
// the first reversing batch releases it atomically.
|
|
92
|
+
const zeroClaims = new Set(entries.filter(r => r.owner_job_id === job.id && !plan.entries[r.ordinal].effects.length).map(r => r.source_identity));
|
|
93
|
+
for (const claim of ownedClaims) {
|
|
94
|
+
if (!expectedClaims.has(claim.identity) && !(zeroClaims.has(claim.identity) && ['undoing', 'rolling_back'].includes(job.state))) throw new Error('PROVENANCE_SOURCE_CLAIM_MISMATCH');
|
|
95
|
+
const entry = plan.entries.find(e => e.identity === claim.identity);
|
|
96
|
+
if (!entry || claim.source_hash !== entry.source.payloadHash || claim.mapping_version !== plan.mappingVersion || claim.effects_hash !== sha256(canonicalJson(entry.effects))) throw new Error('PROVENANCE_SOURCE_CLAIM_MISMATCH');
|
|
97
|
+
}
|
|
98
|
+
for (const identity of expectedClaims) if (!ownedClaims.some(c => c.identity === identity)) throw new Error('PROVENANCE_SOURCE_CLAIM_MISMATCH');
|
|
99
|
+
guards.push(...exactRows('ik_sources', ['identity','source_hash','mapping_version','effects_hash','job_id'], ownedClaims, sql('job_id=?', job.id)));
|
|
100
|
+
guards.push(...exactRows('ik_d1_entries', ['job_id','ordinal','source_identity','owner_job_id','effects_hash'], entries, sql('job_id=?', job.id)));
|
|
101
|
+
// Row comparisons are supplied by provenanceSet; cardinalities close each set.
|
|
102
|
+
if(context.maxBatchBytes!==undefined){
|
|
103
|
+
for(const [table,rows] of [['ik_provenance',provenance],['ik_d1_history',history],['ik_sources',ownedClaims],['ik_d1_entries',entries]] as const) guards.push(...selectedCounts(table,'job_id',[job.id],rows));
|
|
104
|
+
}else{
|
|
105
|
+
guards.push(guard(sql('(SELECT COUNT(*) FROM ik_provenance WHERE job_id=?)=?', job.id, provenance.length)));
|
|
106
|
+
guards.push(guard(sql('(SELECT COUNT(*) FROM ik_d1_history WHERE job_id=?)=?', job.id, history.length)));
|
|
107
|
+
}
|
|
108
|
+
const allBatches = await read('ik_batches', 'job_id', [job.id]);
|
|
109
|
+
if(context.maxBatchBytes!==undefined)guards.push(...exactRows('ik_batches',['job_id','action','position','end_position','fence'],allBatches),...selectedCounts('ik_batches','job_id',[job.id],allBatches));
|
|
110
|
+
const batches = allBatches.filter(row => row.action === 'apply').sort((a, b) => a.position - b.position);
|
|
111
|
+
let checkpoint = 0, fence = 0;
|
|
112
|
+
for (const batch of batches) {
|
|
113
|
+
if (batch.position !== checkpoint || batch.end_position < batch.position || batch.end_position > job.total || batch.fence !== fence + 1 || batch.fence > job.fence) throw new Error('IMPORT_JOURNAL_MISMATCH');
|
|
114
|
+
checkpoint = batch.end_position; fence = batch.fence;
|
|
115
|
+
}
|
|
116
|
+
if (checkpoint !== job.checkpoint) throw new Error('IMPORT_JOURNAL_MISMATCH');
|
|
117
|
+
guards.push(...exactRows('ik_batches', ['job_id','action','position','end_position','fence'], batches, sql("job_id=? AND action='apply'", job.id)));
|
|
118
|
+
if (reversal.policy === 'preserve-conflicts' && ['undoing','undone','rolling_back','rolled_back'].includes(job.state)) {
|
|
119
|
+
const reversedBatches=snapshot?snapshot.rows('ik_batches','job_id',[job.id]).filter(row=>row.action!=='apply').sort((a,b)=>a.position-b.position):(await db.prepare("SELECT * FROM ik_batches WHERE job_id=? AND action<>'apply' ORDER BY position").bind(job.id).all()).results;
|
|
120
|
+
validateReversalJournal(job, reversedBatches, fence, reversal.conversion);
|
|
121
|
+
if ((reversal.cursor > 0 && !reversedBatches.length) || (['undone','rolled_back'].includes(job.state) && !reversedBatches.length)) throw new Error('IMPORT_JOURNAL_MISMATCH');
|
|
122
|
+
guards.push(...exactRows('ik_batches', ['job_id','action','position','end_position','fence'], reversedBatches, sql("job_id=? AND action<>'apply'", job.id)));
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
inventories.set(context, new Map(all.map((raw, i) => [raw.id, { raw: canonicalJson(raw), decoded: state.decoded[i] }])));
|
|
126
|
+
return guards;
|
|
127
|
+
}
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
import type { ImportPlan } from '../types.ts';
|
|
2
|
+
import type { D1Database, D1PlanCodec,D1Sql } from './types.ts';
|
|
3
|
+
import { selectKeys,jsonChunks } from './sets.ts';
|
|
4
|
+
import { loadPlans } from './plans.ts';
|
|
5
|
+
import { decodeProvenance } from '../storage/provenance.ts';
|
|
6
|
+
import type { D1ProvenanceRow } from './types.ts';
|
|
7
|
+
|
|
8
|
+
/** A single publication's real database snapshot, shared by all affected jobs.
|
|
9
|
+
* Validation emits exact row and set guards; nothing survives the invocation. */
|
|
10
|
+
export class InventorySnapshot {
|
|
11
|
+
readonly tables = new Map<string, any[]>();
|
|
12
|
+
readonly plans = new Map<string, ImportPlan>();
|
|
13
|
+
readonly checkedPlans = new Set<string>();
|
|
14
|
+
private indexes = new Map<string, Map<string, any[]>>();
|
|
15
|
+
rows(table: string, key: string, values: string[]): any[] {
|
|
16
|
+
if (!this.tables.has(table)) throw new Error('IMPORT_INVENTORY_SNAPSHOT_MISSING');
|
|
17
|
+
const name = `${table}.${key}`;
|
|
18
|
+
let index = this.indexes.get(name);
|
|
19
|
+
if (!index) {
|
|
20
|
+
index = new Map();
|
|
21
|
+
for (const row of this.tables.get(table)!) { const rows = index.get(row[key]) ?? []; rows.push(row); index.set(row[key], rows); }
|
|
22
|
+
this.indexes.set(name, index);
|
|
23
|
+
}
|
|
24
|
+
return [...new Set(values)].flatMap(value => index!.get(value) ?? []);
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
export async function readInventorySnapshot(ids: string[], db: D1Database, codec?: D1PlanCodec, revisions=false,fence?:D1Sql): Promise<InventorySnapshot> {
|
|
28
|
+
const snapshot = new InventorySnapshot(), owners = new Set(ids);
|
|
29
|
+
const read=async(table:string,key:string,values:string[])=>{const rows=await selectKeys(db,table,key,values,fence);return revisions?orderedRows(rows):rows;};
|
|
30
|
+
let pending = ids, entries: any[] = [];
|
|
31
|
+
while (pending.length) {
|
|
32
|
+
const found = await read('ik_d1_entries', 'job_id', pending); entries.push(...found);
|
|
33
|
+
pending = [...new Set(found.map(row => row.owner_job_id))].filter(id => !owners.has(id));
|
|
34
|
+
for (const id of pending) owners.add(id);
|
|
35
|
+
}
|
|
36
|
+
const allIds = [...owners]; snapshot.tables.set('ik_d1_entries', entries);
|
|
37
|
+
for (const table of ['ik_jobs', 'ik_provenance', 'ik_d1_history', 'ik_sources', 'ik_batches', 'ik_original_sets', 'ik_originals', 'ik_d1_original_chunks']) {
|
|
38
|
+
snapshot.tables.set(table, table==='ik_jobs'&&revisions?await revisionJobs(allIds,db,fence):await read(table, table === 'ik_jobs' ? 'id' : 'job_id', allIds));
|
|
39
|
+
}
|
|
40
|
+
// Preserved undo validates every still-owned predecessor. Those owners need
|
|
41
|
+
// not be entry-reuse owners, so close their durable history independently.
|
|
42
|
+
const jobs = snapshot.tables.get('ik_jobs')!, history = snapshot.tables.get('ik_d1_history')!;
|
|
43
|
+
const preserved = new Set(jobs.filter(job => job.reversal_policy === 'preserve-conflicts').map(job => job.id));
|
|
44
|
+
let frontier = [...snapshot.tables.get('ik_provenance')!, ...history].filter(row => preserved.has(row.job_id));
|
|
45
|
+
const knownHistory = new Set(history.map(row => row.id)), knownOwners = new Set(jobs.map(row => row.id));
|
|
46
|
+
while (frontier.length) {
|
|
47
|
+
const ids = frontier.filter(row => !row.reversed).map(row => (decodeProvenance(row) as D1ProvenanceRow).predecessorId).filter((id): id is string => id !== undefined && !knownHistory.has(id));
|
|
48
|
+
frontier = await read('ik_d1_history', 'id', ids);
|
|
49
|
+
for (const row of frontier) { history.push(row); knownHistory.add(row.id); }
|
|
50
|
+
const missing = [...new Set(frontier.map(row => row.job_id))].filter(id => !knownOwners.has(id));
|
|
51
|
+
for (const owner of await (revisions?revisionJobs(missing,db,fence):read('ik_jobs', 'id', missing))) { jobs.push(owner); knownOwners.add(owner.id); }
|
|
52
|
+
}
|
|
53
|
+
const provenance = [...snapshot.tables.get('ik_provenance')!, ...snapshot.tables.get('ik_d1_history')!];
|
|
54
|
+
const successors = [...new Set(provenance.filter(row => row.successor_id !== undefined).map(row => row.successor_id))];
|
|
55
|
+
for (const table of ['ik_provenance', 'ik_d1_history']) {
|
|
56
|
+
const rows = snapshot.tables.get(table)!, known = new Set(rows.map(row => row.id));
|
|
57
|
+
rows.push(...(await read(table, 'id', successors)).filter(row => !known.has(row.id)));
|
|
58
|
+
}
|
|
59
|
+
const claims = snapshot.tables.get('ik_sources')!, known = new Set(claims.map(row => row.identity));
|
|
60
|
+
claims.push(...(await read('ik_sources', 'identity', [...entries.map(row => row.source_identity), ...provenance.map(row => row.source_identity)])).filter(row => !known.has(row.identity)));
|
|
61
|
+
if(revisions){
|
|
62
|
+
for(const job of jobs)if(!Number.isSafeInteger(job.custody_revision)||!Number.isSafeInteger(job.state_revision))throw new Error('IMPORT_CUSTODY_STATE_REQUIRED');
|
|
63
|
+
const reports:any[]=[];
|
|
64
|
+
for(const chunk of jsonChunks(jobs.map(row=>row.id)))reports.push(...(await db.prepare(`SELECT r.*,p.reversed AS native_reversed FROM ik_d1_reversal_reports r LEFT JOIN ik_provenance p ON p.id=r.effect_id AND p.job_id=r.job_id WHERE r.job_id IN (SELECT value FROM json_each(?))${fence?` AND (${fence.sql})`:''} ORDER BY r.job_id,r.position`).bind(chunk,...fence?.params??[]).all()).results);
|
|
65
|
+
snapshot.tables.set('ik_d1_reversal_reports',reports);
|
|
66
|
+
snapshot.tables.set('ik_d1_stream_pages',await read('ik_d1_stream_pages','job_id',jobs.map(row=>row.id)));
|
|
67
|
+
}
|
|
68
|
+
if(revisions)for(const rows of snapshot.tables.values())orderedRows(rows);
|
|
69
|
+
for (const [id, plan] of await loadPlans(snapshot.tables.get('ik_jobs')!, db, codec,fence)) snapshot.plans.set(id, plan);
|
|
70
|
+
return snapshot;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
async function revisionJobs(ids:string[],db:D1Database,fence?:D1Sql):Promise<any[]>{
|
|
74
|
+
const rows:any[]=[];
|
|
75
|
+
for(const chunk of jsonChunks(ids))rows.push(...(await db.prepare(`SELECT j.*,r.revision AS custody_revision,s.revision AS state_revision FROM ik_jobs j LEFT JOIN ik_d1_custody_revisions r ON r.job_id=j.id LEFT JOIN ik_d1_custody_states s ON s.job_id=j.id WHERE j.id IN (SELECT value FROM json_each(?))${fence?` AND (${fence.sql})`:''}`).bind(chunk,...fence?.params??[]).all()).results);
|
|
76
|
+
return orderedRows(rows);
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
// Only primary-key scalars participate; repeated bounded passes never depend
|
|
80
|
+
// on unspecified database row order or compare complete private JSON bodies.
|
|
81
|
+
function orderedRows(rows:any[]):any[]{
|
|
82
|
+
const columns=['job_id','id','identity','stream_id','ordinal','effect_id','action','position','kind','original_key'];
|
|
83
|
+
return rows.sort((a,b)=>{for(const column of columns){const x=a[column],y=b[column];if(x===y)continue;if(x===undefined)return -1;if(y===undefined)return 1;return x<y?-1:1;}return 0;});
|
|
84
|
+
}
|