@runeditrun/rer 0.2.77 → 0.2.78
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -0
- package/cli/export.mjs +1 -1
- package/cli/portability-migrations.mjs +35 -14
- package/cli/portability-worker.mjs +2 -2
- package/cli/portability.mjs +2 -2
- package/global/CHANGELOG.md +8 -0
- package/global/README.md +32 -0
- package/modules/deploy-kit/owner-cli.mjs +15 -5
- package/modules/deploy-kit/provider.mjs +9 -4
- package/modules/import-kit/src/d1/README.md +1322 -0
- package/modules/import-kit/src/d1/apply.ts +115 -0
- package/modules/import-kit/src/d1/budget.ts +89 -0
- package/modules/import-kit/src/d1/compressed-plan.ts +63 -0
- package/modules/import-kit/src/d1/creation.ts +95 -0
- package/modules/import-kit/src/d1/custody-fence.ts +27 -0
- package/modules/import-kit/src/d1/custody-state-schema.ts +67 -0
- package/modules/import-kit/src/d1/database.ts +163 -0
- package/modules/import-kit/src/d1/entry-locator.ts +61 -0
- package/modules/import-kit/src/d1/erased-artifacts.ts +38 -0
- package/modules/import-kit/src/d1/erased-custody.ts +36 -0
- package/modules/import-kit/src/d1/erased-plan.ts +47 -0
- package/modules/import-kit/src/d1/erased-reversal.ts +19 -0
- package/modules/import-kit/src/d1/erasure-bounded.ts +22 -0
- package/modules/import-kit/src/d1/erasure-claims.ts +27 -0
- package/modules/import-kit/src/d1/erasure-compiler.ts +67 -0
- package/modules/import-kit/src/d1/erasure-disposition.ts +16 -0
- package/modules/import-kit/src/d1/erasure-fragments.ts +86 -0
- package/modules/import-kit/src/d1/erasure-index.ts +53 -0
- package/modules/import-kit/src/d1/erasure-pages.ts +62 -0
- package/modules/import-kit/src/d1/erasure-progress-schema.ts +8 -0
- package/modules/import-kit/src/d1/erasure-progress.ts +61 -0
- package/modules/import-kit/src/d1/erasure-related.ts +124 -0
- package/modules/import-kit/src/d1/erasure-report.ts +31 -0
- package/modules/import-kit/src/d1/erasure-schema.ts +24 -0
- package/modules/import-kit/src/d1/erasure-selection.ts +77 -0
- package/modules/import-kit/src/d1/erasure-snapshot.ts +15 -0
- package/modules/import-kit/src/d1/erasure-stage.ts +146 -0
- package/modules/import-kit/src/d1/erasure-step.ts +12 -0
- package/modules/import-kit/src/d1/erasure-superseded.ts +98 -0
- package/modules/import-kit/src/d1/erasure-values.ts +83 -0
- package/modules/import-kit/src/d1/erasure-writes.ts +46 -0
- package/modules/import-kit/src/d1/erasure.ts +170 -0
- package/modules/import-kit/src/d1/evolutions/0001-preserved-undo.ts +13 -0
- package/modules/import-kit/src/d1/expected-state.ts +91 -0
- package/modules/import-kit/src/d1/field-redaction.ts +86 -0
- package/modules/import-kit/src/d1/guards.ts +37 -0
- package/modules/import-kit/src/d1/history.ts +71 -0
- package/modules/import-kit/src/d1/index.ts +88 -0
- package/modules/import-kit/src/d1/integrity.ts +127 -0
- package/modules/import-kit/src/d1/inventory-snapshot.ts +84 -0
- package/modules/import-kit/src/d1/jobs.ts +97 -0
- package/modules/import-kit/src/d1/native-set.ts +48 -0
- package/modules/import-kit/src/d1/native-staging-admission.ts +19 -0
- package/modules/import-kit/src/d1/native-staging-context.ts +44 -0
- package/modules/import-kit/src/d1/native-staging-lifecycle.ts +33 -0
- package/modules/import-kit/src/d1/native-staging-offset.ts +74 -0
- package/modules/import-kit/src/d1/native-staging-publication.ts +18 -0
- package/modules/import-kit/src/d1/native-staging-rows.ts +72 -0
- package/modules/import-kit/src/d1/native-staging-schema.ts +28 -0
- package/modules/import-kit/src/d1/native-staging-state.ts +22 -0
- package/modules/import-kit/src/d1/native-staging.ts +64 -0
- package/modules/import-kit/src/d1/original-copies.ts +53 -0
- package/modules/import-kit/src/d1/original-storage.ts +56 -0
- package/modules/import-kit/src/d1/originals.ts +74 -0
- package/modules/import-kit/src/d1/plan-dictionary.ts +80 -0
- package/modules/import-kit/src/d1/plans.ts +121 -0
- package/modules/import-kit/src/d1/preparation-fingerprint.ts +57 -0
- package/modules/import-kit/src/d1/preparation-header.ts +33 -0
- package/modules/import-kit/src/d1/preparation-reader.ts +98 -0
- package/modules/import-kit/src/d1/preparation-schema.ts +16 -0
- package/modules/import-kit/src/d1/preparation-state.ts +56 -0
- package/modules/import-kit/src/d1/preparation-step-combine.ts +80 -0
- package/modules/import-kit/src/d1/preparation-step-control.ts +48 -0
- package/modules/import-kit/src/d1/preparation-step-hash.ts +131 -0
- package/modules/import-kit/src/d1/preparation-step-pages.ts +77 -0
- package/modules/import-kit/src/d1/preparation-steps-schema.ts +11 -0
- package/modules/import-kit/src/d1/preparation-steps.ts +102 -0
- package/modules/import-kit/src/d1/preparations.ts +87 -0
- package/modules/import-kit/src/d1/preserved-preview.ts +89 -0
- package/modules/import-kit/src/d1/preserved-reverse.ts +91 -0
- package/modules/import-kit/src/d1/provenance-set.ts +87 -0
- package/modules/import-kit/src/d1/publication-fence.ts +22 -0
- package/modules/import-kit/src/d1/publication-receipt-schema.ts +15 -0
- package/modules/import-kit/src/d1/published-stream.ts +11 -0
- package/modules/import-kit/src/d1/read-fence.ts +64 -0
- package/modules/import-kit/src/d1/read-results.ts +16 -0
- package/modules/import-kit/src/d1/redacted-custody.ts +76 -0
- package/modules/import-kit/src/d1/redaction-step.ts +125 -0
- package/modules/import-kit/src/d1/referenced-plans.ts +136 -0
- package/modules/import-kit/src/d1/retained-values.ts +35 -0
- package/modules/import-kit/src/d1/retirement-bounded.ts +71 -0
- package/modules/import-kit/src/d1/retirement-door.ts +105 -0
- package/modules/import-kit/src/d1/retirement-schema.ts +39 -0
- package/modules/import-kit/src/d1/retirement-scope.ts +94 -0
- package/modules/import-kit/src/d1/retirement-steps.ts +76 -0
- package/modules/import-kit/src/d1/retirement-tally.ts +74 -0
- package/modules/import-kit/src/d1/retirement.ts +136 -0
- package/modules/import-kit/src/d1/reversal-conversion-record.ts +19 -0
- package/modules/import-kit/src/d1/reversal-conversion-schema.ts +11 -0
- package/modules/import-kit/src/d1/reversal-conversion.ts +40 -0
- package/modules/import-kit/src/d1/reversal-fence-schema.ts +14 -0
- package/modules/import-kit/src/d1/reversal-fence.ts +106 -0
- package/modules/import-kit/src/d1/reversal-journal.ts +24 -0
- package/modules/import-kit/src/d1/reverse.ts +103 -0
- package/modules/import-kit/src/d1/reviewed-source.ts +23 -0
- package/modules/import-kit/src/d1/sets.ts +78 -0
- package/modules/import-kit/src/d1/staged-creation.ts +201 -0
- package/modules/import-kit/src/d1/staging-admission.ts +45 -0
- package/modules/import-kit/src/d1/staging-control.ts +43 -0
- package/modules/import-kit/src/d1/staging-erasure.ts +28 -0
- package/modules/import-kit/src/d1/staging-lifecycle.ts +49 -0
- package/modules/import-kit/src/d1/staging-plan.ts +15 -0
- package/modules/import-kit/src/d1/staging-privacy.ts +22 -0
- package/modules/import-kit/src/d1/staging-publication.ts +79 -0
- package/modules/import-kit/src/d1/staging-schema.ts +37 -0
- package/modules/import-kit/src/d1/staging-statements.ts +8 -0
- package/modules/import-kit/src/d1/staging-transport.ts +70 -0
- package/modules/import-kit/src/d1/stream-boundary.ts +11 -0
- package/modules/import-kit/src/d1/stream-create.ts +55 -0
- package/modules/import-kit/src/d1/stream-identity-admission.ts +37 -0
- package/modules/import-kit/src/d1/stream-input.ts +18 -0
- package/modules/import-kit/src/d1/stream-layout.ts +43 -0
- package/modules/import-kit/src/d1/stream-origin.ts +44 -0
- package/modules/import-kit/src/d1/stream-page-integrity.ts +18 -0
- package/modules/import-kit/src/d1/stream-partition.ts +50 -0
- package/modules/import-kit/src/d1/stream-plan.ts +133 -0
- package/modules/import-kit/src/d1/stream-receipt.ts +45 -0
- package/modules/import-kit/src/d1/stream-revision-schema.ts +35 -0
- package/modules/import-kit/src/d1/stream-snapshot.ts +26 -0
- package/modules/import-kit/src/d1/stream-stage.ts +79 -0
- package/modules/import-kit/src/d1/stream-state.ts +25 -0
- package/modules/import-kit/src/d1/stream-working-set.ts +30 -0
- package/modules/import-kit/src/d1/streams.ts +39 -0
- package/modules/import-kit/src/d1/target-guards.ts +9 -0
- package/modules/import-kit/src/d1/target-identities.ts +17 -0
- package/modules/import-kit/src/d1/transport.ts +10 -0
- package/modules/import-kit/src/d1/types.ts +162 -0
- package/modules/import-kit/src/d1/undo-observations.ts +39 -0
- package/modules/import-kit/src/d1/undo-report.ts +22 -0
- package/modules/import-kit/src/d1/undo-snapshot.ts +87 -0
- package/modules/import-kit/src/d1/undo-state.ts +84 -0
- package/modules/import-kit/src/d1/undo-types.ts +110 -0
- package/modules/import-kit/src/journal/authority.ts +12 -0
- package/modules/import-kit/src/plan/budget.ts +60 -0
- package/modules/import-kit/src/plan/index.ts +123 -0
- package/modules/import-kit/src/plan/size.ts +122 -0
- package/modules/import-kit/src/provenance/canonical-hash.ts +130 -0
- package/modules/import-kit/src/provenance/canonical-stream.ts +85 -0
- package/modules/import-kit/src/provenance/credential-names.ts +131 -0
- package/modules/import-kit/src/provenance/credentials.ts +317 -0
- package/modules/import-kit/src/provenance/identity.ts +51 -0
- package/modules/import-kit/src/provenance/json-snapshot.ts +21 -0
- package/modules/import-kit/src/provenance/resumable-sha256.ts +75 -0
- package/modules/import-kit/src/provenance/validate-json.ts +19 -0
- package/modules/import-kit/src/storage/originals.ts +87 -0
- package/modules/import-kit/src/storage/provenance.ts +64 -0
- package/modules/import-kit/src/storage/schema.ts +42 -0
- package/modules/import-kit/src/types.ts +47 -0
- package/modules/portability/README.md +98 -2
- package/modules/portability/backfills.mjs +42 -15
- package/modules/portability/ledgers.mjs +7 -1
- package/modules/portability/transient-tables.mjs +40 -0
- package/package.json +3 -1
- package/scripts/publish-check/reviewed.py +3 -1
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import { Buffer } from 'node:buffer';
|
|
2
|
+
/** SHA-256 (FIPS 180-4) whose running state can be saved and restored, so one
|
|
3
|
+
* digest can span several requests. Node's hash cannot export its state. The
|
|
4
|
+
* saved state is the eight chaining words, the unprocessed tail (< 64 bytes)
|
|
5
|
+
* and the total length: it reveals no more of the input than that tail. */
|
|
6
|
+
const K = new Uint32Array([
|
|
7
|
+
0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174,
|
|
8
|
+
0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967,
|
|
9
|
+
0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070,
|
|
10
|
+
0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2]);
|
|
11
|
+
const INITIAL = [0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19];
|
|
12
|
+
|
|
13
|
+
export interface Sha256State { h: number[]; tail: string; length: number }
|
|
14
|
+
|
|
15
|
+
export class ResumableSha256 {
|
|
16
|
+
private h = new Uint32Array(INITIAL);
|
|
17
|
+
private block = new Uint8Array(64);
|
|
18
|
+
private used = 0;
|
|
19
|
+
private length = 0;
|
|
20
|
+
private w = new Uint32Array(64);
|
|
21
|
+
|
|
22
|
+
static from(state: Sha256State): ResumableSha256 {
|
|
23
|
+
const tail = Buffer.from(state.tail, 'hex');
|
|
24
|
+
if (!Array.isArray(state.h) || state.h.length !== 8 || !state.h.every(word => Number.isInteger(word) && word >= 0 && word <= 0xffffffff)
|
|
25
|
+
|| tail.length >= 64 || tail.toString('hex') !== state.tail || !Number.isSafeInteger(state.length) || state.length % 64 !== tail.length) throw new Error('IMPORT_HASH_STATE_INVALID');
|
|
26
|
+
const hash = new ResumableSha256();
|
|
27
|
+
hash.h.set(state.h); hash.block.set(tail); hash.used = tail.length; hash.length = state.length;
|
|
28
|
+
return hash;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
state(): Sha256State {
|
|
32
|
+
return { h: [...this.h], tail: Buffer.from(this.block.subarray(0, this.used)).toString('hex'), length: this.length };
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
update(bytes: Uint8Array): this {
|
|
36
|
+
let offset = 0;
|
|
37
|
+
this.length += bytes.length;
|
|
38
|
+
if (this.used) {
|
|
39
|
+
const take = Math.min(64 - this.used, bytes.length);
|
|
40
|
+
this.block.set(bytes.subarray(0, take), this.used); this.used += take; offset = take;
|
|
41
|
+
if (this.used < 64) return this;
|
|
42
|
+
this.compress(this.block, 0); this.used = 0;
|
|
43
|
+
}
|
|
44
|
+
for (; offset + 64 <= bytes.length; offset += 64) this.compress(bytes, offset);
|
|
45
|
+
if (offset < bytes.length) { this.block.set(bytes.subarray(offset), 0); this.used = bytes.length - offset; }
|
|
46
|
+
return this;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
digest(encoding: 'hex'): string {
|
|
50
|
+
if (encoding !== 'hex') throw new Error('IMPORT_HASH_ENCODING');
|
|
51
|
+
const bits = this.length * 8, padding = new Uint8Array((this.used < 56 ? 56 : 120) - this.used + 8);
|
|
52
|
+
padding[0] = 0x80;
|
|
53
|
+
const view = new DataView(padding.buffer);
|
|
54
|
+
view.setUint32(padding.length - 8, Math.floor(bits / 0x100000000)); view.setUint32(padding.length - 4, bits >>> 0);
|
|
55
|
+
const length = this.length;
|
|
56
|
+
this.update(padding); this.length = length;
|
|
57
|
+
return Buffer.from(new Uint8Array(this.h.buffer.slice(0))).swap32().toString('hex');
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
private compress(bytes: Uint8Array, offset: number): void {
|
|
61
|
+
const w = this.w, h = this.h;
|
|
62
|
+
for (let i = 0; i < 16; i++) { const j = offset + i * 4; w[i] = (bytes[j] << 24) | (bytes[j + 1] << 16) | (bytes[j + 2] << 8) | bytes[j + 3]; }
|
|
63
|
+
for (let i = 16; i < 64; i++) {
|
|
64
|
+
const a = w[i - 15], b = w[i - 2];
|
|
65
|
+
w[i] = (((a >>> 7) | (a << 25)) ^ ((a >>> 18) | (a << 14)) ^ (a >>> 3)) + w[i - 7] + (((b >>> 17) | (b << 15)) ^ ((b >>> 19) | (b << 13)) ^ (b >>> 10)) + w[i - 16];
|
|
66
|
+
}
|
|
67
|
+
let a = h[0], b = h[1], c = h[2], d = h[3], e = h[4], f = h[5], g = h[6], k = h[7];
|
|
68
|
+
for (let i = 0; i < 64; i++) {
|
|
69
|
+
const t1 = (k + (((e >>> 6) | (e << 26)) ^ ((e >>> 11) | (e << 21)) ^ ((e >>> 25) | (e << 7))) + ((e & f) ^ (~e & g)) + K[i] + w[i]) | 0;
|
|
70
|
+
const t2 = ((((a >>> 2) | (a << 30)) ^ ((a >>> 13) | (a << 19)) ^ ((a >>> 22) | (a << 10))) + ((a & b) ^ (a & c) ^ (b & c))) | 0;
|
|
71
|
+
k = g; g = f; f = e; e = (d + t1) | 0; d = c; c = b; b = a; a = (t1 + t2) | 0;
|
|
72
|
+
}
|
|
73
|
+
h[0] += a; h[1] += b; h[2] += c; h[3] += d; h[4] += e; h[5] += f; h[6] += g; h[7] += k;
|
|
74
|
+
}
|
|
75
|
+
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/** Validate finite JSON in canonical read order without allocating text.
|
|
2
|
+
* Cycles and excessive nesting fail before recursive encoding can overflow. */
|
|
3
|
+
export function validateJson(value: unknown): void {
|
|
4
|
+
const active = new Set<object>();
|
|
5
|
+
const visit = (item: unknown, depth: number): void => {
|
|
6
|
+
if (item === null || typeof item === 'string' || typeof item === 'boolean' || typeof item === 'number' && Number.isFinite(item)) return;
|
|
7
|
+
if (item && typeof item === 'object' && (Array.isArray(item) || Object.getPrototypeOf(item) === Object.prototype || Object.getPrototypeOf(item) === null)) {
|
|
8
|
+
if (depth >= 256 || active.has(item)) throw new Error('INVALID_JSON: cyclic or excessively nested durable value');
|
|
9
|
+
active.add(item);
|
|
10
|
+
if (Array.isArray(item)) {
|
|
11
|
+
const length = item.length;
|
|
12
|
+
for (let index = 0; index < length; index++) visit(item[index], depth + 1);
|
|
13
|
+
} else for (const key of Object.keys(item).sort()) visit((item as Record<string, unknown>)[key], depth + 1);
|
|
14
|
+
active.delete(item); return;
|
|
15
|
+
}
|
|
16
|
+
throw new Error('INVALID_JSON: durable values must be finite JSON');
|
|
17
|
+
};
|
|
18
|
+
visit(value, 0);
|
|
19
|
+
}
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
import { Buffer } from 'node:buffer';
|
|
2
|
+
import type { ImportPlan, PaginationCursorPolicy, RetainedOriginal, SourceArtifact, SqlDatabase } from '../types.ts';
|
|
3
|
+
import { attachmentHash, canonicalJson, assertNoCredentials, sha256 } from '../provenance/identity.ts';
|
|
4
|
+
|
|
5
|
+
/** Inspect JSON envelopes before retaining bytes; reader format-specific checks still apply. */
|
|
6
|
+
function inspectOriginal(bytes: Uint8Array, policy?: PaginationCursorPolicy): void {
|
|
7
|
+
let text: string;
|
|
8
|
+
try { text = new TextDecoder('utf-8', { fatal: true }).decode(bytes); } catch { return; }
|
|
9
|
+
let value: unknown;
|
|
10
|
+
try { value = JSON.parse(text); } catch {
|
|
11
|
+
// An NDJSON original can contain credential metadata outside mapped records.
|
|
12
|
+
for (const line of text.split(/\r?\n/)) {
|
|
13
|
+
try { value = JSON.parse(line); } catch { continue; }
|
|
14
|
+
assertNoCredentials(value, policy);
|
|
15
|
+
}
|
|
16
|
+
return;
|
|
17
|
+
}
|
|
18
|
+
assertNoCredentials(value, policy);
|
|
19
|
+
}
|
|
20
|
+
/** Original bytes already decoded, hashed and inspected, by artifact: the check
|
|
21
|
+
* is a pure function of these exact values, so an unchanged artifact is not
|
|
22
|
+
* decoded and scanned again each time its custody is guarded. */
|
|
23
|
+
const inspected = new WeakMap<SourceArtifact, { base64: string; size: number; sha256: string; policy?: PaginationCursorPolicy }>();
|
|
24
|
+
export function validateArtifactOriginal(artifact: SourceArtifact, policy?: PaginationCursorPolicy): void {
|
|
25
|
+
if (!artifact.original) return;
|
|
26
|
+
if (artifact.original.status === 'unavailable') {
|
|
27
|
+
if (artifact.original.reason === 'unavailable-after-erasure') {
|
|
28
|
+
if (!['withdraw-affected-artifacts', 'withdraw-all-job-artifacts'].includes(artifact.original.policy) || Object.keys(artifact.original).some(k => !['status', 'reason', 'policy'].includes(k))) throw new Error('INVALID_ORIGINAL');
|
|
29
|
+
} else if (!['not-supplied', 'credentials-removed'].includes(artifact.original.reason) || Object.keys(artifact.original).some(k => !['status', 'reason'].includes(k))) throw new Error('INVALID_ORIGINAL');
|
|
30
|
+
return;
|
|
31
|
+
}
|
|
32
|
+
if (artifact.original.status !== 'available' || typeof artifact.original.base64 !== 'string' || Object.keys(artifact.original).some(k => !['status', 'base64'].includes(k))) throw new Error('INVALID_ORIGINAL');
|
|
33
|
+
const seen = inspected.get(artifact), base64 = artifact.original.base64;
|
|
34
|
+
if (seen && seen.base64 === base64 && seen.size === artifact.size && seen.sha256 === artifact.sha256 && seen.policy === policy) return;
|
|
35
|
+
const bytes = Buffer.from(base64, 'base64');
|
|
36
|
+
if (bytes.toString('base64') !== base64 || bytes.length !== artifact.size || sha256(bytes) !== artifact.sha256) throw new Error('ORIGINAL_HASH_SIZE_MISMATCH');
|
|
37
|
+
inspectOriginal(bytes, policy);
|
|
38
|
+
inspected.set(artifact, { base64, size: artifact.size, sha256: artifact.sha256, policy });
|
|
39
|
+
}
|
|
40
|
+
export function originalsFor(plan: ImportPlan, jobId: string): RetainedOriginal[] {
|
|
41
|
+
const result: RetainedOriginal[] = [], seen = new Set<string>();
|
|
42
|
+
for (const artifact of plan.source.artifacts) {
|
|
43
|
+
validateArtifactOriginal(artifact, plan.source.paginationCursorPolicy);
|
|
44
|
+
if (artifact.original?.status === 'unavailable' && artifact.original.reason === 'unavailable-after-erasure' && plan.erasure?.originalPolicy !== artifact.original.policy) throw new Error('ORIGINAL_ERASURE_POLICY_MISMATCH');
|
|
45
|
+
const key = canonicalJson([artifact.name, artifact.sha256]);
|
|
46
|
+
if (seen.has(key)) throw new Error('DUPLICATE_ARTIFACT');
|
|
47
|
+
seen.add(key);
|
|
48
|
+
result.push({ jobId, kind: 'artifact', key, name: artifact.name, sha256: artifact.sha256, size: artifact.size,
|
|
49
|
+
original: artifact.original?.status === 'available' ? { status: 'retained', base64: artifact.original.base64 } : artifact.original ?? { status: 'unavailable', reason: 'not-supplied' } });
|
|
50
|
+
}
|
|
51
|
+
for (const entry of plan.entries) for (const [index, effect] of entry.effects.entries()) if (effect.attachment) {
|
|
52
|
+
attachmentHash(effect.attachment);
|
|
53
|
+
const bytes = Buffer.from(effect.attachment.base64, 'base64');
|
|
54
|
+
result.push({ jobId, kind: 'attachment', key: canonicalJson([entry.identity, index]), name: canonicalJson([effect.targetKind, effect.targetId]), sha256: effect.attachment.sha256, size: bytes.length, original: { status: 'retained', base64: effect.attachment.base64 } });
|
|
55
|
+
}
|
|
56
|
+
return result;
|
|
57
|
+
}
|
|
58
|
+
/** Caller owns the same native transaction that inserts the job. */
|
|
59
|
+
export function retainImportOriginals(db: SqlDatabase, plan: ImportPlan, jobId: string): void {
|
|
60
|
+
const originals = originalsFor(plan, jobId);
|
|
61
|
+
db.prepare('INSERT INTO ik_original_sets(job_id,version) VALUES(?,1)').run(jobId);
|
|
62
|
+
for (const original of originals) db.prepare('INSERT INTO ik_originals(job_id,kind,original_key,name,sha256,size,bytes) VALUES(?,?,?,?,?,?,?)').run(jobId, original.kind, original.key, original.name, original.sha256, original.size, original.original.status === 'retained' ? Buffer.from(original.original.base64, 'base64') : null);
|
|
63
|
+
}
|
|
64
|
+
export function readStoredOriginals(db: SqlDatabase, plan: ImportPlan, jobId: string): RetainedOriginal[] {
|
|
65
|
+
const expected = originalsFor(plan, jobId);
|
|
66
|
+
const version = db.prepare('SELECT version FROM ik_original_sets WHERE job_id=?').get(jobId);
|
|
67
|
+
const rows = db.prepare('SELECT * FROM ik_originals WHERE job_id=? ORDER BY kind,original_key').all(jobId);
|
|
68
|
+
if (!version) {
|
|
69
|
+
// Pre-custody jobs really retain mapped attachment bytes in their plan, but
|
|
70
|
+
// have no file/page originals. Do not reconstruct those from parsed records.
|
|
71
|
+
if (rows.length || plan.source.artifacts.some(a => a.original !== undefined)) throw new Error('ORIGINAL_CUSTODY_MISSING');
|
|
72
|
+
return expected;
|
|
73
|
+
}
|
|
74
|
+
if (version.version !== 1 || rows.length !== expected.length) throw new Error('ORIGINAL_CUSTODY_MISMATCH');
|
|
75
|
+
return expected.map(original => {
|
|
76
|
+
const row = rows.find(r => r.kind === original.kind && r.original_key === original.key);
|
|
77
|
+
if (!row || row.name !== original.name || row.sha256 !== original.sha256 || row.size !== original.size) throw new Error('ORIGINAL_CUSTODY_MISMATCH');
|
|
78
|
+
if (original.original.status === 'unavailable') {
|
|
79
|
+
if (row.bytes !== null) throw new Error('ORIGINAL_CUSTODY_MISMATCH');
|
|
80
|
+
return original;
|
|
81
|
+
}
|
|
82
|
+
if (row.bytes === null) throw new Error('ORIGINAL_CUSTODY_MISSING');
|
|
83
|
+
const bytes = Buffer.from(row.bytes);
|
|
84
|
+
if (bytes.length !== row.size || sha256(bytes) !== row.sha256 || bytes.toString('base64') !== original.original.base64) throw new Error('ORIGINAL_CUSTODY_MISMATCH');
|
|
85
|
+
return { ...original, original: { status: 'retained' as const, base64: bytes.toString('base64') } };
|
|
86
|
+
});
|
|
87
|
+
}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import type { ImportPlan, ProvenanceRow, Reconciliation, SqlDatabase } from '../types.ts';
|
|
2
|
+
import { canonicalJson, sha256, attachmentHash, sourceIdentity, targetIdentity } from '../provenance/identity.ts';
|
|
3
|
+
import { planFingerprint } from '../plan/index.ts';
|
|
4
|
+
|
|
5
|
+
export type Integrity = Omit<Reconciliation, 'ok'>;
|
|
6
|
+
export const emptyIntegrity = (): Integrity => ({ missing: [], duplicate: [], extra: [], mismatched: [] });
|
|
7
|
+
export const effectIdentity = (job: string, identity: string, index: number): string => sha256(canonicalJson([job, identity, index]));
|
|
8
|
+
export interface DurableProvenance { sql: any; record?: ProvenanceRow }
|
|
9
|
+
export function decodeProvenance(row: any): ProvenanceRow {
|
|
10
|
+
const p = JSON.parse(row.record_json) as ProvenanceRow;
|
|
11
|
+
if (!p || typeof p !== 'object' || p.id !== row.id || p.jobId !== row.job_id || p.sourceIdentity !== row.source_identity || p.destination !== row.destination || p.targetKind !== row.target_kind || p.targetId !== row.target_id || typeof p.reversed !== 'boolean' || Number(p.reversed) !== row.reversed) throw new Error('PROVENANCE_STORAGE_MISMATCH');
|
|
12
|
+
return p;
|
|
13
|
+
}
|
|
14
|
+
export interface OwnerPlan { job: any; plan: ImportPlan; effectsHashes: Map<number, string> }
|
|
15
|
+
/** Cache is confined to one immutable transaction snapshot. */
|
|
16
|
+
export function ownerPlan(db: SqlDatabase, id: string, cache: Map<string, OwnerPlan>): OwnerPlan {
|
|
17
|
+
const prior = cache.get(id); if (prior) return prior;
|
|
18
|
+
const job = db.prepare('SELECT * FROM ik_jobs WHERE id=?').get(id);
|
|
19
|
+
if (!job) throw new Error('PROVENANCE_JOB_MISSING');
|
|
20
|
+
const plan = JSON.parse(job.plan_json) as ImportPlan;
|
|
21
|
+
if (planFingerprint(plan) !== job.fingerprint || plan.fingerprint !== job.fingerprint || plan.destination !== job.destination || plan.namespace !== job.namespace || plan.mappingVersion !== job.mapping_version || plan.principalId !== job.principal_id || plan.version !== 1 || job.total !== plan.entries.length || !Number.isSafeInteger(job.checkpoint) || job.checkpoint < 0 || job.checkpoint > job.total) throw new Error('PROVENANCE_JOB_MISMATCH');
|
|
22
|
+
const result = { job, plan, effectsHashes: new Map<number, string>() }; cache.set(id, result); return result;
|
|
23
|
+
}
|
|
24
|
+
/** Source-only entries have the same publishing authority as entries with effects. */
|
|
25
|
+
export function validateSourceClaim(db: SqlDatabase, claim: any, cache = new Map<string, OwnerPlan>()): void {
|
|
26
|
+
const { job, plan } = ownerPlan(db, claim.job_id, cache);
|
|
27
|
+
const ordinal = plan.entries.findIndex(entry => entry.identity === claim.identity), entry = plan.entries[ordinal];
|
|
28
|
+
if (!entry || ordinal >= job.checkpoint || ['undone', 'rolled_back'].includes(job.state) || entry.identity !== sourceIdentity(job.destination, job.namespace, entry.source.kind, entry.source.sourceId) || entry.source.namespace !== job.namespace || sha256(canonicalJson(entry.source.payload)) !== entry.source.payloadHash || claim.source_hash !== entry.source.payloadHash || claim.mapping_version !== plan.mappingVersion || claim.effects_hash !== sha256(canonicalJson(entry.effects))) throw new Error('PROVENANCE_SOURCE_CLAIM_MISMATCH');
|
|
29
|
+
}
|
|
30
|
+
/** Verify a physical provenance record against the plan that actually published it. */
|
|
31
|
+
export function validateProvenance(db: SqlDatabase, row: any, cache = new Map<string, OwnerPlan>()): ProvenanceRow {
|
|
32
|
+
const p = decodeProvenance(row);
|
|
33
|
+
const { job, plan, effectsHashes } = ownerPlan(db, row.job_id, cache);
|
|
34
|
+
const entry = plan.entries[row.ordinal], effect = entry?.effects[row.effect_ordinal];
|
|
35
|
+
if (!Number.isSafeInteger(row.ordinal) || !Number.isSafeInteger(row.effect_ordinal) || !entry || !effect || row.ordinal >= job.checkpoint) throw new Error('PROVENANCE_POSITION_MISMATCH');
|
|
36
|
+
const source = entry.source;
|
|
37
|
+
if (entry.identity !== sourceIdentity(job.destination, job.namespace, source.kind, source.sourceId) || source.namespace !== job.namespace || sha256(canonicalJson(source.payload)) !== source.payloadHash) throw new Error('PROVENANCE_SOURCE_MISMATCH');
|
|
38
|
+
const claim = db.prepare('SELECT * FROM ik_sources WHERE identity=?').get(entry.identity);
|
|
39
|
+
if (!effectsHashes.has(row.ordinal)) effectsHashes.set(row.ordinal, sha256(canonicalJson(entry.effects)));
|
|
40
|
+
if (!p.reversed && (!claim || claim.job_id !== job.id || claim.source_hash !== source.payloadHash || claim.mapping_version !== job.mapping_version || claim.effects_hash !== effectsHashes.get(row.ordinal))) throw new Error('PROVENANCE_SOURCE_CLAIM_MISMATCH');
|
|
41
|
+
if (p.id !== effectIdentity(job.id, entry.identity, row.effect_ordinal) || p.sourceIdentity !== entry.identity || p.sourceKind !== source.kind || p.sourceId !== source.sourceId || p.sourceHash !== source.payloadHash || p.artifactHash !== source.artifactHash || canonicalJson(p.location) !== canonicalJson(source.location) || p.destination !== job.destination || p.targetKind !== effect.targetKind || p.targetId !== effect.targetId || p.valueHash !== sha256(canonicalJson(effect.value)) || p.attachmentHash !== attachmentHash(effect.attachment) || p.appliedRevision !== sha256(canonicalJson([job.id, entry.identity, row.effect_ordinal, 'applied']))) throw new Error('PROVENANCE_ATTRIBUTION_MISMATCH');
|
|
42
|
+
if (p.before === undefined || (p.before !== null && typeof p.before !== 'object')) throw new Error('PROVENANCE_BEFORE_IMAGE_MISMATCH');
|
|
43
|
+
if (p.before ? p.before.destination !== p.destination || p.before.targetKind !== p.targetKind || p.before.targetId !== p.targetId || p.before.revision !== effect.expectedRevision : effect.expectedRevision !== undefined) throw new Error('PROVENANCE_BEFORE_IMAGE_MISMATCH');
|
|
44
|
+
if (!plan.source.artifacts.some(a => a.name === source.location.artifact && a.sha256 === source.artifactHash)) throw new Error('PROVENANCE_ARTIFACT_MISMATCH');
|
|
45
|
+
return p;
|
|
46
|
+
}
|
|
47
|
+
export function inspectDurableProvenance(db: SqlDatabase, destination: string): { entries: DurableProvenance[]; integrity: Integrity } {
|
|
48
|
+
const integrity = emptyIntegrity(), plans = new Map<string, OwnerPlan>();
|
|
49
|
+
// Include rows whose SQL destination was corrupted away from their owning job.
|
|
50
|
+
const entries = db.prepare('SELECT * FROM ik_provenance WHERE reversed=0 AND (destination=? OR job_id IN (SELECT id FROM ik_jobs WHERE destination=?)) ORDER BY id').all(destination, destination).map(sql => {
|
|
51
|
+
let record: ProvenanceRow | undefined;
|
|
52
|
+
try { const parsed = JSON.parse(sql.record_json); if (parsed && typeof parsed === 'object' && ['id', 'jobId', 'destination', 'targetKind', 'targetId'].every(key => typeof parsed[key] === 'string') && typeof parsed.reversed === 'boolean') record = parsed; } catch { /* reported below */ }
|
|
53
|
+
try { validateProvenance(db, sql, plans); } catch { integrity.mismatched.push('provenance-storage:' + sql.id); }
|
|
54
|
+
return { sql, record };
|
|
55
|
+
});
|
|
56
|
+
const ids = new Map<string, string>();
|
|
57
|
+
for (const { sql, record } of entries) {
|
|
58
|
+
if (!record || typeof record.id !== 'string') continue;
|
|
59
|
+
const key = targetIdentity(sql.destination, sql.target_kind, sql.target_id), prior = ids.get(record.id);
|
|
60
|
+
if (prior !== undefined) integrity.duplicate.push('provenance:' + key);
|
|
61
|
+
ids.set(record.id, key);
|
|
62
|
+
}
|
|
63
|
+
return { entries, integrity };
|
|
64
|
+
}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
/** Private module schema; consumers number native migrations at integration. */
|
|
2
|
+
export const schema = `
|
|
3
|
+
CREATE TABLE IF NOT EXISTS ik_jobs (
|
|
4
|
+
id TEXT PRIMARY KEY, destination TEXT NOT NULL, namespace TEXT NOT NULL,
|
|
5
|
+
mapping_version TEXT NOT NULL, principal_id TEXT NOT NULL, fingerprint TEXT NOT NULL,
|
|
6
|
+
operation_key TEXT NOT NULL, plan_json TEXT NOT NULL, state TEXT NOT NULL,
|
|
7
|
+
checkpoint INTEGER NOT NULL DEFAULT 0, total INTEGER NOT NULL, fence INTEGER NOT NULL DEFAULT 0,
|
|
8
|
+
lease_token TEXT, lease_expires INTEGER, error TEXT,
|
|
9
|
+
UNIQUE(destination, principal_id, operation_key)
|
|
10
|
+
);
|
|
11
|
+
CREATE TABLE IF NOT EXISTS ik_original_sets (
|
|
12
|
+
job_id TEXT PRIMARY KEY REFERENCES ik_jobs(id), version INTEGER NOT NULL
|
|
13
|
+
);
|
|
14
|
+
CREATE TABLE IF NOT EXISTS ik_originals (
|
|
15
|
+
job_id TEXT NOT NULL REFERENCES ik_jobs(id), kind TEXT NOT NULL, original_key TEXT NOT NULL,
|
|
16
|
+
name TEXT NOT NULL, sha256 TEXT NOT NULL, size INTEGER NOT NULL, bytes BLOB,
|
|
17
|
+
PRIMARY KEY(job_id,kind,original_key)
|
|
18
|
+
);
|
|
19
|
+
CREATE TABLE IF NOT EXISTS ik_sources (
|
|
20
|
+
identity TEXT PRIMARY KEY, source_hash TEXT NOT NULL, mapping_version TEXT NOT NULL,
|
|
21
|
+
effects_hash TEXT NOT NULL, job_id TEXT NOT NULL REFERENCES ik_jobs(id)
|
|
22
|
+
);
|
|
23
|
+
CREATE TABLE IF NOT EXISTS ik_provenance (
|
|
24
|
+
id TEXT PRIMARY KEY, job_id TEXT NOT NULL REFERENCES ik_jobs(id), ordinal INTEGER NOT NULL, effect_ordinal INTEGER NOT NULL,
|
|
25
|
+
source_identity TEXT NOT NULL, destination TEXT NOT NULL, target_kind TEXT NOT NULL,
|
|
26
|
+
target_id TEXT NOT NULL, reversed INTEGER NOT NULL DEFAULT 0, record_json TEXT NOT NULL
|
|
27
|
+
);
|
|
28
|
+
CREATE UNIQUE INDEX IF NOT EXISTS ik_active_target ON ik_provenance(destination,target_kind,target_id) WHERE reversed=0;
|
|
29
|
+
CREATE TABLE IF NOT EXISTS ik_stages (
|
|
30
|
+
job_id TEXT NOT NULL REFERENCES ik_jobs(id), fence INTEGER NOT NULL, effect_id TEXT NOT NULL,
|
|
31
|
+
sha256 TEXT NOT NULL, bytes BLOB NOT NULL, PRIMARY KEY(job_id,fence,effect_id)
|
|
32
|
+
);
|
|
33
|
+
CREATE TABLE IF NOT EXISTS ik_batches (
|
|
34
|
+
job_id TEXT NOT NULL REFERENCES ik_jobs(id), action TEXT NOT NULL, position INTEGER NOT NULL,
|
|
35
|
+
end_position INTEGER NOT NULL, fence INTEGER NOT NULL, PRIMARY KEY(job_id,action,position)
|
|
36
|
+
);
|
|
37
|
+
CREATE TABLE IF NOT EXISTS ik_targets (
|
|
38
|
+
destination TEXT NOT NULL, kind TEXT NOT NULL, id TEXT NOT NULL,
|
|
39
|
+
revision TEXT NOT NULL, value_json TEXT NOT NULL, attachment_json TEXT,
|
|
40
|
+
PRIMARY KEY(destination,kind,id)
|
|
41
|
+
);
|
|
42
|
+
`;
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
/** All durable values are JSON; credentials belong only to reader call arguments. */
|
|
2
|
+
export type Json = null | boolean | number | string | Json[] | { [key: string]: Json };
|
|
3
|
+
export interface SourceLocation { artifact: string; row?: number; page?: number; pointer?: string }
|
|
4
|
+
export interface Diagnostic { severity: 'warning' | 'error'; code: string; message: string; kind?: string; sourceId?: string; location?: SourceLocation; field?: string }
|
|
5
|
+
export interface SourceRecord { namespace: string; kind: string; sourceId: string; payload: Json; payloadHash: string; artifactHash: string; location: SourceLocation }
|
|
6
|
+
/** Exact input bytes, never reserialized JSON. Omission is legacy metadata-only input. */
|
|
7
|
+
export type OriginalErasurePolicy = 'withdraw-affected-artifacts' | 'withdraw-all-job-artifacts';
|
|
8
|
+
export type UnavailableOriginal = { status: 'unavailable'; reason: 'not-supplied' } | { status: 'unavailable'; reason: 'credentials-removed' } | { status: 'unavailable'; reason: 'unavailable-after-erasure'; policy: OriginalErasurePolicy };
|
|
9
|
+
export type ArtifactOriginal = { status: 'available'; base64: string } | UnavailableOriginal;
|
|
10
|
+
export interface SourceArtifact { name: string; sha256: string; size: number; original?: ArtifactOriginal }
|
|
11
|
+
/** Durable, job-bound custody readback. No reference/URL can claim byte retention. */
|
|
12
|
+
export interface RetainedOriginal { jobId: string; kind: 'artifact' | 'attachment'; key: string; name: string; sha256: string; size: number; original: { status: 'retained'; base64: string } | UnavailableOriginal }
|
|
13
|
+
/** Exact noncredential pagination query key; strict credential policy when omitted. */
|
|
14
|
+
export type PaginationCursorPolicy = 'page-token-v1';
|
|
15
|
+
export interface SourceBundle { paginationCursorPolicy?: PaginationCursorPolicy; records: SourceRecord[]; artifacts: SourceArtifact[]; diagnostics: Diagnostic[] }
|
|
16
|
+
export interface ReaderLimits { maxBytes?: number; maxRecords?: number; maxRecordBytes?: number }
|
|
17
|
+
export interface ParsedRecord { kind: string; sourceId: string; payload: Json; pointer?: string }
|
|
18
|
+
/** 'refuse' (the default) rejects a source holding a credential value; 'remove'
|
|
19
|
+
* drops each plain credential value, keeps the record, reports it per record as
|
|
20
|
+
* a CREDENTIAL_REMOVED diagnostic and withholds the original bytes. */
|
|
21
|
+
export type SourceCredentialPolicy = 'refuse' | 'remove';
|
|
22
|
+
export interface FileSourceInput { paginationCursorPolicy?: PaginationCursorPolicy; credentials?: SourceCredentialPolicy; namespace: string; name: string; bytes: Uint8Array | AsyncIterable<Uint8Array>; format: 'json' | 'ndjson'; parse: (value: Json, index: number) => ParsedRecord | ParsedRecord[] }
|
|
23
|
+
/** Object keys are decoded strings; array indices are numbers. */
|
|
24
|
+
export type JsonPathSegment = string | number;
|
|
25
|
+
export interface ApiPageContext { readonly numericLexemes: Readonly<Record<string, string>> }
|
|
26
|
+
export interface ApiSourceInput { paginationCursorPolicy?: PaginationCursorPolicy; namespace: string; url: string; token?: string; headers?: Record<string, string>; signal?: AbortSignal; maxRequests?: number; maxRetries?: number; maxRetryAfterMs?: number; limits?: ReaderLimits; preserveNumberLexeme?: (pathSegments: readonly JsonPathSegment[]) => boolean; parsePage: (value: Json, page: number, context: ApiPageContext) => { records: ParsedRecord[]; next?: string | null }; sleep?: (ms: number, signal?: AbortSignal) => Promise<void> }
|
|
27
|
+
export type ApiTransport = (request: Request) => Promise<Response>;
|
|
28
|
+
/** Effects are inert retained data, never executable jobs or grants. */
|
|
29
|
+
export interface TargetEffect { targetKind: string; targetId: string; value: Json; expectedRevision?: string; attachment?: { base64: string; sha256: string; mediaType: string } }
|
|
30
|
+
export interface MappingResult { effects: TargetEffect[]; unmapped?: { field?: string; reason: string }[]; rejection?: string }
|
|
31
|
+
export interface ImportMapping { version: string; map: (record: SourceRecord) => MappingResult }
|
|
32
|
+
export interface PlanEntry { source: SourceRecord; identity: string; effects: TargetEffect[]; unmapped: { field?: string; reason: string }[]; rejection?: string; erased?: true; erasedTargetIdentity?: string }
|
|
33
|
+
export interface ImportPlan { version: 1; destination: string; namespace: string; mappingVersion: string; principalId: string; source: SourceBundle; entries: PlanEntry[]; diagnostics: Diagnostic[]; counts: { input: number; unique: number; duplicates: number; mapped: number; unmapped: number; rejected: number; effects: number; byKind: Record<string, number> }; fingerprint: string; erasure?: { reviewedFingerprint: string; originalPolicy?: OriginalErasurePolicy } }
|
|
34
|
+
export type ImportAction = 'plan' | 'create' | 'apply' | 'undo' | 'rollback' | 'read' | 'erase';
|
|
35
|
+
export interface SqlStatement { get(...values: any[]): any; all(...values: any[]): any[]; run(...values: any[]): { changes: number | bigint } }
|
|
36
|
+
export interface SqlDatabase { exec(sql: string): void; prepare(sql: string): SqlStatement }
|
|
37
|
+
export interface TargetRow { destination: string; targetKind: string; targetId: string; value: Json; revision: string; attachment?: { base64: string; sha256: string; mediaType: string } }
|
|
38
|
+
/** Native adapter must use this database ONLY, synchronously, without external effects. */
|
|
39
|
+
export interface TargetAdapter { read(db: SqlDatabase, destination: string, kind: string, id: string): TargetRow | undefined; write(db: SqlDatabase, row: TargetRow): void; remove(db: SqlDatabase, destination: string, kind: string, id: string): void; list(db: SqlDatabase, destination: string): TargetRow[] }
|
|
40
|
+
export interface ImportStorage { database: SqlDatabase; targets: TargetAdapter; transaction<T>(body: () => T): T; close(): void }
|
|
41
|
+
export interface ImportContext { destination: string; namespace: string; mappingVersion: string; authorization: { principalId: string; /** Must synchronously consult current authority, preferably in this database; throw on denial. */ assertAuthorized: (action: ImportAction, destination: string, db: SqlDatabase) => void }; storage: ImportStorage; reviewedFingerprint?: string; operationKey?: string; batchSize?: number; maxBatches?: number; leaseMs?: number }
|
|
42
|
+
export type JobState = 'queued' | 'applying' | 'completed' | 'failed' | 'undoing' | 'undone' | 'rolling_back' | 'rolled_back';
|
|
43
|
+
export interface ImportJob { id: string; fingerprint: string; destination: string; namespace: string; mappingVersion: string; principalId: string; state: JobState; checkpoint: number; total: number; fence: number; error: string | null; erasedRecords?: number }
|
|
44
|
+
export interface ProvenanceRow { id: string; jobId: string; sourceIdentity: string; sourceKind: string; sourceId: string; sourceHash: string; artifactHash: string; location: SourceLocation; destination: string; targetKind: string; targetId: string; valueHash: string; attachmentHash: string | null; appliedRevision: string; before: TargetRow | null; reversed: boolean }
|
|
45
|
+
/** integrity is present only when durable SQL identities/JSON disagree. Independent readbacks may omit it. */
|
|
46
|
+
export interface PersistedImport { rows: TargetRow[]; provenance: ProvenanceRow[]; integrity?: Omit<Reconciliation, 'ok'> }
|
|
47
|
+
export interface Reconciliation { ok: boolean; missing: string[]; duplicate: string[]; extra: string[]; mismatched: string[] }
|
|
@@ -114,8 +114,8 @@ An installation migrated by Platform has `rer_migrations(name, sha256,
|
|
|
114
114
|
applied_at)` instead of Wrangler's ledger. Local transfers match each entry to
|
|
115
115
|
exactly one migration file by SHA-256 and preserve its application order. They
|
|
116
116
|
verify all local files are recorded and skip Wrangler; a pending file refuses
|
|
117
|
-
as `migration_ledger:unapplied
|
|
118
|
-
ambiguous or conflicting entries refuse
|
|
117
|
+
as `migration_ledger:unapplied` until Platform applies it. Unknown,
|
|
118
|
+
ambiguous or conflicting entries refuse with constant codes, without replaying SQL or
|
|
119
119
|
creating a second ledger.
|
|
120
120
|
|
|
121
121
|
For old archives containing only Platform's migration names, the seed adapter
|
|
@@ -131,6 +131,91 @@ Custom replay runners can supply each plan entry's `aliases` and pass
|
|
|
131
131
|
`{dataOnly}` as `migrationReplay`'s third argument. Verified archive ledger
|
|
132
132
|
hashes take precedence over name-only placement where available.
|
|
133
133
|
|
|
134
|
+
### Before an adapter's native migrations
|
|
135
|
+
|
|
136
|
+
Seed adapters sometimes prepare schema before calling `localPortability`.
|
|
137
|
+
`rer import` and `rer export` dispatch to the seed's portability adapter; a
|
|
138
|
+
check inside the later transfer cannot protect that earlier native apply.
|
|
139
|
+
Call `nativeMigrationPreflight` immediately before **each** such Wrangler step,
|
|
140
|
+
for both import and export. The maintained Node entry point is
|
|
141
|
+
`@runeditrun/rer/portability-migrations`; a vendored CLI uses
|
|
142
|
+
`scripts/base-cli/cli/portability-migrations.mjs`.
|
|
143
|
+
|
|
144
|
+
```js
|
|
145
|
+
import { getPlatformProxy } from 'wrangler';
|
|
146
|
+
import { join } from 'node:path';
|
|
147
|
+
import { nativeMigrationPreflight } from '@runeditrun/rer/portability-migrations';
|
|
148
|
+
|
|
149
|
+
// Inside the seed's local portability adapter, before its existing native apply:
|
|
150
|
+
const proxy = await getPlatformProxy({
|
|
151
|
+
configPath: join(root, 'wrangler.jsonc'),
|
|
152
|
+
persist: { path: join(root, '.wrangler/state/v3') },
|
|
153
|
+
remoteBindings: false,
|
|
154
|
+
});
|
|
155
|
+
let nativeApply;
|
|
156
|
+
try {
|
|
157
|
+
({ nativeApply } = await nativeMigrationPreflight({
|
|
158
|
+
root, db: proxy.env.DB, binding: 'DB',
|
|
159
|
+
platformMigrations: { prefix: 'core-', dataOnly: [] },
|
|
160
|
+
}));
|
|
161
|
+
} finally { await proxy.dispose(); }
|
|
162
|
+
if (nativeApply) {
|
|
163
|
+
await store.run('Preparing archive schema', 'pnpm', [
|
|
164
|
+
'exec', 'wrangler', 'd1', 'migrations', 'apply', database,
|
|
165
|
+
'--local', '--persist-to', '.wrangler/state', '--config', 'wrangler.jsonc',
|
|
166
|
+
], { cwd: root, signal });
|
|
167
|
+
}
|
|
168
|
+
// Continue the seed's archive/attachment checks and localPortability transfer.
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
Use the seed's actual binding, database and deployment prefix, with the same
|
|
172
|
+
`platformMigrations` declaration on `localPortability`. Omit the declaration
|
|
173
|
+
when none is needed. The helper reads this binding's `migrations_dir` in the
|
|
174
|
+
root's `wrangler.jsonc` (default `migrations`) and hashes the original file
|
|
175
|
+
bytes. It uses the transfer's `destinationLedger` and `matchMigrationLedger`.
|
|
176
|
+
Platform history and archive transfer retain declared aliases, hash identities
|
|
177
|
+
and data-only entries. Native history additionally requires every recorded name
|
|
178
|
+
to equal a checkout filename: Wrangler skips files by literal name, so an alias,
|
|
179
|
+
hash-only relocation or unmapped data-only row cannot authorize native apply.
|
|
180
|
+
It does not split or execute the migration SQL. Pass the real target's
|
|
181
|
+
D1-compatible binding; inspection of another database or persistence directory
|
|
182
|
+
cannot authorize this apply. Keep
|
|
183
|
+
destination writers stopped and hold the adapter's normal operation lock
|
|
184
|
+
through inspection, apply and transfer. This local example authorizes no remote
|
|
185
|
+
provider operation.
|
|
186
|
+
|
|
187
|
+
| Existing history | Result before native apply |
|
|
188
|
+
| --- | --- |
|
|
189
|
+
| No migration ledger | `{nativeApply:true}`; ordinary fresh Wrangler setup. |
|
|
190
|
+
| Compatible `d1_migrations`, every recorded name exactly equal to its checkout filename | `{nativeApply:true}`; Wrangler may apply its pending files. Plain Wrangler names remain supported; tracked history must also match hashes and applied states. |
|
|
191
|
+
| Native alias-only or hash-only relocation, or an unmapped native data-only row | `migration_ledger:incompatible`; restore the matching checkout or reconcile through the owning runner before native apply. |
|
|
192
|
+
| Sole, fully applied matching `rer_migrations` | `{nativeApply:false}`; preserve Platform's ledger and skip Wrangler. |
|
|
193
|
+
| Any pending checkout file with Platform history | `migration_ledger:unapplied`; apply it through the owning runner first. |
|
|
194
|
+
| Multiple ledgers, even empty | `migration_ledger:multiple`; reconcile history through its owning runner. |
|
|
195
|
+
| Legacy `_rer_deploy_migrations` alone | `migration_ledger:runner`; use its owning runner, which can adopt the history. |
|
|
196
|
+
| Invalid ledger shape, row or binding | `migration_ledger:incompatible` or `migration_ledger:binding`. |
|
|
197
|
+
| Partial Platform batch or non-applied tracked row | `migration_ledger:uncertain`; resolve the incomplete operation first. |
|
|
198
|
+
| Unknown hash/name, ambiguous hash/alias or conflicting name/hash | `migration_ledger:unknown`, `:ambiguous` or `:conflict`; use the matching checkout and declaration. |
|
|
199
|
+
|
|
200
|
+
Refusals throw `PortabilityError` with status 409 and a constant `code`;
|
|
201
|
+
ledger values never enter these diagnostics. Propagate failures and stop before
|
|
202
|
+
native apply. IO and provider failures also propagate. A successful check
|
|
203
|
+
creates no ledger, changes no row and does not attest unrecorded historical
|
|
204
|
+
bytes in a plain Wrangler ledger. It never silently skips pending migrations.
|
|
205
|
+
RER's native `localPortability` invokes this same entry point in its
|
|
206
|
+
`migration-preflight` worker before its own Wrangler step, then reads applied
|
|
207
|
+
history again for transfer.
|
|
208
|
+
|
|
209
|
+
The implementation ships in the RER CLI, not the Worker portability module.
|
|
210
|
+
Package-mode seeds need a future `@runeditrun/rer` release containing this
|
|
211
|
+
export and an exact pin/lock update; version 0.2.77 does not expose it.
|
|
212
|
+
Vendored seeds need a reviewed Base CLI-copy refresh with
|
|
213
|
+
`node scripts/export-cli.mjs SEED_ROOT` from the selected Base source and their
|
|
214
|
+
normal copy/provenance checks. Copy the complete CLI dependency inventory,
|
|
215
|
+
not this file alone. Either route also needs the seed-owned adapter call above
|
|
216
|
+
before its earlier native apply. This Base change releases no package and
|
|
217
|
+
adopts no seed.
|
|
218
|
+
|
|
134
219
|
When a policy includes operator-auth tables, local dev identity still never
|
|
135
220
|
travels, and no policy option changes that: export drops `oa_sessions` rows of
|
|
136
221
|
kind `dev`, every credential recorded in `oa_dev_credentials` (and that table's
|
|
@@ -391,6 +476,16 @@ Eligible constant seed INSERTs replay too. Statements confined to tables added
|
|
|
391
476
|
after the archive retain their existing admission when they need no replay;
|
|
392
477
|
an INSERT reading a view must still resolve entirely to carried tables.
|
|
393
478
|
|
|
479
|
+
A table created and dropped by the applied history remains a read dependency
|
|
480
|
+
even though it is absent from the destination. When its single CREATE, bounded
|
|
481
|
+
INSERT fills and DROP all follow the archive, finalize can reconstruct that
|
|
482
|
+
transient table in migration order, consume it and drop it in the same atomic
|
|
483
|
+
transaction. Its fills may read archived cells and new tables reconstructed by
|
|
484
|
+
the same plan; those dependencies must all qualify. Transient foreign keys,
|
|
485
|
+
writing triggers and other schema mutations do not qualify. Unsupported
|
|
486
|
+
transient reads keep the prior migration-data admission or refusal, and are
|
|
487
|
+
never scheduled against a missing table. No archive format change is needed.
|
|
488
|
+
|
|
394
489
|
The same check runs when the archive's schema identity equals the destination's:
|
|
395
490
|
a later migration that only purges or backfills rows leaves the schema unchanged,
|
|
396
491
|
yet the archive's rows never passed through it. Eligible carried-table backfills
|
|
@@ -691,6 +786,7 @@ modules/portability/ledgers.mjs
|
|
|
691
786
|
modules/portability/migration-statements.mjs
|
|
692
787
|
modules/portability/sql-text.mjs
|
|
693
788
|
modules/portability/backfills.mjs
|
|
789
|
+
modules/portability/transient-tables.mjs
|
|
694
790
|
modules/portability/rebuilds.mjs
|
|
695
791
|
modules/portability/replay-views.mjs
|
|
696
792
|
modules/portability/legacy-safety.mjs
|
|
@@ -4,6 +4,7 @@ import { changesData, dataStatement, reachingTables, triggerBodies, schemaChange
|
|
|
4
4
|
import { splitStatements } from './sql-text.mjs';
|
|
5
5
|
import { preservingRebuilds } from './rebuilds.mjs';
|
|
6
6
|
import { replayViews, viewReads } from './replay-views.mjs';
|
|
7
|
+
import { transientTables } from './transient-tables.mjs';
|
|
7
8
|
|
|
8
9
|
// The data statements of later migrations, judged by the tables they write
|
|
9
10
|
// (bridge.mjs placement). Restored rows never pass through a later migration.
|
|
@@ -37,40 +38,66 @@ export function laterData({applied,unapplied,replay,catalogue,archived,empty,tab
|
|
|
37
38
|
const reached=reachingTables(statements,carried),writable=reachingTables(statements,restorable);
|
|
38
39
|
const later=unapplied.flatMap(migration=>split(migration).map((sql,index)=>({migration,index,sql,data:dataStatement(sql)})));
|
|
39
40
|
const {admitted,tables:rebuilt}=preservingRebuilds(later,manifest,tables,statements),views=replayViews(statements);
|
|
40
|
-
const
|
|
41
|
+
const {missing,plans:transients}=transientTables(statements,later,fold(tables),carried);
|
|
42
|
+
const current=fold(tables),known=[...current,...missing],outside=later.filter(s=>s.data && !writable.includes(s.data.target) && !transients.has(s.data.target)),tainted=new Set();
|
|
41
43
|
// Tables an admitted write fills from archived rows (directly, through its
|
|
42
44
|
// target's triggers, or from another such table) hold rows the restore lacks.
|
|
43
|
-
const taint=name=>{if(
|
|
45
|
+
const taint=name=>{if(current.includes(name) && !tainted.has(name)){tainted.add(name);triggerBodies(statements,name).forEach(taint);return true;}};
|
|
44
46
|
for(let grew=true;grew;)grew=outside.filter(s=>s.data.names.some(name=>carried.includes(name) || tainted.has(name))).map(s=>taint(s.data.target)).some(Boolean);
|
|
45
47
|
const refused=new Set(),writes=new Map();
|
|
46
48
|
for(const s of later) {
|
|
47
49
|
if(outside.includes(s) || admitted.has(s))continue;
|
|
48
50
|
s.today=changesData(s.sql,[...reached,...tainted],empty);
|
|
49
|
-
if(s.data && restorable.includes(s.data.target)){if(!writes.has(s.data.target))writes.set(s.data.target,[]);writes.get(s.data.target).push(s);}
|
|
51
|
+
if(s.data && (restorable.includes(s.data.target) || transients.has(s.data.target))){if(!writes.has(s.data.target))writes.set(s.data.target,[]);writes.get(s.data.target).push(s);}
|
|
50
52
|
else if(s.today)refused.add(s);
|
|
51
53
|
}
|
|
52
|
-
const
|
|
54
|
+
const choices=new Map(),required=new Set();
|
|
53
55
|
for(const [table,list] of writes) {
|
|
54
56
|
// Seed INSERTs into a new table replay too: the rows already present in a
|
|
55
57
|
// fresh destination must be reset and derived from the restored installation.
|
|
56
|
-
const
|
|
58
|
+
const transient=transients.get(table);
|
|
59
|
+
// A transient fill can also read new catalogue tables whose INSERTs the
|
|
60
|
+
// same plan reconstructs (or whose reset leaves them empty at this point).
|
|
61
|
+
// Dependencies are checked below before any of the plan is selected.
|
|
62
|
+
const available=[...transients.keys(),...(transient?[...writes.keys()].filter(name=>!carried.includes(name)):[])];
|
|
63
|
+
const options={table,statements,restorable,writable,carried:[...carried,...available],tables:known,secrets,views,manifest};
|
|
64
|
+
// Carried target rules must use the actual archive, not reconstructed reads.
|
|
65
|
+
const bad=carried.includes(table) && list.length>1?list:list.filter(s=>!replayable(s,{...options,targetCarried:carried.includes(table)}));
|
|
57
66
|
// Writes confined to tables created after the archive retain their
|
|
58
67
|
// admission. Replay eligible seeds, but do not turn a new-table-only
|
|
59
68
|
// upgrade into a refused backfill. View reads always require resolution.
|
|
60
|
-
const
|
|
61
|
-
if(
|
|
62
|
-
|
|
69
|
+
const needed=!transient && list.some(s=>s.today || s.data.names.some(name=>views.has(name)));
|
|
70
|
+
if(needed)required.add(table);
|
|
71
|
+
if(!needed && bad.length)continue;
|
|
72
|
+
const reach=reachingTables(statements,[table]),other=later.filter(s=>s!==transient?.create && s!==transient?.drop && !admitted.has(s) && !list.includes(s) && writesAnother(s,table,reach,list));
|
|
63
73
|
if(bad.length || other.length){[...bad,...other].forEach(s=>refused.add(s));continue;}
|
|
64
|
-
|
|
74
|
+
const dependencies=list.flatMap(s=>viewReads(s.data.names,s.data.words,{...options,table}) || []).filter(name=>name!==table && !carried.includes(name));
|
|
75
|
+
choices.set(table,{list,dependencies});
|
|
76
|
+
}
|
|
77
|
+
// Remove dependent plans together: skipping an unsupported transient must
|
|
78
|
+
// never leave its consumer scheduled against a table that no longer exists.
|
|
79
|
+
for(let changed=true;changed;) {
|
|
80
|
+
changed=false;
|
|
81
|
+
for(const [table,choice] of choices)if(choice.dependencies.some(name=>!choices.has(name))) {
|
|
82
|
+
choices.delete(table);changed=true;
|
|
83
|
+
if(required.has(table))choice.list.forEach(s=>refused.add(s));
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
const replays=[...choices.values()].flatMap(c=>c.list),schema=new Map();
|
|
87
|
+
for(const [table,{create,drop}] of transients)if(choices.has(table)) {
|
|
88
|
+
replays.push(create,drop);schema.set(create,table);schema.set(drop,table);
|
|
65
89
|
}
|
|
66
90
|
const first=later.find(s=>refused.has(s));
|
|
67
91
|
insist(!first,'schema_mismatch:migration_data:'+first?.migration);
|
|
68
92
|
const named=Object.fromEntries(catalogue.map(name=>[folded(name),name]));
|
|
69
|
-
return {rebuilt,replays:later.filter(s=>replays.includes(s)).map(s=>
|
|
93
|
+
return {rebuilt,replays:later.filter(s=>replays.includes(s)).map(s=>{
|
|
94
|
+
const table=s.data?.target??schema.get(s);
|
|
95
|
+
return {migration:s.migration,statement:s.index,table:named[table]??table,sql:s.sql,...(carried.includes(table)?{carried:true}:{}),...(transients.has(table)?{transient:true}:{}),...(schema.has(s)?{schema:true}:{})};
|
|
96
|
+
})};
|
|
70
97
|
}
|
|
71
|
-
function replayable({data},{table,statements,restorable,writable,carried,tables,secrets,views,manifest}) {
|
|
98
|
+
function replayable({data},{table,statements,restorable,writable,carried,targetCarried,tables,secrets,views,manifest}) {
|
|
72
99
|
if(!['insert','replace'].includes(data.verb) || data.star)return false;
|
|
73
|
-
if(
|
|
100
|
+
if(targetCarried) {
|
|
74
101
|
// sqlite_sequence's high-water mark is not archived, so replay cannot
|
|
75
102
|
// reproduce a carried AUTOINCREMENT table's live key allocation.
|
|
76
103
|
if(!data.columns || autoincrement(manifest.find(t=>folded(t.name)===table).sql))return false;
|
|
@@ -83,7 +110,7 @@ function replayable({data},{table,statements,restorable,writable,carried,tables,
|
|
|
83
110
|
}
|
|
84
111
|
}
|
|
85
112
|
// A carried target participates in secret checks, even through a view.
|
|
86
|
-
if(!viewReads(data.names,data.words,{views,tables,carried,table:
|
|
113
|
+
if(!viewReads(data.names,data.words,{views,tables,carried,table:targetCarried?undefined:table,secrets}))return false;
|
|
87
114
|
if(triggerBodies(statements,table).some(name=>writable.includes(name)))return false;
|
|
88
115
|
// Foreign key actions: a row the replay deletes or rekeys (REPLACE, upsert) must reach no other catalogue table.
|
|
89
116
|
return !reachingTables(statements,restorable.filter(name=>name!==table)).includes(table);
|
|
@@ -104,12 +131,12 @@ function writesAnother(s,table,reach,replays) {
|
|
|
104
131
|
// recording rows in the job's bridge record (`replayed[i].rows`) in the same transaction.
|
|
105
132
|
export function replayStatements(db,row) {
|
|
106
133
|
const replays=row.metadata.replays??[];
|
|
107
|
-
const reset=[...new Set(replays.filter(r=>!r.carried).map(r=>r.table))].flatMap(name=>{
|
|
134
|
+
const reset=[...new Set(replays.filter(r=>!r.carried && !r.transient).map(r=>r.table))].flatMap(name=>{
|
|
108
135
|
const target=row.metadata.targets.find(t=>t.name===name);
|
|
109
136
|
insist(target,'schema_mismatch:replay_table:'+name);
|
|
110
137
|
return [statement(db,`DELETE FROM ${quote(name)}`),...(autoincrement(target.sql)?[statement(db,'DELETE FROM sqlite_sequence WHERE name=?',name)]:[])];
|
|
111
138
|
});
|
|
112
139
|
return [...reset,...replays.flatMap((r,i)=>[db.prepare(r.sql),
|
|
113
|
-
statement(db,`UPDATE rer_portability_jobs SET metadata=json_set(metadata,'$.bridge.replayed[${i}].rows'
|
|
140
|
+
statement(db,`UPDATE rer_portability_jobs SET metadata=json_set(metadata,'$.bridge.replayed[${i}].rows',${r.schema?'0':'changes()'}) WHERE id=?`,row.id)])];
|
|
114
141
|
}
|
|
115
142
|
const autoincrement=sql=>tokens(sql).some(t=>!t.quoted && folded(t.text)==='autoincrement');
|