@runeditrun/rer 0.2.84 → 0.2.86
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +156 -50
- package/cli/README.md +78 -2
- package/cli/agent-door.mjs +149 -0
- package/cli/agent-fixtures.mjs +16 -0
- package/cli/agent-mail.mjs +54 -0
- package/cli/agent-network.mjs +64 -0
- package/cli/agent-reviewers.mjs +110 -0
- package/cli/agent-secrets.mjs +23 -0
- package/cli/agent-sponsors.mjs +43 -0
- package/cli/agent-surface.mjs +34 -0
- package/cli/agent-webhooks.mjs +19 -0
- package/cli/authoring.mjs +2 -1
- package/cli/bounded.mjs +1 -1
- package/cli/business/archive-task.mjs +32 -0
- package/cli/business/carry.mjs +151 -0
- package/cli/business/catchup-task.mjs +42 -0
- package/cli/business/choices.mjs +97 -0
- package/cli/business/command.mjs +29 -6
- package/cli/business/connect.mjs +20 -8
- package/cli/business/detector.mjs +4 -1
- package/cli/business/device.mjs +13 -0
- package/cli/business/export-inputs.mjs +69 -0
- package/cli/business/export-task.mjs +17 -0
- package/cli/business/invocation.mjs +8 -1
- package/cli/business/keychain.mjs +8 -1
- package/cli/business/lease.mjs +11 -1
- package/cli/business/local-commands.mjs +80 -30
- package/cli/business/outbound.mjs +9 -4
- package/cli/business/parsers-config.mjs +8 -7
- package/cli/business/parsers-syntax.mjs +7 -3
- package/cli/business/permissions.mjs +2 -0
- package/cli/business/questions.mjs +89 -6
- package/cli/business/setup-command.mjs +49 -0
- package/cli/business/setup-company.json +15 -0
- package/cli/business/setup-demo.mjs +95 -0
- package/cli/business/state.mjs +44 -10
- package/cli/business/supervisor.mjs +97 -32
- package/cli/business/vendor-door.mjs +230 -0
- package/cli/business/vendors/README.md +29 -3
- package/cli/business/vendors/common.mjs +8 -6
- package/cli/business/vendors/index.mjs +22 -11
- package/cli/business/vendors/sorry.mjs +106 -0
- package/cli/container-transfer.mjs +2 -2
- package/cli/contract.mjs +7 -7
- package/cli/demo-deploy-error.mjs +14 -0
- package/cli/demo-native.mjs +6 -2
- package/cli/deploy-command.mjs +6 -4
- package/cli/deploy-config.mjs +1 -1
- package/cli/deploy-owner-input.mjs +3 -1
- package/cli/deploy-variables.mjs +58 -0
- package/cli/deploy.mjs +9 -7
- package/cli/door-clauses.mjs +157 -0
- package/cli/errors.mjs +1 -1
- package/cli/export.mjs +16 -4
- package/cli/hosted-output.mjs +17 -7
- package/cli/human-error.mjs +20 -0
- package/cli/inspect.mjs +4 -0
- package/cli/jsonc.mjs +8 -4
- package/cli/lifecycle.mjs +26 -6
- package/cli/main.mjs +2 -1
- package/cli/managed-data-schema.mjs +127 -0
- package/cli/managed-data-zip.mjs +116 -0
- package/cli/managed-data.mjs +110 -0
- package/cli/managed-server.mjs +25 -14
- package/cli/native-owner-entry.mjs +86 -0
- package/cli/output.mjs +3 -2
- package/cli/parse.mjs +2 -2
- package/cli/portability-empty.mjs +65 -0
- package/cli/portability-migrations.mjs +33 -9
- package/cli/portability-policy.mjs +5 -2
- package/cli/portability-worker.mjs +15 -6
- package/cli/portability.mjs +30 -8
- package/cli/production-auth.mjs +1 -1
- package/cli/production-mail.mjs +8 -2
- package/cli/ready-summary.mjs +9 -0
- package/cli/run-transfers.mjs +37 -0
- package/cli/runtime-front-door.mjs +4 -4
- package/cli/server-proxy.mjs +4 -3
- package/cli/server-registry.mjs +13 -8
- package/cli/site.mjs +6 -1
- package/cli/transfers.mjs +17 -4
- package/edition-authoring/archive-source.mjs +59 -0
- package/edition-authoring/base-copies.mjs +147 -25
- package/edition-authoring/cli.mjs +1 -1
- package/edition-authoring/edition-migrations.mjs +7 -3
- package/edition-authoring/module-closure.mjs +2 -16
- package/edition-authoring/workflow-copies.mjs +1 -1
- package/edition-authoring/workflow-scripts.mjs +12 -3
- package/foundation/runtime/contract-docs.mjs +218 -34
- package/foundation/runtime/contract-prose-patterns.mjs +14 -0
- package/foundation/runtime/contract-prose.mjs +88 -0
- package/foundation/runtime/run-data.d.mts +1 -0
- package/foundation/runtime/run-data.mjs +2 -0
- package/global/CHANGELOG.md +126 -0
- package/global/README.md +156 -50
- package/global/cli/command.mjs +8 -2
- package/global/cli/install.mjs +4 -5
- package/global/cli/path-link.mjs +3 -2
- package/global/cli/repo-command.mjs +3 -7
- package/global/door/command.mjs +3 -1
- package/global/launch/dev.mjs +12 -2
- package/global/launch/facts.mjs +2 -2
- package/global/launch/render.mjs +16 -9
- package/global/launch/state.mjs +4 -4
- package/global/main.mjs +13 -8
- package/install.sh +17 -6
- package/modules/agent-door/CLAUSES.md +215 -0
- package/modules/agent-door/conformance/audit.mjs +66 -0
- package/modules/agent-door/conformance/authority.d.mts +2 -0
- package/modules/agent-door/conformance/authority.mjs +17 -0
- package/modules/agent-door/conformance/crypto.mjs +36 -0
- package/modules/agent-door/conformance/discovery.mjs +61 -0
- package/modules/agent-door/conformance/index.mjs +41 -0
- package/modules/agent-door/conformance/invocation.mjs +32 -0
- package/modules/agent-door/conformance/journey.mjs +112 -0
- package/modules/agent-door/conformance/narrowing.d.mts +1 -0
- package/modules/agent-door/conformance/narrowing.mjs +21 -0
- package/modules/agent-door/conformance/negative.mjs +47 -0
- package/modules/agent-door/conformance/network-modules.mjs +86 -0
- package/modules/agent-door/conformance/network-runtime.mjs +46 -0
- package/modules/agent-door/conformance/policy-phases.mjs +75 -0
- package/modules/agent-door/conformance/protocol.mjs +145 -0
- package/modules/agent-door/conformance/replay.mjs +38 -0
- package/modules/agent-door/conformance/schema.mjs +71 -0
- package/modules/agent-door/conformance/security.mjs +33 -0
- package/modules/agent-door/conformance/sponsors.mjs +68 -0
- package/modules/agent-door/conformance/surface.mjs +89 -0
- package/modules/agent-door/conformance/target.mjs +18 -0
- package/modules/agent-door/conformance/tool-name.d.mts +1 -0
- package/modules/agent-door/conformance/tool-name.mjs +7 -0
- package/modules/agent-door/conformance/webhooks.mjs +30 -0
- package/modules/container/README.md +6 -1
- package/modules/container/adapters/d1-limits.mjs +286 -0
- package/modules/container/adapters/d1-sqlite.mjs +95 -0
- package/modules/container/adapters/d1-workerd-preload.mjs +19 -0
- package/modules/container/adapters/d1-workerd-setup.mjs +26 -0
- package/modules/container/adapters/d1-workerd.mjs +5 -0
- package/modules/container/adapters/sqlite-worker.mjs +5 -3
- package/modules/container/build.mjs +1 -0
- package/modules/container/image-variables.mjs +14 -3
- package/modules/container/transfer.mjs +14 -4
- package/modules/container/worker-engine.mjs +3 -3
- package/modules/demo-host/README.md +6 -0
- package/modules/demo-host/build.mjs +1 -0
- package/modules/demo-host/native/prepare.mjs +2 -1
- package/modules/demo-host/native/sql.mjs +7 -2
- package/modules/demo-host/ownership.mjs +39 -0
- package/modules/demo-host/plan.mjs +2 -1
- package/modules/demo-host/remove.mjs +6 -1
- package/modules/deploy-kit/README.md +22 -2
- package/modules/deploy-kit/d1-import-diagnostics.mjs +56 -0
- package/modules/deploy-kit/d1-import.mjs +17 -5
- package/modules/deploy-kit/diagnostics.mjs +29 -3
- package/modules/deploy-kit/owner-cli.mjs +20 -7
- package/modules/deploy-kit/provider-transport.mjs +19 -4
- package/modules/deploy-kit/provider.mjs +57 -22
- package/modules/deploy-kit/readiness-window.mjs +1 -1
- package/modules/deploy-kit/site.mjs +8 -3
- package/modules/deploy-kit/worker.mjs +4 -4
- package/modules/import-kit/schema/importer-verification-receipt.schema.json +171 -0
- package/modules/import-kit/schema/importers.schema.json +107 -0
- package/modules/import-kit/src/d1/README.md +23 -4
- package/modules/import-kit/src/d1/atomic-prefix.ts +2 -2
- package/modules/import-kit/src/d1/budget.ts +15 -7
- package/modules/import-kit/src/d1/column-layout.ts +5 -0
- package/modules/import-kit/src/d1/creation.ts +15 -8
- package/modules/import-kit/src/d1/database.ts +3 -3
- package/modules/import-kit/src/d1/erasure-pages.ts +2 -1
- package/modules/import-kit/src/d1/erasure.ts +2 -2
- package/modules/import-kit/src/d1/jobs.ts +10 -6
- package/modules/import-kit/src/d1/plans.ts +14 -8
- package/modules/import-kit/src/d1/retirement.ts +2 -2
- package/modules/import-kit/src/d1/sets.ts +1 -3
- package/modules/import-kit/src/d1/staged-creation.ts +3 -3
- package/modules/import-kit/src/door/ADOPTING.md +20 -0
- package/modules/import-kit/src/door/README.md +2 -1
- package/modules/import-kit/src/door/admission.ts +5 -3
- package/modules/import-kit/src/door/archive.ts +5 -2
- package/modules/import-kit/src/door/authority.ts +1 -0
- package/modules/import-kit/src/door/grants.ts +1 -1
- package/modules/import-kit/src/door/handler.ts +14 -8
- package/modules/import-kit/src/door/index.ts +1 -1
- package/modules/import-kit/src/door/jobs.ts +40 -0
- package/modules/import-kit/src/door/limits.ts +1 -1
- package/modules/import-kit/src/door/operator.ts +3 -3
- package/modules/import-kit/src/door/planning.ts +10 -6
- package/modules/import-kit/src/door/protocol.ts +2 -2
- package/modules/import-kit/src/door/raw.ts +25 -0
- package/modules/import-kit/src/door/refusals.ts +2 -1
- package/modules/import-kit/src/door/types.ts +26 -6
- package/modules/import-kit/src/receipt/encoding.ts +37 -0
- package/modules/import-kit/src/receipt/index.ts +69 -0
- package/modules/import-kit/src/receipt/policy.ts +24 -0
- package/modules/import-kit/src/receipt/types.ts +64 -0
- package/modules/import-kit/src/receipt/uri.ts +45 -0
- package/modules/import-kit/src/receipt/validation.ts +87 -0
- package/modules/import-kit/src/status/encoding.ts +13 -0
- package/modules/import-kit/src/status/evidence.ts +218 -0
- package/modules/import-kit/src/status/index.ts +56 -0
- package/modules/import-kit/src/status/schema.ts +208 -0
- package/modules/import-kit/src/status/types.ts +86 -0
- package/modules/operator-auth/accounts/delegated.mjs +48 -0
- package/modules/operator-auth/index.d.mts +7 -0
- package/modules/operator-auth/service.mjs +2 -0
- package/modules/portability/README.md +138 -9
- package/modules/portability/backfills.mjs +9 -8
- package/modules/portability/bridge.mjs +32 -5
- package/modules/portability/catalogue.mjs +12 -5
- package/modules/portability/copies.mjs +8 -1
- package/modules/portability/derived.mjs +33 -0
- package/modules/portability/files.mjs +4 -3
- package/modules/portability/http-errors.mjs +1 -1
- package/modules/portability/http.mjs +3 -3
- package/modules/portability/import.mjs +29 -13
- package/modules/portability/index.d.mts +4 -3
- package/modules/portability/index.mjs +1 -1
- package/modules/portability/ledgers.mjs +3 -2
- package/modules/portability/legacy-safety.mjs +11 -2
- package/modules/portability/migration-history.mjs +2 -2
- package/modules/portability/progress.mjs +1 -0
- package/modules/portability/restore-empty.mjs +12 -0
- package/modules/release/github-verify.mjs +11 -2
- package/onboarding/adapter.mjs +35 -6
- package/onboarding/app-fetch.mjs +6 -4
- package/onboarding/development-runtime.mjs +19 -0
- package/onboarding/local/operation.mjs +45 -27
- package/onboarding/loopback-fetch.mjs +16 -0
- package/onboarding/operator-installer.mjs +2 -4
- package/onboarding/process-diagnostics.mjs +6 -2
- package/onboarding/production-runtime.mjs +4 -2
- package/onboarding/readiness.mjs +1 -0
- package/package.json +4 -2
- package/scripts/publish-check/copy_identities.py +3 -1
- package/scripts/publish-check/github_refs.py +32 -13
- package/scripts/publish-check/public_hygiene.py +1 -1
- package/cli/business/opt-ins.mjs +0 -33
|
@@ -15,8 +15,8 @@ import { transientTables } from './transient-tables.mjs';
|
|
|
15
15
|
// - An eligible seed or backfill of a catalogue table replays at finalize.
|
|
16
16
|
// A carried table keeps its archived rows and admits only one later writer,
|
|
17
17
|
// naming its target columns. An absent table is reset first; each later write is a
|
|
18
|
-
// replayable INSERT
|
|
19
|
-
//
|
|
18
|
+
// replayable INSERT or a bounded UPDATE of a newly reconstructed table. They
|
|
19
|
+
// run in migration order, exactly as written, after archived rows are inserted and with the destination's triggers still
|
|
20
20
|
// dropped, so the table holds what the live upgrade gave it. A replay may read
|
|
21
21
|
// only archived tables (complete by then), views over those tables and the
|
|
22
22
|
// table itself, by named
|
|
@@ -56,11 +56,12 @@ export function laterData({applied,unapplied,replay,catalogue,archived,empty,tab
|
|
|
56
56
|
// Seed INSERTs into a new table replay too: the rows already present in a
|
|
57
57
|
// fresh destination must be reset and derived from the restored installation.
|
|
58
58
|
const transient=transients.get(table);
|
|
59
|
-
// A
|
|
60
|
-
//
|
|
59
|
+
// A new target can read other new catalogue tables that the same plan
|
|
60
|
+
// reconstructs. Carried targets still read only archived tables.
|
|
61
61
|
// Dependencies are checked below before any of the plan is selected.
|
|
62
|
-
const available=[...transients.keys(),...(
|
|
63
|
-
const
|
|
62
|
+
const available=[...transients.keys(),...(!carried.includes(table)?[...writes.keys()].filter(name=>!carried.includes(name)):[])];
|
|
63
|
+
const allowUpdate=!transient && !carried.includes(table) && list.some(s=>['insert','replace'].includes(s.data.verb));
|
|
64
|
+
const options={table,allowUpdate,statements,restorable,writable,carried:[...carried,...available],tables:known,secrets,views,manifest};
|
|
64
65
|
// Carried target rules must use the actual archive, not reconstructed reads.
|
|
65
66
|
const bad=carried.includes(table) && list.length>1?list:list.filter(s=>!replayable(s,{...options,targetCarried:carried.includes(table)}));
|
|
66
67
|
// Writes confined to tables created after the archive retain their
|
|
@@ -95,8 +96,8 @@ export function laterData({applied,unapplied,replay,catalogue,archived,empty,tab
|
|
|
95
96
|
return {migration:s.migration,statement:s.index,table:named[table]??table,sql:s.sql,...(carried.includes(table)?{carried:true}:{}),...(transients.has(table)?{transient:true}:{}),...(schema.has(s)?{schema:true}:{})};
|
|
96
97
|
})};
|
|
97
98
|
}
|
|
98
|
-
function replayable({data},{table,statements,restorable,writable,carried,targetCarried,tables,secrets,views,manifest}) {
|
|
99
|
-
if(!['insert','replace'].includes(data.verb) || data.star)return false;
|
|
99
|
+
function replayable({data},{table,allowUpdate,statements,restorable,writable,carried,targetCarried,tables,secrets,views,manifest}) {
|
|
100
|
+
if(!['insert','replace',...(allowUpdate?['update']:[])].includes(data.verb) || data.star)return false;
|
|
100
101
|
if(targetCarried) {
|
|
101
102
|
// sqlite_sequence's high-water mark is not archived, so replay cannot
|
|
102
103
|
// reproduce a carried AUTOINCREMENT table's live key allocation.
|
|
@@ -22,7 +22,7 @@ export async function appliedMigrations(db) {
|
|
|
22
22
|
// Restored rows never pass through a later migration, so one that writes or
|
|
23
23
|
// rewrites restorable data is refused, whether or not it changes the schema,
|
|
24
24
|
// unless its statement is replayed at finalize (backfills.mjs).
|
|
25
|
-
async function placement(db,manifest,cat,replay,ledger,history) {
|
|
25
|
+
async function placement(db,manifest,cat,replay,ledger,history,legacyRecovery,unverified=false) {
|
|
26
26
|
const files=replay.names.map(name=>({name,sha256:replay.sha256?.(name),aliases:replay.aliases?.(name)}));
|
|
27
27
|
// An aligned replayer keeps its existing contract: hashes are optional.
|
|
28
28
|
// Resolve destination aliases only when its ledger uses different names.
|
|
@@ -32,10 +32,17 @@ async function placement(db,manifest,cat,replay,ledger,history) {
|
|
|
32
32
|
const archived=history?.();
|
|
33
33
|
const historyEntries=manifest.migrations?.map(name=>archived?.find(entry=>entry.name===name) ?? {name}) ?? archived;
|
|
34
34
|
const recorded=historyEntries && matchMigrationLedger(historyEntries,files,replay);
|
|
35
|
+
// Only explicitly acknowledged v1 recovery may use the old projection for
|
|
36
|
+
// archive placement. The live destination check above remains complete, and
|
|
37
|
+
// current v2 imports use their authenticated object inventory. Verified legacy
|
|
38
|
+
// v2 uses the replayed complete identity; acknowledged recovery places by tables only.
|
|
39
|
+
const schemaAfter=names=>replay.schemaAfter(names,legacyRecovery && manifest.schemaObjects===undefined?{legacyExcludedObjects:true}:unverified?{tablesOnly:true}:undefined);
|
|
40
|
+
if(unverified && archived && manifest.migrations)insist(canonicalJson(matchMigrationLedger(archived,files,replay))===canonicalJson(recorded),'schema_mismatch:migration_history',409);
|
|
35
41
|
let earlier,unapplied;
|
|
36
42
|
if(recorded) {
|
|
43
|
+
if(unverified)insist(recorded.every((name,i)=>applied.indexOf(name)>(i?applied.indexOf(recorded[i-1]):-1)),'schema_mismatch:migration_history',409);
|
|
37
44
|
earlier=applied.filter(name=>recorded.includes(name));
|
|
38
|
-
insist(await
|
|
45
|
+
insist(await schemaAfter(earlier)===manifest.schema,'schema_mismatch');
|
|
39
46
|
unapplied=applied.filter(name=>!earlier.includes(name));
|
|
40
47
|
}else{
|
|
41
48
|
// Unrecorded: the latest matching prefix. Migrations that change no archived
|
|
@@ -43,7 +50,7 @@ async function placement(db,manifest,cat,replay,ledger,history) {
|
|
|
43
50
|
// and count as later. The source may have stopped at any matching prefix,
|
|
44
51
|
// so every migration after the earliest one may be missing from its rows.
|
|
45
52
|
const matching=[];
|
|
46
|
-
for(let k=0;k<=applied.length;k++)if(await
|
|
53
|
+
for(let k=0;k<=applied.length;k++)if(await schemaAfter(applied.slice(0,k))===manifest.schema)matching.push(k);
|
|
47
54
|
insist(matching.length,'schema_mismatch');
|
|
48
55
|
earlier=applied.slice(0,matching.at(-1));
|
|
49
56
|
unapplied=applied.slice(matching[0]);
|
|
@@ -62,6 +69,26 @@ async function placement(db,manifest,cat,replay,ledger,history) {
|
|
|
62
69
|
const {replays,rebuilt}=laterData({applied,unapplied,replay,catalogue,archived:archivedTables,empty,tables,secrets:manifest.excludedColumns??{},manifest:manifest.tables});
|
|
63
70
|
return {applied,earlier,replays,rebuilt};
|
|
64
71
|
}
|
|
72
|
+
// Old v2 writers recorded tables and applied migration names, but no behavioral
|
|
73
|
+
// object inventory. Verify both projections: the recorded files must reproduce
|
|
74
|
+
// the complete live schema, and their tables must reproduce the archive. Never
|
|
75
|
+
// guess an unrecorded prefix or install source SQL to fill missing evidence.
|
|
76
|
+
export async function verifiedLegacySchema(db,manifest,cat,migrations,history) {
|
|
77
|
+
if(!migrations)return false;
|
|
78
|
+
const replay=await migrations(),ledger=await destinationLedger(db)??[];
|
|
79
|
+
try {
|
|
80
|
+
const files=replay.names.map(name=>({name,sha256:replay.sha256?.(name),aliases:replay.aliases?.(name)}));
|
|
81
|
+
const archived=history(),entries=manifest.migrations?.map(name=>archived?.find(entry=>entry.name===name)??{name})??archived;
|
|
82
|
+
if(!entries?.length)return false;
|
|
83
|
+
const applied=matchMigrationLedger(ledger,files,replay),recorded=matchMigrationLedger(entries,files,replay);
|
|
84
|
+
if(!recorded.length || canonicalJson(applied)!==canonicalJson(replay.names)
|
|
85
|
+
|| !recorded.every((name,i)=>applied.indexOf(name)>(i?applied.indexOf(recorded[i-1]):-1)))return false;
|
|
86
|
+
if(archived && manifest.migrations && canonicalJson(matchMigrationLedger(archived,files,replay))!==canonicalJson(recorded))return false;
|
|
87
|
+
return await replay.schemaAfter(applied)===cat.identity
|
|
88
|
+
&& await replay.schemaAfter(recorded)===cat.identity
|
|
89
|
+
&& await replay.schemaAfter(recorded,{tablesOnly:true})===manifest.schema;
|
|
90
|
+
}catch{return false;} // Unknown history or a failed replay is not verification.
|
|
91
|
+
}
|
|
65
92
|
// An archive with the destination's schema identity can still predate later
|
|
66
93
|
// migrations that changed only data (a purge or a backfill).
|
|
67
94
|
// Those that change restorable data are refused like any other. Without a
|
|
@@ -85,10 +112,10 @@ export async function sameSchemaMigrations(db,manifest,cat,migrations,history) {
|
|
|
85
112
|
// are additive, or a proven row-preserving CHECK rebuild: new tables, new
|
|
86
113
|
// columns that take a default, or later schema objects established by replay.
|
|
87
114
|
// Anything else stays a schema_mismatch refusal.
|
|
88
|
-
export async function additiveBridge(db,manifest,cat,migrations,history) {
|
|
115
|
+
export async function additiveBridge(db,manifest,cat,migrations,history,legacyRecovery,unverified=false) {
|
|
89
116
|
insist(migrations,'schema_mismatch');
|
|
90
117
|
const replay=await migrations(),ledger=await destinationLedger(db)??[];
|
|
91
|
-
const {applied,earlier,replays,rebuilt}=await placement(db,manifest,cat,replay,ledger,history);
|
|
118
|
+
const {applied,earlier,replays,rebuilt}=await placement(db,manifest,cat,replay,ledger,history,legacyRecovery,unverified);
|
|
92
119
|
const columns={};
|
|
93
120
|
for(const table of manifest.tables) {
|
|
94
121
|
const target=cat.tables.find(t=>t.name===table.name),label=':'+table.name;
|
|
@@ -53,19 +53,26 @@ export function policy(input) {
|
|
|
53
53
|
/** Platform tables are always excluded; explicit includes override only optional exclusions. */
|
|
54
54
|
export const excludedTable=(table,input={})=>platformTable(table) || (!(input.includeTables??[]).includes(table) && [...excludedTables,...(input.excludeTables??[])].some(pattern=>matches(table,pattern)));
|
|
55
55
|
async function archived(db,config) {
|
|
56
|
-
const objects=await all(db,"SELECT name,sql,type,tbl_name FROM sqlite_master WHERE sql IS NOT NULL ORDER BY name");
|
|
56
|
+
const objects=await all(db,"SELECT name,sql,type,tbl_name FROM sqlite_master WHERE sql IS NOT NULL AND substr(lower(name),1,7) <> 'sqlite_' AND substr(lower(name),1,4) <> '_cf_' ORDER BY name");
|
|
57
57
|
const kinds=await all(db,'PRAGMA table_list');
|
|
58
58
|
const derived=config.derivedTables;
|
|
59
59
|
const excluded=objects.filter(x=>x.type==='table' && (excludedTable(x.name,config) || (!config.includeTables.includes(x.name) &&
|
|
60
60
|
derived.some(p=>x.name===p || (kinds.find(k=>k.name===x.name)?.type==='shadow' && x.name.startsWith(p+'_'))))));
|
|
61
61
|
objects.sort((a,b)=>a.name<b.name?-1:a.name>b.name?1:a.type<b.type?-1:a.type>b.type?1:0);
|
|
62
|
-
|
|
63
|
-
|
|
62
|
+
// Row exclusions never erase behavioral schema evidence, even for objects
|
|
63
|
+
// attached to excluded tables. Only literal engine-internal names are exempt.
|
|
64
|
+
const schemaObjects=objects.filter(x=>['index','view','trigger'].includes(x.type)).map(({name,type,tbl_name,sql})=>({name,type,table:tbl_name,sql}));
|
|
64
65
|
return {objects,kinds,excluded,schemaObjects,entries:objects.filter(x=>x.type==='table' && !excluded.some(y=>y.name===x.name))};
|
|
65
66
|
}
|
|
66
67
|
// The archive schema identity alone, for databases that need not satisfy the policy.
|
|
67
|
-
export async function schemaIdentity(db,config) {
|
|
68
|
-
const {entries,schemaObjects}=await archived(db,config);
|
|
68
|
+
export async function schemaIdentity(db,config,{legacyExcludedObjects=false,tablesOnly=false}={}) {
|
|
69
|
+
const {entries,excluded,schemaObjects}=await archived(db,config);
|
|
70
|
+
// Acknowledged v1 recovery places an old backup using its historical
|
|
71
|
+
// projection; acknowledged recovery of a version 2 archive without an object
|
|
72
|
+
// inventory compares tables only. Current archives always retain the
|
|
73
|
+
// complete object inventory.
|
|
74
|
+
const objects=tablesOnly?[]:legacyExcludedObjects?schemaObjects.filter(object=>!excluded.some(table=>table.name===object.table)):schemaObjects;
|
|
75
|
+
return schemaIdentityOf(entries,objects);
|
|
69
76
|
}
|
|
70
77
|
export async function catalogue(db,config) {
|
|
71
78
|
const {objects,kinds,excluded,entries,schemaObjects}=await archived(db,config);
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { all,statement,quote,insist } from './storage.mjs';
|
|
1
|
+
import { all,statement,quote,insist,PortabilityError } from './storage.mjs';
|
|
2
2
|
import { projectedColumns,encodedRow,mapRow,bindCell,writableColumns } from './catalogue.mjs';
|
|
3
3
|
import { canonicalJson, utf8 } from './hash.mjs';
|
|
4
4
|
import { generatedCopySQL } from './generated.mjs';
|
|
@@ -43,6 +43,13 @@ export async function readCopy(db,table,copy,offset,limit,encoded=false) {
|
|
|
43
43
|
return raw.map((row,i)=>({row,meta:meta[i]}));
|
|
44
44
|
}
|
|
45
45
|
export async function normalizeCopy(context,table,copy) {
|
|
46
|
+
try { return await normalize(context,table,copy); }
|
|
47
|
+
catch(error) {
|
|
48
|
+
if(error instanceof PortabilityError)throw error;
|
|
49
|
+
throw new PortabilityError('copy_normalization_failed',409);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
async function normalize(context,table,copy) {
|
|
46
53
|
const {db,config}=context,secrets=config.secretColumns[table.name]??[];
|
|
47
54
|
// Generated reads must reflect exactly the base cells a restore will receive.
|
|
48
55
|
if(table.columns.some(c=>c.generated)&&secrets.length)await db.prepare(`UPDATE ${quote(copy)} SET ${secrets.map(name=>quote(name)+'=NULL').join(',')}`).run();
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import { all, quote } from './storage.mjs';
|
|
2
|
+
import { sqliteTokens, asciiFold } from './sqlite-tokens.mjs';
|
|
3
|
+
|
|
4
|
+
const value = token => {
|
|
5
|
+
const text = token?.text ?? '', delimiter = text[0];
|
|
6
|
+
return ['"', "'", '`', '['].includes(delimiter)
|
|
7
|
+
? text.slice(1, -1).replaceAll(delimiter + delimiter, delimiter) : text;
|
|
8
|
+
};
|
|
9
|
+
|
|
10
|
+
// Only external-content FTS5 has SQLite's native rebuild contract: its rows
|
|
11
|
+
// come from the migrated content table. Other derived data remains seed-owned.
|
|
12
|
+
function externalFts5(sql) {
|
|
13
|
+
const tokens = sqliteTokens(sql), words = tokens.map(token => asciiFold(token.text));
|
|
14
|
+
if (words.slice(0, 3).join(' ') !== 'create virtual table') return false;
|
|
15
|
+
const using = words.indexOf('using');
|
|
16
|
+
if (using < 0 || asciiFold(value(tokens[using + 1])) !== 'fts5') return false;
|
|
17
|
+
let depth = 0;
|
|
18
|
+
for (let i = using + 2; i < tokens.length; i++) {
|
|
19
|
+
if (tokens[i].text === '(') depth++;
|
|
20
|
+
else if (tokens[i].text === ')') depth--;
|
|
21
|
+
else if (depth === 1 && ['(', ','].includes(tokens[i - 1]?.text)
|
|
22
|
+
&& asciiFold(value(tokens[i])) === 'content' && tokens[i + 1]?.text === '=')
|
|
23
|
+
return value(tokens[i + 2]).length > 0;
|
|
24
|
+
}
|
|
25
|
+
return false;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
export async function rebuildDerivedStatements(db, config) {
|
|
29
|
+
if (!config.derivedTables.length) return [];
|
|
30
|
+
const objects = await all(db, "SELECT name,sql FROM sqlite_master WHERE type='table' AND sql IS NOT NULL ORDER BY name");
|
|
31
|
+
return objects.filter(object => config.derivedTables.includes(object.name) && !config.includeTables.includes(object.name) && externalFts5(object.sql))
|
|
32
|
+
.map(object => db.prepare(`INSERT INTO ${quote(object.name)}(${quote(object.name)}) VALUES('rebuild')`));
|
|
33
|
+
}
|
|
@@ -60,7 +60,7 @@ export async function exportArchive(service, path, transient, progress) {
|
|
|
60
60
|
if (id) await service.exports.abort(id);
|
|
61
61
|
}
|
|
62
62
|
}
|
|
63
|
-
export async function importArchive(service, path, transient, journal, progress) {
|
|
63
|
+
export async function importArchive(service, path, transient, journal, progress, options={}) {
|
|
64
64
|
let id, manifest, closing, complete = false, staged = 0;
|
|
65
65
|
try {
|
|
66
66
|
// Validate both complete archives before creating any restore state. An old
|
|
@@ -76,7 +76,7 @@ export async function importArchive(service, path, transient, journal, progress)
|
|
|
76
76
|
const ledgers = ledger.ledgers();
|
|
77
77
|
for await (const record of fileRecords(path)) {
|
|
78
78
|
if (record.kind === 'manifest') {
|
|
79
|
-
manifest = record; id = await service.imports.create(manifest, undefined, ledgers
|
|
79
|
+
manifest = record; id = await service.imports.create(manifest, undefined, {...options,...(ledgers?{ledgers}:{})});
|
|
80
80
|
const { skippedTables } = await service.imports.status(id);
|
|
81
81
|
if (skippedTables?.length) progress?.({ phase: 'skipped', tables: skippedTables });
|
|
82
82
|
}
|
|
@@ -89,6 +89,7 @@ export async function importArchive(service, path, transient, journal, progress)
|
|
|
89
89
|
}
|
|
90
90
|
progress?.({ phase: 'finishing', total: manifest.partCount });
|
|
91
91
|
const status = await service.imports.finalize(id, { contentHash: closing.contentHash, archiveHash: closing.archiveHash, ...(later ? { laterJournal: later } : {}) }); complete = true;
|
|
92
|
-
|
|
92
|
+
if(status.schemaUnverified)progress?.({phase:'schema-unverified',receipt:status.schemaUnverified});
|
|
93
|
+
return { ...receipt(manifest, closing, transient), ...(status.schemaUnverified?{schemaUnverified:status.schemaUnverified}:{}), ...(status.bridged ? { bridged: status.bridged } : {}), ...(status.skippedTables ? { skippedTables: status.skippedTables } : {}), ...(status.safetyChanges ? { safetyChanges: status.safetyChanges } : {}) };
|
|
93
94
|
} finally { if (id && !complete) await service.imports.abort(id); }
|
|
94
95
|
}
|
|
@@ -5,7 +5,7 @@ import { importRunningMetadata } from './import-lease.mjs';
|
|
|
5
5
|
const publicCodes = new Set(`
|
|
6
6
|
D1_RPC_TRANSPORT_LIMIT advance_limit archive_checksum archive_format archive_incomplete archive_json
|
|
7
7
|
binary_checksum binary_inventory binary_order binary_size body_too_large bucket_inventory bucket_not_empty
|
|
8
|
-
closing_missing column_inventory configuration_inventory confirmation_required content_checksum copy_inventory copy_rowid
|
|
8
|
+
closing_missing column_inventory configuration_inventory confirmation_required content_checksum copy_inventory copy_normalization_failed copy_rowid
|
|
9
9
|
data_after_closing dev_data erased_row erasure_epoch excluded_destination_dependents excluded_foreign_key
|
|
10
10
|
export_incomplete export_phase export_row_must_be_pure export_row_not_idempotent extra_part generated_sql
|
|
11
11
|
import_applying import_complete import_lease_expired import_phase import_row_columns import_row_key import_row_must_be_pure
|
|
@@ -11,10 +11,10 @@ export async function boundedJson(request,limit) {
|
|
|
11
11
|
try{return JSON.parse(new TextDecoder('utf-8',{fatal:true}).decode(bytes));}catch{throw new PortabilityError('invalid_json');}
|
|
12
12
|
}
|
|
13
13
|
const json=(value,status=200)=>new Response(JSON.stringify(value),{status,headers:{'Content-Type':'application/json','Cache-Control':'no-store'}});
|
|
14
|
-
export async function createPortabilityHandler({db,buckets,policy,appOrigin,basePath='/api/portability',authorize,maxBodyBytes=MAX_LINE_BYTES}) {
|
|
14
|
+
export async function createPortabilityHandler({db,buckets,policy,migrations,appOrigin,basePath='/api/portability',authorize,maxBodyBytes=MAX_LINE_BYTES}) {
|
|
15
15
|
insist(typeof authorize==='function' && new URL(appOrigin).origin===appOrigin,'invalid_handler_configuration');
|
|
16
16
|
insist(/^\/[A-Za-z0-9/_-]+$/.test(basePath) && !basePath.endsWith('/') && Number.isSafeInteger(maxBodyBytes) && maxBodyBytes>0,'invalid_handler_configuration');
|
|
17
|
-
const service=await createPortability({db,buckets,policy});
|
|
17
|
+
const service=await createPortability({db,buckets,policy,migrations});
|
|
18
18
|
return async request=>{
|
|
19
19
|
const url=new URL(request.url);
|
|
20
20
|
if(url.pathname!==basePath && !url.pathname.startsWith(basePath+'/'))return null;
|
|
@@ -31,7 +31,7 @@ export async function createPortabilityHandler({db,buckets,policy,appOrigin,base
|
|
|
31
31
|
if(collection==='imports' && request.method==='GET' && !id)return json({active:await api.active(actor)});
|
|
32
32
|
if(request.method==='POST' && !id) {
|
|
33
33
|
const body=await boundedJson(request,maxBodyBytes);
|
|
34
|
-
if(collection==='imports'){insist(body.confirmed===true,'confirmation_required');return json({id:await api.create(body.manifest,actor)},201);}
|
|
34
|
+
if(collection==='imports'){insist(body.confirmed===true,'confirmation_required');return json({id:await api.create(body.manifest,actor,{ledgers:body.ledgers,acknowledgeUnverifiedSchemaObjects:body.acknowledgeUnverifiedSchemaObjects===true})},201);}
|
|
35
35
|
return json({id:await api.create(actor)},201);
|
|
36
36
|
}
|
|
37
37
|
if(request.method==='GET' && id && !action)return json(await api.status(id));
|
|
@@ -13,26 +13,39 @@ import { fence, mergeJournal, tombstone } from './journal.mjs';
|
|
|
13
13
|
import { writeObject } from './payloads.mjs';
|
|
14
14
|
import { createCopy,sidecar,readCopy,restoreCopy } from './copies.mjs';
|
|
15
15
|
import { excludedDependentGuards, migrationSeeded, seededCondition } from './restore-safety.mjs';
|
|
16
|
-
import { additiveBridge, sameSchemaMigrations, appliedMigrations } from './bridge.mjs';
|
|
16
|
+
import { additiveBridge, sameSchemaMigrations, appliedMigrations, verifiedLegacySchema } from './bridge.mjs';
|
|
17
17
|
import { replayStatements } from './backfills.mjs';
|
|
18
18
|
import { checkArchivedLedgers, archivedHistory } from './ledgers.mjs';
|
|
19
19
|
import { legacyEmptyGuards, legacyAuditStatements, requireLegacyEmpty } from './legacy-safety.mjs';
|
|
20
20
|
import { checkReferences } from './references.mjs';
|
|
21
|
+
import { rebuildDerivedStatements } from './derived.mjs';
|
|
21
22
|
// ledgers: the rows of each migration ledger table the archive carries, as the
|
|
22
23
|
// caller read them from it ({table:rows}); uploadPart binds them to its parts.
|
|
23
|
-
export async function createImport(context,manifest,actor='direct',legacyRecovery,ledgers) {
|
|
24
|
+
export async function createImport(context,manifest,actor='direct',legacyRecovery,ledgers,acknowledgeUnverifiedSchemaObjects=false) {
|
|
24
25
|
const {db,config,buckets}=context;await checkManifest(manifest);checkArchivedLedgers(manifest,ledgers);
|
|
25
26
|
insist(manifest.product===config.product,'wrong_product');
|
|
26
27
|
// Explicit schema upgrades can reconcile rows, but cannot attest an interrupted deployment.
|
|
27
|
-
await appliedMigrations(db);
|
|
28
|
+
await appliedMigrations(db);
|
|
28
29
|
const original=manifest;manifest=await restorableManifest(original);
|
|
29
30
|
const skippedTables=original.tables.filter(table=>platformTable(table.name)).map(table=>table.name);
|
|
30
31
|
const cat=await catalogue(db,config),upgrade=config.acceptedSchemas[original.schema];
|
|
31
|
-
|
|
32
|
-
//
|
|
33
|
-
|
|
34
|
-
const
|
|
35
|
-
const
|
|
32
|
+
const legacy=original.version===2 && manifest.schemaObjects===undefined;
|
|
33
|
+
// An explicit acknowledgment keeps its existing recovery semantics. Without
|
|
34
|
+
// it, legacy v2 needs positive replay evidence for the complete destination.
|
|
35
|
+
const unverified=legacy && acknowledgeUnverifiedSchemaObjects===true;
|
|
36
|
+
const verifyingLegacy=legacy && !unverified && !legacyRecovery;
|
|
37
|
+
const history=()=>archivedHistory(original,ledgers,{allowOlderShape:unverified || verifyingLegacy});
|
|
38
|
+
const verified=verifyingLegacy && await verifiedLegacySchema(db,manifest,cat,context.migrations,history);
|
|
39
|
+
insist(!legacy || legacyRecovery || unverified || verified,'schema_mismatch:legacy_schema',409);
|
|
40
|
+
if(verified)manifest={...manifest,schema:cat.identity,schemaObjects:cat.schemaObjects};
|
|
41
|
+
if(unverified)for(const table of manifest.tables) {
|
|
42
|
+
const target=cat.tables.find(t=>t.name===table.name);
|
|
43
|
+
insist(target,'schema_mismatch:table:'+table.name,409);
|
|
44
|
+
for(const column of table.columns)insist(target.columns.some(c=>canonicalJson(c)===canonicalJson(column)),'schema_mismatch:column:'+table.name+':'+column.name,409);
|
|
45
|
+
insist(table.dependencies.every(d=>target.dependencies.includes(d)),'schema_mismatch:dependencies:'+table.name,409);
|
|
46
|
+
}
|
|
47
|
+
const same=manifest.schema===cat.identity;
|
|
48
|
+
const {replays,...bridge}=unverified?await additiveBridge(db,manifest,cat,context.migrations,history,legacyRecovery,true):upgrade?{}:same?(await sameSchemaMigrations(db,manifest,cat,context.migrations,history)??{}):await additiveBridge(db,manifest,cat,context.migrations,history,legacyRecovery);
|
|
36
49
|
// Finalize checks object references on the staged rows; a later migration
|
|
37
50
|
// replayed into a reference table would publish rows that check never saw.
|
|
38
51
|
const referenced=replays?.find(r=>Object.values(config.buckets).some(spec=>spec.objectReferences?.table===r.table));
|
|
@@ -49,6 +62,7 @@ export async function createImport(context,manifest,actor='direct',legacyRecover
|
|
|
49
62
|
insist(canonicalJson(manifest.requiredConfiguration)===canonicalJson(config.requiredConfiguration),'configuration_inventory');
|
|
50
63
|
for(const o of manifest.objects) insist(buckets[o.bucket] && config.buckets[o.bucket] && o.key.startsWith(config.buckets[o.bucket].prefix??''),'bucket_inventory');
|
|
51
64
|
insist(canonicalJson(manifest.payloads.map(x=>x.name))===canonicalJson(Object.keys(config.extraPayloads).sort()),'payload_inventory');
|
|
65
|
+
if(unverified)await destinationState(context,{metadata:{targets:cat.tables,manifest:original}});
|
|
52
66
|
await ensureImportLeases(db);
|
|
53
67
|
const previous=(await state(db)).import_job;
|
|
54
68
|
if(previous) {
|
|
@@ -56,10 +70,10 @@ export async function createImport(context,manifest,actor='direct',legacyRecover
|
|
|
56
70
|
await abortImport(context,previous,{staleOnly:true});
|
|
57
71
|
}
|
|
58
72
|
const id=crypto.randomUUID(),copies=Object.fromEntries(targets.map((t,i)=>[t.name,'rer_portability_stage_'+id.replaceAll('-','')+'_'+i]));
|
|
59
|
-
const row={id,kind:'import',actor,phase:'staging',completed:0,metadata:{manifest:original,...(skippedTables.length?{restoreManifest:manifest,skippedTables}:{}),copies,targets,triggers:cat.triggers,...(bridge.fromSchema?{bridge}:{}),...(replays?{replays}:{}),...(legacyRecovery?{legacyRecovery}: {}),...(ledgers?{ledgers}:{})}};
|
|
73
|
+
const row={id,kind:'import',actor,phase:'staging',completed:0,metadata:{manifest:original,...(unverified?{schemaUnverified:{reason:'acknowledged recovery of a version 2 archive without schemaObjects',sourceIdentity:original.sourceIdentity,exportedAt:original.exportedAt,acknowledgedAt:new Date().toISOString()}}:{}),...(skippedTables.length?{restoreManifest:manifest,skippedTables}:{}),copies,targets,triggers:cat.triggers,...(bridge.fromSchema?{bridge}:{}),...(replays?{replays}:{}),...(legacyRecovery?{legacyRecovery}: {}),...(ledgers?{ledgers}:{})}};
|
|
60
74
|
try {
|
|
61
75
|
await db.batch([guard(db,'(SELECT import_job FROM rer_portability_state WHERE id=1) IS NULL'),
|
|
62
|
-
statement(db,'UPDATE rer_portability_state SET import_job=? WHERE id=1',id),
|
|
76
|
+
statement(db,'UPDATE rer_portability_state SET import_job=?,applying=? WHERE id=1',id,unverified?1:0),
|
|
63
77
|
...targets.flatMap(t=>createCopy(db,t,copies[t.name])),
|
|
64
78
|
statement(db,'INSERT INTO rer_portability_jobs(id,kind,actor,phase,metadata) VALUES(?,?,?,?,?)',id,'import',actor,row.phase,canonicalJson(row.metadata)),createImportLease(db,id),db.prepare('DELETE FROM rer_portability_guard')]);
|
|
65
79
|
}catch(e){const running=(await state(db)).import_job;if(running)throw await importRunning(db,running);throw e;}
|
|
@@ -208,14 +222,16 @@ async function finalizeLeasedImport(context,id,{contentHash,archiveHash,laterJou
|
|
|
208
222
|
insist(hash.digest()===object.sha256,'object_write_checksum');
|
|
209
223
|
}
|
|
210
224
|
for(const payload of manifest.payloads)await config.extraPayloads[payload.name].consume(binaryStream(context,id,payload,binaryIndexes),payload);
|
|
225
|
+
const derived=await rebuildDerivedStatements(db,config);
|
|
211
226
|
row.phase='complete';row.metadata.contentHash=digest;
|
|
227
|
+
if(row.metadata.schemaUnverified)row.metadata.schemaUnverified.archiveHash=archiveHash;
|
|
212
228
|
await db.batch([db.prepare('PRAGMA defer_foreign_keys=ON'),...emptyGuards(context,row,seeded),...seeded.map(t=>guard(db,...seededCondition(t))),...dependentGuards,...legacyEmptyGuards(db,row),
|
|
213
229
|
guard(db,'NOT EXISTS(SELECT 1 FROM rer_portability_epochs WHERE source=? AND epoch>?) AND NOT EXISTS(SELECT 1 FROM rer_portability_state WHERE source_identity=? AND epoch>?)',manifest.sourceIdentity,checked.epoch,manifest.sourceIdentity,checked.epoch),
|
|
214
230
|
...row.metadata.targets.map(t=>guard(db,`NOT EXISTS(SELECT 1 FROM ${quote(sidecar(row.metadata.copies[t.name]))} s JOIN rer_portability_journal j ON j.source=? AND j.tombstone=s.rer_portability_tombstone)`,manifest.sourceIdentity)),
|
|
215
231
|
...row.metadata.triggers.map(t=>db.prepare(`DROP TRIGGER ${quote(t.name)}`)),
|
|
216
232
|
...replaced.map(t=>db.prepare(`DELETE FROM ${quote(t)}`)),
|
|
217
233
|
...row.metadata.targets.flatMap(t=>restoreCopy(db,t,row.metadata.copies[t.name])),save(db,row),...replayStatements(db,row),
|
|
218
|
-
...row.metadata.triggers.map(t=>db.prepare(t.sql)),...mergeJournal(db,manifest,checked),...legacyAuditStatements(db,row),
|
|
234
|
+
...row.metadata.triggers.map(t=>db.prepare(t.sql)),...derived,...mergeJournal(db,manifest,checked),...legacyAuditStatements(db,row),
|
|
219
235
|
// Publication and removal of this import's scratch commit together. A lost
|
|
220
236
|
// acknowledgement still finds a complete job with no staging left behind.
|
|
221
237
|
...dropCopyStatements(db,row),
|
|
@@ -224,14 +240,14 @@ async function finalizeLeasedImport(context,id,{contentHash,archiveHash,laterJou
|
|
|
224
240
|
// D1 may have committed even when its acknowledgement failed.
|
|
225
241
|
if((await job(db,id)).phase==='complete')return importStatus(context,id);
|
|
226
242
|
if(!await context.importLease.owned())throw new (await import('./storage.mjs')).PortabilityError('import_lease_expired',409);
|
|
227
|
-
await removeOwned(context,id);row.phase='staging';await db.batch([save(db,row),db
|
|
243
|
+
await removeOwned(context,id);row.phase='staging';await db.batch([save(db,row),statement(db,'UPDATE rer_portability_state SET applying=? WHERE id=1',row.metadata.schemaUnverified?1:0)]);throw e;
|
|
228
244
|
}
|
|
229
245
|
return importStatus(context,id);
|
|
230
246
|
}
|
|
231
247
|
// Upload receipts stay idempotent; the status door exposes the moving lease.
|
|
232
248
|
export async function importStatus(context,id,{includeLease=true}={}) {
|
|
233
249
|
const row=await job(context.db,id,'import');return {id,kind:'import',phase:row.phase,completed:row.completed,total:row.metadata.manifest.partCount,...(includeLease?await importLeaseStatus(context.db,row):{}),
|
|
234
|
-
contentHash:row.metadata.contentHash??null,requiredConfiguration:row.phase==='complete'?row.metadata.manifest.requiredConfiguration:[],...(row.metadata.bridge?{bridged:row.metadata.bridge}:{}),...(row.metadata.skippedTables?{skippedTables:row.metadata.skippedTables}:{}),...(row.metadata.safetyChanges?{safetyChanges:row.metadata.safetyChanges}:{})};
|
|
250
|
+
contentHash:row.metadata.contentHash??null,...(row.phase==='complete' && row.metadata.schemaUnverified?{schemaUnverified:row.metadata.schemaUnverified}:{}),requiredConfiguration:row.phase==='complete'?row.metadata.manifest.requiredConfiguration:[],...(row.metadata.bridge?{bridged:row.metadata.bridge}:{}),...(row.metadata.skippedTables?{skippedTables:row.metadata.skippedTables}:{}),...(row.metadata.safetyChanges?{safetyChanges:row.metadata.safetyChanges}:{})};
|
|
235
251
|
}
|
|
236
252
|
export async function activeImport(context,actor='direct') {
|
|
237
253
|
const id=(await state(context.db)).import_job;
|
|
@@ -39,11 +39,12 @@ export interface ArchivePolicy {
|
|
|
39
39
|
}
|
|
40
40
|
/** Actual changed rows grouped by table, column and safe replacement value; never original values. */
|
|
41
41
|
export interface SafetyChange {table:string;column:string;value:Cell;rows:number}
|
|
42
|
-
export interface
|
|
42
|
+
export interface SchemaUnverifiedReceipt {reason:string;sourceIdentity:string;exportedAt:string;acknowledgedAt:string;archiveHash:string}
|
|
43
|
+
export interface Status {schemaUnverified?:SchemaUnverifiedReceipt;id:string;kind:'export'|'import';phase:'building'|'staging'|'applying'|'complete'|'aborted';completed:number;total:number|null;/** Import only: inactivity deadline and whether it has passed. */abortableAt?:string|null;stale?:boolean;/** Export only: bytes of the parts built so far. */bytes?:number;contentHash:string|null;requiredConfiguration?:string[];bridged?:Bridge;/** Platform bookkeeping ignored in an older archive. */skippedTables?:string[];/** Durable summary of bounded import normalization. */safetyChanges?:SafetyChange[]}
|
|
43
44
|
/** Later additive migrations an older archive was restored across. */
|
|
44
45
|
export interface Bridge {fromSchema:string;toSchema:string;migrations:string[];tables:string[];columns:Record<string,string[]>;/** Later backfills of tables the archive lacks, replayed at finalize in this order; rows is null until the apply commits. */replayed?:{migration:string;statement:number;table:string;rows:number|null}[]}
|
|
45
46
|
/** A runner's replay of the destination's applied migrations: the archive schema identity after a chosen set, and each migration's statements. */
|
|
46
|
-
export interface MigrationReplay {names:string[];/** Seed-declared data-only Platform entries without migration files. */dataOnly?:string[];/** Names generated by the seed's deployment specification for this file. */aliases?(name:string):string[]|undefined;statements(name:string):string[];/** The migration file's SHA-256 hex, where the runner read the file. */sha256?(name:string):string|undefined;schemaAfter(names:string[]):Promise<string>}
|
|
47
|
+
export interface MigrationReplay {names:string[];/** Seed-declared data-only Platform entries without migration files. */dataOnly?:string[];/** Names generated by the seed's deployment specification for this file. */aliases?(name:string):string[]|undefined;statements(name:string):string[];/** The migration file's SHA-256 hex, where the runner read the file. */sha256?(name:string):string|undefined;schemaAfter(names:string[],options?:{tablesOnly?:boolean}):Promise<string>}
|
|
47
48
|
/** Export job creation only; no record data, object keys or file paths. */
|
|
48
49
|
export interface ExportEvent {id:string;/** Opaque caller id; the direct/CLI sentinel is normalized to null. */actor:string|null;scope:'installation'}
|
|
49
50
|
export interface PortabilityOptions {
|
|
@@ -53,7 +54,7 @@ export interface PortabilityOptions {
|
|
|
53
54
|
}
|
|
54
55
|
export interface Portability {
|
|
55
56
|
exports: {create(actor?:string):Promise<string>;advance(id:string,options?:{expectedPart?:number;maxParts?:number}):Promise<Status>;status(id:string):Promise<Status>;download(id:string):Promise<ReadableStream<Uint8Array>>;abort(id:string):Promise<Status>};
|
|
56
|
-
imports: {active(actor?:string):Promise<Status|null>;/** ledgers: the rows of each migration ledger table an old archive carries, as read from it; they place it in the destination's history and must equal its parts. */create(manifest:Manifest,actor?:string,options?:{ledgers?:Record<string,Row[]
|
|
57
|
+
imports: {active(actor?:string):Promise<Status|null>;/** ledgers: the rows of each migration ledger table an old archive carries, as read from it; they place it in the destination's history and must equal its parts. */create(manifest:Manifest,actor?:string,options?:{ledgers?:Record<string,Row[]>;acknowledgeUnverifiedSchemaObjects?:boolean}):Promise<string>;uploadPart(id:string,index:number,part:Part):Promise<Status>;finalize(id:string,options:{archiveHash:string;contentHash?:string;laterJournal?:LaterJournal}):Promise<Status>;status(id:string):Promise<Status>;abort(id:string):Promise<Status>;/** Aborts the import left staged by a crashed restore and returns its id; call only as the destination's sole operator. */abortAbandoned():Promise<string|null>};
|
|
57
58
|
status(id:string):Promise<Status>;actor(id:string):Promise<string>;
|
|
58
59
|
erasureStatements(table:string,primaryKey:Record<string,Cell>):Promise<D1PreparedStatement[]>;
|
|
59
60
|
redactionStatements():Promise<D1PreparedStatement[]>;
|
|
@@ -20,7 +20,7 @@ export async function createPortability({db,buckets={},policy:input,migrations,o
|
|
|
20
20
|
const context={db,buckets,config,migrations,onExport};
|
|
21
21
|
return {
|
|
22
22
|
exports:{create:actor=>createExport(context,actor),advance:(id,options)=>advanceExport(context,id,options),status:id=>exportStatus(context,id),download:id=>downloadExport(context,id),abort:id=>abortExport(context,id)},
|
|
23
|
-
imports:{active:actor=>activeImport(context,actor),create:(manifest,actor,options={})=>createImport(context,manifest,actor,undefined,options.ledgers),uploadPart:(id,index,part)=>uploadPart(context,id,index,part),finalize:(id,options)=>finalizeImport(context,id,options),status:id=>importStatus(context,id),abort:id=>abortImport(context,id),abortAbandoned:()=>abortAbandonedImport(context)},
|
|
23
|
+
imports:{active:actor=>activeImport(context,actor),create:(manifest,actor,options={})=>createImport(context,manifest,actor,undefined,options.ledgers,options.acknowledgeUnverifiedSchemaObjects),uploadPart:(id,index,part)=>uploadPart(context,id,index,part),finalize:(id,options)=>finalizeImport(context,id,options),status:id=>importStatus(context,id),abort:id=>abortImport(context,id),abortAbandoned:()=>abortAbandonedImport(context)},
|
|
24
24
|
status:async id=>(await job(db,id)).kind==='export'?exportStatus(context,id):importStatus(context,id),
|
|
25
25
|
actor:async id=>(await job(db,id)).actor,
|
|
26
26
|
erasureStatements:(table,key)=>erasureStatements(db,table,key),redactionStatements:()=>redactionStatements(db),invalidateTransfersStatements:()=>invalidateTransfersStatements(db),
|
|
@@ -115,7 +115,7 @@ export function checkArchivedLedgers(manifest,ledgers) {
|
|
|
115
115
|
}
|
|
116
116
|
|
|
117
117
|
/** The applied migrations an archive's own ledger records, [{name,sha256?}]; undefined when it carries no ledger. */
|
|
118
|
-
export function archivedHistory(manifest,ledgers) {
|
|
118
|
+
export function archivedHistory(manifest,ledgers,{allowOlderShape=false}={}) {
|
|
119
119
|
const tables=archivedLedgerTables(manifest);
|
|
120
120
|
// A Platform-origin export records its ledger in the manifest, never as a table.
|
|
121
121
|
if(manifest.migrationLedger)insist(!tables.length,'schema_mismatch:migration_history');
|
|
@@ -123,7 +123,8 @@ export function archivedHistory(manifest,ledgers) {
|
|
|
123
123
|
insist(tables.length===1 && ledgers?.[tables[0].name],'schema_mismatch:migration_history');
|
|
124
124
|
const [table]=tables,rows=ledgers[table.name],columns=table.columns.map(c=>c.name);
|
|
125
125
|
if(identifier(table.name)==='rer_migrations') {
|
|
126
|
-
|
|
126
|
+
const older=canonicalJson([{name:'name',type:'TEXT',notnull:0,pk:1},{name:'applied_at',type:'TEXT',notnull:1,pk:0}]);
|
|
127
|
+
insist(shape(table.columns)===platformShape || (allowOlderShape && shape(table.columns)===older),'schema_mismatch:migration_history');
|
|
127
128
|
insist(!rows.some(row=>partial(row.name)),'schema_mismatch:uncertain_migrations');
|
|
128
129
|
}else{
|
|
129
130
|
insist(columns.includes('name') && columns.includes('state')===columns.includes('sha256'),'schema_mismatch:migration_history');
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { all, statement, guard, quote, insist } from './storage.mjs';
|
|
2
2
|
import { canonicalJson } from './hash.mjs';
|
|
3
|
+
import { excludedTable } from './catalogue.mjs';
|
|
3
4
|
import { migrationSeeded, seededCondition } from './restore-safety.mjs';
|
|
4
5
|
|
|
5
6
|
// Recovery requires a new installation, including excluded identity tables and
|
|
@@ -22,14 +23,22 @@ function shadowTables(tables) {
|
|
|
22
23
|
/** What the destination must still hold when the restore commits: `emptyTables` stay empty and
|
|
23
24
|
* each of `seededTables` keeps exactly the rows its migrations put there. A table with other rows
|
|
24
25
|
* is listed as to-be-empty, so the preflight refuses it. */
|
|
25
|
-
export async function legacyTargetState(db,migrations) {
|
|
26
|
+
export async function legacyTargetState(db,migrations,include=()=>true) {
|
|
26
27
|
const tables=await all(db,"SELECT name,sql FROM sqlite_master WHERE type='table' AND sql IS NOT NULL ORDER BY name"),shadows=shadowTables(tables);
|
|
27
|
-
const names=tables.map(t=>t.name).filter(name=>!bookkeeping(name) && !shadows.has(name.toLowerCase()));
|
|
28
|
+
const names=tables.map(t=>t.name).filter(name=>!bookkeeping(name) && !shadows.has(name.toLowerCase()) && include(name));
|
|
28
29
|
const filled=[];
|
|
29
30
|
for(const name of names)if(await statement(db,`SELECT 1 FROM ${quote(name)} LIMIT 1`).first())filled.push(name);
|
|
30
31
|
const seeded=await migrationSeeded(db,filled,migrations);
|
|
31
32
|
return {emptyTables:names.filter(name=>!seeded.some(s=>s.name===name)),seededTables:seeded};
|
|
32
33
|
}
|
|
34
|
+
// Before a runner applies pending migrations, use the normal product emptiness
|
|
35
|
+
// rule against the existing tables. Identity exclusions and replaceable rows
|
|
36
|
+
// retain their normal meaning; only the already-applied history can vouch for seeds.
|
|
37
|
+
export async function requireProductEmpty({db,policy,migrations}) {
|
|
38
|
+
const {emptyTables}=await legacyTargetState(db,migrations,name=>!excludedTable(name,policy)
|
|
39
|
+
&& !(policy.derivedTables??[]).includes(name) && !(policy.replaceableTables??[]).includes(name));
|
|
40
|
+
for(const name of emptyTables)insist(!(await statement(db,`SELECT 1 FROM ${quote(name)} LIMIT 1`).first()),'target_not_empty:'+name,409);
|
|
41
|
+
}
|
|
33
42
|
export async function requireLegacyEmpty({db,buckets},{emptyTables,seededTables=[]}) {
|
|
34
43
|
for(const table of emptyTables)insist(!(await statement(db,`SELECT 1 FROM ${quote(table)} LIMIT 1`).first()),'legacy_target_not_empty:'+table,409);
|
|
35
44
|
for(const table of seededTables){const [condition,...values]=seededCondition(table);insist((await statement(db,`SELECT CASE WHEN (${condition}) THEN 1 ELSE 0 END AS ok`,...values).first()).ok===1,'legacy_target_not_empty:'+table.name,409);}
|
|
@@ -8,12 +8,12 @@ import { policy, schemaIdentity } from './catalogue.mjs';
|
|
|
8
8
|
// sha256 being the migration file's hash where the runner read the file.
|
|
9
9
|
export function migrationReplay(plan,input,{dataOnly=[]}={}) {
|
|
10
10
|
const config=policy(input);
|
|
11
|
-
return {names:plan.map(migration=>migration.name),dataOnly,aliases:name=>plan.find(m=>m.name===name).aliases,statements:name=>plan.find(m=>m.name===name).statements,sha256:name=>plan.find(m=>m.name===name).sha256,async schemaAfter(names) {
|
|
11
|
+
return {names:plan.map(migration=>migration.name),dataOnly,aliases:name=>plan.find(m=>m.name===name).aliases,statements:name=>plan.find(m=>m.name===name).statements,sha256:name=>plan.find(m=>m.name===name).sha256,async schemaAfter(names,options) {
|
|
12
12
|
const scratch=new DatabaseSync(':memory:');
|
|
13
13
|
const db={prepare:sql=>{const query=scratch.prepare(sql);let values=[];return {bind(...args){values=args;return this;},async all(){return {results:query.all(...values)};}};}};
|
|
14
14
|
try {
|
|
15
15
|
for(const migration of plan.filter(m=>names.includes(m.name)))for(const sql of migration.statements)scratch.exec(sql);
|
|
16
|
-
return await schemaIdentity(db,config);
|
|
16
|
+
return await schemaIdentity(db,config,options);
|
|
17
17
|
}finally{scratch.close();}
|
|
18
18
|
},
|
|
19
19
|
// The rows the whole replay leaves in each requested table, as sorted images (legacy-safety.mjs).
|
|
@@ -39,6 +39,7 @@ export function transferProgress(report, { interval = 3000, now = () => performa
|
|
|
39
39
|
}
|
|
40
40
|
};
|
|
41
41
|
return event => {
|
|
42
|
+
if(event.phase==='schema-unverified') { report('Schema unverified: '+JSON.stringify(event.receipt)); return; }
|
|
42
43
|
if (event.phase === 'skipped') { report(skippedTablesLine(event.tables)); return; }
|
|
43
44
|
if (event.phase === 'start') {
|
|
44
45
|
const noun = event.direction === 'export' ? `Exporting about ${formatBytes(event.bytes)} of local data` : `Restoring a ${formatBytes(event.bytes)} archive`;
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { all, quote, insist } from './storage.mjs';
|
|
2
|
+
import { catalogue, policy } from './catalogue.mjs';
|
|
3
|
+
import { migrationSeeded } from './restore-safety.mjs';
|
|
4
|
+
|
|
5
|
+
// Runner preflight before creating a restore job. Finalize repeats its atomic
|
|
6
|
+
// empty-table guards; this early read keeps an occupied destination untouched.
|
|
7
|
+
export async function requireEmptyTables({ db, policy: input, migrations }) {
|
|
8
|
+
const config = policy(input), cat = await catalogue(db, config), filled = [];
|
|
9
|
+
for (const table of cat.tables) if (!config.replaceableTables.includes(table.name) && (await all(db, `SELECT 1 FROM ${quote(table.name)} LIMIT 1`)).length) filled.push(table.name);
|
|
10
|
+
const seeded = await migrationSeeded(db, filled, migrations);
|
|
11
|
+
for (const name of filled) insist(seeded.some(table => table.name === name), 'target_not_empty:' + name, 409);
|
|
12
|
+
}
|
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
import { execFileSync } from 'node:child_process';
|
|
2
2
|
import { git } from './source.mjs';
|
|
3
3
|
import { verifyWorkflow } from './verify-workflow.mjs';
|
|
4
|
+
import { shared } from '../../cli/shared.mjs';
|
|
5
|
+
|
|
6
|
+
const { createRedaction } = await shared('redaction.mjs');
|
|
4
7
|
|
|
5
8
|
const same = (left, right) => typeof left === 'string' && left.toLowerCase() === right.toLowerCase();
|
|
6
9
|
function identity(value) {
|
|
@@ -101,9 +104,15 @@ export function githubVerify(source, repository, selector) {
|
|
|
101
104
|
}
|
|
102
105
|
if (tagUnavailable) throw Error('tag');
|
|
103
106
|
result.details = rejected.size ? [...rejected].join('; ') : 'No runs found for this SHA.';
|
|
104
|
-
} catch {
|
|
107
|
+
} catch (error) {
|
|
105
108
|
result.reason = 'github-verify-unavailable';
|
|
106
|
-
|
|
109
|
+
const fallback = 'GitHub Verify could not be resolved. Check gh authentication and repository access, then try again; no local verification was substituted.';
|
|
110
|
+
const stderr = typeof error.stderr === 'string' || Buffer.isBuffer(error.stderr)
|
|
111
|
+
? createRedaction().redact(error.stderr.toString()).trim() : '';
|
|
112
|
+
// Redact the complete diagnostic before bounding it, including long tokens.
|
|
113
|
+
const diagnostic = stderr ? stderr.slice(0, 8192) : fallback;
|
|
114
|
+
result.details = Number.isInteger(error.status)
|
|
115
|
+
? `gh exited with status ${error.status}: ${diagnostic}` : diagnostic;
|
|
107
116
|
}
|
|
108
117
|
return result;
|
|
109
118
|
}
|