@runeditrun/rer 0.2.126 → 0.2.127
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cli/portability-evidence.mjs +27 -5
- package/cli/portability-progress.mjs +14 -0
- package/cli/portability-worker.mjs +9 -3
- package/cli/portability.mjs +15 -3
- package/global/CHANGELOG.md +20 -0
- package/modules/portability/README.md +11 -2
- package/modules/portability/archive-step.mjs +1 -1
- package/modules/portability/bridge.mjs +26 -10
- package/modules/portability/checkout-history.mjs +7 -3
- package/modules/portability/files.mjs +12 -4
- package/modules/portability/import.mjs +1 -1
- package/modules/portability/worker-history.d.mts +5 -1
- package/modules/portability/worker-history.mjs +11 -9
- package/package.json +1 -1
|
@@ -27,6 +27,11 @@ async function migrationEvidence(root,{binding='DB'}={}) {
|
|
|
27
27
|
const bytes=await readFile(file);
|
|
28
28
|
return {name,sha256:await sha256(bytes),statements:splitStatements(bytes.toString('utf8'))};
|
|
29
29
|
}));
|
|
30
|
+
// Wrangler sorts the leading migration number before breaking ties by name.
|
|
31
|
+
const numbers=files.map(({name})=>Number.parseInt(name,10));
|
|
32
|
+
const nativeOrder=files.map((_,i)=>i).sort((a,b)=>(Number.isFinite(numbers[a])?numbers[a]:Infinity)
|
|
33
|
+
-(Number.isFinite(numbers[b])?numbers[b]:Infinity) || a-b);
|
|
34
|
+
const reordered=nativeOrder.some((index,i)=>index!==i);
|
|
30
35
|
const sources=await ownerMigrationSources(root,binding);
|
|
31
36
|
const ownerFiles = (await Promise.all(sources.filter(source => source.directory).map(async ({directory,prefix,table}) => {
|
|
32
37
|
const owners = await collectMigrations(root, [{binding,path:directory,prefix}], [{kind:'d1',binding}]);
|
|
@@ -40,15 +45,32 @@ async function migrationEvidence(root,{binding='DB'}={}) {
|
|
|
40
45
|
declared.add(file.name);
|
|
41
46
|
files.push(file);
|
|
42
47
|
}
|
|
43
|
-
return {version:2,files,ownerSources};
|
|
48
|
+
return {version:2,files,ownerSources,...(reordered?{nativeOrder:[...nativeOrder,...ownerFiles.map((_,i)=>nativeOrder.length+i)]}:{})};
|
|
44
49
|
}
|
|
45
|
-
const evidence = await bundledPrefixes(files);
|
|
50
|
+
const evidence = await bundledPrefixes(files,reordered?nativeOrder:undefined);
|
|
46
51
|
if (ownerSources.length) evidence.ownerSources = ownerSources;
|
|
47
52
|
return evidence;
|
|
48
53
|
}
|
|
49
54
|
|
|
50
|
-
async function bundledPrefixes(files) {
|
|
51
|
-
const objects=[],indices=new Map()
|
|
55
|
+
async function bundledPrefixes(files,nativeOrder) {
|
|
56
|
+
const objects=[],indices=new Map();
|
|
57
|
+
const prefixes=await capturePrefixes(files,objects,indices);
|
|
58
|
+
if(nativeOrder) {
|
|
59
|
+
const lexicalObjects=objects.length;
|
|
60
|
+
let nativePrefixes;
|
|
61
|
+
try { nativePrefixes=await capturePrefixes(nativeOrder.map(i=>files[i]),objects,indices); }
|
|
62
|
+
catch(error) {
|
|
63
|
+
// A lexical checkout can contain dependencies that make numeric order impossible.
|
|
64
|
+
if(error.code!=='ERR_SQLITE_ERROR')throw error;
|
|
65
|
+
}
|
|
66
|
+
if(nativePrefixes)return {version:1,files,objects,prefixes,nativeOrder,nativePrefixes};
|
|
67
|
+
objects.length=lexicalObjects;
|
|
68
|
+
}
|
|
69
|
+
return {version:1,files,objects,prefixes};
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
async function capturePrefixes(files,objects,indices) {
|
|
73
|
+
const prefixes=[];
|
|
52
74
|
const scratch=new DatabaseSync(':memory:');
|
|
53
75
|
try {
|
|
54
76
|
const recorder=migrationRecorder({exec:sql=>scratch.exec(sql),all:sql=>scratch.prepare(sql).all()});
|
|
@@ -65,7 +87,7 @@ async function bundledPrefixes(files) {
|
|
|
65
87
|
await capture();
|
|
66
88
|
}
|
|
67
89
|
}finally{scratch.close();}
|
|
68
|
-
return
|
|
90
|
+
return prefixes;
|
|
69
91
|
}
|
|
70
92
|
|
|
71
93
|
export async function generateMigrationEvidence(root,options) {
|
|
@@ -17,3 +17,17 @@ export function nativeFailure(error) {
|
|
|
17
17
|
const code = typeof error?.code === 'string' && /^[A-Za-z0-9_.-]{1,64}$/.test(error.code) ? error.code : Number.isSafeInteger(error?.code) ? String(error.code) : undefined;
|
|
18
18
|
return { class: name, ...(code ? { code } : {}) };
|
|
19
19
|
}
|
|
20
|
+
|
|
21
|
+
const transportCodes = new Set([
|
|
22
|
+
'ECONNREFUSED', 'ECONNRESET', 'EPIPE', 'ETIMEDOUT',
|
|
23
|
+
'UND_ERR_CONNECT_TIMEOUT', 'UND_ERR_HEADERS_TIMEOUT', 'UND_ERR_BODY_TIMEOUT', 'UND_ERR_SOCKET',
|
|
24
|
+
]);
|
|
25
|
+
|
|
26
|
+
// The local Wrangler bindings use Undici over loopback. Carry only known
|
|
27
|
+
// transport codes, never socket addresses or driver messages.
|
|
28
|
+
export function localStorageTransport(error) {
|
|
29
|
+
if (!(error instanceof TypeError) || !['fetch failed', 'terminated'].includes(error.message)
|
|
30
|
+
|| !/(?:node:internal\/deps\/undici|[/\\]undici[/\\])/.test(error.stack ?? '')) return;
|
|
31
|
+
const code = error.cause?.code;
|
|
32
|
+
return transportCodes.has(code) ? code : 'unreachable';
|
|
33
|
+
}
|
|
@@ -12,7 +12,7 @@ import { mkdir, readFile, rm, stat, writeFile } from 'node:fs/promises';
|
|
|
12
12
|
import { createRequire } from 'node:module';
|
|
13
13
|
import { createPortability, PortabilityError, readArchive } from '../modules/portability/index.mjs';
|
|
14
14
|
import { transferProgress } from '../modules/portability/progress.mjs';
|
|
15
|
-
import { nativeFailure, stateBytes } from './portability-progress.mjs';
|
|
15
|
+
import { localStorageTransport, nativeFailure, stateBytes } from './portability-progress.mjs';
|
|
16
16
|
import { parseJsonc } from './jsonc.mjs';
|
|
17
17
|
import { loadPolicy } from './portability-policy.mjs';
|
|
18
18
|
import { appliedMigrations, platformMigrationDeclaration, nativeMigrationPreflight } from './portability-migrations.mjs';
|
|
@@ -29,6 +29,7 @@ async function dataConfiguration(root) {
|
|
|
29
29
|
return path;
|
|
30
30
|
}
|
|
31
31
|
|
|
32
|
+
let importStep = 'verifying the archive';
|
|
32
33
|
async function run(direction, { root, path, journal, binding, buckets, policy, policyModule, transient, legacyRecovery, acknowledgeUnverifiedSchemaObjects, platformMigrations }) {
|
|
33
34
|
// Lines for the person go to the CLI over IPC, which prints them as the command's progress.
|
|
34
35
|
const progress = transferProgress(line => process.send({ progress: line }));
|
|
@@ -43,6 +44,7 @@ async function run(direction, { root, path, journal, binding, buckets, policy, p
|
|
|
43
44
|
policy = await loadPolicy(root, inputPolicy, policyModule);
|
|
44
45
|
const declaration=platformMigrationDeclaration(platformMigrations);
|
|
45
46
|
if (direction === 'policy-preflight') return { checkedPolicy: true };
|
|
47
|
+
importStep = 'opening local D1 and R2';
|
|
46
48
|
const require = createRequire(join(root, 'package.json'));
|
|
47
49
|
const { getPlatformProxy, unstable_splitSqlQuery } = require('wrangler');
|
|
48
50
|
installD1Preflight(createRequire(require.resolve('wrangler/package.json'))('miniflare').D1_PLUGIN);
|
|
@@ -69,6 +71,7 @@ async function run(direction, { root, path, journal, binding, buckets, policy, p
|
|
|
69
71
|
await requireLegacyEmpty(options,await legacyTargetState(options.db,options.migrations));
|
|
70
72
|
return {checkedEmpty:true};
|
|
71
73
|
}
|
|
74
|
+
importStep = 'checking local restore storage';
|
|
72
75
|
await ensurePortabilitySchema(proxy.env[binding], await portabilityPlan(unstable_splitSqlQuery));
|
|
73
76
|
if(legacyRecovery)return await recoverLegacyArchive(options,path,legacyRecovery.output,transient,legacyRecovery.acknowledged);
|
|
74
77
|
if (direction === 'import') await requireEmptyTables(options);
|
|
@@ -77,6 +80,7 @@ async function run(direction, { root, path, journal, binding, buckets, policy, p
|
|
|
77
80
|
// the recorded abandoned job, dropping only its exact staging copies and
|
|
78
81
|
// sidecars before the replacement import creates its own scratch tables.
|
|
79
82
|
if (direction === 'import' && await service.imports.abortAbandoned()) process.send({ progress: 'Cleaned up an unfinished earlier import: a restore that stopped part-way left its staged data here. Its staging is removed and this import starts from the beginning.' });
|
|
83
|
+
importStep = 'preparing the restore';
|
|
80
84
|
return await (direction === 'export' ? exportArchive(service,path,transient,progress) : importArchive(service,path,transient,journal,progress,{acknowledgeUnverifiedSchemaObjects}));
|
|
81
85
|
} finally { await proxy?.dispose(); await rm(configPath, { force: true }); }
|
|
82
86
|
}
|
|
@@ -103,8 +107,10 @@ catch (error) {
|
|
|
103
107
|
const identityTable = error instanceof PortabilityError && error.code.startsWith('identity_table_undeclared:') ? error.code.slice('identity_table_undeclared:'.length) : undefined;
|
|
104
108
|
const safety = error instanceof PortabilityError && error.code.startsWith('import_row_') ? error.code.slice(error.code.indexOf(':')+1) : undefined;
|
|
105
109
|
// Anything that is not a portability refusal names its class and code, never its message: that can hold row data.
|
|
106
|
-
const step=failureStep(error) ?? (process.argv[2] === 'import' ? undefined : process.argv[2]);
|
|
107
110
|
const native = error instanceof PortabilityError || error.message === 'archive_json' ? undefined : nativeFailure(error);
|
|
111
|
+
const importing = process.argv[2] === 'import';
|
|
112
|
+
const step = importing ? (native?.class === 'TypeError' ? failureStep(error) ?? importStep : undefined) : failureStep(error) ?? process.argv[2];
|
|
113
|
+
const storageTransport = importing && importStep !== 'verifying the archive' ? localStorageTransport(error) : undefined;
|
|
108
114
|
process.exitCode = exitCode;
|
|
109
|
-
await reply({ ok: false, code: exitCode === 2 ? 'invalid_archive' : exitCode === 4 ? 'restore_refused' : 'transfer_failed', reason, step, ...(reExport ? { reExport: true } : {}), ...(migration ? { migration } : {}), ...(unapplied ? { unapplied } : {}), ...(occupied ? { occupied } : {}), ...(identityTable ? { identityTable } : {}), ...(safety ? { safety } : {}), ...(native ? { native } : {}), exitCode });
|
|
115
|
+
await reply({ ok: false, code: exitCode === 2 ? 'invalid_archive' : exitCode === 4 ? 'restore_refused' : 'transfer_failed', reason, step, ...(reExport ? { reExport: true } : {}), ...(migration ? { migration } : {}), ...(unapplied ? { unapplied } : {}), ...(occupied ? { occupied } : {}), ...(identityTable ? { identityTable } : {}), ...(safety ? { safety } : {}), ...(storageTransport ? { storageTransport } : {}), ...(native ? { native } : {}), exitCode });
|
|
110
116
|
}
|
package/cli/portability.mjs
CHANGED
|
@@ -95,10 +95,20 @@ async function nativeWorker(store,direction,input,signal) {
|
|
|
95
95
|
const child = store.spawn(`${direction === 'migration-preflight' ? 'Checking migration history for' : direction === 'policy-preflight' ? 'Checking archive policy' : ['legacy-preflight','schema-recovery-preflight'].includes(direction) ? 'Checking empty recovery destination' : direction === 'export' ? 'Exporting' : 'Restoring'} local installation data`, process.execPath,
|
|
96
96
|
['--import', notice, worker, direction, JSON.stringify(input)], { cwd: input.root, signal, env: { ...process.env, WRANGLER_SEND_METRICS: 'false' }, stdio: ['ignore', 'pipe', 'pipe', 'ipc'] });
|
|
97
97
|
let receipt;
|
|
98
|
-
//
|
|
99
|
-
|
|
98
|
+
// A failed startup can leave Wrangler's restart process running after the
|
|
99
|
+
// worker reports the error. Start normal cleanup as soon as it reports failure.
|
|
100
|
+
const failed = new Promise(resolve => {
|
|
101
|
+
child.on('message', message => {
|
|
102
|
+
if (typeof message?.progress === 'string') store.report(message.progress);
|
|
103
|
+
else {
|
|
104
|
+
receipt = message;
|
|
105
|
+
if (receipt?.ok === false) resolve();
|
|
106
|
+
}
|
|
107
|
+
});
|
|
108
|
+
});
|
|
100
109
|
const closed = new Promise(resolve => child.once('close', (code, signal) => resolve({ code, signal })));
|
|
101
|
-
|
|
110
|
+
const exited = new Promise(resolve => { child.once('exit', resolve); child.once('error', resolve); });
|
|
111
|
+
await Promise.race([exited, failed]);
|
|
102
112
|
await store.stop(child);
|
|
103
113
|
const status = await closed;
|
|
104
114
|
if (!receipt || typeof receipt !== 'object') throw new RerError('transfer_failed', `Local portability ended before it finished (${status.signal ? `signal ${status.signal}` : `exit code ${status.code}`}). Run the same command again; it starts over${direction === 'import' ? ' and first cleans up anything this attempt staged' : ''}.`, 1);
|
|
@@ -112,6 +122,8 @@ async function nativeWorker(store,direction,input,signal) {
|
|
|
112
122
|
: receipt.reason?.startsWith('migration_ledger:') ? ledgerRefusal(receipt)
|
|
113
123
|
: receipt.migration ? `Restore refused (schema_mismatch:migration_data): migration ${receipt.migration} changed data after this archive was made, so its rows would skip that change. Restore an archive made after it.`
|
|
114
124
|
: receipt.occupied ? occupiedRefusal(input,receipt.reason)
|
|
125
|
+
: receipt.storageTransport ? `The connection to the local Wrangler storage runtime failed (${receipt.storageTransport}) while ${receipt.step}. Keep the archive and retry the import once local Wrangler can start and respond.`
|
|
126
|
+
: direction === 'import' && receipt.native?.class === 'TypeError' ? `Local restore stopped while ${receipt.step} (TypeError). Keep the archive. Report this stage and the rer version if the same command fails again.`
|
|
115
127
|
: `The portability module refused or failed the archive operation (${receipt.reason ?? 'incomplete'}${receipt.native ? `: ${receipt.native.class}${receipt.native.code && receipt.native.code !== receipt.reason ? `, code ${receipt.native.code}` : ''}` : ''})${receipt.step ? ` during ${receipt.step}` : ''}.${receipt.native && receipt.exitCode === 1 ? ' Run the same command again; it starts over.' : ''}`, receipt.exitCode ?? 1);
|
|
116
128
|
delete receipt.ok;
|
|
117
129
|
return receipt;
|
package/global/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,25 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.2.127
|
|
4
|
+
|
|
5
|
+
Signed: Run Edit Run fleet, 2026-11-21.
|
|
6
|
+
|
|
7
|
+
### Hosted archives into native destinations
|
|
8
|
+
|
|
9
|
+
- A hosted export with a legacy name-only migration ledger can restore into a native checkout whose Wrangler applied the core filenames in a different order. The recorded filenames, as a set, must be a contiguous prefix of the destination's applied history. Restore replays that prefix in the destination's order, checks the complete archived schema, and bridges later additive core or owner migrations only after the whole prefix. This covers the hosted-to-native path that rer 0.2.126 refused with `schema_mismatch:migration_history`.
|
|
10
|
+
- An interior migration missing from the archive, changed, extra or unavailable filenames, or a reorder that changes the archived schema still refuse before importing product rows or creating import jobs (CLI exit 4; zero product rows and zero import jobs). The CLI and Worker use the same placement decision. Core-only Worker bundles grow by about 0.7 KB in the qualified fixture; SQLite WASM remains excluded without owner migration files.
|
|
11
|
+
|
|
12
|
+
### Local restore storage failures
|
|
13
|
+
|
|
14
|
+
- A dropped connection to the local Wrangler storage runtime now names the restore stage, including staging or preparing the restore, and the storage connection failure. Diagnostics retain only bounded transport codes and fixed stage names, with no native driver message or payload. Failed restores publish no product rows; the same archive can be fully restored on a re-run once local storage responds.
|
|
15
|
+
- Maintained CLI copies and inventory are regenerated.
|
|
16
|
+
|
|
17
|
+
Archives written by rer 0.2.124, 0.2.125 or 0.2.126 import into 0.2.127, and 0.2.127 archives import into those versions for the same compatible core-only and owner-migration fixtures: the archive format identity remains `runeditrun-portability` version 2, unchanged from 0.2.126. This does not give older readers the new hosted-to-native restore behavior.
|
|
18
|
+
|
|
19
|
+
HappyToHelp and any seed restoring a hosted export need a re-pin to 0.2.127. This also applies to Dial, Nook and Penny if they import hosted archive bytes. Seeds on older pins need a re-pin to receive these fixes.
|
|
20
|
+
|
|
21
|
+
These changes require adoption through the seed's pinned package or refreshed source. This preparation establishes no seed adoption, live deployment, provider delivery or production result. The source range changes portability and import error naming and their maintained copies, with no import-kit or product migration change.
|
|
22
|
+
|
|
3
23
|
## 0.2.126
|
|
4
24
|
|
|
5
25
|
Signed: Run Edit Run fleet, 2026-11-20.
|
|
@@ -628,8 +628,10 @@ The same explicit build-evidence adoption supports native Wrangler's plain
|
|
|
628
628
|
the binding's declared directories when building. Each ledger's live names must
|
|
629
629
|
select the exact ordered prefix of its files, and combined replay must reproduce
|
|
630
630
|
the complete live schema, including indexes and triggers on excluded tables.
|
|
631
|
-
|
|
632
|
-
|
|
631
|
+
For complete name-only v2 archives, the core ledger may also follow Wrangler's
|
|
632
|
+
numeric filename order. Core-only Workers bundle snapshots of that order when
|
|
633
|
+
it differs. Unknown names, gaps, tracked but missing hashes and schema drift
|
|
634
|
+
refuse before restore writes. Existing hashes must still match; the
|
|
633
635
|
Node automatic checkout path still requires hashed history. This binds the
|
|
634
636
|
reviewed build to the verified current installation; it does not attest bytes
|
|
635
637
|
Wrangler never recorded. Do not adopt an installation whose applied source
|
|
@@ -677,6 +679,13 @@ them into a scratch database to find the archive's place in that history:
|
|
|
677
679
|
- Archives record the applied migration names in `manifest.migrations` (since
|
|
678
680
|
additive restore). Every recorded migration must be applied here, and
|
|
679
681
|
replaying exactly those must give the archive's schema identity.
|
|
682
|
+
A hosted export from the legacy deployment ledger can list filenames in
|
|
683
|
+
lexical order, which differs from Wrangler's application order for
|
|
684
|
+
mixed-width migration numbers. For an archive with a complete schema object
|
|
685
|
+
inventory, the recorded filenames must equal the set of files in a single
|
|
686
|
+
applied prefix. The destination replays that prefix in its own order and
|
|
687
|
+
checks the complete archive schema. Core and owner migrations after the
|
|
688
|
+
prefix can be bridged if additive.
|
|
680
689
|
- Archives exported from an installation RER Platform migrated (a Managed
|
|
681
690
|
installation leaving for native) also carry `manifest.migrationLedger`:
|
|
682
691
|
`{origin:'rer_migrations',entries:[{name,sha256}]}` in application order,
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// Stage names are fixed by the implementation. Never send native messages,
|
|
2
2
|
// SQL, object keys, row values or filesystem paths to the CLI.
|
|
3
|
-
const steps=new Set(['export_object_inventory','export_snapshot','export_part']);
|
|
3
|
+
const steps=new Set(['export_object_inventory','export_snapshot','export_part','preparing the restore','staging the restore','finishing the restore']);
|
|
4
4
|
export async function archiveStep(step,operation) {
|
|
5
5
|
try{return await operation();}
|
|
6
6
|
catch(error){
|
|
@@ -30,7 +30,7 @@ function recordedPlacement(manifest, recorded, applied) {
|
|
|
30
30
|
return recorded;
|
|
31
31
|
}
|
|
32
32
|
|
|
33
|
-
async function verifyRecordedSchema(manifest, earlier, schemaAfter) {
|
|
33
|
+
async function verifyRecordedSchema(manifest, earlier, schemaAfter, legacyNameOrder) {
|
|
34
34
|
const owner=manifest.migrationLedger?.origin==='d1_migrations';
|
|
35
35
|
const platform=platformPlacement(manifest);
|
|
36
36
|
let schema;
|
|
@@ -41,7 +41,7 @@ async function verifyRecordedSchema(manifest, earlier, schemaAfter) {
|
|
|
41
41
|
if(!owner || !(error?.code==='ERR_SQLITE_ERROR' || error instanceof PortabilityError && error.code==='schema_mismatch:legacy_evidence'))throw error;
|
|
42
42
|
throw new PortabilityError('schema_mismatch:owner_migration_history',409);
|
|
43
43
|
}
|
|
44
|
-
insist(schema===manifest.schema,owner?'schema_mismatch:owner_migration_history':'schema_mismatch',owner || platform?409:400);
|
|
44
|
+
insist(schema===manifest.schema,owner?'schema_mismatch:owner_migration_history':'schema_mismatch',owner || platform || legacyNameOrder?409:400);
|
|
45
45
|
}
|
|
46
46
|
// Where an archive sits in the destination's applied migrations: those its
|
|
47
47
|
// rows passed through (earlier) and those applied after it (unapplied).
|
|
@@ -56,8 +56,9 @@ async function placement(db,manifest,cat,replay,ledger,history,legacyRecovery,un
|
|
|
56
56
|
const files=replay.names.map(name=>({name,sha256:replay.sha256?.(name),aliases:replay.aliases?.(name)}));
|
|
57
57
|
// An aligned replayer keeps its existing contract: hashes are optional.
|
|
58
58
|
// Resolve destination aliases only when its ledger uses different names.
|
|
59
|
-
|
|
60
|
-
|
|
59
|
+
const entries=ledger?.entries??[];
|
|
60
|
+
let applied=entries.map(m=>m.name);
|
|
61
|
+
if(canonicalJson(replay.names)!==canonicalJson(applied))applied=matchMigrationLedger(entries,files,replay);
|
|
61
62
|
insist(canonicalJson(replay.names)===canonicalJson(applied) && await replay.schemaAfter(applied)===cat.identity,'schema_mismatch');
|
|
62
63
|
const archived=history?.();
|
|
63
64
|
const recorded=recordedMigrations(manifest,archived,files,replay);
|
|
@@ -67,11 +68,25 @@ async function placement(db,manifest,cat,replay,ledger,history,legacyRecovery,un
|
|
|
67
68
|
// v2 uses the replayed complete identity; acknowledged recovery places by tables only.
|
|
68
69
|
const schemaAfter=(names,allowSubsequence)=>replay.schemaAfter(names,legacyRecovery && manifest.schemaObjects===undefined?{legacyExcludedObjects:true}:unverified?{tablesOnly:true}:undefined,allowSubsequence);
|
|
69
70
|
if(unverified && archived && manifest.migrations)insist(canonicalJson(matchMigrationLedger(archived,files,replay))===canonicalJson(recorded),'schema_mismatch:migration_history',409);
|
|
71
|
+
// Name-only v2 archives can list the same files in a different order from
|
|
72
|
+
// Wrangler. Every recorded file must belong to the applied prefix, so a
|
|
73
|
+
// migration omitted from its interior cannot be treated as a later change.
|
|
74
|
+
const hosted=archived===undefined && manifest.version===2 && manifest.schemaObjects!==undefined
|
|
75
|
+
&& ledger?.table==='d1_migrations' && recorded!==undefined
|
|
76
|
+
&& (!ledger.owners || !entries.some(row=>row.table!==ledger.table && recorded.includes(row.name)));
|
|
77
|
+
insist(!replay.hostedOrderOnly || hosted,'schema_mismatch:legacy_evidence',409);
|
|
70
78
|
let earlier,unapplied;
|
|
71
79
|
if(recorded) {
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
80
|
+
if(hosted) {
|
|
81
|
+
earlier=applied.slice(0,recorded.length);
|
|
82
|
+
insist(earlier.length===manifest.migrations.length && earlier.every(name=>manifest.migrations.includes(name)),
|
|
83
|
+
'schema_mismatch:migration_history',409);
|
|
84
|
+
unapplied=applied.slice(earlier.length);
|
|
85
|
+
}else{
|
|
86
|
+
earlier=recordedPlacement(manifest,recorded,applied);
|
|
87
|
+
unapplied=applied.filter(name=>!earlier.includes(name));
|
|
88
|
+
}
|
|
89
|
+
await verifyRecordedSchema(manifest,earlier,schemaAfter,hosted && earlier.some((name,i)=>name!==recorded[i]));
|
|
75
90
|
}else{
|
|
76
91
|
// Unrecorded: the latest matching prefix. Migrations that change no archived
|
|
77
92
|
// definition (excluded tables or data-only changes) cannot be told apart
|
|
@@ -107,6 +122,7 @@ async function placement(db,manifest,cat,replay,ledger,history,legacyRecovery,un
|
|
|
107
122
|
export async function verifiedLegacySchema(ledger,manifest,cat,migrations,history) {
|
|
108
123
|
if(!migrations)return false;
|
|
109
124
|
const replay=await migrations();
|
|
125
|
+
if(replay.hostedOrderOnly)return false;
|
|
110
126
|
try {
|
|
111
127
|
const files=replay.names.map(name=>({name,sha256:replay.sha256?.(name),aliases:replay.aliases?.(name)}));
|
|
112
128
|
const archived=history(),recorded=recordedMigrations(manifest,archived,files,replay);
|
|
@@ -127,11 +143,11 @@ export async function verifiedLegacySchema(ledger,manifest,cat,migrations,histor
|
|
|
127
143
|
// records a migration the destination never applied is newer than it, even
|
|
128
144
|
// when that migration left the schema as it was (an index, a data change).
|
|
129
145
|
export async function sameSchemaMigrations(db,ledger,manifest,cat,migrations,history) {
|
|
130
|
-
const applied=ledger?.map(m=>m.name);
|
|
146
|
+
const applied=ledger?.entries.map(m=>m.name);
|
|
131
147
|
if(manifest.migrationLedger?.origin==='d1_migrations')insist(migrations && applied,'schema_mismatch:owner_migration_history',409);
|
|
132
148
|
if(!applied)return;
|
|
133
149
|
if(!migrations){
|
|
134
|
-
if(manifest.migrations)matchMigrationLedger(manifest.migrations.map(name=>({name})),ledger);
|
|
150
|
+
if(manifest.migrations)matchMigrationLedger(manifest.migrations.map(name=>({name})),ledger.entries);
|
|
135
151
|
insist(!manifest.migrations || applied.every(name=>manifest.migrations.includes(name)),'schema_mismatch:migrations');return;
|
|
136
152
|
}
|
|
137
153
|
// A data-only migration leaves the schema unchanged but still needs its
|
|
@@ -146,7 +162,7 @@ export async function sameSchemaMigrations(db,ledger,manifest,cat,migrations,his
|
|
|
146
162
|
export async function additiveBridge(db,ledger,manifest,cat,migrations,history,legacyRecovery,unverified=false) {
|
|
147
163
|
insist(migrations,'schema_mismatch');
|
|
148
164
|
const replay=await migrations();
|
|
149
|
-
const {applied,earlier,replays,rebuilt}=await placement(db,manifest,cat,replay,ledger
|
|
165
|
+
const {applied,earlier,replays,rebuilt}=await placement(db,manifest,cat,replay,ledger,history,legacyRecovery,unverified);
|
|
150
166
|
const columns={};
|
|
151
167
|
for(const table of manifest.tables) {
|
|
152
168
|
const target=cat.tables.find(t=>t.name===table.name),label=':'+table.name;
|
|
@@ -10,16 +10,20 @@ function bundledPrefix(history,files) {
|
|
|
10
10
|
// Both evidence adapters bind the seed's files to live history and the complete
|
|
11
11
|
// schema. Plain Wrangler history must select a literal ordered prefix within
|
|
12
12
|
// each ledger. The merged chronology chooses how those prefixes are replayed.
|
|
13
|
-
export async function checkoutHistory(db,input,files,replayFor,{nativeWrangler=false,ownerSources=[],nativePrefix=bundledPrefix}={}) {
|
|
13
|
+
export async function checkoutHistory(db,input,files,replayFor,{nativeWrangler=false,ownerSources=[],nativePrefix=bundledPrefix,nativeOrder}={}) {
|
|
14
14
|
const history=await (ownerSources.length?nativeMigrationHistory(db,ownerSources):readLedger(db)),ledger=history?.entries??[];
|
|
15
15
|
insist(ledger.length,'schema_mismatch:legacy_evidence',409);
|
|
16
16
|
requireAppliedOwnerFiles(history,files);
|
|
17
|
+
let hostedOrderOnly=false;
|
|
17
18
|
if(!ledger.every(row=>row.sha256)) {
|
|
18
|
-
insist(nativeWrangler && history.table==='d1_migrations'
|
|
19
|
+
insist(nativeWrangler && history.table==='d1_migrations','schema_mismatch:legacy_evidence',409);
|
|
20
|
+
hostedOrderOnly=!nativePrefix(history,files);
|
|
21
|
+
insist(!hostedOrderOnly || nativeOrder && nativePrefix(history,nativeOrder.map(index=>files[index])),
|
|
19
22
|
'schema_mismatch:legacy_evidence',409);
|
|
20
23
|
}
|
|
21
24
|
const applied=matchMigrationLedger(ledger,files);
|
|
22
|
-
const config=policy(ownerMigrationPolicy(input,ownerSources)),replay=await replayFor(applied,config);
|
|
25
|
+
const config=policy(ownerMigrationPolicy(input,ownerSources)),replay=await replayFor(applied,config,hostedOrderOnly);
|
|
26
|
+
if(hostedOrderOnly)replay.hostedOrderOnly=true;
|
|
23
27
|
insist(await replay.schemaAfter(replay.names)===(await catalogue(db,config)).identity,'schema_mismatch');
|
|
24
28
|
return ownerSources.length?{...replay,ownerSources,history}:replay;
|
|
25
29
|
}
|
|
@@ -6,6 +6,7 @@ import { randomUUID } from 'node:crypto';
|
|
|
6
6
|
import { readArchive } from './index.mjs';
|
|
7
7
|
import { canonicalJson } from './hash.mjs';
|
|
8
8
|
import { ledgerReader } from './ledgers.mjs';
|
|
9
|
+
import { archiveStep } from './archive-step.mjs';
|
|
9
10
|
export async function* fileRecords(path,reader=readArchive) {
|
|
10
11
|
const file = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW);
|
|
11
12
|
try { yield* reader(Readable.toWeb(file.createReadStream({ autoClose: false }))); }
|
|
@@ -76,20 +77,27 @@ export async function importArchive(service, path, transient, journal, progress,
|
|
|
76
77
|
const ledgers = ledger.ledgers();
|
|
77
78
|
for await (const record of fileRecords(path)) {
|
|
78
79
|
if (record.kind === 'manifest') {
|
|
79
|
-
manifest = record;
|
|
80
|
+
manifest = record;
|
|
81
|
+
id = await archiveStep('preparing the restore', () => service.imports.create(manifest, undefined, {...options,...(ledgers?{ledgers}:{})}));
|
|
80
82
|
const { skippedTables } = await service.imports.status(id);
|
|
81
83
|
if (skippedTables?.length) progress?.({ phase: 'skipped', tables: skippedTables });
|
|
82
84
|
}
|
|
83
85
|
else if (record.kind === 'part') {
|
|
84
|
-
await service.imports.uploadPart(id, record.index, record);
|
|
86
|
+
await archiveStep('staging the restore', () => service.imports.uploadPart(id, record.index, record));
|
|
85
87
|
staged += Buffer.byteLength(canonicalJson(record)) + 1;
|
|
86
88
|
progress?.({ phase: 'staging', parts: record.index + 1, total: manifest.partCount, bytes: staged });
|
|
87
89
|
}
|
|
88
90
|
else closing = record;
|
|
89
91
|
}
|
|
90
92
|
progress?.({ phase: 'finishing', total: manifest.partCount });
|
|
91
|
-
const status = await service.imports.finalize(id, { contentHash: closing.contentHash, archiveHash: closing.archiveHash, ...(later ? { laterJournal: later } : {}) });
|
|
93
|
+
const status = await archiveStep('finishing the restore', () => service.imports.finalize(id, { contentHash: closing.contentHash, archiveHash: closing.archiveHash, ...(later ? { laterJournal: later } : {}) }));
|
|
94
|
+
complete = true;
|
|
92
95
|
if(status.schemaUnverified)progress?.({phase:'schema-unverified',receipt:status.schemaUnverified});
|
|
93
96
|
return { ...receipt(manifest, closing, transient), ...(status.schemaUnverified?{schemaUnverified:status.schemaUnverified}:{}), ...(status.bridged ? { bridged: status.bridged } : {}), ...(status.skippedTables ? { skippedTables: status.skippedTables } : {}), ...(status.safetyChanges ? { safetyChanges: status.safetyChanges } : {}) };
|
|
94
|
-
}
|
|
97
|
+
} catch (error) {
|
|
98
|
+
// A dead storage connection may also prevent cleanup. Preserve the original
|
|
99
|
+
// failure and stage; the next local import can clean up the abandoned job.
|
|
100
|
+
if (id && !complete) await service.imports.abort(id).catch(() => {});
|
|
101
|
+
throw error;
|
|
102
|
+
}
|
|
95
103
|
}
|
|
@@ -55,7 +55,7 @@ export async function createImport(context,manifest,actor='direct',legacyRecover
|
|
|
55
55
|
insist(table.dependencies.every(d=>target.dependencies.includes(d)),'schema_mismatch:dependencies:'+table.name,409);
|
|
56
56
|
}
|
|
57
57
|
const same=manifest.schema===cat.identity;
|
|
58
|
-
const {replays,...bridge}=unverified?await additiveBridge(db,
|
|
58
|
+
const {replays,...bridge}=unverified?await additiveBridge(db,historyLedger,manifest,cat,migrations,history,legacyRecovery,true):upgrade?{}:same?(await sameSchemaMigrations(db,historyLedger,manifest,cat,migrations,history)??{}):await additiveBridge(db,historyLedger,manifest,cat,migrations,history,legacyRecovery);
|
|
59
59
|
// Finalize checks object references on the staged rows; a later migration
|
|
60
60
|
// replayed into a reference table would publish rows that check never saw.
|
|
61
61
|
const referenced=replays?.find(r=>Object.values(config.buckets).some(spec=>spec.objectReferences?.table===r.table));
|
|
@@ -3,18 +3,22 @@ type MigrationFile = {name:string;sha256:string;statements:string[]};
|
|
|
3
3
|
type OwnerSource = {prefix:string;table:string};
|
|
4
4
|
type SchemaObject = {name:string;sql:string;type:string;tbl_name:string};
|
|
5
5
|
type SeedRows = {name:string;columns:string[];hashes:string[]};
|
|
6
|
+
type Prefix = {objects:number[];shadows:string[];seeds:SeedRows[]};
|
|
6
7
|
interface BundledEvidence {
|
|
7
8
|
version:1;
|
|
8
9
|
files:MigrationFile[];
|
|
9
10
|
ownerSources?:OwnerSource[];
|
|
10
11
|
objects:SchemaObject[];
|
|
11
|
-
prefixes:
|
|
12
|
+
prefixes:Prefix[];
|
|
13
|
+
nativeOrder?:number[];
|
|
14
|
+
nativePrefixes?:Prefix[];
|
|
12
15
|
}
|
|
13
16
|
type OrderedFile = MigrationFile & {table?:string};
|
|
14
17
|
interface OrderedEvidence {
|
|
15
18
|
version:2;
|
|
16
19
|
files:OrderedFile[];
|
|
17
20
|
ownerSources:OwnerSource[];
|
|
21
|
+
nativeOrder?:number[];
|
|
18
22
|
nativePrefix(history:{table:string;owners?:boolean;entries:{name:string;table?:string;sha256?:string}[]},files:OrderedFile[]):boolean;
|
|
19
23
|
snapshots(names:string[]):Promise<{
|
|
20
24
|
inventory(index:number):{objects:SchemaObject[];kinds:{name:string;type:string}[]};
|
|
@@ -3,15 +3,17 @@ import { canonicalJson } from './hash.mjs';
|
|
|
3
3
|
import { checkoutHistory } from './checkout-history.mjs';
|
|
4
4
|
import { insist, PortabilityError } from './storage.mjs';
|
|
5
5
|
|
|
6
|
-
function bundledSnapshots(evidence,names) {
|
|
7
|
-
|
|
8
|
-
|
|
6
|
+
function bundledSnapshots(evidence,names,hostedOrderOnly) {
|
|
7
|
+
const {files,objects,nativeOrder}=evidence;
|
|
8
|
+
const prefixes=hostedOrderOnly?evidence.nativePrefixes:evidence.prefixes;
|
|
9
|
+
insist(prefixes?.length===files.length+1
|
|
10
|
+
&& names.every((name,i)=>name===files[hostedOrderOnly?nativeOrder[i]:i]?.name),'schema_mismatch:legacy_evidence',409);
|
|
9
11
|
return {
|
|
10
12
|
inventory(index){
|
|
11
|
-
const snapshot=
|
|
12
|
-
return {objects:snapshot.objects.map(i=>
|
|
13
|
+
const snapshot=prefixes[index];
|
|
14
|
+
return {objects:snapshot.objects.map(i=>objects[i]),kinds:snapshot.shadows.map(name=>({name,type:'shadow'}))};
|
|
13
15
|
},
|
|
14
|
-
seeds:
|
|
16
|
+
seeds:prefixes[names.length].seeds,
|
|
15
17
|
};
|
|
16
18
|
}
|
|
17
19
|
|
|
@@ -22,8 +24,8 @@ export async function workerMigrations(db,input,evidence) {
|
|
|
22
24
|
insist(evidence?.version===1 || evidence?.version===2 && evidence.ownerSources?.length
|
|
23
25
|
&& typeof evidence.snapshots==='function' && typeof evidence.nativePrefix==='function','schema_mismatch:legacy_evidence',409);
|
|
24
26
|
const {files,ownerSources}=evidence;
|
|
25
|
-
return await checkoutHistory(db,input,files,async (names,config)=>{
|
|
26
|
-
const {inventory,seeds,replayInventory}=evidence.version===2?await evidence.snapshots(names):bundledSnapshots(evidence,names);
|
|
27
|
+
return await checkoutHistory(db,input,files,async (names,config,hostedOrderOnly)=>{
|
|
28
|
+
const {inventory,seeds,replayInventory}=evidence.version===2?await evidence.snapshots(names):bundledSnapshots(evidence,names,hostedOrderOnly);
|
|
27
29
|
return {names,statements:name=>files.find(file=>file.name===name)?.statements,sha256:name=>files.find(file=>file.name===name)?.sha256,
|
|
28
30
|
async schemaAfter(selected,options,allowSubsequence){
|
|
29
31
|
if(selected.length<=names.length && selected.every((name,i)=>name===names[i]))return inventorySchemaIdentity(inventory(selected.length),config,options);
|
|
@@ -33,6 +35,6 @@ export async function workerMigrations(db,input,evidence) {
|
|
|
33
35
|
async rowImages(requests){return new Map(requests.filter(r=>seeds.some(s=>s.name===r.name && !s.hashes.length)).map(r=>[r.name,[]]));},
|
|
34
36
|
async rowHashes(requests,limit){return new Map(requests.flatMap(r=>{const found=seeds.find(s=>s.name===r.name && s.hashes.length<=limit && canonicalJson(s.columns)===canonicalJson(r.columns));return found?[[r.name,found.hashes]]:[];}));},
|
|
35
37
|
};
|
|
36
|
-
},{nativeWrangler:true,ownerSources,nativePrefix:evidence.nativePrefix});
|
|
38
|
+
},{nativeWrangler:true,ownerSources,nativePrefix:evidence.nativePrefix,nativeOrder:evidence.nativeOrder});
|
|
37
39
|
}catch(error){if(error instanceof PortabilityError)throw error;throw new PortabilityError('schema_mismatch:legacy_evidence',409);}
|
|
38
40
|
}
|