@runeditrun/rer 0.2.123 → 0.2.125
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cli/portability-empty.mjs +4 -2
- package/cli/portability-evidence.mjs +49 -34
- package/cli/portability-migrations.mjs +30 -8
- package/cli/portability-owners.mjs +54 -0
- package/cli/portability-sqlite.mjs +72 -0
- package/cli/portability-worker.mjs +13 -5
- package/cli/portability.mjs +6 -4
- package/cli/vendor/wa-sqlite/LICENSE +21 -0
- package/cli/vendor/wa-sqlite/README.md +13 -0
- package/cli/vendor/wa-sqlite/wa-sqlite.mjs +14 -0
- package/cli/vendor/wa-sqlite/wa-sqlite.wasm +0 -0
- package/global/CHANGELOG.md +36 -0
- package/modules/portability/README.md +22 -11
- package/modules/portability/bridge.mjs +25 -23
- package/modules/portability/checkout-history.mjs +13 -9
- package/modules/portability/export.mjs +6 -3
- package/modules/portability/format.mjs +2 -2
- package/modules/portability/http-errors.mjs +1 -1
- package/modules/portability/http.mjs +2 -2
- package/modules/portability/import.mjs +13 -11
- package/modules/portability/index.d.mts +4 -2
- package/modules/portability/index.mjs +2 -2
- package/modules/portability/ledgers.mjs +16 -3
- package/modules/portability/legacy-files.mjs +3 -3
- package/modules/portability/migration-evidence.mjs +47 -0
- package/modules/portability/migration-history.mjs +4 -3
- package/modules/portability/owner-history.mjs +62 -0
- package/modules/portability/worker-history.d.mts +21 -4
- package/modules/portability/worker-history.mjs +24 -16
- package/modules/portability/worker-owner-history.mjs +38 -0
- package/package.json +1 -1
|
@@ -11,10 +11,10 @@ export async function boundedJson(request,limit) {
|
|
|
11
11
|
try{return JSON.parse(new TextDecoder('utf-8',{fatal:true}).decode(bytes));}catch{throw new PortabilityError('invalid_json');}
|
|
12
12
|
}
|
|
13
13
|
const json=(value,status=200)=>new Response(JSON.stringify(value),{status,headers:{'Content-Type':'application/json','Cache-Control':'no-store'}});
|
|
14
|
-
export async function createPortabilityHandler({db,buckets,policy,migrations,appOrigin,basePath='/api/portability',authorize,maxBodyBytes=MAX_LINE_BYTES}) {
|
|
14
|
+
export async function createPortabilityHandler({db,buckets,policy,migrations,migrationHistory,appOrigin,basePath='/api/portability',authorize,maxBodyBytes=MAX_LINE_BYTES}) {
|
|
15
15
|
insist(typeof authorize==='function' && new URL(appOrigin).origin===appOrigin,'invalid_handler_configuration');
|
|
16
16
|
insist(/^\/[A-Za-z0-9/_-]+$/.test(basePath) && !basePath.endsWith('/') && Number.isSafeInteger(maxBodyBytes) && maxBodyBytes>0,'invalid_handler_configuration');
|
|
17
|
-
const service=await createPortability({db,buckets,policy,migrations});
|
|
17
|
+
const service=await createPortability({db,buckets,policy,migrations,migrationHistory});
|
|
18
18
|
return async request=>{
|
|
19
19
|
const url=new URL(request.url);
|
|
20
20
|
if(url.pathname!==basePath && !url.pathname.startsWith(basePath+'/'))return null;
|
|
@@ -16,25 +16,27 @@ import { checkEpoch, fence, mergeJournal, tombstone } from './journal.mjs';
|
|
|
16
16
|
import { writeObject } from './payloads.mjs';
|
|
17
17
|
import { createCopy,sidecar,readCopy,restoreCopy } from './copies.mjs';
|
|
18
18
|
import { excludedDependentGuards, migrationSeeded, seededCondition } from './restore-safety.mjs';
|
|
19
|
-
import { additiveBridge, sameSchemaMigrations,
|
|
19
|
+
import { additiveBridge, sameSchemaMigrations, verifiedLegacySchema } from './bridge.mjs';
|
|
20
20
|
import { replayStatements } from './backfills.mjs';
|
|
21
|
-
import { checkArchivedLedgers, archivedHistory, archivedLedgerTables, ledgerReader, matchMigrationLedger } from './ledgers.mjs';
|
|
21
|
+
import { checkArchivedLedgers, archivedHistory, archivedLedgerTables, ledgerReader, matchMigrationLedger, readLedger } from './ledgers.mjs';
|
|
22
|
+
import { transferHistory } from './owner-history.mjs';
|
|
22
23
|
import { legacyEmptyGuards, legacyAuditStatements, requireLegacyEmpty } from './legacy-safety.mjs';
|
|
23
24
|
import { checkReferences } from './references.mjs';
|
|
24
25
|
import { rebuildDerivedStatements } from './derived.mjs';
|
|
26
|
+
const acceptedUpgrade=(config,manifest)=>manifest.migrationLedger?.origin==='d1_migrations'?undefined:config.acceptedSchemas[manifest.schema];
|
|
25
27
|
// ledgers: the rows of each migration ledger table the archive carries, as the
|
|
26
28
|
// caller read them from it ({table:rows}); uploadPart binds them to its parts.
|
|
27
29
|
export async function createImport(context,manifest,actor='direct',legacyRecovery,ledgers,acknowledgeUnverifiedSchemaObjects=false,id=crypto.randomUUID()) {
|
|
28
|
-
const {db,
|
|
29
|
-
insist(manifest.product===config.product,'wrong_product');
|
|
30
|
+
const {db,buckets}=context;await checkManifest(manifest);checkArchivedLedgers(manifest,ledgers);
|
|
31
|
+
insist(manifest.product===context.config.product,'wrong_product');
|
|
30
32
|
// An already-stale backup cannot become restorable. Refuse before schema
|
|
31
33
|
// inventory, staging copies and part uploads; fence checks again at finalize.
|
|
32
34
|
await checkEpoch(db,manifest);
|
|
33
35
|
// Explicit schema upgrades can reconcile rows, but cannot attest an interrupted deployment.
|
|
34
|
-
await
|
|
36
|
+
const {config,ledger:historyLedger,migrations}=await transferHistory(context),ledger=historyLedger?.entries;
|
|
35
37
|
const original=manifest;manifest=await restorableManifest(original,{canonical:true});
|
|
36
38
|
const skippedTables=original.tables.filter(table=>platformTable(table.name)).map(table=>table.name);
|
|
37
|
-
const cat=await catalogue(db,config),upgrade=config
|
|
39
|
+
const cat=await catalogue(db,config),upgrade=acceptedUpgrade(config,original);
|
|
38
40
|
manifest=await withoutAbsentExcludedObjects(manifest,cat,config);
|
|
39
41
|
const legacy=original.version===2 && manifest.schemaObjects===undefined;
|
|
40
42
|
// An explicit acknowledgment keeps its existing recovery semantics. Without
|
|
@@ -43,7 +45,7 @@ export async function createImport(context,manifest,actor='direct',legacyRecover
|
|
|
43
45
|
const unverified=legacy && acknowledgeUnverifiedSchemaObjects===true;
|
|
44
46
|
const verifyingLegacy=legacy && !unverified && !legacyRecovery;
|
|
45
47
|
const history=()=>archivedHistory(original,ledgers,{allowOlderShape:unverified || verifyingLegacy});
|
|
46
|
-
const verified=(verifyingLegacy || (!legacy && manifest.schema!==cat.identity)) && await verifiedLegacySchema(
|
|
48
|
+
const verified=(verifyingLegacy || (!legacy && manifest.schema!==cat.identity)) && await verifiedLegacySchema(ledger??[],manifest,cat,migrations,history);
|
|
47
49
|
insist(!legacy || legacyRecovery || unverified || verified,'schema_mismatch:legacy_schema',409);
|
|
48
50
|
if(verified)manifest={...manifest,schema:verified};
|
|
49
51
|
if(unverified)for(const table of manifest.tables) {
|
|
@@ -53,7 +55,7 @@ export async function createImport(context,manifest,actor='direct',legacyRecover
|
|
|
53
55
|
insist(table.dependencies.every(d=>target.dependencies.includes(d)),'schema_mismatch:dependencies:'+table.name,409);
|
|
54
56
|
}
|
|
55
57
|
const same=manifest.schema===cat.identity;
|
|
56
|
-
const {replays,...bridge}=unverified?await additiveBridge(db,manifest,cat,
|
|
58
|
+
const {replays,...bridge}=unverified?await additiveBridge(db,ledger,manifest,cat,migrations,history,legacyRecovery,true):upgrade?{}:same?(await sameSchemaMigrations(db,ledger,manifest,cat,migrations,history)??{}):await additiveBridge(db,ledger,manifest,cat,migrations,history,legacyRecovery);
|
|
57
59
|
// Finalize checks object references on the staged rows; a later migration
|
|
58
60
|
// replayed into a reference table would publish rows that check never saw.
|
|
59
61
|
const referenced=replays?.find(r=>Object.values(config.buckets).some(spec=>spec.objectReferences?.table===r.table));
|
|
@@ -70,7 +72,7 @@ export async function createImport(context,manifest,actor='direct',legacyRecover
|
|
|
70
72
|
insist(canonicalJson(manifest.requiredConfiguration)===canonicalJson(config.requiredConfiguration),'configuration_inventory');
|
|
71
73
|
for(const o of manifest.objects) insist(buckets[o.bucket] && config.buckets[o.bucket] && o.key.startsWith(config.buckets[o.bucket].prefix??''),'bucket_inventory');
|
|
72
74
|
insist(canonicalJson(manifest.payloads.map(x=>x.name))===canonicalJson(Object.keys(config.extraPayloads).sort()),'payload_inventory');
|
|
73
|
-
if(unverified)await destinationState(context,{metadata:{targets:cat.tables,manifest:original}});
|
|
75
|
+
if(unverified)await destinationState({...context,config,migrations},{metadata:{targets:cat.tables,manifest:original}});
|
|
74
76
|
await ensureImportLeases(db);
|
|
75
77
|
const previous=(await state(db)).import_job;
|
|
76
78
|
if(previous) {
|
|
@@ -96,7 +98,7 @@ async function uploadLeasedPart(context,id,index,part) {
|
|
|
96
98
|
const previous=await statement(db,'SELECT sha256 FROM rer_portability_parts WHERE job=? AND idx=?',id,index).first();
|
|
97
99
|
if(previous){insist(previous.sha256===part.sha256,'part_conflict',409);return importStatus(context,id,{includeLease:false});}
|
|
98
100
|
insist(row.phase==='staging','import_phase',409);insist(index===row.completed && index<row.metadata.manifest.partCount,'part_order',409);
|
|
99
|
-
const inserts=[],groups=new Map(),manifest=row.metadata.manifest,upgrade=config
|
|
101
|
+
const inserts=[],groups=new Map(),manifest=row.metadata.manifest,upgrade=acceptedUpgrade(config,manifest);
|
|
100
102
|
for(const item of part.items) {
|
|
101
103
|
if(item.kind==='row') {
|
|
102
104
|
const table=manifest.tables.find(x=>x.name===item.table);insist(table && Number.isSafeInteger(item.index) && item.index>=0 && item.index<table.count,'row_inventory');
|
|
@@ -303,7 +305,7 @@ export function libraryImports(context,direct,migrationRoot) {
|
|
|
303
305
|
await checkManifest(manifest);
|
|
304
306
|
if(!archivedLedgerTables(manifest).length)return direct.create(manifest,actor,options);
|
|
305
307
|
insist(manifest.product===context.config.product,'wrong_product');
|
|
306
|
-
await
|
|
308
|
+
await (context.migrationHistory?.()??readLedger(context.db));
|
|
307
309
|
insist(!(await statement(context.db,'SELECT import_job FROM rer_portability_state WHERE id=1').first())?.import_job && ![...pending.values()].some(row=>row.phase==='staging'),'import_running',409);
|
|
308
310
|
context.migrations??=await libraryMigrations(context.db,context.inputPolicy,migrationRoot);
|
|
309
311
|
const id=crypto.randomUUID();pending.set(id,{id,manifest:structuredClone(manifest),actor,parts:[],phase:'staging'});return id;
|
|
@@ -13,8 +13,8 @@ export interface Manifest {
|
|
|
13
13
|
excludedTables: string[]; excludedColumns: Record<string,string[]>; requiredConfiguration: string[]; partCount: number; epoch: number; journal: string[];
|
|
14
14
|
/** Applied product migrations; absent from archives made before additive restore. */
|
|
15
15
|
migrations?: string[];
|
|
16
|
-
/**
|
|
17
|
-
migrationLedger?: {origin:'rer_migrations';entries:{name:string;sha256:string}[]};
|
|
16
|
+
/** Hash-bound application order from RER Platform or the local core and owner ledgers. */
|
|
17
|
+
migrationLedger?: {origin:'rer_migrations'|'d1_migrations';entries:{name:string;sha256:string}[]};
|
|
18
18
|
}
|
|
19
19
|
export type Item = {kind:'row';table:string;index:number;row:Row;rowid?:number} | {kind:'object';bucket:string;key:string;offset:number;data:string} | {kind:'payload';name:string;offset:number;data:string};
|
|
20
20
|
export interface Part {kind:'part';index:number;items:Item[];sha256:string}
|
|
@@ -51,6 +51,8 @@ export interface PortabilityOptions {
|
|
|
51
51
|
/** Node checkout root for automatic legacy evidence; defaults to cwd. */
|
|
52
52
|
migrationRoot?:string;
|
|
53
53
|
db:D1Database;buckets?:Record<string,R2Bucket>;policy:ArchivePolicy;migrations?:()=>Promise<MigrationReplay>;
|
|
54
|
+
/** Runtime-owned history for databases with separate owner migration ledgers. */
|
|
55
|
+
migrationHistory?:()=>Promise<{table:string;entries:{name:string;sha256?:string}[];owners?:boolean}|undefined>;
|
|
54
56
|
/** Seed-owned audit SQL, appended to the snapshot/job batch. Throwing or SQL failure refuses creation. */
|
|
55
57
|
onExport?:(event:ExportEvent)=>[sql:string,args:unknown[]][];
|
|
56
58
|
}
|
|
@@ -12,13 +12,13 @@ export { readArchive, makePart, MAX_LINE_BYTES, MAX_PART_BYTES } from './format.
|
|
|
12
12
|
export { canonicalJson, sha256 } from './hash.mjs';
|
|
13
13
|
export { PortabilityError } from './storage.mjs';
|
|
14
14
|
export { createPortabilityHandler } from './http.mjs';
|
|
15
|
-
export async function createPortability({db,buckets={},policy:input,migrations,onExport,migrationRoot}) {
|
|
15
|
+
export async function createPortability({db,buckets={},policy:input,migrations,migrationHistory,onExport,migrationRoot}) {
|
|
16
16
|
insist(onExport===undefined || typeof onExport==='function','invalid_on_export');
|
|
17
17
|
db=wireDatabase(db);
|
|
18
18
|
const config=policy(input);await ensureObjectInventory(db);await catalogue(db,config);
|
|
19
19
|
for(const name of Object.keys(config.buckets))insist(buckets[name] && typeof buckets[name].get==='function','missing_bucket:'+name);
|
|
20
20
|
await ensureImportLeases(db);
|
|
21
|
-
const context={db,buckets,config,migrations,onExport};
|
|
21
|
+
const context={db,buckets,config,migrations,migrationHistory,onExport};
|
|
22
22
|
const importContext={...context,inputPolicy:input};
|
|
23
23
|
const {lookup,actor,...imports}=libraryImports(importContext,{
|
|
24
24
|
active:actor=>activeImport(importContext,actor),
|
|
@@ -76,8 +76,7 @@ export function matchMigrationLedger(entries,files,{dataOnly=[]}={}) {
|
|
|
76
76
|
* {origin:'rer_migrations',entries:[{name,sha256}]} in application order, each
|
|
77
77
|
* entry named by the checkout file with that hash (the native identity) where
|
|
78
78
|
* the exporter can read the checkout's files. Declared data-only entries and
|
|
79
|
-
* hosted exporters, which have no files, keep the ledger's own names.
|
|
80
|
-
* sources record nothing here, so their archives are unchanged.
|
|
79
|
+
* hosted exporters, which have no files, keep the ledger's own names.
|
|
81
80
|
*/
|
|
82
81
|
export async function platformOrigin(ledger,migrations) {
|
|
83
82
|
if(ledger?.table!=='rer_migrations')return;
|
|
@@ -88,6 +87,20 @@ export async function platformOrigin(ledger,migrations) {
|
|
|
88
87
|
return {origin:'rer_migrations',entries};
|
|
89
88
|
}
|
|
90
89
|
|
|
90
|
+
// Separate native owner ledgers are also bookkeeping. Bind the merged applied
|
|
91
|
+
// history to the checkout's files without moving any of those tables' rows.
|
|
92
|
+
export async function nativeOrigin(ledger,migrations,schema) {
|
|
93
|
+
const replay=await migrations?.();
|
|
94
|
+
insist(replay && canonicalJson(replay.names)===canonicalJson(ledger.entries.map(row=>row.name)),'schema_mismatch:migration_history',409);
|
|
95
|
+
const entries=replay.names.map(name=>{
|
|
96
|
+
const sha256=replay.sha256?.(name);
|
|
97
|
+
insist(typeof sha256==='string' && /^[a-f0-9]{64}$/.test(sha256),'schema_mismatch:migration_history',409);
|
|
98
|
+
return {name,sha256};
|
|
99
|
+
});
|
|
100
|
+
insist(await replay.schemaAfter(replay.names)===schema,'schema_mismatch',409);
|
|
101
|
+
return {origin:'d1_migrations',entries};
|
|
102
|
+
}
|
|
103
|
+
|
|
91
104
|
/** The ledger tables an archive carries (old archives carried rer_migrations before it became bookkeeping). */
|
|
92
105
|
export const archivedLedgerTables=manifest=>manifest.tables.filter(t=>ledgerNames.includes(identifier(t.name)));
|
|
93
106
|
|
|
@@ -117,7 +130,7 @@ export function checkArchivedLedgers(manifest,ledgers) {
|
|
|
117
130
|
/** The applied migrations an archive's own ledger records, [{name,sha256?}]; undefined when it carries no ledger. */
|
|
118
131
|
export function archivedHistory(manifest,ledgers,{allowOlderShape=false}={}) {
|
|
119
132
|
const tables=archivedLedgerTables(manifest);
|
|
120
|
-
//
|
|
133
|
+
// Hash-bearing exports record their ledger in the manifest.
|
|
121
134
|
if(manifest.migrationLedger)insist(!tables.length,'schema_mismatch:migration_history');
|
|
122
135
|
if(!tables.length)return manifest.migrationLedger?.entries.map(entry);
|
|
123
136
|
insist(tables.length===1 && ledgers?.[tables[0].name],'schema_mismatch:migration_history');
|
|
@@ -8,7 +8,7 @@ import { ensureLegacyAudit, legacyTargetState, requireLegacyEmpty } from './lega
|
|
|
8
8
|
import { insist, PortabilityError } from './storage.mjs';
|
|
9
9
|
import { ledgerReader } from './ledgers.mjs';
|
|
10
10
|
|
|
11
|
-
export async function recoverLegacyArchive({db,buckets,policy:input,migrations},path,output,transient,acknowledged) {
|
|
11
|
+
export async function recoverLegacyArchive({db,buckets,policy:input,migrations,migrationHistory},path,output,transient,acknowledged) {
|
|
12
12
|
insist(acknowledged===true,'legacy_acknowledgement_required',409);
|
|
13
13
|
insist(typeof output==='string' && output!==path,'legacy_output_required',409);
|
|
14
14
|
insist(!(await lstat(output).catch(error=>{if(error.code==='ENOENT')return null;throw error;})),'legacy_output_exists',409);
|
|
@@ -16,7 +16,7 @@ export async function recoverLegacyArchive({db,buckets,policy:input,migrations},
|
|
|
16
16
|
// reads the archive's own migration ledger, if any, for placement (bridge.mjs).
|
|
17
17
|
const ledger=ledgerReader();
|
|
18
18
|
for await(const record of fileRecords(path,readLegacyArchive))ledger.add(record);
|
|
19
|
-
const context={db,buckets,config:policy(input),migrations},target=await legacyTargetState(db,migrations);
|
|
19
|
+
const context={db,buckets,config:policy(input),migrations,migrationHistory},target=await legacyTargetState(db,migrations);
|
|
20
20
|
await requireLegacyEmpty(context,target);
|
|
21
21
|
let id,manifest,closing,integrity,status,complete=false;
|
|
22
22
|
try {
|
|
@@ -40,7 +40,7 @@ export async function recoverLegacyArchive({db,buckets,policy:input,migrations},
|
|
|
40
40
|
// A fresh export binds the recovered installation from this point forward.
|
|
41
41
|
// A failure here must be reported; the durable restore audit remains intact.
|
|
42
42
|
try {
|
|
43
|
-
const recovered=await exportArchive(await createPortability({db,buckets,policy:input}),output,transient);
|
|
43
|
+
const recovered=await exportArchive(await createPortability({db,buckets,policy:input,migrations,migrationHistory}),output,transient);
|
|
44
44
|
return {...recovered,legacyRecovery:true,unverifiedMetadata:true,restoredContentHash:closing.contentHash,...(status.bridged?{bridged:status.bridged}:{}),...(status.safetyChanges?{safetyChanges:status.safetyChanges}:{})};
|
|
45
45
|
}catch{throw new PortabilityError('legacy_restored_export_failed',500);}
|
|
46
46
|
}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import { sha256 } from './hash.mjs';
|
|
2
|
+
import { quote } from './storage.mjs';
|
|
3
|
+
import { dataStatement, triggerBodies } from './migration-statements.mjs';
|
|
4
|
+
import { sqliteTokens } from './sqlite-tokens.mjs';
|
|
5
|
+
|
|
6
|
+
// Date functions can default to 'now', so even fixed arguments are
|
|
7
|
+
// conservatively excluded from seeded-row evidence.
|
|
8
|
+
const variable = sql => /\b(?:random|randomblob|date|time|datetime|julianday|strftime|unixepoch|sqlite_version|sqlite_source_id)\s*\(|\bcurrent_(?:date|time|timestamp)\b/i
|
|
9
|
+
.test(sqliteTokens(sql).map(token => token.value).join(' '));
|
|
10
|
+
|
|
11
|
+
export function migrationRecorder(scratch) {
|
|
12
|
+
const unstable = new Set(), applied = [];
|
|
13
|
+
return {
|
|
14
|
+
apply(file) {
|
|
15
|
+
for (const sql of file.statements) {
|
|
16
|
+
const data = dataStatement(sql);
|
|
17
|
+
if (data) {
|
|
18
|
+
const writers = new Set([data.target]);
|
|
19
|
+
for (const name of writers) for (const target of triggerBodies(applied, name)) writers.add(target);
|
|
20
|
+
const definitions = scratch.all('SELECT name,sql FROM sqlite_master WHERE sql IS NOT NULL');
|
|
21
|
+
if (variable(sql) || data.names.some(name => unstable.has(name))
|
|
22
|
+
|| definitions.some(row => writers.has(row.name.toLowerCase()) && variable(row.sql))
|
|
23
|
+
|| applied.some(statement => variable(statement) && triggerBodies([statement], data.target).some(name => writers.has(name)))) {
|
|
24
|
+
for (const name of writers) unstable.add(name);
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
scratch.exec(sql);
|
|
28
|
+
applied.push(sql);
|
|
29
|
+
}
|
|
30
|
+
},
|
|
31
|
+
inventory() {
|
|
32
|
+
const objects = scratch.all("SELECT name,sql,type,tbl_name FROM sqlite_master WHERE sql IS NOT NULL AND substr(lower(name),1,7)<>'sqlite_' AND substr(lower(name),1,4)<>'_cf_' ORDER BY name,type");
|
|
33
|
+
const kinds = scratch.all('PRAGMA table_list').filter(table => table.type === 'shadow').map(({ name, type }) => ({ name, type }));
|
|
34
|
+
return { objects, kinds };
|
|
35
|
+
},
|
|
36
|
+
async seeds(objects) {
|
|
37
|
+
const seeds = [];
|
|
38
|
+
for (const { name } of objects.filter(object => object.type === 'table')) {
|
|
39
|
+
const columns = scratch.all(`PRAGMA table_info(${quote(name)})`).map(column => column.name);
|
|
40
|
+
const rows = scratch.all(`SELECT json_array(${columns.map(column => `quote(${quote(column)})`).join(',')}) image FROM ${quote(name)} LIMIT 10001`);
|
|
41
|
+
if (rows.length > 10000 || unstable.has(name.toLowerCase()) && rows.length) continue;
|
|
42
|
+
seeds.push({ name, columns, hashes: (await Promise.all(rows.map(row => sha256(row.image)))).sort() });
|
|
43
|
+
}
|
|
44
|
+
return seeds;
|
|
45
|
+
},
|
|
46
|
+
};
|
|
47
|
+
}
|
|
@@ -1,18 +1,19 @@
|
|
|
1
1
|
import { policy, schemaIdentity } from './catalogue.mjs';
|
|
2
2
|
|
|
3
|
-
// Replays a chosen
|
|
3
|
+
// Replays a chosen sequence of the destination's migrations, in recorded order,
|
|
4
4
|
// into a scratch database and returns the resulting archive schema identity.
|
|
5
5
|
// Node runners only: restore uses it to place an older archive within the
|
|
6
6
|
// destination's history (see bridge.mjs). plan: [{name,statements,sha256?}],
|
|
7
7
|
// sha256 being the migration file's hash where the runner read the file.
|
|
8
8
|
export function migrationReplay(plan,input,{dataOnly=[]}={}) {
|
|
9
9
|
const config=policy(input),sqlite='node:sqlite';
|
|
10
|
-
|
|
10
|
+
const files=new Map(plan.map(migration=>[migration.name,migration]));
|
|
11
|
+
return {names:plan.map(migration=>migration.name),dataOnly,aliases:name=>files.get(name).aliases,statements:name=>files.get(name).statements,sha256:name=>files.get(name).sha256,async schemaAfter(names,options) {
|
|
11
12
|
const {DatabaseSync}=await import(sqlite);
|
|
12
13
|
const scratch=new DatabaseSync(':memory:');
|
|
13
14
|
const db={prepare:sql=>{const query=scratch.prepare(sql);let values=[];return {bind(...args){values=args;return this;},async all(){return {results:query.all(...values)};}};}};
|
|
14
15
|
try {
|
|
15
|
-
for(const
|
|
16
|
+
for(const name of names)for(const sql of files.get(name).statements)scratch.exec(sql);
|
|
16
17
|
return await schemaIdentity(db,config,options);
|
|
17
18
|
}finally{scratch.close();}
|
|
18
19
|
},
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import { ledgerNames, readLedger } from './ledgers.mjs';
|
|
2
|
+
import { all, insist, quote } from './storage.mjs';
|
|
3
|
+
|
|
4
|
+
export function ownerMigrationPolicy(policy, sources) {
|
|
5
|
+
if (!sources.length) return policy;
|
|
6
|
+
const tables = sources.map(source => source.table);
|
|
7
|
+
insist(!policy.includeTables?.some(name => tables.includes(name.toLowerCase())), 'migration_ledger:incompatible', 409);
|
|
8
|
+
return { ...policy, excludeTables: [...new Set([...(policy.excludeTables ?? []), ...tables])] };
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
const ownerColumns = columns => columns.length === 3 && columns[0].name === 'id' && columns[0].pk === 1
|
|
12
|
+
&& columns[1].name === 'name' && columns[2].name === 'applied_at';
|
|
13
|
+
const timestamp = /^\d{4}-\d\d-\d\d \d\d:\d\d:\d\d$/;
|
|
14
|
+
async function nativeRows(db, table, owner = false) {
|
|
15
|
+
const columns = await all(db, `PRAGMA table_info(${quote(table)})`);
|
|
16
|
+
if (!columns.length) return [];
|
|
17
|
+
insist(owner ? ownerColumns(columns) : ownerColumns(columns.slice(0, 3)), 'migration_ledger:incompatible', 409);
|
|
18
|
+
const rows = await all(db, `SELECT name, applied_at FROM ${quote(table)} ORDER BY id`);
|
|
19
|
+
insist(new Set(rows.map(row => row.name)).size === rows.length
|
|
20
|
+
&& rows.every((row, i) => typeof row.name === 'string' && timestamp.test(row.applied_at)
|
|
21
|
+
&& (!i || rows[i - 1].applied_at <= row.applied_at)), 'migration_ledger:incompatible', 409);
|
|
22
|
+
return rows;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export async function nativeMigrationHistory(db, sources) {
|
|
26
|
+
const ledger = await readLedger(db);
|
|
27
|
+
const owners = (await Promise.all(sources.map(async ({ table, prefix }) =>
|
|
28
|
+
(await nativeRows(db, table, true)).map(row => ({ ...row, name: prefix + row.name, table }))))).flat();
|
|
29
|
+
if (!owners.length) return ledger;
|
|
30
|
+
insist(ledger?.table === 'd1_migrations', 'migration_ledger:multiple', 409);
|
|
31
|
+
const core = (await nativeRows(db, ledger.table)).map((row, i) => ({ ...ledger.entries[i], applied_at: row.applied_at, table: ledger.table }));
|
|
32
|
+
const rows = [...core, ...owners].sort((a, b) => a.applied_at.localeCompare(b.applied_at));
|
|
33
|
+
// IDs order each ledger. Across ledgers equal timestamps carry no order.
|
|
34
|
+
insist(rows.every((row, i) => !i || row.applied_at !== rows[i - 1].applied_at || row.table === rows[i - 1].table), 'migration_ledger:uncertain', 409);
|
|
35
|
+
const entries = rows.map(({ name, sha256, table }) => ({ name, table, ...(sha256 ? { sha256 } : {}) }));
|
|
36
|
+
insist(new Set(entries.map(row => row.name)).size === entries.length, 'migration_ledger:ambiguous', 409);
|
|
37
|
+
return { table: ledger.table, entries, owners: true };
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
// The existing service callback is also used for core-only replay. Consult it
|
|
41
|
+
// early only when native owner bookkeeping may exist. The replay supplies the
|
|
42
|
+
// declared table identities; a similar product table never becomes bookkeeping.
|
|
43
|
+
async function hasOwnerLedger(db) {
|
|
44
|
+
const candidates = await all(db, "SELECT name FROM sqlite_master WHERE type='table' AND name GLOB '[a-z]*_*_migrations'");
|
|
45
|
+
for (const { name } of candidates) {
|
|
46
|
+
if (!ledgerNames.includes(name) && ownerColumns(await all(db, `PRAGMA table_info(${quote(name)})`))) return true;
|
|
47
|
+
}
|
|
48
|
+
return false;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
export async function transferHistory({ db, config, migrations, migrationHistory }) {
|
|
52
|
+
if (migrationHistory) return { config, migrations, ledger: await migrationHistory() };
|
|
53
|
+
const ledger = await readLedger(db);
|
|
54
|
+
if (migrations && ledger?.table === 'd1_migrations' && await hasOwnerLedger(db)) {
|
|
55
|
+
const replay = await migrations();
|
|
56
|
+
if (replay?.ownerSources) return {
|
|
57
|
+
config: ownerMigrationPolicy(config, replay.ownerSources), migrations: async () => replay, ledger: replay.history,
|
|
58
|
+
};
|
|
59
|
+
return { config, migrations: async () => replay, ledger };
|
|
60
|
+
}
|
|
61
|
+
return { config, migrations, ledger };
|
|
62
|
+
}
|
|
@@ -1,9 +1,26 @@
|
|
|
1
1
|
import type { ArchivePolicy, MigrationReplay } from './index.mjs';
|
|
2
|
-
|
|
2
|
+
type MigrationFile = {name:string;sha256:string;statements:string[]};
|
|
3
|
+
type OwnerSource = {prefix:string;table:string};
|
|
4
|
+
type SchemaObject = {name:string;sql:string;type:string;tbl_name:string};
|
|
5
|
+
type SeedRows = {name:string;columns:string[];hashes:string[]};
|
|
6
|
+
interface BundledEvidence {
|
|
3
7
|
version:1;
|
|
4
|
-
files:
|
|
5
|
-
|
|
6
|
-
|
|
8
|
+
files:MigrationFile[];
|
|
9
|
+
ownerSources?:OwnerSource[];
|
|
10
|
+
objects:SchemaObject[];
|
|
11
|
+
prefixes:{objects:number[];shadows:string[];seeds:SeedRows[]}[];
|
|
7
12
|
}
|
|
13
|
+
type OrderedFile = MigrationFile & {table?:string};
|
|
14
|
+
interface OrderedEvidence {
|
|
15
|
+
version:2;
|
|
16
|
+
files:OrderedFile[];
|
|
17
|
+
ownerSources:OwnerSource[];
|
|
18
|
+
nativePrefix(history:{table:string;owners?:boolean;entries:{name:string;table?:string;sha256?:string}[]},files:OrderedFile[]):boolean;
|
|
19
|
+
snapshots(names:string[]):Promise<{
|
|
20
|
+
inventory(index:number):{objects:SchemaObject[];kinds:{name:string;type:string}[]};
|
|
21
|
+
seeds:SeedRows[];
|
|
22
|
+
}>;
|
|
23
|
+
}
|
|
24
|
+
export type MigrationEvidence = BundledEvidence | OrderedEvidence;
|
|
8
25
|
/** Return a replay bound to the destination's hashed history or exact native Wrangler prefix, and complete schema. */
|
|
9
26
|
export function workerMigrations(db:D1Database,policy:ArchivePolicy,evidence:MigrationEvidence):Promise<MigrationReplay>;
|
|
@@ -1,29 +1,37 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { inventorySchemaIdentity } from './catalogue.mjs';
|
|
2
2
|
import { canonicalJson } from './hash.mjs';
|
|
3
3
|
import { checkoutHistory } from './checkout-history.mjs';
|
|
4
4
|
import { insist, PortabilityError } from './storage.mjs';
|
|
5
5
|
|
|
6
|
+
function bundledSnapshots(evidence,names) {
|
|
7
|
+
insist(evidence.prefixes.length===evidence.files.length+1
|
|
8
|
+
&& names.every((name,i)=>name===evidence.files[i]?.name),'schema_mismatch:legacy_evidence',409);
|
|
9
|
+
return {
|
|
10
|
+
inventory(index){
|
|
11
|
+
const snapshot=evidence.prefixes[index];
|
|
12
|
+
return {objects:snapshot.objects.map(i=>evidence.objects[i]),kinds:snapshot.shadows.map(name=>({name,type:'shadow'}))};
|
|
13
|
+
},
|
|
14
|
+
seeds:evidence.prefixes[names.length].seeds,
|
|
15
|
+
};
|
|
16
|
+
}
|
|
17
|
+
|
|
6
18
|
// This adapter supplies replay evidence, never archive validation. The same
|
|
7
19
|
// collector, ledger matching, schema bridge and restore guards serve both doors.
|
|
8
20
|
export async function workerMigrations(db,input,evidence) {
|
|
9
21
|
try {
|
|
10
|
-
insist(evidence?.version===1
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
insist(found,'schema_mismatch:legacy_evidence',409);return found;
|
|
17
|
-
};
|
|
18
|
-
const final=prefix(names);
|
|
19
|
-
return {names,statements:name=>files.find(f=>f.name===name)?.statements,sha256:name=>files.find(f=>f.name===name)?.sha256,
|
|
22
|
+
insist(evidence?.version===1 || evidence?.version===2 && evidence.ownerSources?.length
|
|
23
|
+
&& typeof evidence.snapshots==='function' && typeof evidence.nativePrefix==='function','schema_mismatch:legacy_evidence',409);
|
|
24
|
+
const {files,ownerSources}=evidence;
|
|
25
|
+
return await checkoutHistory(db,input,files,async (names,config)=>{
|
|
26
|
+
const {inventory,seeds}=evidence.version===2?await evidence.snapshots(names):bundledSnapshots(evidence,names);
|
|
27
|
+
return {names,statements:name=>files.find(file=>file.name===name)?.statements,sha256:name=>files.find(file=>file.name===name)?.sha256,
|
|
20
28
|
schemaAfter(selected,options){
|
|
21
|
-
|
|
22
|
-
return inventorySchemaIdentity(
|
|
29
|
+
insist(selected.length<=names.length && selected.every((name,i)=>name===names[i]),'schema_mismatch:legacy_evidence',409);
|
|
30
|
+
return inventorySchemaIdentity(inventory(selected.length),config,options);
|
|
23
31
|
},
|
|
24
|
-
async rowImages(requests){return new Map(requests.filter(r=>
|
|
25
|
-
async rowHashes(requests,limit){return new Map(requests.flatMap(r=>{const found=
|
|
32
|
+
async rowImages(requests){return new Map(requests.filter(r=>seeds.some(s=>s.name===r.name && !s.hashes.length)).map(r=>[r.name,[]]));},
|
|
33
|
+
async rowHashes(requests,limit){return new Map(requests.flatMap(r=>{const found=seeds.find(s=>s.name===r.name && s.hashes.length<=limit && canonicalJson(s.columns)===canonicalJson(r.columns));return found?[[r.name,found.hashes]]:[];}));},
|
|
26
34
|
};
|
|
27
|
-
},{nativeWrangler:true});
|
|
35
|
+
},{nativeWrangler:true,ownerSources,nativePrefix:evidence.nativePrefix});
|
|
28
36
|
}catch(error){if(error instanceof PortabilityError)throw error;throw new PortabilityError('schema_mismatch:legacy_evidence',409);}
|
|
29
37
|
}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import { migrationRecorder } from './migration-evidence.mjs';
|
|
2
|
+
|
|
3
|
+
function nativePrefix(history,files) {
|
|
4
|
+
if(!history.owners)return history.entries.every((row,i)=>
|
|
5
|
+
row.sha256===undefined && row.name===files[i]?.name && files[i].table===undefined);
|
|
6
|
+
const byTable=new Map();
|
|
7
|
+
for(const file of files) {
|
|
8
|
+
const table=file.table??history.table;
|
|
9
|
+
if(!byTable.has(table))byTable.set(table,[]);
|
|
10
|
+
byTable.get(table).push(file.name);
|
|
11
|
+
}
|
|
12
|
+
const offsets=new Map();
|
|
13
|
+
return history.entries.every(({name,table})=>{
|
|
14
|
+
const offset=offsets.get(table)??0;
|
|
15
|
+
offsets.set(table,offset+1);
|
|
16
|
+
return name===byTable.get(table)?.[offset];
|
|
17
|
+
});
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
// Only the build evidence for a checkout with owner SQL links this module.
|
|
21
|
+
export function workerOwnerEvidence(evidence,openDatabase) {
|
|
22
|
+
const files=new Map(evidence.files.map(file=>[file.name,file]));
|
|
23
|
+
return {
|
|
24
|
+
...evidence,
|
|
25
|
+
nativePrefix,
|
|
26
|
+
async snapshots(names) {
|
|
27
|
+
const scratch=await openDatabase();
|
|
28
|
+
try {
|
|
29
|
+
const recorder=migrationRecorder(scratch),prefixes=[recorder.inventory()];
|
|
30
|
+
for(const name of names) {
|
|
31
|
+
recorder.apply(files.get(name));
|
|
32
|
+
prefixes.push(recorder.inventory());
|
|
33
|
+
}
|
|
34
|
+
return {inventory:index=>prefixes[index],seeds:await recorder.seeds(prefixes.at(-1).objects)};
|
|
35
|
+
}finally{scratch.close();}
|
|
36
|
+
},
|
|
37
|
+
};
|
|
38
|
+
}
|