@runeditrun/rer 0.2.67 → 0.2.68
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cli/business/vendors/README.md +16 -4
- package/cli/business/vendors/common.mjs +49 -28
- package/cli/business/vendors/credential-names.mjs +124 -0
- package/cli/import-summary.mjs +7 -0
- package/cli/output.mjs +2 -0
- package/cli/portability-worker.mjs +10 -4
- package/cli/portability.mjs +1 -0
- package/global/CHANGELOG.md +8 -0
- package/modules/portability/README.md +73 -8
- package/modules/portability/bridge.mjs +44 -29
- package/modules/portability/catalogue.mjs +33 -3
- package/modules/portability/files.mjs +9 -3
- package/modules/portability/import-safety.mjs +80 -0
- package/modules/portability/import.mjs +13 -7
- package/modules/portability/index.d.mts +9 -3
- package/modules/portability/index.mjs +1 -1
- package/modules/portability/ledgers.mjs +83 -0
- package/modules/portability/legacy-files.mjs +7 -4
- package/modules/portability/migration-history.mjs +3 -2
- package/modules/portability/migration-statements.mjs +83 -9
- package/modules/release/README.md +14 -6
- package/modules/release/publication-scan.mjs +13 -7
- package/package.json +1 -1
|
@@ -1,35 +1,31 @@
|
|
|
1
1
|
import { all, quote, insist } from './storage.mjs';
|
|
2
2
|
import { canonicalJson } from './hash.mjs';
|
|
3
3
|
import { tokens, identifier, folded } from './generated.mjs';
|
|
4
|
-
import { changesData } from './migration-statements.mjs';
|
|
4
|
+
import { changesData, reachingTables } from './migration-statements.mjs';
|
|
5
|
+
import { destinationLedger } from './ledgers.mjs';
|
|
5
6
|
|
|
6
|
-
// Applied product migrations in the ledger's own order. A
|
|
7
|
-
// ledger may hold an unacknowledged migration whose effects are
|
|
8
|
-
// an archive nor a restore can attest its history until that
|
|
7
|
+
// Applied product migrations in the ledger's own order (ledgers.mjs). A
|
|
8
|
+
// state-bearing ledger may hold an unacknowledged migration whose effects are
|
|
9
|
+
// unknown; neither an archive nor a restore can attest its history until that
|
|
10
|
+
// is resolved.
|
|
9
11
|
export async function appliedMigrations(db) {
|
|
10
|
-
|
|
11
|
-
if(ledgers.includes('_rer_deploy_migrations'))return (await all(db,"SELECT name FROM _rer_deploy_migrations WHERE state='applied' ORDER BY name")).map(x=>x.name);
|
|
12
|
-
if(!ledgers.includes('d1_migrations'))return;
|
|
13
|
-
const columns=(await all(db,'PRAGMA table_info(d1_migrations)')).map(x=>x.name);
|
|
14
|
-
const tracked=columns.includes('state') || columns.includes('sha256');
|
|
15
|
-
if(tracked)insist(columns.includes('state') && columns.includes('sha256'),'schema_mismatch:migration_history',409);
|
|
16
|
-
const rows=await all(db,`SELECT name${tracked?',state':''} FROM d1_migrations ORDER BY id`);
|
|
17
|
-
if(tracked)insist(rows.every(row=>row.state==='applied'),'schema_mismatch:uncertain_migrations',409);
|
|
18
|
-
return rows.map(row=>row.name);
|
|
12
|
+
return (await destinationLedger(db))?.map(m=>m.name);
|
|
19
13
|
}
|
|
20
14
|
// Where an archive sits in the destination's applied migrations: those its
|
|
21
15
|
// rows passed through (earlier) and those applied after it (unapplied).
|
|
22
16
|
// migrations() is the runner's replay of the destination's applied migrations
|
|
23
17
|
// (migration-history.mjs). The archive's schema must be the replay of the
|
|
24
|
-
// migrations it records or, for archives that predate that record, of
|
|
25
|
-
//
|
|
26
|
-
//
|
|
27
|
-
// the schema.
|
|
28
|
-
async function placement(manifest,cat,replay,
|
|
18
|
+
// migrations it records or, for archives that predate that record, of those
|
|
19
|
+
// its own migration ledger records or, without one, of an earlier prefix.
|
|
20
|
+
// Restored rows never pass through a later migration, so one that writes or
|
|
21
|
+
// rewrites restorable data is refused, whether or not it changes the schema.
|
|
22
|
+
async function placement(manifest,cat,replay,ledger,history) {
|
|
23
|
+
const applied=ledger.map(m=>m.name);
|
|
29
24
|
insist(canonicalJson(replay.names)===canonicalJson(applied) && await replay.schemaAfter(applied)===cat.identity,'schema_mismatch');
|
|
25
|
+
const recorded=manifest.migrations ?? archivedApplied(history?.(),ledger,replay);
|
|
30
26
|
let earlier,unapplied;
|
|
31
|
-
if(
|
|
32
|
-
earlier=applied.filter(name=>
|
|
27
|
+
if(recorded) {
|
|
28
|
+
earlier=applied.filter(name=>recorded.includes(name));
|
|
33
29
|
insist(await replay.schemaAfter(earlier)===manifest.schema,'schema_mismatch');
|
|
34
30
|
unapplied=applied.filter(name=>!earlier.includes(name));
|
|
35
31
|
}else{
|
|
@@ -44,12 +40,31 @@ async function placement(manifest,cat,replay,applied) {
|
|
|
44
40
|
unapplied=applied.slice(matching[0]);
|
|
45
41
|
}
|
|
46
42
|
// Only rows the archive carries can skip a later migration's change. A table
|
|
47
|
-
// the archive does not hold is restored empty, so
|
|
48
|
-
//
|
|
43
|
+
// the archive does not hold is restored empty, so backfills that name only
|
|
44
|
+
// such tables (a newly added module's own) change no restored row, unless a
|
|
45
|
+
// trigger from any applied migration carries their writes into one that is
|
|
46
|
+
// restorable. Trigger DDL itself changes no row, nor does an UPDATE or DELETE
|
|
47
|
+
// of a table the archive holds no rows of (migration-statements.mjs).
|
|
49
48
|
const restorable=cat.tables.map(t=>t.name).filter(name=>manifest.tables.some(t=>t.name===name));
|
|
50
|
-
|
|
49
|
+
const reached=reachingTables(applied.flatMap(name=>replay.statements(name)),restorable);
|
|
50
|
+
const empty=restorable.filter(name=>!manifest.tables.find(t=>t.name===name).count);
|
|
51
|
+
for(const name of unapplied)insist(!replay.statements(name).some(sql=>changesData(sql,reached,empty)),'schema_mismatch:migration_data:'+name);
|
|
51
52
|
return {earlier};
|
|
52
53
|
}
|
|
54
|
+
// An archive without a migration record that carries its source's migration
|
|
55
|
+
// ledger (an old archive from before that ledger became bookkeeping) records
|
|
56
|
+
// exactly which migrations its rows passed through. Each must be one the
|
|
57
|
+
// destination applied, with the same file hash where the ledger records one:
|
|
58
|
+
// the destination ledger's, else the replayed file's. The ledger itself is
|
|
59
|
+
// never restored (bookkeeping.mjs). history: [{name,sha256?}] from
|
|
60
|
+
// ledgers.mjs archivedHistory, undefined when the archive carries no ledger.
|
|
61
|
+
function archivedApplied(history,ledger,replay) {
|
|
62
|
+
for(const m of history??[]) {
|
|
63
|
+
const applied=ledger.find(x=>x.name===m.name);
|
|
64
|
+
insist(applied && (m.sha256===undefined || m.sha256===(applied.sha256??replay.sha256?.(m.name))),'schema_mismatch:migration_history:'+m.name);
|
|
65
|
+
}
|
|
66
|
+
return history?.map(m=>m.name);
|
|
67
|
+
}
|
|
53
68
|
// An archive with the destination's schema identity can still predate later
|
|
54
69
|
// migrations that changed only data (a purge or a backfill) or added an index.
|
|
55
70
|
// Those that change restorable data are refused like any other. Without a
|
|
@@ -57,21 +72,21 @@ async function placement(manifest,cat,replay,applied) {
|
|
|
57
72
|
// migrations the destination's ledger has outgrown is refused. An archive that
|
|
58
73
|
// records a migration the destination never applied is newer than it, even
|
|
59
74
|
// when that migration left the schema as it was (an index, a data change).
|
|
60
|
-
export async function sameSchemaMigrations(db,manifest,cat,migrations) {
|
|
61
|
-
const
|
|
75
|
+
export async function sameSchemaMigrations(db,manifest,cat,migrations,history) {
|
|
76
|
+
const ledger=await destinationLedger(db),applied=ledger?.map(m=>m.name);
|
|
62
77
|
if(!applied)return;
|
|
63
78
|
insist(!manifest.migrations || manifest.migrations.every(name=>applied.includes(name)),'schema_mismatch:migrations');
|
|
64
79
|
if(!migrations){insist(!manifest.migrations || applied.every(name=>manifest.migrations.includes(name)),'schema_mismatch:migrations');return;}
|
|
65
|
-
await placement(manifest,cat,await migrations(),
|
|
80
|
+
await placement(manifest,cat,await migrations(),ledger,history);
|
|
66
81
|
}
|
|
67
82
|
// An archive made before later migrations restores only when those migrations
|
|
68
83
|
// are additive: new tables, new columns that take a default, or indexes (which
|
|
69
84
|
// archives never carry). Anything else stays a schema_mismatch refusal.
|
|
70
|
-
export async function additiveBridge(db,manifest,cat,migrations) {
|
|
85
|
+
export async function additiveBridge(db,manifest,cat,migrations,history) {
|
|
71
86
|
insist(migrations,'schema_mismatch');
|
|
72
|
-
const replay=await migrations(),
|
|
87
|
+
const replay=await migrations(),ledger=await destinationLedger(db)??[],applied=ledger.map(m=>m.name);
|
|
73
88
|
insist(!manifest.migrations || manifest.migrations.every(name=>applied.includes(name)),'schema_mismatch:migrations');
|
|
74
|
-
const {earlier}=await placement(manifest,cat,replay,
|
|
89
|
+
const {earlier}=await placement(manifest,cat,replay,ledger,history);
|
|
75
90
|
const columns={};
|
|
76
91
|
for(const table of manifest.tables) {
|
|
77
92
|
const target=cat.tables.find(t=>t.name===table.name),label=':'+table.name;
|
|
@@ -3,18 +3,40 @@ import { canonicalJson, sha256, base64, unbase64, utf8 } from './hash.mjs';
|
|
|
3
3
|
import { platformTables, platformTable, identifier, matches } from './bookkeeping.mjs';
|
|
4
4
|
/** Platform bookkeeping is mandatory; operator identity remains a seed choice. */
|
|
5
5
|
export const excludedTables=Object.freeze([...platformTables,'oa_%','oc_%']);
|
|
6
|
-
const fields=['product','excludeTables','includeTables','derivedTables','secretColumns','exportRow','replaceableTables','requiredConfiguration','validateStaged','acceptedSchemas','buckets','extraPayloads','partBytes'];
|
|
6
|
+
const fields=['product','excludeTables','includeTables','derivedTables','secretColumns','exportRow','importSafety','importRow','replaceableTables','requiredConfiguration','validateStaged','acceptedSchemas','buckets','extraPayloads','partBytes'];
|
|
7
7
|
const name=value=>typeof value==='string' && value.length>0 && value.length<=512 && !/[\x00-\x1f]/.test(value);
|
|
8
8
|
const names=value=>Array.isArray(value) && value.every(name) && new Set(value).size===value.length;
|
|
9
|
+
const record=value=>value && typeof value==='object' && !Array.isArray(value) && [Object.prototype,null].includes(Object.getPrototypeOf(value));
|
|
10
|
+
const safeCell=value=>{
|
|
11
|
+
if(value===null || typeof value==='string' || (typeof value==='number' && Number.isFinite(value)))return true;
|
|
12
|
+
if(!record(value) || Object.keys(value).length!==1 || typeof value.blob!=='string')return false;
|
|
13
|
+
try{unbase64(value.blob);return true;}catch{return false;}
|
|
14
|
+
};
|
|
9
15
|
export function policy(input) {
|
|
10
16
|
insist(input && typeof input==='object' && Object.keys(input).every(k=>fields.includes(k)) && name(input.product),'invalid_policy');
|
|
11
|
-
const result={excludeTables:[],includeTables:[],derivedTables:[],secretColumns:{},replaceableTables:[],requiredConfiguration:[],acceptedSchemas:{},buckets:{},extraPayloads:{},partBytes:16*1024*1024,...input};
|
|
17
|
+
const result={excludeTables:[],includeTables:[],derivedTables:[],secretColumns:{},importSafety:{},replaceableTables:[],requiredConfiguration:[],acceptedSchemas:{},buckets:{},extraPayloads:{},partBytes:16*1024*1024,...input};
|
|
12
18
|
for(const key of ['excludeTables','includeTables','derivedTables','replaceableTables','requiredConfiguration']) insist(names(result[key]),'invalid_policy_'+key);
|
|
13
19
|
for(const table of result.includeTables) if(platformTable(table)) {
|
|
14
20
|
const error=new PortabilityError('platform_table:'+table);
|
|
15
21
|
error.message='includeTables cannot include platform-owned table "'+table+'".';throw error;
|
|
16
22
|
}
|
|
17
|
-
for(const key of ['exportRow','validateStaged']) insist(result[key]===undefined || typeof result[key]==='function','invalid_policy_'+key);
|
|
23
|
+
for(const key of ['exportRow','importRow','validateStaged']) insist(result[key]===undefined || typeof result[key]==='function','invalid_policy_'+key);
|
|
24
|
+
insist(record(result.importSafety),'invalid_policy_importSafety');
|
|
25
|
+
const safety=Object.create(null);
|
|
26
|
+
for(const [table,columns] of Object.entries(result.importSafety)) {
|
|
27
|
+
insist(name(table) && record(columns) && Object.keys(columns).length>0,'invalid_import_safety:'+table);
|
|
28
|
+
const declared=Object.create(null);
|
|
29
|
+
for(const [column,values] of Object.entries(columns)) {
|
|
30
|
+
insist(name(column) && Array.isArray(values) && values.length>0,'invalid_import_safety:'+table+':'+column);
|
|
31
|
+
const cells=Array.from(values,value=>record(value)?{...value}:value);
|
|
32
|
+
insist(cells.every(safeCell) && new Set(cells.map(canonicalJson)).size===cells.length,'invalid_import_safety:'+table+':'+column);
|
|
33
|
+
// The hook's closure cannot expand the declaration while it runs.
|
|
34
|
+
declared[column]=Object.freeze(cells.map(value=>value && typeof value==='object'?Object.freeze({...value}):value));
|
|
35
|
+
}
|
|
36
|
+
safety[table]=Object.freeze(declared);
|
|
37
|
+
}
|
|
38
|
+
insist(Boolean(result.importRow)===(Object.keys(safety).length>0),'invalid_policy_importSafety_pair');
|
|
39
|
+
result.importSafety=Object.freeze(safety);
|
|
18
40
|
insist(Number.isSafeInteger(result.partBytes) && result.partBytes>=1024 && result.partBytes<=16*1024*1024,'invalid_part_limit');
|
|
19
41
|
for(const [table,columns] of Object.entries(result.secretColumns)) insist(name(table) && names(columns),'invalid_secret_columns');
|
|
20
42
|
for(const [identity,entry] of Object.entries(result.acceptedSchemas)) insist(/^[a-f0-9]{64}$/.test(identity) && typeof entry.upgradeRow==='function','invalid_accepted_schema');
|
|
@@ -63,6 +85,14 @@ export async function catalogue(db,config) {
|
|
|
63
85
|
const entry=tables.find(t=>t.name===table); insist(entry,'secret_table_missing:'+table);
|
|
64
86
|
for(const key of list) {const column=entry.columns.find(c=>c.name===key);insist(column && !column.notnull && !column.pk && !column.generated,'invalid_secret_column:'+table+':'+key);}
|
|
65
87
|
}
|
|
88
|
+
for(const [table,columns] of Object.entries(config.importSafety)) {
|
|
89
|
+
const entry=tables.find(t=>t.name===table);insist(entry,'import_safety_table_missing:'+table);
|
|
90
|
+
for(const [key,values] of Object.entries(columns)) {
|
|
91
|
+
const column=entry.columns.find(c=>c.name===key);
|
|
92
|
+
insist(column && !column.generated && !column.pk,'invalid_import_safety_column:'+table+':'+key);
|
|
93
|
+
for(const value of values)validateRow({...entry,columns:[column]},{[key]:value});
|
|
94
|
+
}
|
|
95
|
+
}
|
|
66
96
|
for(const table of [...config.replaceableTables,...config.includeTables]) insist(tables.some(x=>x.name===table),'policy_table_missing:'+table);
|
|
67
97
|
for(const spec of Object.values(config.buckets)) if(spec.objectReferences) {
|
|
68
98
|
const {table,keyColumn}=spec.objectReferences;
|
|
@@ -5,6 +5,7 @@ import { Readable } from 'node:stream';
|
|
|
5
5
|
import { randomUUID } from 'node:crypto';
|
|
6
6
|
import { readArchive } from './index.mjs';
|
|
7
7
|
import { canonicalJson } from './hash.mjs';
|
|
8
|
+
import { ledgerReader } from './ledgers.mjs';
|
|
8
9
|
export async function* fileRecords(path,reader=readArchive) {
|
|
9
10
|
const file = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW);
|
|
10
11
|
try { yield* reader(Readable.toWeb(file.createReadStream({ autoClose: false }))); }
|
|
@@ -62,15 +63,20 @@ export async function exportArchive(service, path, transient, progress) {
|
|
|
62
63
|
export async function importArchive(service, path, transient, journal, progress) {
|
|
63
64
|
let id, manifest, closing, complete = false, staged = 0;
|
|
64
65
|
try {
|
|
65
|
-
// Validate both complete archives before creating any restore state.
|
|
66
|
+
// Validate both complete archives before creating any restore state. An old
|
|
67
|
+
// archive's own migration ledger places it in the destination's history
|
|
68
|
+
// (bridge.mjs); import binds the rows read here to the parts it stages.
|
|
66
69
|
let later;
|
|
70
|
+
const ledger = ledgerReader();
|
|
67
71
|
progress?.({ phase: 'checking', direction: 'import' });
|
|
68
72
|
for (const file of [path, journal].filter(Boolean)) for await (const record of fileRecords(file)) {
|
|
69
73
|
if (file === journal && record.kind === 'manifest') later = { sourceIdentity: record.sourceIdentity, epoch: record.epoch, journal: record.journal };
|
|
74
|
+
if (file === path) ledger.add(record);
|
|
70
75
|
}
|
|
76
|
+
const ledgers = ledger.ledgers();
|
|
71
77
|
for await (const record of fileRecords(path)) {
|
|
72
78
|
if (record.kind === 'manifest') {
|
|
73
|
-
manifest = record; id = await service.imports.create(manifest);
|
|
79
|
+
manifest = record; id = await service.imports.create(manifest, undefined, ledgers ? { ledgers } : {});
|
|
74
80
|
const { skippedTables } = await service.imports.status(id);
|
|
75
81
|
if (skippedTables?.length) progress?.({ phase: 'skipped', tables: skippedTables });
|
|
76
82
|
}
|
|
@@ -83,6 +89,6 @@ export async function importArchive(service, path, transient, journal, progress)
|
|
|
83
89
|
}
|
|
84
90
|
progress?.({ phase: 'finishing', total: manifest.partCount });
|
|
85
91
|
const status = await service.imports.finalize(id, { contentHash: closing.contentHash, archiveHash: closing.archiveHash, ...(later ? { laterJournal: later } : {}) }); complete = true;
|
|
86
|
-
return { ...receipt(manifest, closing, transient), ...(status.bridged ? { bridged: status.bridged } : {}), ...(status.skippedTables ? { skippedTables: status.skippedTables } : {}) };
|
|
92
|
+
return { ...receipt(manifest, closing, transient), ...(status.bridged ? { bridged: status.bridged } : {}), ...(status.skippedTables ? { skippedTables: status.skippedTables } : {}), ...(status.safetyChanges ? { safetyChanges: status.safetyChanges } : {}) };
|
|
87
93
|
} finally { if (id && !complete) await service.imports.abort(id); }
|
|
88
94
|
}
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
import { encodedRow, validateRow, bindCell } from './catalogue.mjs';
|
|
2
|
+
import { readCopy, copyRowid } from './copies.mjs';
|
|
3
|
+
import { canonicalJson } from './hash.mjs';
|
|
4
|
+
import { statement, insist, quote } from './storage.mjs';
|
|
5
|
+
import { RPC_LIMIT, rpcBytes } from '../d1-transport/index.mjs';
|
|
6
|
+
|
|
7
|
+
const same=(a,b)=>canonicalJson(a)===canonicalJson(b);
|
|
8
|
+
const location=(table,column)=>table.name+(column===undefined?'':':'+column);
|
|
9
|
+
function checkResult(config,table,before,after) {
|
|
10
|
+
insist(after && typeof after==='object' && !Array.isArray(after) && typeof after.then!=='function','import_row_must_be_pure:'+table.name,409);
|
|
11
|
+
const keys=Reflect.ownKeys(after),original=Object.keys(before);
|
|
12
|
+
const structural=[...keys.filter(k=>!original.includes(k)),...original.filter(k=>!Object.prototype.propertyIsEnumerable.call(after,k))];
|
|
13
|
+
insist(!structural.length,'import_row_columns:'+location(table,String(structural[0])),409);
|
|
14
|
+
validateRow(table,after);
|
|
15
|
+
for(const column of original) {
|
|
16
|
+
insist(after[column]!==undefined,'import_row_columns:'+location(table,column),409);
|
|
17
|
+
if(same(before[column],after[column]))continue;
|
|
18
|
+
insist(!table.columns.some(c=>c.name===column && c.pk),'import_row_key:'+location(table,column),409);
|
|
19
|
+
const allowed=config.importSafety[table.name][column];
|
|
20
|
+
insist(allowed,'import_row_undeclared:'+location(table,column),409);
|
|
21
|
+
insist(allowed.some(value=>same(value,after[column])),'import_row_unsafe_value:'+location(table,column),409);
|
|
22
|
+
}
|
|
23
|
+
return structuredClone(after);
|
|
24
|
+
}
|
|
25
|
+
function mapImportRow(config,table,before) {
|
|
26
|
+
const apply=row=>checkResult(config,table,row,config.importRow(table.name,structuredClone(row)));
|
|
27
|
+
const after=apply(before),again=apply(before),twice=apply(after);
|
|
28
|
+
for(const [result,reason] of [[again,'deterministic'],[twice,'idempotent']]) {
|
|
29
|
+
const column=Object.keys(before).find(key=>!same(after[key],result[key]));
|
|
30
|
+
insist(column===undefined,'import_row_not_'+reason+':'+location(table,column),409);
|
|
31
|
+
}
|
|
32
|
+
return after;
|
|
33
|
+
}
|
|
34
|
+
function countChanges(summary,table,before,after) {
|
|
35
|
+
for(const column of Object.keys(before))if(!same(before[column],after[column])) {
|
|
36
|
+
let entry=summary.find(x=>x.table===table.name && x.column===column && same(x.value,after[column]));
|
|
37
|
+
if(!entry){entry={table:table.name,column,value:after[column],rows:0};summary.push(entry);}
|
|
38
|
+
entry.rows++;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
const guardSql=`INSERT INTO rer_portability_guard(value) SELECT CASE WHEN (
|
|
42
|
+
EXISTS(SELECT 1 FROM rer_portability_jobs WHERE id=? AND metadata=?)
|
|
43
|
+
AND (SELECT import_job FROM rer_portability_state WHERE id=1)=?
|
|
44
|
+
) THEN 1 ELSE 0 END`;
|
|
45
|
+
const checkpoint=(row,before,metadata,queries)=>[
|
|
46
|
+
{sql:guardSql,params:[row.id,before,row.id]},...queries,
|
|
47
|
+
{sql:'UPDATE rer_portability_jobs SET phase=?,completed=?,metadata=? WHERE id=?',params:[row.phase,row.completed,canonicalJson(metadata),row.id]},
|
|
48
|
+
{sql:'DELETE FROM rer_portability_guard'},
|
|
49
|
+
];
|
|
50
|
+
// Each page's staged updates, cursor and value-free change counts commit together.
|
|
51
|
+
// A resumed finalize never transforms an already-normalized row or recounts it.
|
|
52
|
+
export async function normalizeStaged(context,row) {
|
|
53
|
+
const {db,config}=context;if(!config.importRow)return;
|
|
54
|
+
for(const table of row.metadata.targets.filter(t=>Object.hasOwn(config.importSafety,t.name))) {
|
|
55
|
+
const copy=row.metadata.copies[table.name],count=row.metadata.stagedCounts?.[table.name]??0;
|
|
56
|
+
let offset=row.metadata.safetyProgress?.[table.name]??0;
|
|
57
|
+
while(offset<count) {
|
|
58
|
+
let before=canonicalJson(row.metadata),queries=[];
|
|
59
|
+
const commit=async()=>{
|
|
60
|
+
await db.batch(checkpoint(row,before,row.metadata,queries).map(q=>statement(db,q.sql,...(q.params??[]))));
|
|
61
|
+
before=canonicalJson(row.metadata);queries=[];
|
|
62
|
+
};
|
|
63
|
+
for(const {row:raw,meta} of await readCopy(db,table,copy,offset,100,true)) {
|
|
64
|
+
const original=encodedRow(table,raw),after=mapImportRow(config,table,original);
|
|
65
|
+
const columns=Object.keys(original).filter(key=>!same(original[key],after[key]));
|
|
66
|
+
insist(Number.isSafeInteger(meta.rer_portability_copy_rowid),'copy_rowid');
|
|
67
|
+
const query=columns.length?{sql:`UPDATE ${quote(copy)} SET ${columns.map(c=>quote(c)+'=?').join(',')} WHERE ${copyRowid(table)}=${meta.rer_portability_copy_rowid}`,params:columns.map(c=>bindCell(after[c]))}:undefined;
|
|
68
|
+
// Reserve room for a metadata checkpoint and its concurrency guard.
|
|
69
|
+
const summary=structuredClone(row.metadata.safetyChanges??[]);countChanges(summary,table,original,after);
|
|
70
|
+
const metadata={...row.metadata,safetyChanges:summary,safetyProgress:{...row.metadata.safetyProgress,[table.name]:offset+1}};
|
|
71
|
+
const pending=[...queries,...(query?[query]:[])];
|
|
72
|
+
if(rpcBytes(checkpoint(row,before,metadata,pending))>RPC_LIMIT && queries.length)await commit();
|
|
73
|
+
insist(rpcBytes(checkpoint(row,before,metadata,[...queries,...(query?[query]:[])]))<=RPC_LIMIT,'D1_RPC_TRANSPORT_LIMIT',413);
|
|
74
|
+
if(query)queries.push(query);
|
|
75
|
+
row.metadata=metadata;offset++;
|
|
76
|
+
}
|
|
77
|
+
await commit();
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { all, statement, state, guard, job, save, partQueries, parts, readPart, cleanCopies, quote, stream, insist, run } from './storage.mjs';
|
|
2
2
|
import { stagePart } from './import-staging.mjs';
|
|
3
|
+
import { normalizeStaged } from './import-safety.mjs';
|
|
3
4
|
import { refuseProviderData } from './provider-data.mjs';
|
|
4
5
|
import { refuseDevData } from './dev-data.mjs';
|
|
5
6
|
import { catalogue, validateRow, bindCell, writableColumns } from './catalogue.mjs';
|
|
@@ -11,18 +12,21 @@ import { writeObject } from './payloads.mjs';
|
|
|
11
12
|
import { createCopy,sidecar,readCopy,restoreCopy } from './copies.mjs';
|
|
12
13
|
import { excludedDependentGuards, migrationSeeded, seededCondition } from './restore-safety.mjs';
|
|
13
14
|
import { additiveBridge, sameSchemaMigrations, appliedMigrations } from './bridge.mjs';
|
|
15
|
+
import { checkArchivedLedgers, archivedHistory } from './ledgers.mjs';
|
|
14
16
|
import { legacyEmptyGuards, legacyAuditStatements, requireLegacyEmpty } from './legacy-safety.mjs';
|
|
15
|
-
|
|
16
|
-
|
|
17
|
+
// ledgers: the rows of each migration ledger table the archive carries, as the
|
|
18
|
+
// caller read them from it ({table:rows}); uploadPart binds them to its parts.
|
|
19
|
+
export async function createImport(context,manifest,actor='direct',legacyRecovery,ledgers) {
|
|
20
|
+
const {db,config,buckets}=context;await checkManifest(manifest);checkArchivedLedgers(manifest,ledgers);
|
|
17
21
|
insist(manifest.product===config.product,'wrong_product');
|
|
18
22
|
// Explicit schema upgrades can reconcile rows, but cannot attest an interrupted deployment.
|
|
19
23
|
await appliedMigrations(db);await state(db);
|
|
20
24
|
const original=manifest;manifest=await restorableManifest(original);
|
|
21
25
|
const skippedTables=original.tables.filter(table=>platformTable(table.name)).map(table=>table.name);
|
|
22
26
|
const cat=await catalogue(db,config),upgrade=config.acceptedSchemas[original.schema];
|
|
23
|
-
const same=manifest.schema===cat.identity;
|
|
24
|
-
const bridge=same || upgrade?undefined:await additiveBridge(db,manifest,cat,context.migrations);
|
|
25
|
-
if(same && !upgrade)await sameSchemaMigrations(db,manifest,cat,context.migrations);
|
|
27
|
+
const same=manifest.schema===cat.identity,history=()=>archivedHistory(original,ledgers);
|
|
28
|
+
const bridge=same || upgrade?undefined:await additiveBridge(db,manifest,cat,context.migrations,history);
|
|
29
|
+
if(same && !upgrade)await sameSchemaMigrations(db,manifest,cat,context.migrations,history);
|
|
26
30
|
if(!bridge)insist(canonicalJson(manifest.tables.map(x=>x.name))===canonicalJson(cat.tables.map(x=>x.name)),'table_inventory');
|
|
27
31
|
if(!upgrade && !bridge)for(const table of manifest.tables) {
|
|
28
32
|
const target=cat.tables.find(t=>t.name===table.name);
|
|
@@ -36,7 +40,7 @@ export async function createImport(context,manifest,actor='direct',legacyRecover
|
|
|
36
40
|
for(const o of manifest.objects) insist(buckets[o.bucket] && config.buckets[o.bucket] && o.key.startsWith(config.buckets[o.bucket].prefix??''),'bucket_inventory');
|
|
37
41
|
insist(canonicalJson(manifest.payloads.map(x=>x.name))===canonicalJson(Object.keys(config.extraPayloads).sort()),'payload_inventory');
|
|
38
42
|
const id=crypto.randomUUID(),copies=Object.fromEntries(targets.map((t,i)=>[t.name,'rer_portability_stage_'+id.replaceAll('-','')+'_'+i]));
|
|
39
|
-
const row={id,kind:'import',actor,phase:'staging',completed:0,metadata:{manifest:original,...(skippedTables.length?{restoreManifest:manifest,skippedTables}:{}),copies,targets,triggers:cat.triggers,...(bridge?{bridge}:{}),...(legacyRecovery?{legacyRecovery}: {})}};
|
|
43
|
+
const row={id,kind:'import',actor,phase:'staging',completed:0,metadata:{manifest:original,...(skippedTables.length?{restoreManifest:manifest,skippedTables}:{}),copies,targets,triggers:cat.triggers,...(bridge?{bridge}:{}),...(legacyRecovery?{legacyRecovery}: {}),...(ledgers?{ledgers}:{})}};
|
|
40
44
|
try {
|
|
41
45
|
await db.batch([guard(db,'(SELECT import_job FROM rer_portability_state WHERE id=1) IS NULL'),
|
|
42
46
|
statement(db,'UPDATE rer_portability_state SET import_job=? WHERE id=1',id),
|
|
@@ -55,6 +59,7 @@ export async function uploadPart(context,id,index,part) {
|
|
|
55
59
|
if(item.kind==='row') {
|
|
56
60
|
const table=manifest.tables.find(x=>x.name===item.table);insist(table && Number.isSafeInteger(item.index) && item.index>=0 && item.index<table.count,'row_inventory');
|
|
57
61
|
validateRow(table,item.row,manifest.excludedColumns[table.name]??[]);
|
|
62
|
+
if(row.metadata.ledgers?.[table.name])insist(canonicalJson(item.row)===canonicalJson(row.metadata.ledgers[table.name][item.index]),'ledger_inventory');
|
|
58
63
|
if(platformTable(table.name))continue;
|
|
59
64
|
const target=row.metadata.targets.find(x=>x.name===table.name);
|
|
60
65
|
const data=upgrade?upgrade.upgradeRow(table.name,structuredClone(item.row)):item.row;
|
|
@@ -158,6 +163,7 @@ export async function finalizeImport(context,id,{contentHash,archiveHash,laterJo
|
|
|
158
163
|
await bucketsEmpty(context,row);
|
|
159
164
|
await refuseDevData(db,row.metadata.copies,row.metadata.targets);
|
|
160
165
|
await refuseProviderData(db,row.metadata.copies,row.metadata.targets);
|
|
166
|
+
await normalizeStaged(context,row);
|
|
161
167
|
if(config.validateStaged) {
|
|
162
168
|
const read=async table=>{insist(row.metadata.copies[table],'staged_table');const target=row.metadata.targets.find(t=>t.name===table);return (await readCopy(db,target,row.metadata.copies[table],0,row.metadata.manifest.tables.find(t=>t.name===table)?.count??0)).map(x=>x.row);};
|
|
163
169
|
insist(await config.validateStaged({db,tables:{...row.metadata.copies},read,manifest})!==false,'staged_validation');
|
|
@@ -193,7 +199,7 @@ export async function finalizeImport(context,id,{contentHash,archiveHash,laterJo
|
|
|
193
199
|
}
|
|
194
200
|
export async function importStatus(context,id) {
|
|
195
201
|
const row=await job(context.db,id,'import');return {id,kind:'import',phase:row.phase,completed:row.completed,total:row.metadata.manifest.partCount,
|
|
196
|
-
contentHash:row.metadata.contentHash??null,requiredConfiguration:row.phase==='complete'?row.metadata.manifest.requiredConfiguration:[],...(row.metadata.bridge?{bridged:row.metadata.bridge}:{}),...(row.metadata.skippedTables?{skippedTables:row.metadata.skippedTables}:{})};
|
|
202
|
+
contentHash:row.metadata.contentHash??null,requiredConfiguration:row.phase==='complete'?row.metadata.manifest.requiredConfiguration:[],...(row.metadata.bridge?{bridged:row.metadata.bridge}:{}),...(row.metadata.skippedTables?{skippedTables:row.metadata.skippedTables}:{}),...(row.metadata.safetyChanges?{safetyChanges:row.metadata.safetyChanges}:{})};
|
|
197
203
|
}
|
|
198
204
|
export async function abortImport(context,id) {
|
|
199
205
|
const row=await job(context.db,id,'import');insist(row.phase!=='complete','import_complete',409);
|
|
@@ -22,21 +22,27 @@ export interface Staged {
|
|
|
22
22
|
export interface ArchivePolicy {
|
|
23
23
|
product: string; excludeTables?: string[]; includeTables?: string[]; derivedTables?: string[];
|
|
24
24
|
secretColumns?: Record<string,string[]>; exportRow?: (table:string,row:Row)=>Row;
|
|
25
|
+
/** Nonempty finite safe-value sets for writable non-key columns; requires importRow. */
|
|
26
|
+
importSafety?: Record<string,Record<string,Cell[]>>;
|
|
27
|
+
/** Pure, synchronous, deterministic and idempotent; called only for importSafety tables before validateStaged. */
|
|
28
|
+
importRow?: (table:string,row:Row)=>Row;
|
|
25
29
|
replaceableTables?: string[]; requiredConfiguration?: string[]; validateStaged?: (staged:Staged)=>boolean|void|Promise<boolean|void>;
|
|
26
30
|
acceptedSchemas?: Record<string,{upgradeRow:(table:string,row:Row)=>Row}>;
|
|
27
31
|
buckets?: Record<string,{prefix?:string;objectReferences?:{table:string;keyColumn:string}}>;
|
|
28
32
|
extraPayloads?: Record<string,{produce:()=>Uint8Array|ReadableStream<Uint8Array>|Promise<Uint8Array|ReadableStream<Uint8Array>>;consume:(source:ReadableStream<Uint8Array>,descriptor:PayloadDescriptor)=>void|Promise<void>}>;
|
|
29
33
|
partBytes?: number;
|
|
30
34
|
}
|
|
31
|
-
|
|
35
|
+
/** Actual changed rows grouped by table, column and safe replacement value; never original values. */
|
|
36
|
+
export interface SafetyChange {table:string;column:string;value:Cell;rows:number}
|
|
37
|
+
export interface Status {id:string;kind:'export'|'import';phase:'building'|'staging'|'applying'|'complete'|'aborted';completed:number;total:number|null;/** Export only: bytes of the parts built so far. */bytes?:number;contentHash:string|null;requiredConfiguration?:string[];bridged?:Bridge;/** Platform bookkeeping ignored in an older archive. */skippedTables?:string[];/** Durable summary of bounded import normalization. */safetyChanges?:SafetyChange[]}
|
|
32
38
|
/** Later additive migrations an older archive was restored across. */
|
|
33
39
|
export interface Bridge {fromSchema:string;toSchema:string;migrations:string[];tables:string[];columns:Record<string,string[]>}
|
|
34
40
|
/** A runner's replay of the destination's applied migrations: the archive schema identity after a chosen set, and each migration's statements. */
|
|
35
|
-
export interface MigrationReplay {names:string[];statements(name:string):string[];schemaAfter(names:string[]):Promise<string>}
|
|
41
|
+
export interface MigrationReplay {names:string[];statements(name:string):string[];/** The migration file's SHA-256 hex, where the runner read the file. */sha256?(name:string):string|undefined;schemaAfter(names:string[]):Promise<string>}
|
|
36
42
|
export interface PortabilityOptions {db:D1Database;buckets?:Record<string,R2Bucket>;policy:ArchivePolicy;migrations?:()=>Promise<MigrationReplay>}
|
|
37
43
|
export interface Portability {
|
|
38
44
|
exports: {create(actor?:string):Promise<string>;advance(id:string,options?:{expectedPart?:number;maxParts?:number}):Promise<Status>;status(id:string):Promise<Status>;download(id:string):Promise<ReadableStream<Uint8Array>>;abort(id:string):Promise<Status>};
|
|
39
|
-
imports: {create(manifest:Manifest,actor?:string):Promise<string>;uploadPart(id:string,index:number,part:Part):Promise<Status>;finalize(id:string,options:{archiveHash:string;contentHash?:string;laterJournal?:LaterJournal}):Promise<Status>;status(id:string):Promise<Status>;abort(id:string):Promise<Status>;/** Aborts the import left staged by a crashed restore and returns its id; call only as the destination's sole operator. */abortAbandoned():Promise<string|null>};
|
|
45
|
+
imports: {/** ledgers: the rows of each migration ledger table an old archive carries, as read from it; they place it in the destination's history and must equal its parts. */create(manifest:Manifest,actor?:string,options?:{ledgers?:Record<string,Row[]>}):Promise<string>;uploadPart(id:string,index:number,part:Part):Promise<Status>;finalize(id:string,options:{archiveHash:string;contentHash?:string;laterJournal?:LaterJournal}):Promise<Status>;status(id:string):Promise<Status>;abort(id:string):Promise<Status>;/** Aborts the import left staged by a crashed restore and returns its id; call only as the destination's sole operator. */abortAbandoned():Promise<string|null>};
|
|
40
46
|
status(id:string):Promise<Status>;actor(id:string):Promise<string>;
|
|
41
47
|
erasureStatements(table:string,primaryKey:Record<string,Cell>):Promise<D1PreparedStatement[]>;
|
|
42
48
|
redactionStatements():Promise<D1PreparedStatement[]>;
|
|
@@ -17,7 +17,7 @@ export async function createPortability({db,buckets={},policy:input,migrations})
|
|
|
17
17
|
const context={db,buckets,config,migrations};
|
|
18
18
|
return {
|
|
19
19
|
exports:{create:actor=>createExport(context,actor),advance:(id,options)=>advanceExport(context,id,options),status:id=>exportStatus(context,id),download:id=>downloadExport(context,id),abort:id=>abortExport(context,id)},
|
|
20
|
-
imports:{create:(manifest,actor)=>createImport(context,manifest,actor),uploadPart:(id,index,part)=>uploadPart(context,id,index,part),finalize:(id,options)=>finalizeImport(context,id,options),status:id=>importStatus(context,id),abort:id=>abortImport(context,id),abortAbandoned:()=>abortAbandonedImport(context)},
|
|
20
|
+
imports:{create:(manifest,actor,options={})=>createImport(context,manifest,actor,undefined,options.ledgers),uploadPart:(id,index,part)=>uploadPart(context,id,index,part),finalize:(id,options)=>finalizeImport(context,id,options),status:id=>importStatus(context,id),abort:id=>abortImport(context,id),abortAbandoned:()=>abortAbandonedImport(context)},
|
|
21
21
|
status:async id=>(await job(db,id)).kind==='export'?exportStatus(context,id):importStatus(context,id),
|
|
22
22
|
actor:async id=>(await job(db,id)).actor,
|
|
23
23
|
erasureStatements:(table,key)=>erasureStatements(db,table,key),redactionStatements:()=>redactionStatements(db),invalidateTransfersStatements:()=>invalidateTransfersStatements(db),
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import { all, insist } from './storage.mjs';
|
|
2
|
+
import { identifier } from './bookkeeping.mjs';
|
|
3
|
+
import { canonicalJson } from './hash.mjs';
|
|
4
|
+
import { validateRow } from './catalogue.mjs';
|
|
5
|
+
|
|
6
|
+
// Migration ledgers, by the runner that writes them: Base deploy and container
|
|
7
|
+
// runners (_rer_deploy_migrations), Wrangler and the Base deploy kit
|
|
8
|
+
// (d1_migrations, tracked or plain) and RER Platform (rer_migrations).
|
|
9
|
+
export const ledgerNames=['_rer_deploy_migrations','d1_migrations','rer_migrations'];
|
|
10
|
+
// Platform's ledger exactly as its runner creates it
|
|
11
|
+
// (platform src/integrations/cloudflare-migrations.ts:33).
|
|
12
|
+
const platformShape=canonicalJson([{name:'name',type:'TEXT',notnull:0,pk:1},{name:'sha256',type:'TEXT',notnull:1,pk:0},{name:'applied_at',type:'TEXT',notnull:1,pk:0}]);
|
|
13
|
+
const shape=columns=>canonicalJson(columns.map(({name,type,notnull,pk})=>({name,type,notnull,pk})));
|
|
14
|
+
// Platform records a file applied in part as <name>#<start>-<end> until the
|
|
15
|
+
// file completes (platform src/integrations/migration-batch.ts:31): its effects
|
|
16
|
+
// are uncertain, like a tracked ledger's 'applying' row.
|
|
17
|
+
const partial=name=>name.includes('#');
|
|
18
|
+
|
|
19
|
+
/** The destination's applied product migrations, [{name,sha256?}] in the ledger's own order, or undefined without a ledger. */
|
|
20
|
+
export async function destinationLedger(db) {
|
|
21
|
+
const found=(await all(db,`SELECT name FROM sqlite_master WHERE type='table' AND name IN (${ledgerNames.map(n=>`'${n}'`).join(',')})`)).map(x=>x.name);
|
|
22
|
+
if(found.includes('_rer_deploy_migrations'))return (await all(db,"SELECT * FROM _rer_deploy_migrations WHERE state='applied' ORDER BY name")).map(entry);
|
|
23
|
+
if(found.includes('d1_migrations')) {
|
|
24
|
+
const columns=(await all(db,'PRAGMA table_info(d1_migrations)')).map(x=>x.name);
|
|
25
|
+
const tracked=columns.includes('state') || columns.includes('sha256');
|
|
26
|
+
if(tracked)insist(columns.includes('state') && columns.includes('sha256'),'schema_mismatch:migration_history',409);
|
|
27
|
+
const rows=await all(db,`SELECT name${tracked?',state,sha256':''} FROM d1_migrations ORDER BY id`);
|
|
28
|
+
if(tracked)insist(rows.every(row=>row.state==='applied'),'schema_mismatch:uncertain_migrations',409);
|
|
29
|
+
return rows.map(entry);
|
|
30
|
+
}
|
|
31
|
+
if(!found.includes('rer_migrations'))return;
|
|
32
|
+
// Platform applies pending files in its plan's order and inserts each ledger
|
|
33
|
+
// row in the same request as the file's SQL (migration-batch.ts:35-38,43-64;
|
|
34
|
+
// cloudflare-migrations.ts:34-44), so rowid order is application order.
|
|
35
|
+
insist(shape(await all(db,'PRAGMA table_info(rer_migrations)'))===platformShape,'schema_mismatch:migration_history',409);
|
|
36
|
+
const rows=await all(db,'SELECT name,sha256 FROM rer_migrations ORDER BY rowid');
|
|
37
|
+
insist(!rows.some(row=>partial(row.name)),'schema_mismatch:uncertain_migrations',409);
|
|
38
|
+
return rows.map(entry);
|
|
39
|
+
}
|
|
40
|
+
const entry=row=>({name:row.name,...(typeof row.sha256==='string'?{sha256:row.sha256}:{})});
|
|
41
|
+
|
|
42
|
+
/** The ledger tables an archive carries (old archives carried rer_migrations before it became bookkeeping). */
|
|
43
|
+
export const archivedLedgerTables=manifest=>manifest.tables.filter(t=>ledgerNames.includes(identifier(t.name)));
|
|
44
|
+
|
|
45
|
+
/** Collects the rows of an archive's ledger tables while its records are read: add(record) each, then ledgers() is {table:rows}, or undefined without one. */
|
|
46
|
+
export function ledgerReader() {
|
|
47
|
+
let ledgers;
|
|
48
|
+
return {
|
|
49
|
+
add(record) {
|
|
50
|
+
if(record.kind==='manifest') {const tables=archivedLedgerTables(record);if(tables.length)ledgers=Object.fromEntries(tables.map(t=>[t.name,[]]));}
|
|
51
|
+
if(record.kind==='part' && ledgers)for(const item of record.items)if(item.kind==='row' && ledgers[item.table])ledgers[item.table].push(item.row);
|
|
52
|
+
},
|
|
53
|
+
ledgers:()=>ledgers,
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
/** Checks rows a caller read from the archive for its ledger tables ({table:rows}), so uploadPart can bind each to the authenticated parts. */
|
|
57
|
+
export function checkArchivedLedgers(manifest,ledgers) {
|
|
58
|
+
if(ledgers===undefined)return;
|
|
59
|
+
const tables=archivedLedgerTables(manifest);
|
|
60
|
+
insist(ledgers && typeof ledgers==='object' && Object.keys(ledgers).every(name=>tables.some(t=>t.name===name)),'ledger_inventory');
|
|
61
|
+
for(const [name,rows] of Object.entries(ledgers)) {
|
|
62
|
+
const table=tables.find(t=>t.name===name);
|
|
63
|
+
insist(Array.isArray(rows) && rows.length===table.count,'ledger_inventory');
|
|
64
|
+
for(const row of rows)validateRow(table,row,manifest.excludedColumns[name]??[]);
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/** The applied migrations an archive's own ledger records, [{name,sha256?}]; undefined when it carries no ledger. */
|
|
69
|
+
export function archivedHistory(manifest,ledgers) {
|
|
70
|
+
const tables=archivedLedgerTables(manifest);
|
|
71
|
+
if(!tables.length)return;
|
|
72
|
+
insist(tables.length===1 && ledgers?.[tables[0].name],'schema_mismatch:migration_history');
|
|
73
|
+
const [table]=tables,rows=ledgers[table.name],columns=table.columns.map(c=>c.name);
|
|
74
|
+
if(identifier(table.name)==='rer_migrations') {
|
|
75
|
+
insist(shape(table.columns)===platformShape,'schema_mismatch:migration_history');
|
|
76
|
+
insist(!rows.some(row=>partial(row.name)),'schema_mismatch:uncertain_migrations');
|
|
77
|
+
}else{
|
|
78
|
+
insist(columns.includes('name') && columns.includes('state')===columns.includes('sha256'),'schema_mismatch:migration_history');
|
|
79
|
+
insist(rows.every(row=>row.state===undefined || row.state==='applied'),'schema_mismatch:uncertain_migrations');
|
|
80
|
+
}
|
|
81
|
+
insist(rows.every(row=>typeof row.name==='string') && new Set(rows.map(row=>row.name)).size===rows.length,'schema_mismatch:migration_history');
|
|
82
|
+
return rows.map(entry);
|
|
83
|
+
}
|
|
@@ -6,13 +6,16 @@ import { readLegacyArchive, ArchiveVerifier } from './format.mjs';
|
|
|
6
6
|
import { fileRecords, exportArchive } from './files.mjs';
|
|
7
7
|
import { ensureLegacyAudit, legacyTargetState, requireLegacyEmpty } from './legacy-safety.mjs';
|
|
8
8
|
import { insist, PortabilityError } from './storage.mjs';
|
|
9
|
+
import { ledgerReader } from './ledgers.mjs';
|
|
9
10
|
|
|
10
11
|
export async function recoverLegacyArchive({db,buckets,policy:input,migrations},path,output,transient,acknowledged) {
|
|
11
12
|
insist(acknowledged===true,'legacy_acknowledgement_required',409);
|
|
12
13
|
insist(typeof output==='string' && output!==path,'legacy_output_required',409);
|
|
13
14
|
insist(!(await lstat(output).catch(error=>{if(error.code==='ENOENT')return null;throw error;})),'legacy_output_exists',409);
|
|
14
|
-
// Whole-file verification precedes even staging/audit schema writes.
|
|
15
|
-
|
|
15
|
+
// Whole-file verification precedes even staging/audit schema writes. It also
|
|
16
|
+
// reads the archive's own migration ledger, if any, for placement (bridge.mjs).
|
|
17
|
+
const ledger=ledgerReader();
|
|
18
|
+
for await(const record of fileRecords(path,readLegacyArchive))ledger.add(record);
|
|
16
19
|
const context={db,buckets,config:policy(input),migrations},target=await legacyTargetState(db,migrations);
|
|
17
20
|
await requireLegacyEmpty(context,target);
|
|
18
21
|
let id,manifest,closing,integrity,status,complete=false;
|
|
@@ -26,7 +29,7 @@ export async function recoverLegacyArchive({db,buckets,policy:input,migrations},
|
|
|
26
29
|
event:'legacy_v1_unverified_restore',acknowledgedUnverifiedMetadata:true,
|
|
27
30
|
sourceIdentity:record.sourceIdentity,exportedAt:record.exportedAt,
|
|
28
31
|
recoveryOutput:output,...target,
|
|
29
|
-
});
|
|
32
|
+
},ledger.ledgers());
|
|
30
33
|
// A refused schema writes nothing, not even the audit table.
|
|
31
34
|
await ensureLegacyAudit(db);
|
|
32
35
|
}else if(record.kind==='part') {integrity.add(record);await uploadPart(context,id,record.index,record);}
|
|
@@ -38,6 +41,6 @@ export async function recoverLegacyArchive({db,buckets,policy:input,migrations},
|
|
|
38
41
|
// A failure here must be reported; the durable restore audit remains intact.
|
|
39
42
|
try {
|
|
40
43
|
const recovered=await exportArchive(await createPortability({db,buckets,policy:input}),output,transient);
|
|
41
|
-
return {...recovered,legacyRecovery:true,unverifiedMetadata:true,restoredContentHash:closing.contentHash,...(status.bridged?{bridged:status.bridged}:{})};
|
|
44
|
+
return {...recovered,legacyRecovery:true,unverifiedMetadata:true,restoredContentHash:closing.contentHash,...(status.bridged?{bridged:status.bridged}:{}),...(status.safetyChanges?{safetyChanges:status.safetyChanges}:{})};
|
|
42
45
|
}catch{throw new PortabilityError('legacy_restored_export_failed',500);}
|
|
43
46
|
}
|
|
@@ -4,10 +4,11 @@ import { policy, schemaIdentity } from './catalogue.mjs';
|
|
|
4
4
|
// Replays a chosen set of the destination's migrations, in its applied order,
|
|
5
5
|
// into a scratch database and returns the resulting archive schema identity.
|
|
6
6
|
// Node runners only: restore uses it to place an older archive within the
|
|
7
|
-
// destination's history (see bridge.mjs). plan: [{name,statements}]
|
|
7
|
+
// destination's history (see bridge.mjs). plan: [{name,statements,sha256?}],
|
|
8
|
+
// sha256 being the migration file's hash where the runner read the file.
|
|
8
9
|
export function migrationReplay(plan,input) {
|
|
9
10
|
const config=policy(input);
|
|
10
|
-
return {names:plan.map(migration=>migration.name),statements:name=>plan.find(m=>m.name===name).statements,async schemaAfter(names) {
|
|
11
|
+
return {names:plan.map(migration=>migration.name),statements:name=>plan.find(m=>m.name===name).statements,sha256:name=>plan.find(m=>m.name===name).sha256,async schemaAfter(names) {
|
|
11
12
|
const scratch=new DatabaseSync(':memory:');
|
|
12
13
|
const db={prepare:sql=>{const query=scratch.prepare(sql);let values=[];return {bind(...args){values=args;return this;},async all(){return {results:query.all(...values)};}};}};
|
|
13
14
|
try {
|