@runeditrun/rer 0.2.123 → 0.2.124

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -11,6 +11,7 @@ import { splitStatements } from '../modules/portability/sql-text.mjs';
11
11
  import { requireEmptyTables } from '../modules/portability/restore-empty.mjs';
12
12
  import { insist } from '../modules/portability/storage.mjs';
13
13
  import { shared } from './shared.mjs';
14
+ import { ownerMigrationPolicy, ownerMigrationSources } from './portability-owners.mjs';
14
15
 
15
16
  const status = path => lstat(path).catch(error => { if (error.code === 'ENOENT') return null; throw error; });
16
17
  // Miniflare's named D1 identity: the declared binding determines its file.
@@ -58,8 +59,9 @@ export async function emptyNativePreflight({ root, binding, policy, policyModule
58
59
  async all() { return { results: query.all(...values) }; },
59
60
  async first() { return query.get(...values) ?? null; },
60
61
  }; } };
61
- const rules = await loadPolicy(root, policy, policyModule), declaration = platformMigrationDeclaration(platformMigrations);
62
- const migrations = async () => migrationReplay(await appliedMigrations(root, db, binding, declaration, splitStatements), rules, declaration);
62
+ const owners = await ownerMigrationSources(root, binding);
63
+ const rules = ownerMigrationPolicy(await loadPolicy(root, policy, policyModule), owners), declaration = platformMigrationDeclaration(platformMigrations);
64
+ const migrations = async () => migrationReplay(await appliedMigrations(root, db, binding, declaration, splitStatements, owners), rules, declaration);
63
65
  await requireEmptyTables({ db, policy: rules, migrations });
64
66
  } finally { try { snapshot?.close(); } finally { await rm(directory, { recursive: true, force: true }); } }
65
67
  }
@@ -2,6 +2,8 @@ import { DatabaseSync } from 'node:sqlite';
2
2
  import { realpath, readFile, readdir } from 'node:fs/promises';
3
3
  import { resolve, relative, isAbsolute, sep } from 'node:path';
4
4
  import { migrationDirectory } from './portability-migrations.mjs';
5
+ import { ownerMigrationSources } from './portability-owners.mjs';
6
+ import { collectMigrations } from '../modules/delivery-producer/migrations.mjs';
5
7
  import { splitStatements } from '../modules/portability/sql-text.mjs';
6
8
  import { canonicalJson, sha256 } from '../modules/portability/hash.mjs';
7
9
  import { quote, insist } from '../modules/portability/storage.mjs';
@@ -25,6 +27,12 @@ export async function generateMigrationEvidence(root,{binding='DB'}={}) {
25
27
  const bytes=await readFile(file);
26
28
  return {name,sha256:await sha256(bytes),statements:splitStatements(bytes.toString('utf8'))};
27
29
  }));
30
+ const sources=await ownerMigrationSources(root,binding);
31
+ const owners=await collectMigrations(root,sources.filter(source=>source.directory).map(({directory,prefix})=>({binding,path:directory,prefix})),[{kind:'d1',binding}]);
32
+ for(const {name,sha256,sql} of owners) {
33
+ insist(!files.some(file=>file.name===name),'migration_ledger:ambiguous',409);
34
+ files.push({name,sha256,statements:splitStatements(sql)});
35
+ }
28
36
  const objects=[],indices=new Map(),prefixes=[];
29
37
  const scratch=new DatabaseSync(':memory:');
30
38
  try {
@@ -64,7 +72,7 @@ export async function generateMigrationEvidence(root,{binding='DB'}={}) {
64
72
  await capture();
65
73
  }
66
74
  }finally{scratch.close();}
67
- return {version:1,files,objects,prefixes};
75
+ return {version:1,files,objects,prefixes,...(sources.length?{ownerSources:sources.map(({prefix,table})=>({prefix,table}))}:{})};
68
76
  }
69
77
 
70
78
  // Explicit seed opt-in. The module is consumed only by the server/Worker graph;
@@ -5,6 +5,8 @@ import { migrationStatements } from '../modules/portability/migration-sql.mjs';
5
5
  import { destinationLedger, ledgerNames, matchMigrationLedger } from '../modules/portability/ledgers.mjs';
6
6
  import { all, insist, PortabilityError } from '../modules/portability/storage.mjs';
7
7
  import {migrationDirectory} from '../modules/deploy-kit/migration-directory.mjs';
8
+ import { collectMigrations } from '../modules/delivery-producer/migrations.mjs';
9
+ import { nativeMigrationHistory } from './portability-owners.mjs';
8
10
  export {migrationDirectory};
9
11
 
10
12
  // Seed-owned declarations, never archive metadata: the deploymentSpec prefix
@@ -19,18 +21,26 @@ export function platformMigrationDeclaration(input={}) {
19
21
 
20
22
  // Reproduce the runner's stored SQL text, then recover SQLite's statement
21
23
  // boundaries (migration-sql.mjs).
22
- export async function migrationInventory(root,binding,{prefix},split) {
24
+ export async function migrationInventory(root,binding,{prefix},split,sources=[]) {
23
25
  const directory=await migrationDirectory(root,binding);
24
- return Promise.all((await readdir(directory)).filter(name=>name.endsWith('.sql')).sort().map(async name=>{
26
+ const files=await Promise.all((await readdir(directory)).filter(name=>name.endsWith('.sql')).sort().map(async name=>{
25
27
  const bytes=await readFile(join(directory,name));
26
28
  return {name,...(split?{statements:migrationStatements(bytes.toString('utf8'),split)}:{}),sha256:createHash('sha256').update(bytes).digest('hex'),...(prefix===undefined?{}:{aliases:[prefix+name]})};
27
29
  }));
30
+ if(sources.length) {
31
+ const owners=await collectMigrations(root,sources.filter(source=>source.directory).map(({directory,prefix})=>({binding,path:directory,prefix})),[{kind:'d1',binding}]);
32
+ for(const {name,sha256,sql} of owners) {
33
+ insist(!files.some(file=>file.name===name || file.aliases?.includes(name)),'migration_ledger:ambiguous',409);
34
+ files.push({name,sha256,...(split?{statements:migrationStatements(sql,split)}:{})});
35
+ }
36
+ }
37
+ return files;
28
38
  }
29
39
 
30
40
  // Preserve the runner's applied order, resolving file identities by hash.
31
- export async function appliedMigrations(root,db,binding,declaration,split) {
32
- const ledger=await destinationLedger(db) ?? [],files=await migrationInventory(root,binding,declaration,split);
33
- return matchMigrationLedger(ledger,files,declaration).map(name=>files.find(file=>file.name===name));
41
+ export async function appliedMigrations(root,db,binding,declaration,split,sources=[]) {
42
+ const history=await nativeMigrationHistory(db,sources),files=await migrationInventory(root,binding,declaration,split,sources);
43
+ return matchMigrationLedger(history?.entries ?? [],files,declaration).map(name=>files.find(file=>file.name===name));
34
44
  }
35
45
 
36
46
  // Ledger contents can contain private values. Report constant categories,
@@ -87,17 +97,23 @@ async function legacyPairPreflight(db) {
87
97
  // Adapter entry point, before *any* native Wrangler apply. The caller supplies
88
98
  // the real target binding and keeps its writers stopped through apply/transfer.
89
99
  // This only reads: it never repairs history or creates a second ledger.
90
- export async function nativeMigrationPreflight({root,db,binding='DB',platformMigrations}) {
100
+ export async function nativeMigrationPreflight({root,db,binding='DB',platformMigrations,ownerSources=[]}) {
91
101
  const declaration=platformMigrationDeclaration(platformMigrations);
92
102
  const tables=await all(db,`SELECT type,name FROM sqlite_master WHERE lower(name) IN (${ledgerNames.map(name=>`'${name}'`).join(',')})`);
93
103
  const legacyPair=tables.length===2 && ['_rer_deploy_migrations','d1_migrations'].every(name=>tables.some(row=>row.name===name));
94
104
  if(tables.length>1 && !legacyPair)throw Object.assign(new PortabilityError('migration_ledger:multiple',409),tables.some(row=>row.name==='rer_migrations')?{identity:'rer_migrations'}:{});
95
105
  insist(tables.every(row=>row.type==='table' && ledgerNames.includes(row.name)),'migration_ledger:incompatible',409);
96
106
  const table=legacyPair?'_rer_deploy_migrations':tables[0]?.name;
97
- if(!table)return {nativeApply:true};
107
+ if(!table) {
108
+ if(ownerSources.length) {
109
+ await nativeMigrationHistory(db,ownerSources);
110
+ await migrationInventory(root,binding,declaration,undefined,ownerSources);
111
+ }
112
+ return {nativeApply:true};
113
+ }
98
114
  // This runner's history cannot authorize Wrangler to apply the same files.
99
115
  insist(table!=='_rer_deploy_migrations' || legacyPair,'migration_ledger:runner',409);
100
- const files=await migrationInventory(root,binding,declaration);
116
+ const files=await migrationInventory(root,binding,declaration,undefined,table==='rer_migrations'?ownerSources:[]);
101
117
  let nativeApply=false;
102
118
  try {
103
119
  if(legacyPair)await legacyPairPreflight(db);
@@ -108,5 +124,11 @@ export async function nativeMigrationPreflight({root,db,binding='DB',platformMig
108
124
  // The checkout file is named: it is the seed's own, never a ledger value.
109
125
  if(table!=='d1_migrations')for(const file of files)if(!applied.includes(file.name))throw Object.assign(new PortabilityError('migration_ledger:unapplied',409),{migration:file.name});
110
126
  } catch(error) { throw constantRefusal(error,files); }
127
+ if(ownerSources.length && table!=='rer_migrations') {
128
+ const history=await nativeMigrationHistory(db,ownerSources);
129
+ const allFiles=await migrationInventory(root,binding,declaration,undefined,ownerSources);
130
+ try { matchMigrationLedger(history.entries,allFiles,declaration); }
131
+ catch(error) { throw constantRefusal(error,allFiles); }
132
+ }
111
133
  return {nativeApply};
112
134
  }
@@ -0,0 +1,54 @@
1
+ import { randomUUID } from 'node:crypto';
2
+ import { readFile, readdir, rm, writeFile } from 'node:fs/promises';
3
+ import { join } from 'node:path';
4
+ import { sourcePath } from '../modules/delivery-producer/migrations.mjs';
5
+ import { ledgerNames } from '../modules/portability/ledgers.mjs';
6
+ import { insist } from '../modules/portability/storage.mjs';
7
+ import { deliveryDeclarationSources } from './delivery-declarations.mjs';
8
+ import { parseJsonc } from './jsonc.mjs';
9
+ export { nativeMigrationHistory, ownerMigrationPolicy } from '../modules/portability/owner-history.mjs';
10
+
11
+ // Delivery declares the extra directories and their prefixes. Native Wrangler
12
+ // uses the edition's permanent <project>_<prefix>_migrations ledger for each.
13
+ export async function ownerMigrationSources(root, binding) {
14
+ let directories;
15
+ for (const [path, select] of deliveryDeclarationSources) {
16
+ const file = await sourcePath(root, path, { optional: true });
17
+ if (!file) continue;
18
+ directories = select(JSON.parse(await readFile(file, 'utf8')))?.extraMigrationDirectories;
19
+ if (directories !== undefined) break;
20
+ }
21
+ directories ??= [];
22
+ insist(Array.isArray(directories), 'migration_ledger:incompatible', 409);
23
+ const sources = directories.filter(source => source.binding === binding);
24
+ if (!sources.length) return [];
25
+ const { name } = JSON.parse(await readFile(join(root, 'package.json'), 'utf8'));
26
+ const project = name?.split('/').at(-1).replaceAll('-', '_');
27
+ insist(typeof project === 'string' && /^[a-z][a-z0-9_]*$/.test(project), 'migration_ledger:incompatible', 409);
28
+ const selected = await Promise.all(sources.map(async ({ path, prefix, optional }) => {
29
+ insist(typeof prefix === 'string' && /^[a-z][a-z0-9_-]*-$/.test(prefix), 'migration_ledger:incompatible', 409);
30
+ const table = project + '_' + prefix.slice(0, -1).replaceAll('-', '_') + '_migrations';
31
+ insist(!ledgerNames.includes(table), 'migration_ledger:ambiguous', 409);
32
+ const directory = await sourcePath(root, path, { kind: 'directory', optional: optional === true });
33
+ return { directory, prefix, table };
34
+ }));
35
+ insist(new Set(selected.map(source => source.table)).size === selected.length, 'migration_ledger:ambiguous', 409);
36
+ return selected;
37
+ }
38
+
39
+ export async function applyNativeOwnerMigrations({ root, binding, sources, store, signal, env }) {
40
+ if (!sources.some(source => source.directory)) return;
41
+ const config = parseJsonc(await readFile(join(root, 'wrangler.jsonc'), 'utf8'));
42
+ const target = config.d1_databases.find(entry => entry.binding === binding);
43
+ for (const { table, directory } of sources) {
44
+ if (!directory || !(await readdir(directory)).some(name => name.endsWith('.sql'))) continue;
45
+ const path = join(root, '.wrangler', `rer-owner-migrations-${randomUUID()}.json`);
46
+ try {
47
+ await writeFile(path, JSON.stringify({
48
+ name: config.name, compatibility_date: config.compatibility_date,
49
+ d1_databases: [{ ...target, migrations_dir: directory, migrations_table: table }],
50
+ }), { mode: 0o600, flag: 'wx' });
51
+ await store.run('Applying local owner migrations', 'pnpm', ['exec', 'wrangler', 'd1', 'migrations', 'apply', binding, '--local', '--persist-to', '.wrangler/state', '--config', path], { cwd: root, signal, env });
52
+ } finally { await rm(path, { force: true }); }
53
+ }
54
+ }
@@ -16,6 +16,7 @@ import { nativeFailure, stateBytes } from './portability-progress.mjs';
16
16
  import { parseJsonc } from './jsonc.mjs';
17
17
  import { loadPolicy } from './portability-policy.mjs';
18
18
  import { appliedMigrations, platformMigrationDeclaration, nativeMigrationPreflight } from './portability-migrations.mjs';
19
+ import { nativeMigrationHistory, ownerMigrationPolicy, ownerMigrationSources } from './portability-owners.mjs';
19
20
 
20
21
  // Transfers read only the D1 database and R2 buckets. A configuration with just
21
22
  // those keeps the app's other bindings out of the proxy: a Durable Object class
@@ -51,11 +52,18 @@ async function run(direction, { root, path, journal, binding, buckets, policy, p
51
52
  let proxy;
52
53
  try {
53
54
  proxy = await getPlatformProxy({ configPath, persist: { path: join(root, '.wrangler/state/v3') }, remoteBindings: false });
54
- if(direction==='migration-preflight')return await nativeMigrationPreflight({root,db:proxy.env[binding],binding,platformMigrations});
55
- const migrations=async()=>migrationReplay(await appliedMigrations(root,proxy.env[binding],binding,declaration,unstable_splitSqlQuery),policy,declaration);
55
+ const db = proxy.env[binding], owners = await ownerMigrationSources(root, binding);
56
+ const migrationHistory = () => nativeMigrationHistory(db, owners);
57
+ if(direction==='migration-preflight') {
58
+ const result=await nativeMigrationPreflight({root,db,binding,platformMigrations,ownerSources:owners});
59
+ if(!owners.length)return result;
60
+ const history=await migrationHistory();
61
+ return {...result, nativeOwners: !history || history.table==='d1_migrations'};
62
+ }
63
+ const migrations=async()=>migrationReplay(await appliedMigrations(root,db,binding,declaration,unstable_splitSqlQuery,owners),policy,declaration);
56
64
  const boundBuckets = Object.fromEntries(Object.entries(buckets).map(([name, binding]) => [name, proxy.env[binding]]));
57
- policy = await loadPolicy(root, inputPolicy, policyModule, { db: proxy.env[binding], buckets: boundBuckets });
58
- const options={ db: proxy.env[binding], buckets: boundBuckets, policy, migrations };
65
+ policy = ownerMigrationPolicy(await loadPolicy(root, inputPolicy, policyModule, { db, buckets: boundBuckets }), owners);
66
+ const options={ db, buckets: boundBuckets, policy, migrations, migrationHistory };
59
67
  if(direction==='schema-recovery-preflight') { await requireProductEmpty(options);return {checkedEmpty:true}; }
60
68
  if(direction==='legacy-preflight') {
61
69
  await requireLegacyEmpty(options,await legacyTargetState(options.db,options.migrations));
@@ -83,7 +91,7 @@ catch (error) {
83
91
  // Report symbolic codes and identities verified against the checkout or the
84
92
  // known Platform ledger name; native database errors can contain private data.
85
93
  const ledgerIdentity = error instanceof PortabilityError && error.code.startsWith('migration_ledger:') ? error.migration ?? error.identity : undefined;
86
- const reason = error instanceof PortabilityError ? (error.code.startsWith('migration_ledger:') ? error.code + (ledgerIdentity ? ':' + ledgerIdentity : '') : error.code.split(':')[0]) : /^[A-Z_]+$/.test(error.code ?? '') ? error.code : error.message === 'archive_json' ? 'archive_json' : 'native_error';
94
+ const reason = error instanceof PortabilityError ? (error.code.startsWith('migration_ledger:') ? error.code + (ledgerIdentity ? ':' + ledgerIdentity : '') : error.code === 'schema_mismatch:owner_migration_history' ? error.code : error.code.split(':')[0]) : /^[A-Z_]+$/.test(error.code ?? '') ? error.code : error.message === 'archive_json' ? 'archive_json' : 'native_error';
87
95
  // A later migration that changes restorable data is named: it is a file of this installation.
88
96
  const migration = error instanceof PortabilityError && error.code.startsWith('schema_mismatch:migration_data:') ? error.code.slice('schema_mismatch:migration_data:'.length) : undefined;
89
97
  // An unapplied migration is a file of this checkout: it is named so it can be applied.
@@ -10,6 +10,7 @@ import { shared } from './shared.mjs';
10
10
  import { emptyNativePreflight } from './portability-empty.mjs';
11
11
  import { withManagedDataImport } from './managed-data.mjs';
12
12
  import { ledgerRefusal } from './ledger-refusal.mjs';
13
+ import { applyNativeOwnerMigrations, ownerMigrationSources } from './portability-owners.mjs';
13
14
 
14
15
  export { withManagedDataImport };
15
16
 
@@ -71,14 +72,14 @@ export function localPortability({ database, binding = 'DB', buckets = {}, polic
71
72
  }
72
73
  }
73
74
  if(unverifiedSchemaRecovery)await nativeWorker(store,'schema-recovery-preflight',input,signal);
74
- const {nativeApply}=await nativeWorker(store,'migration-preflight',input,signal);
75
+ const {nativeApply,nativeOwners}=await nativeWorker(store,'migration-preflight',input,signal);
75
76
  // Local migrations need no provider account: wrangler runs without credentials.
76
77
  // The shared runner shows its progress (a count every ten migrations) and leaves out its
77
78
  // own info lines, which only confuse a local backup; warnings and errors print.
78
79
  const { dependencyEnvironment } = await shared('dependency-environment.mjs');
79
- if(nativeApply)await store.run('Applying local data migrations', 'pnpm', ['exec', 'wrangler', 'd1', 'migrations', 'apply', database, '--local', '--persist-to', '.wrangler/state', '--config', 'wrangler.jsonc'], {
80
- cwd: root, signal, env: { ...dependencyEnvironment(), RER_LOCAL_OPERATION: operationToken, WRANGLER_SEND_METRICS: 'false' },
81
- });
80
+ const env = { ...dependencyEnvironment(), RER_LOCAL_OPERATION: operationToken, WRANGLER_SEND_METRICS: 'false' };
81
+ if(nativeApply)await store.run('Applying local data migrations', 'pnpm', ['exec', 'wrangler', 'd1', 'migrations', 'apply', database, '--local', '--persist-to', '.wrangler/state', '--config', 'wrangler.jsonc'], { cwd: root, signal, env });
82
+ if(direction==='import' && nativeOwners)await applyNativeOwnerMigrations({root,binding,sources:await ownerMigrationSources(root,binding),store,signal,env});
82
83
  const receipt=await nativeWorker(store,direction,input,signal);
83
84
  if(legacyRecovery)receipt.output=relative(root,legacyRecovery.output);
84
85
  return { target: 'local', [direction === 'export' ? 'output' : 'input']: relative(root, path), ...receipt };
@@ -107,6 +108,7 @@ async function nativeWorker(store,direction,input,signal) {
107
108
  : receipt.reExport ? `Restore refused (schema_mismatch): the schema definitions cannot be safely verified by this version. ${reExport}`
108
109
  : receipt.identityTable ? `Transfer refused (identity_table_undeclared:${receipt.identityTable}): declare this table in includeTables to transfer its rows or excludeTables to omit them.`
109
110
  : receipt.safety ? `Restore refused (${receipt.reason}): safety normalization for ${receipt.safety} violates the declared import policy.`
111
+ : receipt.reason === 'schema_mismatch:owner_migration_history' ? 'Restore refused (schema_mismatch:owner_migration_history): the recorded migration order or file contents differ from this checkout.'
110
112
  : receipt.reason?.startsWith('migration_ledger:') ? ledgerRefusal(receipt)
111
113
  : receipt.migration ? `Restore refused (schema_mismatch:migration_data): migration ${receipt.migration} changed data after this archive was made, so its rows would skip that change. Restore an archive made after it.`
112
114
  : receipt.occupied ? occupiedRefusal(input,receipt.reason)
@@ -1,5 +1,23 @@
1
1
  # Changelog
2
2
 
3
+ ## 0.2.124
4
+
5
+ Signed: Run Edit Run fleet, 2026-11-19.
6
+
7
+ ### Archive restore with owner migrations
8
+
9
+ - Local `rer import` predicts the archived schema by replaying the recorded core and owner migrations, including an owner migration that alters a core table. Native exports record the combined applied history with file hashes and omit owner migration bookkeeping tables from product data. Worker migration evidence and the shared restore path also include declared owner migrations, covering the Spool restore adapter when core migrations precede owner migrations.
10
+ - Changed, missing, reordered or archive-extra owner migrations refuse with `schema_mismatch:owner_migration_history` before importing product rows. Later additive migrations in the destination remain supported by the additive-restore contract.
11
+ - The portability adoption list now includes `owner-history.mjs`. Maintained CLI copies and inventory are regenerated.
12
+
13
+ Archives written by this version for a seed WITH owner migrations carry a combined migration ledger and are not readable by rer 0.2.123 or older: those readers refuse `manifest_inventory`. Core-only and legacy archives are unchanged both ways. The archive format remains `runeditrun-portability` version 2; the existing optional migration ledger now accepts the native `d1_migrations` origin.
14
+
15
+ KNOWN LIMIT: a Worker restore replays core files before owner files. An owner who applied an ALTER and later installed a core upgrade can have a matching archive refused (HTTP 409, zero rows, nothing published). A follow-up fixes the applied-order replay.
16
+
17
+ ChargeDesk checkouts pin rer 0.2.118 and other seeds have older pins, and need a re-pin to receive this fix.
18
+
19
+ These changes require adoption through the seed's pinned package or refreshed source. This preparation establishes no seed adoption, live Spool deployment, provider delivery or production result. The source range changes portability and its maintained copies, with no import-kit or product migration change.
20
+
3
21
  ## 0.2.123
4
22
 
5
23
  Signed: Run Edit Run fleet, 2026-11-18.
@@ -682,8 +682,12 @@ them into a scratch database to find the archive's place in that history:
682
682
  ledger's own names are kept beside their hashes. Import resolves each entry
683
683
  by hash exactly as it does an archived ledger table (next item), so a
684
684
  Platform prefix needs no declaration and a changed file refuses. Platform's
685
- ledger is never imported as product tables. Native sources record no
686
- `migrationLedger`; their archives are unchanged, and v1 and earlier v2
685
+ ledger is never imported as product tables. Native sources with applied
686
+ owner migrations carry `{origin:'d1_migrations',entries:[{name,sha256}]}`
687
+ in the same field. The entries combine core names and prefixed owner names
688
+ in recorded application order. Restore checks those names and hashes against
689
+ the destination's applied history and keeps both ledgers in place. Core-only
690
+ native archives use `manifest.migrations` as before, and v1 and earlier v2
687
691
  archives remain importable.
688
692
  - Older archives that still carry their source's migration ledger as a table
689
693
  (`rer_migrations` and the other ledgers above, from before they became
@@ -1145,6 +1149,7 @@ modules/portability/restore-safety.mjs
1145
1149
  modules/portability/bridge.mjs
1146
1150
  modules/portability/library-history.mjs
1147
1151
  modules/portability/checkout-history.mjs
1152
+ modules/portability/owner-history.mjs
1148
1153
  modules/portability/migration-history.mjs
1149
1154
  modules/portability/ledgers.mjs
1150
1155
  modules/portability/migration-statements.mjs
@@ -2,16 +2,18 @@ import { all, quote, insist } from './storage.mjs';
2
2
  import { canonicalJson } from './hash.mjs';
3
3
  import { tokens, identifier, folded } from './generated.mjs';
4
4
  import { laterData } from './backfills.mjs';
5
- import { destinationLedger, matchMigrationLedger } from './ledgers.mjs';
5
+ import { matchMigrationLedger } from './ledgers.mjs';
6
6
  import { checkRebuild } from './rebuilds.mjs';
7
7
  import { schemaSQL } from './schema-identity.mjs';
8
8
 
9
- // Applied product migrations in the ledger's own order (ledgers.mjs). A
10
- // state-bearing ledger may hold an unacknowledged migration whose effects are
11
- // unknown; neither an archive nor a restore can attest its history until that
12
- // is resolved.
13
- export async function appliedMigrations(db) {
14
- return (await destinationLedger(db))?.map(m=>m.name);
9
+ function recordedMigrations(manifest, archived, files, replay) {
10
+ const entries=manifest.migrations?.map(name=>archived?.find(entry=>entry.name===name)??{name})??archived;
11
+ if(manifest.migrationLedger?.origin!=='d1_migrations')return entries && matchMigrationLedger(entries,files,replay);
12
+ // Native owner identities come from the checkout's declared directories.
13
+ // A matching digest under another name cannot stand in for a changed file.
14
+ insist(entries?.every(entry=>files.some(file=>file.name===entry.name && file.sha256===entry.sha256)),
15
+ 'schema_mismatch:owner_migration_history',409);
16
+ return entries.map(entry=>entry.name);
15
17
  }
16
18
  // Where an archive sits in the destination's applied migrations: those its
17
19
  // rows passed through (earlier) and those applied after it (unapplied).
@@ -30,8 +32,7 @@ async function placement(db,manifest,cat,replay,ledger,history,legacyRecovery,un
30
32
  if(canonicalJson(replay.names)!==canonicalJson(applied))applied=matchMigrationLedger(ledger,files,replay);
31
33
  insist(canonicalJson(replay.names)===canonicalJson(applied) && await replay.schemaAfter(applied)===cat.identity,'schema_mismatch');
32
34
  const archived=history?.();
33
- const historyEntries=manifest.migrations?.map(name=>archived?.find(entry=>entry.name===name) ?? {name}) ?? archived;
34
- const recorded=historyEntries && matchMigrationLedger(historyEntries,files,replay);
35
+ const recorded=recordedMigrations(manifest,archived,files,replay);
35
36
  // Only explicitly acknowledged v1 recovery may use the old projection for
36
37
  // archive placement. The live destination check above remains complete, and
37
38
  // current v2 imports use their authenticated object inventory. Verified legacy
@@ -40,8 +41,9 @@ async function placement(db,manifest,cat,replay,ledger,history,legacyRecovery,un
40
41
  if(unverified && archived && manifest.migrations)insist(canonicalJson(matchMigrationLedger(archived,files,replay))===canonicalJson(recorded),'schema_mismatch:migration_history',409);
41
42
  let earlier,unapplied;
42
43
  if(recorded) {
43
- if(unverified)insist(recorded.every((name,i)=>applied.indexOf(name)>(i?applied.indexOf(recorded[i-1]):-1)),'schema_mismatch:migration_history',409);
44
- earlier=applied.filter(name=>recorded.includes(name));
44
+ const mismatch=manifest.migrationLedger?.origin==='d1_migrations'?'schema_mismatch:owner_migration_history':'schema_mismatch:migration_history';
45
+ insist(recorded.every((name,i)=>applied.indexOf(name)>(i?applied.indexOf(recorded[i-1]):-1)),mismatch,409);
46
+ earlier=recorded;
45
47
  insist(await schemaAfter(earlier)===manifest.schema,'schema_mismatch');
46
48
  unapplied=applied.filter(name=>!earlier.includes(name));
47
49
  }else{
@@ -76,15 +78,14 @@ async function placement(db,manifest,cat,replay,ledger,history,legacyRecovery,un
76
78
  // Verify the archive's historical projection and the complete live replay, then
77
79
  // return the recorded schema's complete identity for ordinary bridge placement.
78
80
  // Never guess an unrecorded prefix or install source SQL to fill missing evidence.
79
- export async function verifiedLegacySchema(db,manifest,cat,migrations,history) {
81
+ export async function verifiedLegacySchema(ledger,manifest,cat,migrations,history) {
80
82
  if(!migrations)return false;
81
- const replay=await migrations(),ledger=await destinationLedger(db)??[];
83
+ const replay=await migrations();
82
84
  try {
83
85
  const files=replay.names.map(name=>({name,sha256:replay.sha256?.(name),aliases:replay.aliases?.(name)}));
84
- const archived=history(),entries=manifest.migrations?.map(name=>archived?.find(entry=>entry.name===name)??{name})??archived;
85
- if(!entries?.length)return false;
86
- const applied=matchMigrationLedger(ledger,files,replay),recorded=matchMigrationLedger(entries,files,replay);
87
- if(!recorded.length || canonicalJson(applied)!==canonicalJson(replay.names)
86
+ const archived=history(),recorded=recordedMigrations(manifest,archived,files,replay);
87
+ const applied=matchMigrationLedger(ledger,files,replay);
88
+ if(!recorded?.length || canonicalJson(applied)!==canonicalJson(replay.names)
88
89
  || !recorded.every((name,i)=>applied.indexOf(name)>(i?applied.indexOf(recorded[i-1]):-1)))return false;
89
90
  if(archived && manifest.migrations && canonicalJson(matchMigrationLedger(archived,files,replay))!==canonicalJson(recorded))return false;
90
91
  if(await replay.schemaAfter(applied)!==cat.identity
@@ -99,8 +100,9 @@ export async function verifiedLegacySchema(db,manifest,cat,migrations,history) {
99
100
  // migrations the destination's ledger has outgrown is refused. An archive that
100
101
  // records a migration the destination never applied is newer than it, even
101
102
  // when that migration left the schema as it was (an index, a data change).
102
- export async function sameSchemaMigrations(db,manifest,cat,migrations,history) {
103
- const ledger=await destinationLedger(db),applied=ledger?.map(m=>m.name);
103
+ export async function sameSchemaMigrations(db,ledger,manifest,cat,migrations,history) {
104
+ const applied=ledger?.map(m=>m.name);
105
+ if(manifest.migrationLedger?.origin==='d1_migrations')insist(migrations && applied,'schema_mismatch:owner_migration_history',409);
104
106
  if(!applied)return;
105
107
  if(!migrations){
106
108
  if(manifest.migrations)matchMigrationLedger(manifest.migrations.map(name=>({name})),ledger);
@@ -108,17 +110,17 @@ export async function sameSchemaMigrations(db,manifest,cat,migrations,history) {
108
110
  }
109
111
  // A data-only migration leaves the schema unchanged but still needs its
110
112
  // carried-table replay plan retained for finalize.
111
- const bridge=await additiveBridge(db,manifest,cat,migrations,history);
113
+ const bridge=await additiveBridge(db,ledger,manifest,cat,migrations,history);
112
114
  return bridge.replays?.length?bridge:undefined;
113
115
  }
114
116
  // An archive made before later migrations restores only when those migrations
115
117
  // are additive, or a proven row-preserving CHECK rebuild: new tables, new
116
118
  // columns that take a default, or later schema objects established by replay.
117
119
  // Anything else stays a schema_mismatch refusal.
118
- export async function additiveBridge(db,manifest,cat,migrations,history,legacyRecovery,unverified=false) {
120
+ export async function additiveBridge(db,ledger,manifest,cat,migrations,history,legacyRecovery,unverified=false) {
119
121
  insist(migrations,'schema_mismatch');
120
- const replay=await migrations(),ledger=await destinationLedger(db)??[];
121
- const {applied,earlier,replays,rebuilt}=await placement(db,manifest,cat,replay,ledger,history,legacyRecovery,unverified);
122
+ const replay=await migrations();
123
+ const {applied,earlier,replays,rebuilt}=await placement(db,manifest,cat,replay,ledger??[],history,legacyRecovery,unverified);
122
124
  const columns={};
123
125
  for(const table of manifest.tables) {
124
126
  const target=cat.tables.find(t=>t.name===table.name),label=':'+table.name;
@@ -1,20 +1,21 @@
1
1
  import { catalogue, policy } from './catalogue.mjs';
2
2
  import { readLedger, matchMigrationLedger } from './ledgers.mjs';
3
+ import { nativeMigrationHistory, ownerMigrationPolicy } from './owner-history.mjs';
3
4
  import { insist } from './storage.mjs';
4
5
 
5
6
  // Both evidence adapters bind the seed's files to live history and the complete
6
7
  // schema. Only an explicitly bundled Worker may bind plain Wrangler history:
7
8
  // literal names must select an ordered checkout prefix, never aliases or gaps.
8
- export async function checkoutHistory(db,input,files,replayFor,{nativeWrangler=false}={}) {
9
- const history=await readLedger(db),ledger=history?.entries??[];
9
+ export async function checkoutHistory(db,input,files,replayFor,{nativeWrangler=false,ownerSources=[]}={}) {
10
+ const history=await (ownerSources.length?nativeMigrationHistory(db,ownerSources):readLedger(db)),ledger=history?.entries??[];
10
11
  insist(ledger.length,'schema_mismatch:legacy_evidence',409);
11
12
  if(!ledger.every(row=>row.sha256)) {
12
13
  insist(nativeWrangler && history.table==='d1_migrations'
13
- && ledger.every((row,i)=>row.sha256===undefined && row.name===files[i]?.name),
14
+ && ledger.every((row,i)=>row.name===files[i]?.name && (history.owners || row.sha256===undefined)),
14
15
  'schema_mismatch:legacy_evidence',409);
15
16
  }
16
17
  const applied=matchMigrationLedger(ledger,files);
17
- const replay=await replayFor(applied);
18
- insist(await replay.schemaAfter(replay.names)===(await catalogue(db,policy(input))).identity,'schema_mismatch');
19
- return replay;
18
+ const config=policy(ownerMigrationPolicy(input,ownerSources)),replay=await replayFor(applied,config);
19
+ insist(await replay.schemaAfter(replay.names)===(await catalogue(db,config)).identity,'schema_mismatch');
20
+ return ownerSources.length?{...replay,ownerSources,history}:replay;
20
21
  }
@@ -7,13 +7,15 @@ import { createCopy,readCopy,normalizeCopy } from './copies.mjs';
7
7
  import { canonicalJson, utf8, base64, StreamHash } from './hash.mjs';
8
8
  import { FORMAT, makePart, checkPart, ContentVerifier, ArchiveVerifier } from './format.mjs';
9
9
  import { journal } from './journal.mjs';
10
- import { readLedger, platformOrigin } from './ledgers.mjs';
10
+ import { platformOrigin, nativeOrigin } from './ledgers.mjs';
11
+ import { transferHistory } from './owner-history.mjs';
11
12
  import { capturePayload, payloadRange } from './payloads.mjs';
12
13
  import { dropProviderData } from './provider-data.mjs';
13
14
  import { dropDevData } from './dev-data.mjs';
14
15
  import { revokePolicyApprovals, refusePolicyApprovals } from './policy-data.mjs';
15
16
  export async function createExport(context,actor='direct') {
16
- const {db,config,buckets}=context,s=await state(db);insist(!s.applying,'import_applying',409);
17
+ const {db,buckets}=context,s=await state(db);insist(!s.applying,'import_applying',409);
18
+ const {config,ledger,migrations:replay}=await transferHistory(context);
17
19
  const cat=await catalogue(db,config),id=crypto.randomUUID(),objects=[];
18
20
  for(const [binding,spec] of Object.entries(config.buckets).sort()) {
19
21
  let cursor;
@@ -25,7 +27,8 @@ export async function createExport(context,actor='direct') {
25
27
  }
26
28
  objects.sort((a,b)=>a.bucket<b.bucket?-1:a.bucket>b.bucket?1:a.key<b.key?-1:a.key>b.key?1:0);
27
29
  const copies=Object.fromEntries(cat.tables.map((t,i)=>[t.name,'rer_portability_snap_'+id.replaceAll('-','')+'_'+i]));
28
- const ledger=await readLedger(db),origin=await platformOrigin(ledger,context.migrations),migrations=origin?origin.entries.map(entry=>entry.name):ledger?.entries.map(entry=>entry.name);
30
+ const origin=await (ledger?.owners?nativeOrigin(ledger,replay,cat.identity):platformOrigin(ledger,replay));
31
+ const migrations=origin?origin.entries.map(entry=>entry.name):ledger?.entries.map(entry=>entry.name);
29
32
  const manifest={kind:'manifest',format:FORMAT,version:2,product:config.product,exportedAt:new Date().toISOString(),sourceIdentity:s.source_identity,schema:cat.identity,tables:cat.tables,schemaObjects:cat.schemaObjects,objects,payloads:[],excludedTables:cat.excludedTables,excludedColumns:config.secretColumns,requiredConfiguration:config.requiredConfiguration,partCount:0,epoch:s.epoch,journal:await journal(db,s.source_identity),...(migrations?{migrations}:{}),...(origin?{migrationLedger:origin}:{})};
30
33
  const row={id,kind:'export',actor,phase:'building',completed:0,metadata:{manifest,copies,cursor:{table:0,row:0,binary:0,offset:0},partBytes:config.partBytes}};
31
34
  const audit=context.onExport?context.onExport({id,actor:actor==='direct'?null:actor,scope:'installation'}):[];
@@ -39,9 +39,9 @@ async function checkManifestFields(m) {
39
39
  insist(m.schemaObjects===undefined || m.excludedTables===undefined || (Array.isArray(m.excludedTables) && m.excludedTables.every(name=>typeof name==='string') && new Set(m.excludedTables).size===m.excludedTables.length),'manifest_inventory');
40
40
  // Applied product migrations, recorded by archives since additive restore.
41
41
  insist(m.migrations===undefined || (Array.isArray(m.migrations) && m.migrations.every(x=>typeof x==='string')),'manifest_inventory');
42
- // The ledger an export from a RER Platform installation records, by migration identity.
42
+ // A hash-bearing ledger from Platform or from a native core and owner history.
43
43
  const ledger=m.migrationLedger;
44
- insist(ledger===undefined || (ledger && typeof ledger==='object' && Object.keys(ledger).sort().join()==='entries,origin' && ledger.origin==='rer_migrations' && Array.isArray(ledger.entries)
44
+ insist(ledger===undefined || (ledger && typeof ledger==='object' && Object.keys(ledger).sort().join()==='entries,origin' && ['rer_migrations','d1_migrations'].includes(ledger.origin) && Array.isArray(ledger.entries)
45
45
  && ledger.entries.every(e=>e && typeof e==='object' && Object.keys(e).sort().join()==='name,sha256' && typeof e.name==='string' && /^[a-f0-9]{64}$/.test(e.sha256))
46
46
  && canonicalJson(m.migrations)===canonicalJson(ledger.entries.map(e=>e.name)) && new Set(m.migrations).size===m.migrations.length),'manifest_inventory');
47
47
  insist(m.tables.every((t,i)=>typeof t.name==='string' && typeof t.sql==='string' && Array.isArray(t.columns) && Array.isArray(t.dependencies) && Number.isSafeInteger(t.count) && t.count>=0 && (!i || m.tables[i-1].name<t.name)),'table_inventory');
@@ -27,7 +27,7 @@ const publicCodes = new Set(`
27
27
  upgrade_row_must_be_pure virtual_table wrong_product
28
28
  unauthenticated forbidden owner_required authority_changed step_up_required rate_limited operation_failed
29
29
  `.trim().split(/\s+/));
30
- const publicCode = code => typeof code === 'string' && publicCodes.has(code.split(':')[0]) ? code.split(':')[0] : 'operation_failed';
30
+ const publicCode = code => code === 'schema_mismatch:owner_migration_history' ? code : typeof code === 'string' && publicCodes.has(code.split(':')[0]) ? code.split(':')[0] : 'operation_failed';
31
31
 
32
32
  // Public errors use the existing {error:code} vocabulary. Native messages can
33
33
  // contain SQL, row values or credentials; inspect causes only to choose status.
@@ -11,10 +11,10 @@ export async function boundedJson(request,limit) {
11
11
  try{return JSON.parse(new TextDecoder('utf-8',{fatal:true}).decode(bytes));}catch{throw new PortabilityError('invalid_json');}
12
12
  }
13
13
  const json=(value,status=200)=>new Response(JSON.stringify(value),{status,headers:{'Content-Type':'application/json','Cache-Control':'no-store'}});
14
- export async function createPortabilityHandler({db,buckets,policy,migrations,appOrigin,basePath='/api/portability',authorize,maxBodyBytes=MAX_LINE_BYTES}) {
14
+ export async function createPortabilityHandler({db,buckets,policy,migrations,migrationHistory,appOrigin,basePath='/api/portability',authorize,maxBodyBytes=MAX_LINE_BYTES}) {
15
15
  insist(typeof authorize==='function' && new URL(appOrigin).origin===appOrigin,'invalid_handler_configuration');
16
16
  insist(/^\/[A-Za-z0-9/_-]+$/.test(basePath) && !basePath.endsWith('/') && Number.isSafeInteger(maxBodyBytes) && maxBodyBytes>0,'invalid_handler_configuration');
17
- const service=await createPortability({db,buckets,policy,migrations});
17
+ const service=await createPortability({db,buckets,policy,migrations,migrationHistory});
18
18
  return async request=>{
19
19
  const url=new URL(request.url);
20
20
  if(url.pathname!==basePath && !url.pathname.startsWith(basePath+'/'))return null;
@@ -16,25 +16,27 @@ import { checkEpoch, fence, mergeJournal, tombstone } from './journal.mjs';
16
16
  import { writeObject } from './payloads.mjs';
17
17
  import { createCopy,sidecar,readCopy,restoreCopy } from './copies.mjs';
18
18
  import { excludedDependentGuards, migrationSeeded, seededCondition } from './restore-safety.mjs';
19
- import { additiveBridge, sameSchemaMigrations, appliedMigrations, verifiedLegacySchema } from './bridge.mjs';
19
+ import { additiveBridge, sameSchemaMigrations, verifiedLegacySchema } from './bridge.mjs';
20
20
  import { replayStatements } from './backfills.mjs';
21
- import { checkArchivedLedgers, archivedHistory, archivedLedgerTables, ledgerReader, matchMigrationLedger } from './ledgers.mjs';
21
+ import { checkArchivedLedgers, archivedHistory, archivedLedgerTables, ledgerReader, matchMigrationLedger, readLedger } from './ledgers.mjs';
22
+ import { transferHistory } from './owner-history.mjs';
22
23
  import { legacyEmptyGuards, legacyAuditStatements, requireLegacyEmpty } from './legacy-safety.mjs';
23
24
  import { checkReferences } from './references.mjs';
24
25
  import { rebuildDerivedStatements } from './derived.mjs';
26
+ const acceptedUpgrade=(config,manifest)=>manifest.migrationLedger?.origin==='d1_migrations'?undefined:config.acceptedSchemas[manifest.schema];
25
27
  // ledgers: the rows of each migration ledger table the archive carries, as the
26
28
  // caller read them from it ({table:rows}); uploadPart binds them to its parts.
27
29
  export async function createImport(context,manifest,actor='direct',legacyRecovery,ledgers,acknowledgeUnverifiedSchemaObjects=false,id=crypto.randomUUID()) {
28
- const {db,config,buckets}=context;await checkManifest(manifest);checkArchivedLedgers(manifest,ledgers);
29
- insist(manifest.product===config.product,'wrong_product');
30
+ const {db,buckets}=context;await checkManifest(manifest);checkArchivedLedgers(manifest,ledgers);
31
+ insist(manifest.product===context.config.product,'wrong_product');
30
32
  // An already-stale backup cannot become restorable. Refuse before schema
31
33
  // inventory, staging copies and part uploads; fence checks again at finalize.
32
34
  await checkEpoch(db,manifest);
33
35
  // Explicit schema upgrades can reconcile rows, but cannot attest an interrupted deployment.
34
- await appliedMigrations(db);
36
+ const {config,ledger:historyLedger,migrations}=await transferHistory(context),ledger=historyLedger?.entries;
35
37
  const original=manifest;manifest=await restorableManifest(original,{canonical:true});
36
38
  const skippedTables=original.tables.filter(table=>platformTable(table.name)).map(table=>table.name);
37
- const cat=await catalogue(db,config),upgrade=config.acceptedSchemas[original.schema];
39
+ const cat=await catalogue(db,config),upgrade=acceptedUpgrade(config,original);
38
40
  manifest=await withoutAbsentExcludedObjects(manifest,cat,config);
39
41
  const legacy=original.version===2 && manifest.schemaObjects===undefined;
40
42
  // An explicit acknowledgment keeps its existing recovery semantics. Without
@@ -43,7 +45,7 @@ export async function createImport(context,manifest,actor='direct',legacyRecover
43
45
  const unverified=legacy && acknowledgeUnverifiedSchemaObjects===true;
44
46
  const verifyingLegacy=legacy && !unverified && !legacyRecovery;
45
47
  const history=()=>archivedHistory(original,ledgers,{allowOlderShape:unverified || verifyingLegacy});
46
- const verified=(verifyingLegacy || (!legacy && manifest.schema!==cat.identity)) && await verifiedLegacySchema(db,manifest,cat,context.migrations,history);
48
+ const verified=(verifyingLegacy || (!legacy && manifest.schema!==cat.identity)) && await verifiedLegacySchema(ledger??[],manifest,cat,migrations,history);
47
49
  insist(!legacy || legacyRecovery || unverified || verified,'schema_mismatch:legacy_schema',409);
48
50
  if(verified)manifest={...manifest,schema:verified};
49
51
  if(unverified)for(const table of manifest.tables) {
@@ -53,7 +55,7 @@ export async function createImport(context,manifest,actor='direct',legacyRecover
53
55
  insist(table.dependencies.every(d=>target.dependencies.includes(d)),'schema_mismatch:dependencies:'+table.name,409);
54
56
  }
55
57
  const same=manifest.schema===cat.identity;
56
- const {replays,...bridge}=unverified?await additiveBridge(db,manifest,cat,context.migrations,history,legacyRecovery,true):upgrade?{}:same?(await sameSchemaMigrations(db,manifest,cat,context.migrations,history)??{}):await additiveBridge(db,manifest,cat,context.migrations,history,legacyRecovery);
58
+ const {replays,...bridge}=unverified?await additiveBridge(db,ledger,manifest,cat,migrations,history,legacyRecovery,true):upgrade?{}:same?(await sameSchemaMigrations(db,ledger,manifest,cat,migrations,history)??{}):await additiveBridge(db,ledger,manifest,cat,migrations,history,legacyRecovery);
57
59
  // Finalize checks object references on the staged rows; a later migration
58
60
  // replayed into a reference table would publish rows that check never saw.
59
61
  const referenced=replays?.find(r=>Object.values(config.buckets).some(spec=>spec.objectReferences?.table===r.table));
@@ -70,7 +72,7 @@ export async function createImport(context,manifest,actor='direct',legacyRecover
70
72
  insist(canonicalJson(manifest.requiredConfiguration)===canonicalJson(config.requiredConfiguration),'configuration_inventory');
71
73
  for(const o of manifest.objects) insist(buckets[o.bucket] && config.buckets[o.bucket] && o.key.startsWith(config.buckets[o.bucket].prefix??''),'bucket_inventory');
72
74
  insist(canonicalJson(manifest.payloads.map(x=>x.name))===canonicalJson(Object.keys(config.extraPayloads).sort()),'payload_inventory');
73
- if(unverified)await destinationState(context,{metadata:{targets:cat.tables,manifest:original}});
75
+ if(unverified)await destinationState({...context,config,migrations},{metadata:{targets:cat.tables,manifest:original}});
74
76
  await ensureImportLeases(db);
75
77
  const previous=(await state(db)).import_job;
76
78
  if(previous) {
@@ -96,7 +98,7 @@ async function uploadLeasedPart(context,id,index,part) {
96
98
  const previous=await statement(db,'SELECT sha256 FROM rer_portability_parts WHERE job=? AND idx=?',id,index).first();
97
99
  if(previous){insist(previous.sha256===part.sha256,'part_conflict',409);return importStatus(context,id,{includeLease:false});}
98
100
  insist(row.phase==='staging','import_phase',409);insist(index===row.completed && index<row.metadata.manifest.partCount,'part_order',409);
99
- const inserts=[],groups=new Map(),manifest=row.metadata.manifest,upgrade=config.acceptedSchemas[manifest.schema];
101
+ const inserts=[],groups=new Map(),manifest=row.metadata.manifest,upgrade=acceptedUpgrade(config,manifest);
100
102
  for(const item of part.items) {
101
103
  if(item.kind==='row') {
102
104
  const table=manifest.tables.find(x=>x.name===item.table);insist(table && Number.isSafeInteger(item.index) && item.index>=0 && item.index<table.count,'row_inventory');
@@ -303,7 +305,7 @@ export function libraryImports(context,direct,migrationRoot) {
303
305
  await checkManifest(manifest);
304
306
  if(!archivedLedgerTables(manifest).length)return direct.create(manifest,actor,options);
305
307
  insist(manifest.product===context.config.product,'wrong_product');
306
- await appliedMigrations(context.db);
308
+ await (context.migrationHistory?.()??readLedger(context.db));
307
309
  insist(!(await statement(context.db,'SELECT import_job FROM rer_portability_state WHERE id=1').first())?.import_job && ![...pending.values()].some(row=>row.phase==='staging'),'import_running',409);
308
310
  context.migrations??=await libraryMigrations(context.db,context.inputPolicy,migrationRoot);
309
311
  const id=crypto.randomUUID();pending.set(id,{id,manifest:structuredClone(manifest),actor,parts:[],phase:'staging'});return id;
@@ -13,8 +13,8 @@ export interface Manifest {
13
13
  excludedTables: string[]; excludedColumns: Record<string,string[]>; requiredConfiguration: string[]; partCount: number; epoch: number; journal: string[];
14
14
  /** Applied product migrations; absent from archives made before additive restore. */
15
15
  migrations?: string[];
16
- /** Present when RER Platform's runner migrated the source: each applied migration by native file name and SHA-256, in application order. */
17
- migrationLedger?: {origin:'rer_migrations';entries:{name:string;sha256:string}[]};
16
+ /** Hash-bound application order from RER Platform or the local core and owner ledgers. */
17
+ migrationLedger?: {origin:'rer_migrations'|'d1_migrations';entries:{name:string;sha256:string}[]};
18
18
  }
19
19
  export type Item = {kind:'row';table:string;index:number;row:Row;rowid?:number} | {kind:'object';bucket:string;key:string;offset:number;data:string} | {kind:'payload';name:string;offset:number;data:string};
20
20
  export interface Part {kind:'part';index:number;items:Item[];sha256:string}
@@ -51,6 +51,8 @@ export interface PortabilityOptions {
51
51
  /** Node checkout root for automatic legacy evidence; defaults to cwd. */
52
52
  migrationRoot?:string;
53
53
  db:D1Database;buckets?:Record<string,R2Bucket>;policy:ArchivePolicy;migrations?:()=>Promise<MigrationReplay>;
54
+ /** Runtime-owned history for databases with separate owner migration ledgers. */
55
+ migrationHistory?:()=>Promise<{table:string;entries:{name:string;sha256?:string}[];owners?:boolean}|undefined>;
54
56
  /** Seed-owned audit SQL, appended to the snapshot/job batch. Throwing or SQL failure refuses creation. */
55
57
  onExport?:(event:ExportEvent)=>[sql:string,args:unknown[]][];
56
58
  }
@@ -12,13 +12,13 @@ export { readArchive, makePart, MAX_LINE_BYTES, MAX_PART_BYTES } from './format.
12
12
  export { canonicalJson, sha256 } from './hash.mjs';
13
13
  export { PortabilityError } from './storage.mjs';
14
14
  export { createPortabilityHandler } from './http.mjs';
15
- export async function createPortability({db,buckets={},policy:input,migrations,onExport,migrationRoot}) {
15
+ export async function createPortability({db,buckets={},policy:input,migrations,migrationHistory,onExport,migrationRoot}) {
16
16
  insist(onExport===undefined || typeof onExport==='function','invalid_on_export');
17
17
  db=wireDatabase(db);
18
18
  const config=policy(input);await ensureObjectInventory(db);await catalogue(db,config);
19
19
  for(const name of Object.keys(config.buckets))insist(buckets[name] && typeof buckets[name].get==='function','missing_bucket:'+name);
20
20
  await ensureImportLeases(db);
21
- const context={db,buckets,config,migrations,onExport};
21
+ const context={db,buckets,config,migrations,migrationHistory,onExport};
22
22
  const importContext={...context,inputPolicy:input};
23
23
  const {lookup,actor,...imports}=libraryImports(importContext,{
24
24
  active:actor=>activeImport(importContext,actor),
@@ -76,8 +76,7 @@ export function matchMigrationLedger(entries,files,{dataOnly=[]}={}) {
76
76
  * {origin:'rer_migrations',entries:[{name,sha256}]} in application order, each
77
77
  * entry named by the checkout file with that hash (the native identity) where
78
78
  * the exporter can read the checkout's files. Declared data-only entries and
79
- * hosted exporters, which have no files, keep the ledger's own names. Native
80
- * sources record nothing here, so their archives are unchanged.
79
+ * hosted exporters, which have no files, keep the ledger's own names.
81
80
  */
82
81
  export async function platformOrigin(ledger,migrations) {
83
82
  if(ledger?.table!=='rer_migrations')return;
@@ -88,6 +87,20 @@ export async function platformOrigin(ledger,migrations) {
88
87
  return {origin:'rer_migrations',entries};
89
88
  }
90
89
 
90
+ // Separate native owner ledgers are also bookkeeping. Bind the merged applied
91
+ // history to the checkout's files without moving any of those tables' rows.
92
+ export async function nativeOrigin(ledger,migrations,schema) {
93
+ const replay=await migrations?.();
94
+ insist(replay && canonicalJson(replay.names)===canonicalJson(ledger.entries.map(row=>row.name)),'schema_mismatch:migration_history',409);
95
+ const entries=replay.names.map(name=>{
96
+ const sha256=replay.sha256?.(name);
97
+ insist(typeof sha256==='string' && /^[a-f0-9]{64}$/.test(sha256),'schema_mismatch:migration_history',409);
98
+ return {name,sha256};
99
+ });
100
+ insist(await replay.schemaAfter(replay.names)===schema,'schema_mismatch',409);
101
+ return {origin:'d1_migrations',entries};
102
+ }
103
+
91
104
  /** The ledger tables an archive carries (old archives carried rer_migrations before it became bookkeeping). */
92
105
  export const archivedLedgerTables=manifest=>manifest.tables.filter(t=>ledgerNames.includes(identifier(t.name)));
93
106
 
@@ -117,7 +130,7 @@ export function checkArchivedLedgers(manifest,ledgers) {
117
130
  /** The applied migrations an archive's own ledger records, [{name,sha256?}]; undefined when it carries no ledger. */
118
131
  export function archivedHistory(manifest,ledgers,{allowOlderShape=false}={}) {
119
132
  const tables=archivedLedgerTables(manifest);
120
- // A Platform-origin export records its ledger in the manifest, never as a table.
133
+ // Hash-bearing exports record their ledger in the manifest.
121
134
  if(manifest.migrationLedger)insist(!tables.length,'schema_mismatch:migration_history');
122
135
  if(!tables.length)return manifest.migrationLedger?.entries.map(entry);
123
136
  insist(tables.length===1 && ledgers?.[tables[0].name],'schema_mismatch:migration_history');
@@ -8,7 +8,7 @@ import { ensureLegacyAudit, legacyTargetState, requireLegacyEmpty } from './lega
8
8
  import { insist, PortabilityError } from './storage.mjs';
9
9
  import { ledgerReader } from './ledgers.mjs';
10
10
 
11
- export async function recoverLegacyArchive({db,buckets,policy:input,migrations},path,output,transient,acknowledged) {
11
+ export async function recoverLegacyArchive({db,buckets,policy:input,migrations,migrationHistory},path,output,transient,acknowledged) {
12
12
  insist(acknowledged===true,'legacy_acknowledgement_required',409);
13
13
  insist(typeof output==='string' && output!==path,'legacy_output_required',409);
14
14
  insist(!(await lstat(output).catch(error=>{if(error.code==='ENOENT')return null;throw error;})),'legacy_output_exists',409);
@@ -16,7 +16,7 @@ export async function recoverLegacyArchive({db,buckets,policy:input,migrations},
16
16
  // reads the archive's own migration ledger, if any, for placement (bridge.mjs).
17
17
  const ledger=ledgerReader();
18
18
  for await(const record of fileRecords(path,readLegacyArchive))ledger.add(record);
19
- const context={db,buckets,config:policy(input),migrations},target=await legacyTargetState(db,migrations);
19
+ const context={db,buckets,config:policy(input),migrations,migrationHistory},target=await legacyTargetState(db,migrations);
20
20
  await requireLegacyEmpty(context,target);
21
21
  let id,manifest,closing,integrity,status,complete=false;
22
22
  try {
@@ -40,7 +40,7 @@ export async function recoverLegacyArchive({db,buckets,policy:input,migrations},
40
40
  // A fresh export binds the recovered installation from this point forward.
41
41
  // A failure here must be reported; the durable restore audit remains intact.
42
42
  try {
43
- const recovered=await exportArchive(await createPortability({db,buckets,policy:input}),output,transient);
43
+ const recovered=await exportArchive(await createPortability({db,buckets,policy:input,migrations,migrationHistory}),output,transient);
44
44
  return {...recovered,legacyRecovery:true,unverifiedMetadata:true,restoredContentHash:closing.contentHash,...(status.bridged?{bridged:status.bridged}:{}),...(status.safetyChanges?{safetyChanges:status.safetyChanges}:{})};
45
45
  }catch{throw new PortabilityError('legacy_restored_export_failed',500);}
46
46
  }
@@ -1,18 +1,19 @@
1
1
  import { policy, schemaIdentity } from './catalogue.mjs';
2
2
 
3
- // Replays a chosen set of the destination's migrations, in its applied order,
3
+ // Replays a chosen sequence of the destination's migrations, in recorded order,
4
4
  // into a scratch database and returns the resulting archive schema identity.
5
5
  // Node runners only: restore uses it to place an older archive within the
6
6
  // destination's history (see bridge.mjs). plan: [{name,statements,sha256?}],
7
7
  // sha256 being the migration file's hash where the runner read the file.
8
8
  export function migrationReplay(plan,input,{dataOnly=[]}={}) {
9
9
  const config=policy(input),sqlite='node:sqlite';
10
- return {names:plan.map(migration=>migration.name),dataOnly,aliases:name=>plan.find(m=>m.name===name).aliases,statements:name=>plan.find(m=>m.name===name).statements,sha256:name=>plan.find(m=>m.name===name).sha256,async schemaAfter(names,options) {
10
+ const files=new Map(plan.map(migration=>[migration.name,migration]));
11
+ return {names:plan.map(migration=>migration.name),dataOnly,aliases:name=>files.get(name).aliases,statements:name=>files.get(name).statements,sha256:name=>files.get(name).sha256,async schemaAfter(names,options) {
11
12
  const {DatabaseSync}=await import(sqlite);
12
13
  const scratch=new DatabaseSync(':memory:');
13
14
  const db={prepare:sql=>{const query=scratch.prepare(sql);let values=[];return {bind(...args){values=args;return this;},async all(){return {results:query.all(...values)};}};}};
14
15
  try {
15
- for(const migration of plan.filter(m=>names.includes(m.name)))for(const sql of migration.statements)scratch.exec(sql);
16
+ for(const name of names)for(const sql of files.get(name).statements)scratch.exec(sql);
16
17
  return await schemaIdentity(db,config,options);
17
18
  }finally{scratch.close();}
18
19
  },
@@ -0,0 +1,62 @@
1
+ import { ledgerNames, readLedger } from './ledgers.mjs';
2
+ import { all, insist, quote } from './storage.mjs';
3
+
4
+ export function ownerMigrationPolicy(policy, sources) {
5
+ if (!sources.length) return policy;
6
+ const tables = sources.map(source => source.table);
7
+ insist(!policy.includeTables?.some(name => tables.includes(name.toLowerCase())), 'migration_ledger:incompatible', 409);
8
+ return { ...policy, excludeTables: [...new Set([...(policy.excludeTables ?? []), ...tables])] };
9
+ }
10
+
11
+ const ownerColumns = columns => columns.length === 3 && columns[0].name === 'id' && columns[0].pk === 1
12
+ && columns[1].name === 'name' && columns[2].name === 'applied_at';
13
+ const timestamp = /^\d{4}-\d\d-\d\d \d\d:\d\d:\d\d$/;
14
+ async function nativeRows(db, table, owner = false) {
15
+ const columns = await all(db, `PRAGMA table_info(${quote(table)})`);
16
+ if (!columns.length) return [];
17
+ insist(owner ? ownerColumns(columns) : ownerColumns(columns.slice(0, 3)), 'migration_ledger:incompatible', 409);
18
+ const rows = await all(db, `SELECT name, applied_at FROM ${quote(table)} ORDER BY id`);
19
+ insist(new Set(rows.map(row => row.name)).size === rows.length
20
+ && rows.every((row, i) => typeof row.name === 'string' && timestamp.test(row.applied_at)
21
+ && (!i || rows[i - 1].applied_at <= row.applied_at)), 'migration_ledger:incompatible', 409);
22
+ return rows;
23
+ }
24
+
25
+ export async function nativeMigrationHistory(db, sources) {
26
+ const ledger = await readLedger(db);
27
+ const owners = (await Promise.all(sources.map(async ({ table, prefix }) =>
28
+ (await nativeRows(db, table, true)).map(row => ({ ...row, name: prefix + row.name, table }))))).flat();
29
+ if (!owners.length) return ledger;
30
+ insist(ledger?.table === 'd1_migrations', 'migration_ledger:multiple', 409);
31
+ const core = (await nativeRows(db, ledger.table)).map((row, i) => ({ ...ledger.entries[i], applied_at: row.applied_at, table: ledger.table }));
32
+ const rows = [...core, ...owners].sort((a, b) => a.applied_at.localeCompare(b.applied_at));
33
+ // IDs order each ledger. Across ledgers equal timestamps carry no order.
34
+ insist(rows.every((row, i) => !i || row.applied_at !== rows[i - 1].applied_at || row.table === rows[i - 1].table), 'migration_ledger:uncertain', 409);
35
+ const entries = rows.map(({ name, sha256 }) => ({ name, ...(sha256 ? { sha256 } : {}) }));
36
+ insist(new Set(entries.map(row => row.name)).size === entries.length, 'migration_ledger:ambiguous', 409);
37
+ return { table: ledger.table, entries, owners: true };
38
+ }
39
+
40
+ // The existing service callback is also used for core-only replay. Consult it
41
+ // early only when native owner bookkeeping may exist. The replay supplies the
42
+ // declared table identities; a similar product table never becomes bookkeeping.
43
+ async function hasOwnerLedger(db) {
44
+ const candidates = await all(db, "SELECT name FROM sqlite_master WHERE type='table' AND name GLOB '[a-z]*_*_migrations'");
45
+ for (const { name } of candidates) {
46
+ if (!ledgerNames.includes(name) && ownerColumns(await all(db, `PRAGMA table_info(${quote(name)})`))) return true;
47
+ }
48
+ return false;
49
+ }
50
+
51
+ export async function transferHistory({ db, config, migrations, migrationHistory }) {
52
+ if (migrationHistory) return { config, migrations, ledger: await migrationHistory() };
53
+ const ledger = await readLedger(db);
54
+ if (migrations && ledger?.table === 'd1_migrations' && await hasOwnerLedger(db)) {
55
+ const replay = await migrations();
56
+ if (replay?.ownerSources) return {
57
+ config: ownerMigrationPolicy(config, replay.ownerSources), migrations: async () => replay, ledger: replay.history,
58
+ };
59
+ return { config, migrations: async () => replay, ledger };
60
+ }
61
+ return { config, migrations, ledger };
62
+ }
@@ -2,6 +2,7 @@ import type { ArchivePolicy, MigrationReplay } from './index.mjs';
2
2
  export interface MigrationEvidence {
3
3
  version:1;
4
4
  files:{name:string;sha256:string;statements:string[]}[];
5
+ ownerSources?:{prefix:string;table:string}[];
5
6
  objects:{name:string;sql:string;type:string;tbl_name:string}[];
6
7
  prefixes:{objects:number[];shadows:string[];seeds:{name:string;columns:string[];hashes:string[]}[]}[];
7
8
  }
@@ -1,4 +1,4 @@
1
- import { policy, inventorySchemaIdentity } from './catalogue.mjs';
1
+ import { inventorySchemaIdentity } from './catalogue.mjs';
2
2
  import { canonicalJson } from './hash.mjs';
3
3
  import { checkoutHistory } from './checkout-history.mjs';
4
4
  import { insist, PortabilityError } from './storage.mjs';
@@ -8,8 +8,8 @@ import { insist, PortabilityError } from './storage.mjs';
8
8
  export async function workerMigrations(db,input,evidence) {
9
9
  try {
10
10
  insist(evidence?.version===1 && evidence.prefixes.length===evidence.files.length+1,'schema_mismatch:legacy_evidence',409);
11
- const config=policy(input),files=evidence.files;
12
- return await checkoutHistory(db,input,files,names=>{
11
+ const {files,ownerSources}=evidence;
12
+ return await checkoutHistory(db,input,files,(names,config)=>{
13
13
  const prefix=selected=>{
14
14
  insist(selected.length<=names.length && canonicalJson(selected)===canonicalJson(files.slice(0,selected.length).map(f=>f.name)),'schema_mismatch:legacy_evidence',409);
15
15
  const found=evidence.prefixes[selected.length];
@@ -24,6 +24,6 @@ export async function workerMigrations(db,input,evidence) {
24
24
  async rowImages(requests){return new Map(requests.filter(r=>final.seeds.some(s=>s.name===r.name && !s.hashes.length)).map(r=>[r.name,[]]));},
25
25
  async rowHashes(requests,limit){return new Map(requests.flatMap(r=>{const found=final.seeds.find(s=>s.name===r.name && s.hashes.length<=limit && canonicalJson(s.columns)===canonicalJson(r.columns));return found?[[r.name,found.hashes]]:[];}));},
26
26
  };
27
- },{nativeWrangler:true});
27
+ },{nativeWrangler:true,ownerSources});
28
28
  }catch(error){if(error instanceof PortabilityError)throw error;throw new PortabilityError('schema_mismatch:legacy_evidence',409);}
29
29
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@runeditrun/rer",
3
- "version": "0.2.123",
3
+ "version": "0.2.124",
4
4
  "description": "The rer command: install, set up, run and launch Run Edit Run checkouts.",
5
5
  "license": "MIT",
6
6
  "type": "module",