@runeditrun/rer 0.2.123 → 0.2.125
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cli/portability-empty.mjs +4 -2
- package/cli/portability-evidence.mjs +49 -34
- package/cli/portability-migrations.mjs +30 -8
- package/cli/portability-owners.mjs +54 -0
- package/cli/portability-sqlite.mjs +72 -0
- package/cli/portability-worker.mjs +13 -5
- package/cli/portability.mjs +6 -4
- package/cli/vendor/wa-sqlite/LICENSE +21 -0
- package/cli/vendor/wa-sqlite/README.md +13 -0
- package/cli/vendor/wa-sqlite/wa-sqlite.mjs +14 -0
- package/cli/vendor/wa-sqlite/wa-sqlite.wasm +0 -0
- package/global/CHANGELOG.md +36 -0
- package/modules/portability/README.md +22 -11
- package/modules/portability/bridge.mjs +25 -23
- package/modules/portability/checkout-history.mjs +13 -9
- package/modules/portability/export.mjs +6 -3
- package/modules/portability/format.mjs +2 -2
- package/modules/portability/http-errors.mjs +1 -1
- package/modules/portability/http.mjs +2 -2
- package/modules/portability/import.mjs +13 -11
- package/modules/portability/index.d.mts +4 -2
- package/modules/portability/index.mjs +2 -2
- package/modules/portability/ledgers.mjs +16 -3
- package/modules/portability/legacy-files.mjs +3 -3
- package/modules/portability/migration-evidence.mjs +47 -0
- package/modules/portability/migration-history.mjs +4 -3
- package/modules/portability/owner-history.mjs +62 -0
- package/modules/portability/worker-history.d.mts +21 -4
- package/modules/portability/worker-history.mjs +24 -16
- package/modules/portability/worker-owner-history.mjs +38 -0
- package/package.json +1 -1
|
@@ -11,6 +11,7 @@ import { splitStatements } from '../modules/portability/sql-text.mjs';
|
|
|
11
11
|
import { requireEmptyTables } from '../modules/portability/restore-empty.mjs';
|
|
12
12
|
import { insist } from '../modules/portability/storage.mjs';
|
|
13
13
|
import { shared } from './shared.mjs';
|
|
14
|
+
import { ownerMigrationPolicy, ownerMigrationSources } from './portability-owners.mjs';
|
|
14
15
|
|
|
15
16
|
const status = path => lstat(path).catch(error => { if (error.code === 'ENOENT') return null; throw error; });
|
|
16
17
|
// Miniflare's named D1 identity: the declared binding determines its file.
|
|
@@ -58,8 +59,9 @@ export async function emptyNativePreflight({ root, binding, policy, policyModule
|
|
|
58
59
|
async all() { return { results: query.all(...values) }; },
|
|
59
60
|
async first() { return query.get(...values) ?? null; },
|
|
60
61
|
}; } };
|
|
61
|
-
const
|
|
62
|
-
const
|
|
62
|
+
const owners = await ownerMigrationSources(root, binding);
|
|
63
|
+
const rules = ownerMigrationPolicy(await loadPolicy(root, policy, policyModule), owners), declaration = platformMigrationDeclaration(platformMigrations);
|
|
64
|
+
const migrations = async () => migrationReplay(await appliedMigrations(root, db, binding, declaration, splitStatements, owners), rules, declaration);
|
|
63
65
|
await requireEmptyTables({ db, policy: rules, migrations });
|
|
64
66
|
} finally { try { snapshot?.close(); } finally { await rm(directory, { recursive: true, force: true }); } }
|
|
65
67
|
}
|
|
@@ -1,20 +1,22 @@
|
|
|
1
1
|
import { DatabaseSync } from 'node:sqlite';
|
|
2
2
|
import { realpath, readFile, readdir } from 'node:fs/promises';
|
|
3
3
|
import { resolve, relative, isAbsolute, sep } from 'node:path';
|
|
4
|
+
import { fileURLToPath } from 'node:url';
|
|
4
5
|
import { migrationDirectory } from './portability-migrations.mjs';
|
|
6
|
+
import { ownerMigrationSources } from './portability-owners.mjs';
|
|
7
|
+
import { collectMigrations } from '../modules/delivery-producer/migrations.mjs';
|
|
5
8
|
import { splitStatements } from '../modules/portability/sql-text.mjs';
|
|
6
9
|
import { canonicalJson, sha256 } from '../modules/portability/hash.mjs';
|
|
7
|
-
import {
|
|
8
|
-
import {
|
|
9
|
-
import { sqliteTokens } from '../modules/portability/sqlite-tokens.mjs';
|
|
10
|
+
import { insist } from '../modules/portability/storage.mjs';
|
|
11
|
+
import { migrationRecorder } from '../modules/portability/migration-evidence.mjs';
|
|
10
12
|
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
const
|
|
13
|
+
const wasmPath = fileURLToPath(new URL('./vendor/wa-sqlite/wa-sqlite.wasm', import.meta.url));
|
|
14
|
+
const sqlitePath = fileURLToPath(new URL('./portability-sqlite.mjs', import.meta.url));
|
|
15
|
+
const ownerHistoryPath = fileURLToPath(new URL('../modules/portability/worker-owner-history.mjs', import.meta.url));
|
|
14
16
|
|
|
15
17
|
// Only migration-owned schema and hashed seeded-row identities enter the bundle.
|
|
16
18
|
// Never open a live database, settings file, archive, or archive-supplied path.
|
|
17
|
-
|
|
19
|
+
async function migrationEvidence(root,{binding='DB'}={}) {
|
|
18
20
|
root=await realpath(root);
|
|
19
21
|
const directory=await realpath(await migrationDirectory(root,binding));
|
|
20
22
|
const path=relative(root,directory);
|
|
@@ -25,48 +27,56 @@ export async function generateMigrationEvidence(root,{binding='DB'}={}) {
|
|
|
25
27
|
const bytes=await readFile(file);
|
|
26
28
|
return {name,sha256:await sha256(bytes),statements:splitStatements(bytes.toString('utf8'))};
|
|
27
29
|
}));
|
|
30
|
+
const sources=await ownerMigrationSources(root,binding);
|
|
31
|
+
const ownerFiles = (await Promise.all(sources.filter(source => source.directory).map(async ({directory,prefix,table}) => {
|
|
32
|
+
const owners = await collectMigrations(root, [{binding,path:directory,prefix}], [{kind:'d1',binding}]);
|
|
33
|
+
return owners.map(({name,sha256,sql}) => ({name,sha256,statements:splitStatements(sql),table}));
|
|
34
|
+
}))).flat();
|
|
35
|
+
const ownerSources = sources.map(({prefix,table})=>({prefix,table}));
|
|
36
|
+
if (ownerFiles.length) {
|
|
37
|
+
const declared = new Set(files.map(file => file.name));
|
|
38
|
+
for (const file of ownerFiles) {
|
|
39
|
+
insist(!declared.has(file.name),'migration_ledger:ambiguous',409);
|
|
40
|
+
declared.add(file.name);
|
|
41
|
+
files.push(file);
|
|
42
|
+
}
|
|
43
|
+
return {version:2,files,ownerSources};
|
|
44
|
+
}
|
|
45
|
+
const evidence = await bundledPrefixes(files);
|
|
46
|
+
if (ownerSources.length) evidence.ownerSources = ownerSources;
|
|
47
|
+
return evidence;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
async function bundledPrefixes(files) {
|
|
28
51
|
const objects=[],indices=new Map(),prefixes=[];
|
|
29
52
|
const scratch=new DatabaseSync(':memory:');
|
|
30
53
|
try {
|
|
31
|
-
|
|
32
|
-
// Such rows cannot equal a fresh library replay reliably either. Preserve
|
|
33
|
-
// deterministic seeds in unrelated tables, and empty evidence everywhere.
|
|
34
|
-
const unstable=new Set(),applied=[];
|
|
54
|
+
const recorder=migrationRecorder({exec:sql=>scratch.exec(sql),all:sql=>scratch.prepare(sql).all()});
|
|
35
55
|
const capture=async()=>{
|
|
36
|
-
const schema=
|
|
56
|
+
const {objects:schema,kinds}=recorder.inventory();
|
|
37
57
|
const refs=schema.map(object=>{
|
|
38
58
|
const key=canonicalJson(object);if(!indices.has(key)){indices.set(key,objects.length);objects.push(object);}return indices.get(key);
|
|
39
59
|
});
|
|
40
|
-
|
|
41
|
-
const seeds=[];
|
|
42
|
-
for(const {name} of schema.filter(o=>o.type==='table')) {
|
|
43
|
-
const columns=scratch.prepare(`PRAGMA table_info(${quote(name)})`).all().map(c=>c.name);
|
|
44
|
-
const rows=scratch.prepare(`SELECT json_array(${columns.map(c=>`quote(${quote(c)})`).join(',')}) image FROM ${quote(name)} LIMIT 10001`).all().map(r=>r.image);
|
|
45
|
-
if(rows.length>10000 || unstable.has(name.toLowerCase()) && rows.length)continue;
|
|
46
|
-
seeds.push({name,columns,hashes:(await Promise.all(rows.map(sha256))).sort()});
|
|
47
|
-
}
|
|
48
|
-
prefixes.push({objects:refs,shadows,seeds});
|
|
60
|
+
prefixes.push({objects:refs,shadows:kinds.map(kind=>kind.name).sort(),seeds:await recorder.seeds(schema)});
|
|
49
61
|
};
|
|
50
62
|
await capture();
|
|
51
63
|
for(const file of files) {
|
|
52
|
-
|
|
53
|
-
const data=dataStatement(sql);
|
|
54
|
-
if(data) {
|
|
55
|
-
const writers=new Set([data.target]);
|
|
56
|
-
for(const name of writers)for(const target of triggerBodies(applied,name))writers.add(target);
|
|
57
|
-
const definitions=scratch.prepare("SELECT name,sql FROM sqlite_master WHERE sql IS NOT NULL").all();
|
|
58
|
-
if(variable(sql) || data.names.some(n=>unstable.has(n)) || definitions.some(d=>writers.has(d.name.toLowerCase()) && variable(d.sql))
|
|
59
|
-
|| applied.some(s=>variable(s) && [...writers].some(n=>triggerBodies([s],data.target).includes(n))))for(const name of writers)unstable.add(name);
|
|
60
|
-
}
|
|
61
|
-
scratch.exec(sql);
|
|
62
|
-
applied.push(sql);
|
|
63
|
-
}
|
|
64
|
+
recorder.apply(file);
|
|
64
65
|
await capture();
|
|
65
66
|
}
|
|
66
67
|
}finally{scratch.close();}
|
|
67
68
|
return {version:1,files,objects,prefixes};
|
|
68
69
|
}
|
|
69
70
|
|
|
71
|
+
export async function generateMigrationEvidence(root,options) {
|
|
72
|
+
const evidence = await migrationEvidence(root,options);
|
|
73
|
+
if (evidence.version === 1) return evidence;
|
|
74
|
+
const [{sqliteMemory}, {workerOwnerEvidence}] = await Promise.all([
|
|
75
|
+
import('./portability-sqlite.mjs'), import('../modules/portability/worker-owner-history.mjs'),
|
|
76
|
+
]);
|
|
77
|
+
return workerOwnerEvidence(evidence, sqliteMemory(await WebAssembly.compile(await readFile(wasmPath))));
|
|
78
|
+
}
|
|
79
|
+
|
|
70
80
|
// Explicit seed opt-in. The module is consumed only by the server/Worker graph;
|
|
71
81
|
// it is bundled and hashed by the existing Vite and deploy package paths.
|
|
72
82
|
export function legacyMigrationEvidence({binding='DB'}={}) {
|
|
@@ -79,7 +89,12 @@ export function legacyMigrationEvidence({binding='DB'}={}) {
|
|
|
79
89
|
async load(source){
|
|
80
90
|
if(source!==internal)return;
|
|
81
91
|
if(this.environment?.name==='client')throw new Error('Legacy migration evidence is server-only.');
|
|
82
|
-
|
|
92
|
+
const evidence = await migrationEvidence(root,{binding});
|
|
93
|
+
if (evidence.version === 1) return 'export default '+canonicalJson(evidence)+';';
|
|
94
|
+
return `import wasm from ${JSON.stringify(wasmPath)};
|
|
95
|
+
import { sqliteMemory } from ${JSON.stringify(sqlitePath)};
|
|
96
|
+
import { workerOwnerEvidence } from ${JSON.stringify(ownerHistoryPath)};
|
|
97
|
+
export default workerOwnerEvidence(${canonicalJson(evidence)}, sqliteMemory(wasm));`;
|
|
83
98
|
},
|
|
84
99
|
};
|
|
85
100
|
}
|
|
@@ -5,6 +5,8 @@ import { migrationStatements } from '../modules/portability/migration-sql.mjs';
|
|
|
5
5
|
import { destinationLedger, ledgerNames, matchMigrationLedger } from '../modules/portability/ledgers.mjs';
|
|
6
6
|
import { all, insist, PortabilityError } from '../modules/portability/storage.mjs';
|
|
7
7
|
import {migrationDirectory} from '../modules/deploy-kit/migration-directory.mjs';
|
|
8
|
+
import { collectMigrations } from '../modules/delivery-producer/migrations.mjs';
|
|
9
|
+
import { nativeMigrationHistory } from './portability-owners.mjs';
|
|
8
10
|
export {migrationDirectory};
|
|
9
11
|
|
|
10
12
|
// Seed-owned declarations, never archive metadata: the deploymentSpec prefix
|
|
@@ -19,18 +21,26 @@ export function platformMigrationDeclaration(input={}) {
|
|
|
19
21
|
|
|
20
22
|
// Reproduce the runner's stored SQL text, then recover SQLite's statement
|
|
21
23
|
// boundaries (migration-sql.mjs).
|
|
22
|
-
export async function migrationInventory(root,binding,{prefix},split) {
|
|
24
|
+
export async function migrationInventory(root,binding,{prefix},split,sources=[]) {
|
|
23
25
|
const directory=await migrationDirectory(root,binding);
|
|
24
|
-
|
|
26
|
+
const files=await Promise.all((await readdir(directory)).filter(name=>name.endsWith('.sql')).sort().map(async name=>{
|
|
25
27
|
const bytes=await readFile(join(directory,name));
|
|
26
28
|
return {name,...(split?{statements:migrationStatements(bytes.toString('utf8'),split)}:{}),sha256:createHash('sha256').update(bytes).digest('hex'),...(prefix===undefined?{}:{aliases:[prefix+name]})};
|
|
27
29
|
}));
|
|
30
|
+
if(sources.length) {
|
|
31
|
+
const owners=await collectMigrations(root,sources.filter(source=>source.directory).map(({directory,prefix})=>({binding,path:directory,prefix})),[{kind:'d1',binding}]);
|
|
32
|
+
for(const {name,sha256,sql} of owners) {
|
|
33
|
+
insist(!files.some(file=>file.name===name || file.aliases?.includes(name)),'migration_ledger:ambiguous',409);
|
|
34
|
+
files.push({name,sha256,...(split?{statements:migrationStatements(sql,split)}:{})});
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
return files;
|
|
28
38
|
}
|
|
29
39
|
|
|
30
40
|
// Preserve the runner's applied order, resolving file identities by hash.
|
|
31
|
-
export async function appliedMigrations(root,db,binding,declaration,split) {
|
|
32
|
-
const
|
|
33
|
-
return matchMigrationLedger(
|
|
41
|
+
export async function appliedMigrations(root,db,binding,declaration,split,sources=[]) {
|
|
42
|
+
const history=await nativeMigrationHistory(db,sources),files=await migrationInventory(root,binding,declaration,split,sources);
|
|
43
|
+
return matchMigrationLedger(history?.entries ?? [],files,declaration).map(name=>files.find(file=>file.name===name));
|
|
34
44
|
}
|
|
35
45
|
|
|
36
46
|
// Ledger contents can contain private values. Report constant categories,
|
|
@@ -87,17 +97,23 @@ async function legacyPairPreflight(db) {
|
|
|
87
97
|
// Adapter entry point, before *any* native Wrangler apply. The caller supplies
|
|
88
98
|
// the real target binding and keeps its writers stopped through apply/transfer.
|
|
89
99
|
// This only reads: it never repairs history or creates a second ledger.
|
|
90
|
-
export async function nativeMigrationPreflight({root,db,binding='DB',platformMigrations}) {
|
|
100
|
+
export async function nativeMigrationPreflight({root,db,binding='DB',platformMigrations,ownerSources=[]}) {
|
|
91
101
|
const declaration=platformMigrationDeclaration(platformMigrations);
|
|
92
102
|
const tables=await all(db,`SELECT type,name FROM sqlite_master WHERE lower(name) IN (${ledgerNames.map(name=>`'${name}'`).join(',')})`);
|
|
93
103
|
const legacyPair=tables.length===2 && ['_rer_deploy_migrations','d1_migrations'].every(name=>tables.some(row=>row.name===name));
|
|
94
104
|
if(tables.length>1 && !legacyPair)throw Object.assign(new PortabilityError('migration_ledger:multiple',409),tables.some(row=>row.name==='rer_migrations')?{identity:'rer_migrations'}:{});
|
|
95
105
|
insist(tables.every(row=>row.type==='table' && ledgerNames.includes(row.name)),'migration_ledger:incompatible',409);
|
|
96
106
|
const table=legacyPair?'_rer_deploy_migrations':tables[0]?.name;
|
|
97
|
-
if(!table)
|
|
107
|
+
if(!table) {
|
|
108
|
+
if(ownerSources.length) {
|
|
109
|
+
await nativeMigrationHistory(db,ownerSources);
|
|
110
|
+
await migrationInventory(root,binding,declaration,undefined,ownerSources);
|
|
111
|
+
}
|
|
112
|
+
return {nativeApply:true};
|
|
113
|
+
}
|
|
98
114
|
// This runner's history cannot authorize Wrangler to apply the same files.
|
|
99
115
|
insist(table!=='_rer_deploy_migrations' || legacyPair,'migration_ledger:runner',409);
|
|
100
|
-
const files=await migrationInventory(root,binding,declaration);
|
|
116
|
+
const files=await migrationInventory(root,binding,declaration,undefined,table==='rer_migrations'?ownerSources:[]);
|
|
101
117
|
let nativeApply=false;
|
|
102
118
|
try {
|
|
103
119
|
if(legacyPair)await legacyPairPreflight(db);
|
|
@@ -108,5 +124,11 @@ export async function nativeMigrationPreflight({root,db,binding='DB',platformMig
|
|
|
108
124
|
// The checkout file is named: it is the seed's own, never a ledger value.
|
|
109
125
|
if(table!=='d1_migrations')for(const file of files)if(!applied.includes(file.name))throw Object.assign(new PortabilityError('migration_ledger:unapplied',409),{migration:file.name});
|
|
110
126
|
} catch(error) { throw constantRefusal(error,files); }
|
|
127
|
+
if(ownerSources.length && table!=='rer_migrations') {
|
|
128
|
+
const history=await nativeMigrationHistory(db,ownerSources);
|
|
129
|
+
const allFiles=await migrationInventory(root,binding,declaration,undefined,ownerSources);
|
|
130
|
+
try { matchMigrationLedger(history.entries,allFiles,declaration); }
|
|
131
|
+
catch(error) { throw constantRefusal(error,allFiles); }
|
|
132
|
+
}
|
|
111
133
|
return {nativeApply};
|
|
112
134
|
}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto';
|
|
2
|
+
import { readFile, readdir, rm, writeFile } from 'node:fs/promises';
|
|
3
|
+
import { join } from 'node:path';
|
|
4
|
+
import { sourcePath } from '../modules/delivery-producer/migrations.mjs';
|
|
5
|
+
import { ledgerNames } from '../modules/portability/ledgers.mjs';
|
|
6
|
+
import { insist } from '../modules/portability/storage.mjs';
|
|
7
|
+
import { deliveryDeclarationSources } from './delivery-declarations.mjs';
|
|
8
|
+
import { parseJsonc } from './jsonc.mjs';
|
|
9
|
+
export { nativeMigrationHistory, ownerMigrationPolicy } from '../modules/portability/owner-history.mjs';
|
|
10
|
+
|
|
11
|
+
// Delivery declares the extra directories and their prefixes. Native Wrangler
|
|
12
|
+
// uses the edition's permanent <project>_<prefix>_migrations ledger for each.
|
|
13
|
+
export async function ownerMigrationSources(root, binding) {
|
|
14
|
+
let directories;
|
|
15
|
+
for (const [path, select] of deliveryDeclarationSources) {
|
|
16
|
+
const file = await sourcePath(root, path, { optional: true });
|
|
17
|
+
if (!file) continue;
|
|
18
|
+
directories = select(JSON.parse(await readFile(file, 'utf8')))?.extraMigrationDirectories;
|
|
19
|
+
if (directories !== undefined) break;
|
|
20
|
+
}
|
|
21
|
+
directories ??= [];
|
|
22
|
+
insist(Array.isArray(directories), 'migration_ledger:incompatible', 409);
|
|
23
|
+
const sources = directories.filter(source => source.binding === binding);
|
|
24
|
+
if (!sources.length) return [];
|
|
25
|
+
const { name } = JSON.parse(await readFile(join(root, 'package.json'), 'utf8'));
|
|
26
|
+
const project = name?.split('/').at(-1).replaceAll('-', '_');
|
|
27
|
+
insist(typeof project === 'string' && /^[a-z][a-z0-9_]*$/.test(project), 'migration_ledger:incompatible', 409);
|
|
28
|
+
const selected = await Promise.all(sources.map(async ({ path, prefix, optional }) => {
|
|
29
|
+
insist(typeof prefix === 'string' && /^[a-z][a-z0-9_-]*-$/.test(prefix), 'migration_ledger:incompatible', 409);
|
|
30
|
+
const table = project + '_' + prefix.slice(0, -1).replaceAll('-', '_') + '_migrations';
|
|
31
|
+
insist(!ledgerNames.includes(table), 'migration_ledger:ambiguous', 409);
|
|
32
|
+
const directory = await sourcePath(root, path, { kind: 'directory', optional: optional === true });
|
|
33
|
+
return { directory, prefix, table };
|
|
34
|
+
}));
|
|
35
|
+
insist(new Set(selected.map(source => source.table)).size === selected.length, 'migration_ledger:ambiguous', 409);
|
|
36
|
+
return selected;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export async function applyNativeOwnerMigrations({ root, binding, sources, store, signal, env }) {
|
|
40
|
+
if (!sources.some(source => source.directory)) return;
|
|
41
|
+
const config = parseJsonc(await readFile(join(root, 'wrangler.jsonc'), 'utf8'));
|
|
42
|
+
const target = config.d1_databases.find(entry => entry.binding === binding);
|
|
43
|
+
for (const { table, directory } of sources) {
|
|
44
|
+
if (!directory || !(await readdir(directory)).some(name => name.endsWith('.sql'))) continue;
|
|
45
|
+
const path = join(root, '.wrangler', `rer-owner-migrations-${randomUUID()}.json`);
|
|
46
|
+
try {
|
|
47
|
+
await writeFile(path, JSON.stringify({
|
|
48
|
+
name: config.name, compatibility_date: config.compatibility_date,
|
|
49
|
+
d1_databases: [{ ...target, migrations_dir: directory, migrations_table: table }],
|
|
50
|
+
}), { mode: 0o600, flag: 'wx' });
|
|
51
|
+
await store.run('Applying local owner migrations', 'pnpm', ['exec', 'wrangler', 'd1', 'migrations', 'apply', binding, '--local', '--persist-to', '.wrangler/state', '--config', path], { cwd: root, signal, env });
|
|
52
|
+
} finally { await rm(path, { force: true }); }
|
|
53
|
+
}
|
|
54
|
+
}
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
import initSQLite from './vendor/wa-sqlite/wa-sqlite.mjs';
|
|
2
|
+
|
|
3
|
+
const SQLITE_ROW = 100, SQLITE_DONE = 101;
|
|
4
|
+
|
|
5
|
+
// A compiled module is supplied by the Worker bundle. Each replay opens its
|
|
6
|
+
// own database in memory and executes only the checkout's bundled statements.
|
|
7
|
+
export function sqliteMemory(module) {
|
|
8
|
+
let ready;
|
|
9
|
+
return async function openDatabase() {
|
|
10
|
+
const sqlite = await (ready ??= initSQLite({
|
|
11
|
+
instantiateWasm(imports, accept) {
|
|
12
|
+
const instance = new WebAssembly.Instance(module, imports);
|
|
13
|
+
accept(instance, module);
|
|
14
|
+
return instance.exports;
|
|
15
|
+
},
|
|
16
|
+
}));
|
|
17
|
+
const pointer = sqlite._malloc(4);
|
|
18
|
+
if (!pointer) throw Error('Cannot allocate SQLite handle');
|
|
19
|
+
let db;
|
|
20
|
+
const check = code => {
|
|
21
|
+
if (code !== 0) throw Error(sqlite.UTF8ToString(sqlite._sqlite3_errmsg(db)));
|
|
22
|
+
};
|
|
23
|
+
const withSql = (sql, operation) => {
|
|
24
|
+
const size = sqlite.lengthBytesUTF8(sql) + 1, text = sqlite._malloc(size);
|
|
25
|
+
if (!text) throw Error('Cannot allocate SQLite statement');
|
|
26
|
+
try {
|
|
27
|
+
sqlite.stringToUTF8(sql, text, size);
|
|
28
|
+
check(operation(text));
|
|
29
|
+
} finally {
|
|
30
|
+
sqlite._free(text);
|
|
31
|
+
}
|
|
32
|
+
};
|
|
33
|
+
const exec = sql => withSql(sql, text => sqlite._sqlite3_exec(db, text, 0, 0, 0));
|
|
34
|
+
try {
|
|
35
|
+
const code = sqlite.ccall('sqlite3_open', 'number', ['string', 'number'], [':memory:', pointer]);
|
|
36
|
+
db = sqlite.getValue(pointer, '*');
|
|
37
|
+
check(code);
|
|
38
|
+
exec('PRAGMA foreign_keys=ON');
|
|
39
|
+
} catch (error) {
|
|
40
|
+
if (db) sqlite._sqlite3_close(db);
|
|
41
|
+
sqlite._free(pointer);
|
|
42
|
+
throw error;
|
|
43
|
+
}
|
|
44
|
+
return {
|
|
45
|
+
exec,
|
|
46
|
+
all(sql) {
|
|
47
|
+
withSql(sql, text => sqlite._sqlite3_prepare_v2(db, text, -1, pointer, 0));
|
|
48
|
+
const statement = sqlite.getValue(pointer, '*');
|
|
49
|
+
try {
|
|
50
|
+
const columns = Array.from({ length: sqlite._sqlite3_column_count(statement) }, (_, index) =>
|
|
51
|
+
sqlite.UTF8ToString(sqlite._sqlite3_column_name(statement, index)));
|
|
52
|
+
const rows = [];
|
|
53
|
+
let code;
|
|
54
|
+
while ((code = sqlite._sqlite3_step(statement)) === SQLITE_ROW) {
|
|
55
|
+
rows.push(Object.fromEntries(columns.map((name, index) => {
|
|
56
|
+
const value = sqlite._sqlite3_column_text(statement, index);
|
|
57
|
+
return [name, value ? sqlite.UTF8ToString(value) : null];
|
|
58
|
+
})));
|
|
59
|
+
}
|
|
60
|
+
if (code !== SQLITE_DONE) check(code);
|
|
61
|
+
return rows;
|
|
62
|
+
} finally {
|
|
63
|
+
sqlite._sqlite3_finalize(statement);
|
|
64
|
+
}
|
|
65
|
+
},
|
|
66
|
+
close() {
|
|
67
|
+
sqlite._sqlite3_close(db);
|
|
68
|
+
sqlite._free(pointer);
|
|
69
|
+
},
|
|
70
|
+
};
|
|
71
|
+
};
|
|
72
|
+
}
|
|
@@ -16,6 +16,7 @@ import { nativeFailure, stateBytes } from './portability-progress.mjs';
|
|
|
16
16
|
import { parseJsonc } from './jsonc.mjs';
|
|
17
17
|
import { loadPolicy } from './portability-policy.mjs';
|
|
18
18
|
import { appliedMigrations, platformMigrationDeclaration, nativeMigrationPreflight } from './portability-migrations.mjs';
|
|
19
|
+
import { nativeMigrationHistory, ownerMigrationPolicy, ownerMigrationSources } from './portability-owners.mjs';
|
|
19
20
|
|
|
20
21
|
// Transfers read only the D1 database and R2 buckets. A configuration with just
|
|
21
22
|
// those keeps the app's other bindings out of the proxy: a Durable Object class
|
|
@@ -51,11 +52,18 @@ async function run(direction, { root, path, journal, binding, buckets, policy, p
|
|
|
51
52
|
let proxy;
|
|
52
53
|
try {
|
|
53
54
|
proxy = await getPlatformProxy({ configPath, persist: { path: join(root, '.wrangler/state/v3') }, remoteBindings: false });
|
|
54
|
-
|
|
55
|
-
const
|
|
55
|
+
const db = proxy.env[binding], owners = await ownerMigrationSources(root, binding);
|
|
56
|
+
const migrationHistory = () => nativeMigrationHistory(db, owners);
|
|
57
|
+
if(direction==='migration-preflight') {
|
|
58
|
+
const result=await nativeMigrationPreflight({root,db,binding,platformMigrations,ownerSources:owners});
|
|
59
|
+
if(!owners.length)return result;
|
|
60
|
+
const history=await migrationHistory();
|
|
61
|
+
return {...result, nativeOwners: !history || history.table==='d1_migrations'};
|
|
62
|
+
}
|
|
63
|
+
const migrations=async()=>migrationReplay(await appliedMigrations(root,db,binding,declaration,unstable_splitSqlQuery,owners),policy,declaration);
|
|
56
64
|
const boundBuckets = Object.fromEntries(Object.entries(buckets).map(([name, binding]) => [name, proxy.env[binding]]));
|
|
57
|
-
policy = await loadPolicy(root, inputPolicy, policyModule, { db
|
|
58
|
-
const options={ db
|
|
65
|
+
policy = ownerMigrationPolicy(await loadPolicy(root, inputPolicy, policyModule, { db, buckets: boundBuckets }), owners);
|
|
66
|
+
const options={ db, buckets: boundBuckets, policy, migrations, migrationHistory };
|
|
59
67
|
if(direction==='schema-recovery-preflight') { await requireProductEmpty(options);return {checkedEmpty:true}; }
|
|
60
68
|
if(direction==='legacy-preflight') {
|
|
61
69
|
await requireLegacyEmpty(options,await legacyTargetState(options.db,options.migrations));
|
|
@@ -83,7 +91,7 @@ catch (error) {
|
|
|
83
91
|
// Report symbolic codes and identities verified against the checkout or the
|
|
84
92
|
// known Platform ledger name; native database errors can contain private data.
|
|
85
93
|
const ledgerIdentity = error instanceof PortabilityError && error.code.startsWith('migration_ledger:') ? error.migration ?? error.identity : undefined;
|
|
86
|
-
const reason = error instanceof PortabilityError ? (error.code.startsWith('migration_ledger:') ? error.code + (ledgerIdentity ? ':' + ledgerIdentity : '') : error.code.split(':')[0]) : /^[A-Z_]+$/.test(error.code ?? '') ? error.code : error.message === 'archive_json' ? 'archive_json' : 'native_error';
|
|
94
|
+
const reason = error instanceof PortabilityError ? (error.code.startsWith('migration_ledger:') ? error.code + (ledgerIdentity ? ':' + ledgerIdentity : '') : error.code === 'schema_mismatch:owner_migration_history' ? error.code : error.code.split(':')[0]) : /^[A-Z_]+$/.test(error.code ?? '') ? error.code : error.message === 'archive_json' ? 'archive_json' : 'native_error';
|
|
87
95
|
// A later migration that changes restorable data is named: it is a file of this installation.
|
|
88
96
|
const migration = error instanceof PortabilityError && error.code.startsWith('schema_mismatch:migration_data:') ? error.code.slice('schema_mismatch:migration_data:'.length) : undefined;
|
|
89
97
|
// An unapplied migration is a file of this checkout: it is named so it can be applied.
|
package/cli/portability.mjs
CHANGED
|
@@ -10,6 +10,7 @@ import { shared } from './shared.mjs';
|
|
|
10
10
|
import { emptyNativePreflight } from './portability-empty.mjs';
|
|
11
11
|
import { withManagedDataImport } from './managed-data.mjs';
|
|
12
12
|
import { ledgerRefusal } from './ledger-refusal.mjs';
|
|
13
|
+
import { applyNativeOwnerMigrations, ownerMigrationSources } from './portability-owners.mjs';
|
|
13
14
|
|
|
14
15
|
export { withManagedDataImport };
|
|
15
16
|
|
|
@@ -71,14 +72,14 @@ export function localPortability({ database, binding = 'DB', buckets = {}, polic
|
|
|
71
72
|
}
|
|
72
73
|
}
|
|
73
74
|
if(unverifiedSchemaRecovery)await nativeWorker(store,'schema-recovery-preflight',input,signal);
|
|
74
|
-
const {nativeApply}=await nativeWorker(store,'migration-preflight',input,signal);
|
|
75
|
+
const {nativeApply,nativeOwners}=await nativeWorker(store,'migration-preflight',input,signal);
|
|
75
76
|
// Local migrations need no provider account: wrangler runs without credentials.
|
|
76
77
|
// The shared runner shows its progress (a count every ten migrations) and leaves out its
|
|
77
78
|
// own info lines, which only confuse a local backup; warnings and errors print.
|
|
78
79
|
const { dependencyEnvironment } = await shared('dependency-environment.mjs');
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
});
|
|
80
|
+
const env = { ...dependencyEnvironment(), RER_LOCAL_OPERATION: operationToken, WRANGLER_SEND_METRICS: 'false' };
|
|
81
|
+
if(nativeApply)await store.run('Applying local data migrations', 'pnpm', ['exec', 'wrangler', 'd1', 'migrations', 'apply', database, '--local', '--persist-to', '.wrangler/state', '--config', 'wrangler.jsonc'], { cwd: root, signal, env });
|
|
82
|
+
if(direction==='import' && nativeOwners)await applyNativeOwnerMigrations({root,binding,sources:await ownerMigrationSources(root,binding),store,signal,env});
|
|
82
83
|
const receipt=await nativeWorker(store,direction,input,signal);
|
|
83
84
|
if(legacyRecovery)receipt.output=relative(root,legacyRecovery.output);
|
|
84
85
|
return { target: 'local', [direction === 'export' ? 'output' : 'input']: relative(root, path), ...receipt };
|
|
@@ -107,6 +108,7 @@ async function nativeWorker(store,direction,input,signal) {
|
|
|
107
108
|
: receipt.reExport ? `Restore refused (schema_mismatch): the schema definitions cannot be safely verified by this version. ${reExport}`
|
|
108
109
|
: receipt.identityTable ? `Transfer refused (identity_table_undeclared:${receipt.identityTable}): declare this table in includeTables to transfer its rows or excludeTables to omit them.`
|
|
109
110
|
: receipt.safety ? `Restore refused (${receipt.reason}): safety normalization for ${receipt.safety} violates the declared import policy.`
|
|
111
|
+
: receipt.reason === 'schema_mismatch:owner_migration_history' ? 'Restore refused (schema_mismatch:owner_migration_history): the recorded migration order or file contents differ from this checkout.'
|
|
110
112
|
: receipt.reason?.startsWith('migration_ledger:') ? ledgerRefusal(receipt)
|
|
111
113
|
: receipt.migration ? `Restore refused (schema_mismatch:migration_data): migration ${receipt.migration} changed data after this archive was made, so its rows would skip that change. Restore an archive made after it.`
|
|
112
114
|
: receipt.occupied ? occupiedRefusal(input,receipt.reason)
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2023 Roy T. Hashimoto
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
# SQLite for Worker migration replay
|
|
2
|
+
|
|
3
|
+
The sync ES module and Wasm binary come from `dist/` in
|
|
4
|
+
[`@journeyapps/wa-sqlite@2.0.10`](https://www.npmjs.com/package/@journeyapps/wa-sqlite/v/2.0.10).
|
|
5
|
+
The glue uses its existing virtual temporary directory as the sandbox home
|
|
6
|
+
and omits the upstream simulated user home. The package builds SQLite 3.53.0
|
|
7
|
+
with FTS5 and is distributed under the MIT license in this directory. Only
|
|
8
|
+
the owner migration evidence module imports this engine.
|
|
9
|
+
|
|
10
|
+
To retrieve the sources, run `npm pack @journeyapps/wa-sqlite@2.0.10` and
|
|
11
|
+
extract the two named files from `package/dist/` and `package/LICENSE`.
|
|
12
|
+
The tarball has npm integrity
|
|
13
|
+
`sha512-6Wo5ZijOc4/6M9u+MDsJdW8zDCdrct/p+7LcG7S7CT5YKu42j6lshMfUvgtGYMaZeuH7rpQHULZupPTQFqbS0w==`.
|