@runeditrun/rer 0.0.0-stage → 0.1.1

package/modules/portability/catalogue.mjs

@@ -0,0 +1,118 @@
+import { all, insist, quote, PortabilityError } from './storage.mjs';
+import { canonicalJson, sha256, base64, unbase64, utf8 } from './hash.mjs';
+const defaults=['sqlite_%','_cf_%','d1_migrations','_rer_deploy_migrations','_rer_portability_migrations','rer_portability_%','rer_mail_development_outbox','rer_ai_development_outbox','oa_%','oc_%','rer_dependency_keys','rer_dependency_pending','rer_dependency_codes'];
+const fields=['product','excludeTables','includeTables','derivedTables','secretColumns','exportRow','replaceableTables','requiredConfiguration','validateStaged','acceptedSchemas','buckets','extraPayloads','partBytes'];
+const name=value=>typeof value==='string' && value.length>0 && value.length<=512 && !/[\x00-\x1f]/.test(value);
+const names=value=>Array.isArray(value) && value.every(name) && new Set(value).size===value.length;
+export function policy(input) {
+  insist(input && typeof input==='object' && Object.keys(input).every(k=>fields.includes(k)) && name(input.product),'invalid_policy');
+  const result={excludeTables:[],includeTables:[],derivedTables:[],secretColumns:{},replaceableTables:[],requiredConfiguration:[],acceptedSchemas:{},buckets:{},extraPayloads:{},partBytes:16*1024*1024,...input};
+  for(const key of ['excludeTables','includeTables','derivedTables','replaceableTables','requiredConfiguration']) insist(names(result[key]),'invalid_policy_'+key);
+  for(const key of ['exportRow','validateStaged']) insist(result[key]===undefined || typeof result[key]==='function','invalid_policy_'+key);
+  insist(Number.isSafeInteger(result.partBytes) && result.partBytes>=1024 && result.partBytes<=16*1024*1024,'invalid_part_limit');
+  for(const [table,columns] of Object.entries(result.secretColumns)) insist(name(table) && names(columns),'invalid_secret_columns');
+  for(const [identity,entry] of Object.entries(result.acceptedSchemas)) insist(/^[a-f0-9]{64}$/.test(identity) && typeof entry.upgradeRow==='function','invalid_accepted_schema');
+  for(const [binding,entry] of Object.entries(result.buckets)) {
+    insist(name(binding) && entry && Object.keys(entry).every(k=>['prefix','objectReferences'].includes(k)),'invalid_bucket_policy');
+    insist(entry.prefix===undefined || typeof entry.prefix==='string','invalid_bucket_prefix');
+    if(entry.objectReferences) insist(name(entry.objectReferences.table) && name(entry.objectReferences.keyColumn),'invalid_object_references');
+  }
+  for(const [key,entry] of Object.entries(result.extraPayloads)) insist(name(key) && typeof entry.produce==='function' && typeof entry.consume==='function','invalid_extra_payload');
+  for(const key of ['secretColumns','acceptedSchemas','buckets','extraPayloads'])result[key]=Object.assign(Object.create(null),result[key]);
+  return {...result,excludeTables:[...defaults,...result.excludeTables]};
+}
+const identifier=value=>value.replace(/[A-Z]/g,letter=>letter.toLowerCase());
+const matches=(table,pattern)=>pattern.endsWith('%') ? identifier(table).startsWith(identifier(pattern.slice(0,-1))) : identifier(table)===identifier(pattern);
+async function archived(db,config) {
+  const objects=await all(db,"SELECT name,sql,type,tbl_name FROM sqlite_master WHERE sql IS NOT NULL ORDER BY name");
+  const kinds=await all(db,'PRAGMA table_list');
+  const derived=config.derivedTables;
+  const excluded=objects.filter(x=>x.type==='table' && !config.includeTables.includes(x.name) &&
+    (config.excludeTables.some(p=>matches(x.name,p)) || derived.some(p=>x.name===p || (kinds.find(k=>k.name===x.name)?.type==='shadow' && x.name.startsWith(p+'_')))));
+  objects.sort((a,b)=>a.name<b.name?-1:a.name>b.name?1:0);
+  return {objects,kinds,excluded,entries:objects.filter(x=>x.type==='table' && !excluded.some(y=>y.name===x.name))};
+}
+const identityOf=tables=>sha256(canonicalJson(tables.map(({name,sql})=>({name,sql}))));
+// The archive schema identity alone, for databases that need not satisfy the policy.
+export async function schemaIdentity(db,config) { return identityOf((await archived(db,config)).entries); }
+export async function catalogue(db,config) {
+  const {objects,kinds,excluded,entries}=await archived(db,config);
+  const tables=[];
+  for(const entry of entries) {
+    insist(!/^CREATE\s+VIRTUAL\s+TABLE/i.test(entry.sql) && kinds.find(x=>x.name===entry.name)?.type!=='shadow','virtual_table:'+entry.name);
+    const columns=(await all(db,`PRAGMA table_xinfo(${quote(entry.name)})`)).map(c=>({name:c.name,type:c.type,notnull:c.notnull,pk:c.pk,hidden:c.hidden}));
+    insist(columns.every(c=>[0,2,3].includes(c.hidden) && !c.name.startsWith('rer_portability_')),'unsupported_column:'+entry.name);
+    columns.forEach(c=>{if(c.hidden)c.generated=c.hidden===2?'virtual':'stored';delete c.hidden;});
+    // Both archived base cells and visible staging reads must fit D1's SQL
+    // limit, with room for copy names and joins.
+    insist([false,true].every(encoded=>utf8(projectedColumns({columns},encoded)).length+1024<=100000),'transfer_sql_limit:'+entry.name);
+    const dependencies=[...new Set((await all(db,`PRAGMA foreign_key_list(${quote(entry.name)})`)).map(ref=>objects.find(x=>x.type==='table' && identifier(x.name)===identifier(ref.table))?.name??ref.table))].sort();
+    for(const parent of dependencies) insist(!excluded.some(x=>x.name===parent),'excluded_foreign_key:'+entry.name+':'+parent);
+    const rowidAlias=columns.some(c=>c.pk)?null:['rowid','_rowid_','oid'].find(alias=>!columns.some(c=>c.name.toLowerCase()===alias));
+    insist(rowidAlias!==undefined,'unsupported_rowid:'+entry.name);
+    insist(['rowid','_rowid_','oid'].some(alias=>!columns.some(c=>c.name.toLowerCase()===alias)),'unsupported_copy_rowid:'+entry.name);
+    tables.push({name:entry.name,sql:entry.sql,columns,dependencies,rowidAlias});
+  }
+  for(const [table,list] of Object.entries(config.secretColumns)) {
+    const entry=tables.find(t=>t.name===table); insist(entry,'secret_table_missing:'+table);
+    for(const key of list) {const column=entry.columns.find(c=>c.name===key);insist(column && !column.notnull && !column.pk && !column.generated,'invalid_secret_column:'+table+':'+key);}
+  }
+  for(const table of [...config.replaceableTables,...config.includeTables]) insist(tables.some(x=>x.name===table),'policy_table_missing:'+table);
+  for(const spec of Object.values(config.buckets)) if(spec.objectReferences) {
+    const {table,keyColumn}=spec.objectReferences;
+    insist(tables.find(x=>x.name===table)?.columns.some(c=>c.name===keyColumn),'reference_column_missing');
+    insist(!config.secretColumns[table]?.includes(keyColumn),'reference_column_excluded');
+  }
+  const identity=await identityOf(tables);
+  return {identity,tables,excludedTables:excluded.map(x=>x.name).sort(),triggers:objects.filter(x=>x.type==='trigger' && tables.some(t=>t.name===x.tbl_name))};
+}
+export function affinity(type) {
+  const text=type.toUpperCase();
+  if(text.includes('INT')) return 'INTEGER';
+  if(/CHAR|CLOB|TEXT/.test(text)) return 'TEXT';
+  if(text==='' || text.includes('BLOB')) return 'BLOB';
+  return 'NUMBER';
+}
+export function projectedColumns(table,encoded=false) {
+  return (encoded?writableColumns(table):table.columns).map(c=>encoded && affinity(c.type)==='INTEGER'?`CAST(s.${quote(c.name)} AS TEXT) AS ${quote(c.name)}`:`s.${quote(c.name)}`).join(',');
+}
+export const writableColumns=table=>table.columns.filter(c=>!c.generated);
+export function validateRow(table,row,secrets=[]) {
+  const columns=writableColumns(table).filter(c=>!secrets.includes(c.name));
+  insist(row && typeof row==='object' && !Array.isArray(row) && canonicalJson(Object.keys(row).sort())===canonicalJson(columns.map(c=>c.name).sort()),'row_columns:'+table.name);
+  for(const column of columns) {
+    const value=row[column.name],type=affinity(column.type), label=table.name+':'+column.name;
+    if(value===null) {insist(!column.notnull,'row_not_null:'+label);continue;}
+    if(typeof value==='object') {
+      insist(Object.keys(value).length===1 && typeof value.blob==='string','row_type:'+label); unbase64(value.blob);
+      insist(type==='BLOB','row_type:'+label);continue;
+    }
+    insist(typeof value==='string' || (typeof value==='number' && Number.isFinite(value)),'row_type:'+label);
+    if(type==='INTEGER') insist(typeof value==='number' && Number.isSafeInteger(value),'unsafe_integer:'+label);
+    if(type==='NUMBER') insist(typeof value==='number','row_type:'+label);
+    if(type==='TEXT') insist(typeof value==='string','row_type:'+label);
+  }
+}
+export function encodedRow(table,raw,secrets=[]) {
+  const row=Object.create(null);
+  for(const column of writableColumns(table)) {
+    if(secrets.includes(column.name)) continue;
+    let value=raw[column.name];
+    if(affinity(column.type)==='INTEGER' && value!==null) {
+      insist(/^-?\d+$/.test(String(value)) && Number.isSafeInteger(Number(value)),'unsafe_integer:'+table.name+':'+column.name);
+      value=Number(value);
+    }
+    if(Array.isArray(value) || value instanceof ArrayBuffer || ArrayBuffer.isView(value)) value={blob:base64(Array.isArray(value)?Uint8Array.from(value):new Uint8Array(value.buffer??value,value.byteOffset??0,value.byteLength))};
+    row[column.name]=value;
+  }
+  validateRow(table,row,secrets); return row;
+}
+export function bindCell(value) { return value && typeof value==='object' ? unbase64(value.blob).buffer : value; }
+export function mapRow(config,table,row) {
+  if(!config.exportRow) return row;
+  const result=config.exportRow(table.name,structuredClone(row));
+  insist(!(result instanceof Promise),'export_row_must_be_pure');
+  validateRow(table,result,config.secretColumns[table.name]??[]);
+  insist(canonicalJson(config.exportRow(table.name,structuredClone(result)))===canonicalJson(result),'export_row_not_idempotent:'+table.name);
+  return result;
+}

package/modules/portability/cli-options.mjs

@@ -0,0 +1,38 @@
+import { lstat, readFile, mkdir, open, rename } from 'node:fs/promises';
+import { isAbsolute, dirname } from 'node:path';
+import { constants } from 'node:fs';
+export function options(args) {
+  const command=args.shift(),result={command};
+  if(!['export','import','status','abort'].includes(command))throw new Error('Choose export, import, status or abort.');
+  const allowed=['origin','session-file','out','from','resume','base-path'];
+  while(args.length){const key=args.shift();if(!key?.startsWith('--') || !allowed.includes(key.slice(2)) || result[key.slice(2)]!==undefined || !args.length)throw new Error('Invalid command options.');result[key.slice(2)]=args.shift();}
+  let url;try{url=new URL(result.origin);}catch{throw new Error('Supply a valid origin.');}
+  if(url.username || url.password || url.pathname!=='/' || url.search || url.hash)throw new Error('The origin must have no credentials, path, query or fragment.');
+  if(url.protocol!=='https:' && !(url.protocol==='http:' && /^http:\/\/(localhost|127\.0\.0\.1|\[::1\])(?::\d+)?\/?$/.test(result.origin)))throw new Error('Use HTTPS or an exact loopback HTTP origin.');
+  result.origin=url.origin;
+  for(const key of ['session-file',...(command==='export'?['out']:command==='import'?['from']:[])])if(!isAbsolute(result[key]??''))throw new Error('File paths must be absolute.');
+  if(['status','abort'].includes(command) && !result.resume)throw new Error('Supply --resume with the job id.');
+  if(result.resume && !/^[a-zA-Z0-9-]{1,100}$/.test(result.resume))throw new Error('Invalid job id.');
+  result.basePath=result['base-path']??'/api/portability';
+  if(!/^\/[A-Za-z0-9/_-]+$/.test(result.basePath) || result.basePath.endsWith('/'))throw new Error('Invalid API base path.');
+  return result;
+}
+export async function cookieFile(path) {
+  const info=await lstat(path);if(!info.isFile() || info.size>4096)throw new Error('The session file must be a small regular file.');
+  const file=await open(path,constants.O_RDONLY|constants.O_NOFOLLOW);
+  let value;try{value=(await file.readFile('utf8')).trim();}finally{await file.close();}
+  if(!/^[!#$%&'*+.^_`|~0-9A-Za-z-]+=[!#-:<-~]+$/.test(value) || value.includes(';'))throw new Error('The session file must hold one cookie.');
+  return value;
+}
+export async function absent(path) {
+  try{await lstat(path);}catch(e){if(e.code==='ENOENT')return;throw e;}throw new Error('The output already exists.');
+}
+export async function privateFile(path,resume=false) {
+  await mkdir(dirname(path),{recursive:true,mode:0o700});
+  const file=await open(path,constants.O_WRONLY|constants.O_CREAT|constants.O_NOFOLLOW|(resume?constants.O_TRUNC:constants.O_EXCL),0o600);
+  await file.chmod(0o600);return file;
+}
+export async function checkpoint(path,value) {
+  const file=await privateFile(path,true);try{await file.writeFile(JSON.stringify(value)+'\n');await file.sync();}finally{await file.close();}
+}
+export async function promote(partial,path) {await rename(partial,path);}

package/modules/portability/cli.mjs

@@ -0,0 +1,56 @@
+#!/usr/bin/env node
+import { createReadStream,existsSync,realpathSync } from 'node:fs';
+import { Readable } from 'node:stream';
+import { pathToFileURL, fileURLToPath } from 'node:url';
+import { options, cookieFile, absent, privateFile, checkpoint, promote } from './cli-options.mjs';
+import { readArchive } from './format.mjs';
+import { canonicalJson } from './hash.mjs';
+export async function main(args) {
+  const opts=options([...args]),cookie=await cookieFile(opts['session-file']);
+  const endpoint=opts.origin+opts.basePath;
+  async function request(path,method='GET',body) {
+    let response;
+    try{response=await fetch(endpoint+'/'+path,{method,redirect:'error',headers:{Cookie:cookie,Origin:opts.origin,...(body===undefined?{}:{'Content-Type':'application/json'})},body:body===undefined?undefined:JSON.stringify(body)});}
+    catch{throw new Error('The transfer request failed.');}
+    if(!response.ok) {await response.body?.cancel();throw new Error('The transfer was refused (HTTP '+response.status+').');}
+    return response;
+  }
+  const get=async(...args)=>(await request(...args)).json();
+  const shell=value=>"'"+value.replaceAll("'","'\\''")+"'";
+  const resumeCommand=(id)=>'node '+shell(fileURLToPath(import.meta.url))+' '+opts.command+' --origin '+shell(opts.origin)+' --session-file '+shell(opts['session-file'])+' '+(opts.command==='export'?'--out '+shell(opts.out):'--from '+shell(opts.from))+' --resume '+id;
+  if(opts.command==='export') {
+    await absent(opts.out);
+    const id=opts.resume??(await get('exports','POST',{})).id;
+    await checkpoint(opts.out+'.transfer.json',{version:1,kind:'export',id,origin:opts.origin});
+    process.stdout.write('Job '+id+'\nResume: '+resumeCommand(id)+'\n');
+    let status=await get('exports/'+id);
+    while(status.phase==='building')status=await get('exports/'+id+'/advance','POST',{expectedPart:status.completed,maxParts:16});
+    if(status.phase!=='complete')throw new Error('The export is not complete.');
+    const response=await request('exports/'+id+'/download'),file=await privateFile(opts.out+'.partial',Boolean(opts.resume));
+    try{for await(const record of readArchive(response.body))await file.writeFile(canonicalJson(record)+'\n');await file.sync();}
+    finally{await file.close();}
+    await promote(opts.out+'.partial',opts.out);process.stdout.write('Export complete.\n');return;
+  }
+  if(opts.command==='import') {
+    const source=Readable.toWeb(createReadStream(opts.from));let id=opts.resume,closing;
+    // Verify every record even when earlier uploads are already acknowledged.
+    for await(const record of readArchive(source)) {
+      if(record.kind==='manifest') {
+        id??=(await get('imports','POST',{confirmed:true,manifest:record})).id;
+        await checkpoint(opts.from+'.transfer.json',{version:1,kind:'import',id,origin:opts.origin});
+        process.stdout.write('Job '+id+'\nResume: '+resumeCommand(id)+'\n');
+      }else if(record.kind==='part')await get('imports/'+id+'/parts/'+record.index,'PUT',record);
+      else closing=record;
+    }
+    const result=await get('imports/'+id+'/finalize','POST',{contentHash:closing.contentHash,archiveHash:closing.archiveHash});
+    process.stdout.write('Import '+result.phase+'.\nRequired configuration: '+result.requiredConfiguration.join(', ')+'\n');return;
+  }
+  let collection='exports';
+  let status;try{status=await get(collection+'/'+opts.resume);}catch{collection='imports';status=await get(collection+'/'+opts.resume);}
+  if(opts.command==='abort')status=await get(collection+'/'+opts.resume+'/abort','POST',{});
+  process.stdout.write(JSON.stringify(status)+'\n');
+}
+if(process.argv[1] && existsSync(process.argv[1]) && import.meta.url===pathToFileURL(realpathSync(process.argv[1])).href)main(process.argv.slice(2)).catch(()=>{
+  // Errors may contain server diagnostics or filesystem paths. Do not echo them.
+  process.stderr.write('Transfer failed: check the origin, absolute paths, output, session file and job status.\n');process.exitCode=1;
+});

package/modules/portability/copies.mjs

@@ -0,0 +1,68 @@
+import { all,statement,quote,insist } from './storage.mjs';
+import { projectedColumns,encodedRow,mapRow,bindCell,writableColumns } from './catalogue.mjs';
+import { canonicalJson } from './hash.mjs';
+import { generatedCopySQL } from './generated.mjs';
+
+export const sidecar=copy=>copy+'_meta';
+export function copyRowid(table) {
+  const alias=['rowid','_rowid_','oid'].find(a=>!table.columns.some(c=>c.name.toLowerCase()===a));
+  insist(alias,'unsupported_copy_rowid:'+table.name);return quote(alias);
+}
+const keys=table=>table.columns.filter(c=>c.pk).sort((a,b)=>a.pk-b.pk);
+const order=table=>keys(table).length?keys(table).map(c=>quote(c.name)).join(','):quote(table.rowidAlias);
+export function createCopy(db,table,copy,snapshot=false) {
+  const ordering=order(table),meta=quote(sidecar(copy));
+  const columns=writableColumns(table).map(c=>quote(c.name)).join(',');
+  const product=table.columns.some(c=>c.generated)
+    ?[db.prepare(generatedCopySQL(table,copy)),...(snapshot?[db.prepare(`INSERT INTO ${quote(copy)} SELECT ${columns} FROM ${quote(table.name)} ORDER BY ${ordering}`)]:[])]
+    :[db.prepare(`CREATE TABLE ${quote(copy)} AS SELECT ${table.added?columns:'*'} FROM ${quote(table.name)} ${snapshot?'ORDER BY '+ordering:'WHERE 0'}`)];
+  return [...product,
+    db.prepare(`CREATE TABLE ${meta}(rer_portability_row_order INTEGER PRIMARY KEY,rer_portability_copy_rowid INTEGER NOT NULL UNIQUE,rer_portability_source_rowid INTEGER,rer_portability_tombstone TEXT)`),
+    ...(snapshot?[db.prepare(`INSERT INTO ${meta} SELECT ROW_NUMBER() OVER(ORDER BY ${ordering})-1,ROW_NUMBER() OVER(ORDER BY ${ordering}),${table.rowidAlias?quote(table.rowidAlias):'NULL'},NULL FROM ${quote(table.name)}`)]:[])];
+}
+export async function readCopy(db,table,copy,offset,limit,encoded=false) {
+  const columns=projectedColumns(table,encoded);
+  const raw=await all(db,`SELECT ${columns} FROM ${quote(copy)} s JOIN ${quote(sidecar(copy))} m ON s.${copyRowid(table)}=m.rer_portability_copy_rowid ORDER BY m.rer_portability_row_order LIMIT ? OFFSET ?`,limit,offset);
+  const meta=await all(db,`SELECT * FROM ${quote(sidecar(copy))} ORDER BY rer_portability_row_order LIMIT ? OFFSET ?`,limit,offset);
+  insist(raw.length===meta.length,'copy_inventory:'+table.name);
+  return raw.map((row,i)=>({row,meta:meta[i]}));
+}
+export async function normalizeCopy(context,table,copy) {
+  const {db,config}=context,secrets=config.secretColumns[table.name]??[];
+  // Generated reads must reflect exactly the base cells a restore will receive.
+  if(table.columns.some(c=>c.generated)&&secrets.length)await db.prepare(`UPDATE ${quote(copy)} SET ${secrets.map(name=>quote(name)+'=NULL').join(',')}`).run();
+  if(config.exportRow)for(let offset=0;offset<table.count;offset+=500) {
+    const statements=[];
+    for(const {row,meta} of await readCopy(db,table,copy,offset,500,true)) {
+      const before=encodedRow(table,row,secrets),after=mapRow(config,table,before);
+      if(canonicalJson(before)===canonicalJson(after))continue;
+      const columns=writableColumns(table).filter(c=>!secrets.includes(c.name));
+      // The rowid is internal, checked, and literal so a 100-column update needs only 100 bindings.
+      insist(Number.isSafeInteger(meta.rer_portability_copy_rowid),'copy_rowid');
+      statements.push(statement(db,`UPDATE ${quote(copy)} SET ${columns.map(c=>quote(c.name)+'=?').join(',')} WHERE ${copyRowid(table)}=${meta.rer_portability_copy_rowid}`,...columns.map(c=>bindCell(after[c.name]))));
+    }
+    if(statements.length)await db.batch(statements);
+  }
+  const pk=keys(table);if(!pk.length)return;
+  const normalizedOrder=pk.map(c=>'s.'+quote(c.name)).join(',')+',m.rer_portability_copy_rowid';
+  const temporary=sidecar(copy)+'_sorted';
+  await db.batch([db.prepare(`CREATE TABLE ${quote(temporary)} AS SELECT ROW_NUMBER() OVER(ORDER BY ${normalizedOrder})-1 AS rer_portability_row_order,m.rer_portability_copy_rowid,m.rer_portability_source_rowid,m.rer_portability_tombstone FROM ${quote(copy)} s JOIN ${quote(sidecar(copy))} m ON s.${copyRowid(table)}=m.rer_portability_copy_rowid`),
+    db.prepare(`DROP TABLE ${quote(sidecar(copy))}`),db.prepare(`ALTER TABLE ${quote(temporary)} RENAME TO ${quote(sidecar(copy))}`)]);
+}
+export function restoreCopy(db,table,copy) {
+  const columns=writableColumns(table).map(c=>quote(c.name)),names=columns.join(','),projection=table.columns.some(c=>c.generated)?columns.map(c=>'s.'+c).join(','):'s.*',meta=quote(sidecar(copy));
+  const from=`FROM ${quote(copy)} s JOIN ${meta} m ON s.${copyRowid(table)}=m.rer_portability_copy_rowid ORDER BY m.rer_portability_row_order`;
+  // SQLite's implicit insert list contains only writable columns. Keeping it
+  // implicit also avoids repeating long column names within D1's SQL limit.
+  // A bridged table names its archived columns so later ones take defaults.
+  const into=quote(table.name)+(table.added?` (${names})`:'');
+  if(!table.rowidAlias)return [db.prepare(`INSERT INTO ${into} SELECT ${projection} ${from}`)];
+  if(columns.length<100&&!table.columns.some(c=>c.generated))return [db.prepare(`INSERT INTO ${quote(table.name)} (${names},${quote(table.rowidAlias)}) SELECT ${projection},m.rer_portability_source_rowid ${from}`)];
+  // Avoid selecting 101 columns or repeating generated-table base column names.
+  // Restore rowids in the same atomic batch using temporary negative ids below
+  // both the destination and source ranges.
+  const alias=quote(table.rowidAlias),base=`(SELECT MIN(0,MIN(rer_portability_source_rowid))-COUNT(*) FROM ${meta})`;
+  return [db.prepare(`INSERT INTO ${into} SELECT ${projection} ${from}`),
+    db.prepare(`UPDATE ${quote(table.name)} SET ${alias}=${base}-${alias}`),
+    db.prepare(`UPDATE ${quote(table.name)} AS t SET ${alias}=(SELECT rer_portability_source_rowid FROM ${meta} WHERE rer_portability_row_order=${base}-t.${alias}-1)`)];
+}

package/modules/portability/dev-data.mjs

@@ -0,0 +1,32 @@
+import { quote, all, insist } from './storage.mjs';
+import { sidecar, copyRowid } from './copies.mjs';
+
+// Local dev identity never travels. When a policy includes operator-auth tables,
+// export drops dev sessions and every credential minted while using one, and
+// suspends dev personas (<as>@dev.localhost); import refuses archives holding
+// dev sessions or dev-minted credentials. These rules are not policy options.
+const PERSONAS = "lower(email) LIKE '%@dev.localhost'";
+export async function dropDevData(db, tables, copies) {
+  const live = (await all(db, "SELECT name FROM sqlite_master WHERE type='table' AND name='oa_dev_credentials'")).length;
+  const minted = copies.oa_dev_credentials ? quote(copies.oa_dev_credentials) : live ? 'oa_dev_credentials' : null;
+  // Tokens read the credential record, so the record itself is emptied last.
+  const rules = [['oa_sessions', "kind='dev'"], ['oa_credentials', "kind='dev'"], ...(minted ? [['oa_tokens', `id IN (SELECT id FROM ${minted} WHERE credential='oa_tokens')`]] : []), ['oa_dev_credentials', '1']];
+  // Only operator-auth's own tables: a product table may share a name prefix.
+  const has = (table, ...names) => names.every(name => tables.find(t => t.name === table)?.columns.some(c => c.name === name));
+  const shaped = { oa_credentials: has('oa_credentials', 'kind'), oa_sessions: has('oa_sessions', 'kind'), oa_tokens: has('oa_tokens', 'id'), oa_dev_credentials: has('oa_dev_credentials', 'credential', 'id') };
+  const statements = [];
+  for (const [table, condition] of rules.filter(([table]) => copies[table] && shaped[table])) {
+    const copy = quote(copies[table]), entry = tables.find(t => t.name === table);
+    statements.push(db.prepare(`DELETE FROM ${quote(sidecar(copies[table]))} WHERE rer_portability_copy_rowid IN (SELECT ${copyRowid(entry)} FROM ${copy} WHERE ${condition})`));
+    statements.push(db.prepare(`DELETE FROM ${copy} WHERE ${condition}`));
+  }
+  if (copies.oa_operators && has('oa_operators', 'email', 'status', 'version')) statements.push(db.prepare(`UPDATE ${quote(copies.oa_operators)} SET status='inactive',version=version+1 WHERE ${PERSONAS} AND status!='inactive'`));
+  if (statements.length) await db.batch(statements);
+}
+
+export async function refuseDevData(db, copies, tables) {
+  const has = (table, name) => tables.find(t => t.name === table)?.columns.some(c => c.name === name);
+  if(copies.oa_credentials && has('oa_credentials','kind'))insist(!(await all(db, `SELECT 1 FROM ${quote(copies.oa_credentials)} WHERE kind='dev' LIMIT 1`)).length,'dev_data:oa_credentials',409);
+  if (copies.oa_sessions && has('oa_sessions', 'kind')) insist(!(await all(db, `SELECT 1 FROM ${quote(copies.oa_sessions)} WHERE kind='dev' LIMIT 1`)).length, 'dev_data:oa_sessions', 409);
+  if (copies.oa_dev_credentials) insist(!(await all(db, `SELECT 1 FROM ${quote(copies.oa_dev_credentials)} LIMIT 1`)).length, 'dev_data:oa_dev_credentials', 409);
+}

package/modules/portability/diff.mjs

@@ -0,0 +1,33 @@
+import { readArchive } from './format.mjs';
+import { canonicalJson } from './hash.mjs';
+async function inventory(source) {
+ const tables = new Map(); let manifest, closing;
+ for await (const record of readArchive(source)) {
+  if (record.kind === 'manifest') { manifest = record; for (const table of manifest.tables) tables.set(table.name,new Map()); }
+  if (record.kind === 'closing') closing = record;
+  if (record.kind === 'part') for (const item of record.items) if (item.kind === 'row') {
+   const table = manifest.tables.find(table => table.name === item.table);
+   const primary = table.columns.filter(column => column.pk).sort((a,b) => a.pk-b.pk).map(column => column.name);
+   const key = primary.length ? canonicalJson(primary.map(name => item.row[name])) : table.rowidAlias ? String(item.rowid) : canonicalJson(item.row);
+   const rows = tables.get(item.table); if (rows.has(key)) throw new Error('Archive contains duplicate row keys'); rows.set(key,item.row);
+  }
+ }
+ return {manifest,closing,tables};
+}
+export async function diffArchives(before,after) {
+ const [a,b] = await Promise.all([inventory(before),inventory(after)]);
+ if (a.manifest.product !== b.manifest.product) throw new Error('Archives describe different products');
+ const tables = [];
+ for (const name of [...new Set([...a.tables.keys(),...b.tables.keys()])].sort()) {
+  const left = a.tables.get(name) ?? new Map(),right = b.tables.get(name) ?? new Map(),added=[],removed=[],changed=[];
+  for (const [key,row] of left) { if (!right.has(key)) removed.push(key); else if(canonicalJson(row)!==canonicalJson(right.get(key))) changed.push(key); }
+  for (const key of right.keys()) if(!left.has(key))added.push(key);
+  if(added.length||removed.length||changed.length)tables.push({name,added,removed,changed});
+ }
+ const binary = kind => {
+  const key = item => canonicalJson(kind==='objects'?[item.bucket,item.key]:[item.name]);
+  const left=new Map(a.manifest[kind].map(item=>[key(item),item])),right=new Map(b.manifest[kind].map(item=>[key(item),item]));
+  return {added:[...right.keys()].filter(key=>!left.has(key)),removed:[...left.keys()].filter(key=>!right.has(key)),changed:[...left.keys()].filter(key=>right.has(key)&&canonicalJson(left.get(key))!==canonicalJson(right.get(key)))};
+ };
+ return {equal:a.closing.contentHash===b.closing.contentHash,product:a.manifest.product,schemaChanged:a.manifest.schema!==b.manifest.schema,tables,objects:binary('objects'),payloads:binary('payloads'),configurationChanged:canonicalJson(a.manifest.requiredConfiguration)!==canonicalJson(b.manifest.requiredConfiguration)};
+}

package/modules/portability/export.mjs

@@ -0,0 +1,129 @@
+import { all, statement, state, guard, job, save, partStatements, parts, readPart, cleanCopies, quote, stream, insist, run } from './storage.mjs';
+import { catalogue, encodedRow } from './catalogue.mjs';
+import { createCopy,readCopy,normalizeCopy } from './copies.mjs';
+import { canonicalJson, utf8, base64, StreamHash } from './hash.mjs';
+import { FORMAT, makePart, checkPart, ContentVerifier, ArchiveVerifier } from './format.mjs';
+import { journal } from './journal.mjs';
+import { appliedMigrations } from './bridge.mjs';
+import { capturePayload, payloadRange } from './payloads.mjs';
+import { dropProviderData } from './provider-data.mjs';
+import { dropDevData } from './dev-data.mjs';
+export async function createExport(context,actor='direct') {
+  const {db,config,buckets}=context,s=await state(db);insist(!s.applying,'import_applying',409);
+  const cat=await catalogue(db,config),id=crypto.randomUUID(),objects=[];
+  for(const [binding,spec] of Object.entries(config.buckets).sort()) {
+    let cursor;
+    do {
+      const page=await buckets[binding].list({prefix:spec.prefix??'',cursor,include:['httpMetadata','customMetadata']});
+      for(const object of page.objects) objects.push({bucket:binding,key:object.key,size:object.size,etag:object.etag,httpMetadata:object.httpMetadata??{},customMetadata:object.customMetadata??{}});
+      cursor=page.truncated?page.cursor:undefined;
+    }while(cursor);
+  }
+  objects.sort((a,b)=>a.bucket<b.bucket?-1:a.bucket>b.bucket?1:a.key<b.key?-1:a.key>b.key?1:0);
+  const copies=Object.fromEntries(cat.tables.map((t,i)=>[t.name,'rer_portability_snap_'+id.replaceAll('-','')+'_'+i]));
+  const migrations=await appliedMigrations(db);
+  const manifest={kind:'manifest',format:FORMAT,version:2,product:config.product,exportedAt:new Date().toISOString(),sourceIdentity:s.source_identity,schema:cat.identity,tables:cat.tables,objects,payloads:[],excludedTables:cat.excludedTables,excludedColumns:config.secretColumns,requiredConfiguration:config.requiredConfiguration,partCount:0,epoch:s.epoch,journal:await journal(db,s.source_identity),...(migrations?{migrations}:{})};
+  const row={id,kind:'export',actor,phase:'building',completed:0,metadata:{manifest,copies,cursor:{table:0,row:0,binary:0,offset:0},partBytes:config.partBytes}};
+  await db.batch([guard(db,'EXISTS(SELECT 1 FROM rer_portability_state WHERE id=1 AND applying=0 AND source_identity=? AND epoch=?)',s.source_identity,s.epoch),
+    ...cat.tables.flatMap(t=>createCopy(db,t,copies[t.name],true)),statement(db,'INSERT INTO rer_portability_jobs(id,kind,actor,phase,metadata) VALUES(?,?,?,?,?)',id,'export',actor,row.phase,canonicalJson(row.metadata)),db.prepare('DELETE FROM rer_portability_guard')]);
+  try {
+    await dropDevData(db,cat.tables,copies);
+  await dropProviderData(db,cat.tables,copies);
+    for(const t of manifest.tables) t.count=(await statement(db,`SELECT COUNT(*) AS n FROM ${quote(copies[t.name])}`).first()).n;
+    for(const [name,producer] of Object.entries(config.extraPayloads).sort()) {
+      const descriptor=await capturePayload(db,id,manifest.payloads.length,await producer.produce());manifest.payloads.push({name,...descriptor});
+    }
+    await run(db,'UPDATE rer_portability_jobs SET metadata=? WHERE id=?',canonicalJson(row.metadata),id);
+    await checkReferences(context,row);
+    return id;
+  }catch(e){await abortExport(context,id);throw e;}
+}
+async function checkReferences({db,config},row) {
+  for(const [binding,spec] of Object.entries(config.buckets)) if(spec.objectReferences) {
+    const {table,keyColumn}=spec.objectReferences,keys=new Set((await all(db,`SELECT DISTINCT ${quote(keyColumn)} AS key FROM ${quote(row.metadata.copies[table])} WHERE ${quote(keyColumn)} IS NOT NULL`)).map(x=>x.key));
+    const objects=row.metadata.manifest.objects.filter(x=>x.bucket===binding);
+    insist(objects.length===keys.size && objects.every(x=>keys.has(x.key)),'object_references:'+binding);
+  }
+}
+async function nextItems(context,row) {
+  const {db,config,buckets}=context,{manifest,cursor,copies,partBytes}=row.metadata;
+  while(cursor.table<manifest.tables.length && cursor.row===manifest.tables[cursor.table].count){cursor.table++;cursor.row=0;}
+  if(cursor.table<manifest.tables.length) {
+    const table=manifest.tables[cursor.table];
+    if(!row.metadata.normalized?.includes(table.name)) {
+      await normalizeCopy(context,table,copies[table.name]);
+      row.metadata.normalized??=[];row.metadata.normalized.push(table.name);await save(db,row).run();
+    }
+    const records=await readCopy(db,table,copies[table.name],cursor.row,Math.min(1000,Math.ceil(partBytes/128)),true);
+    const items=[];let size=200;
+    for(const {row:record,meta} of records) {
+      const value=encodedRow(table,record,config.secretColumns[table.name]??[]);
+      const item={kind:'row',table:table.name,index:cursor.row,row:value};
+      if(table.rowidAlias){insist(Number.isSafeInteger(meta.rer_portability_source_rowid),'unsafe_integer:'+table.name+':rowid');item.rowid=meta.rer_portability_source_rowid;}
+      const bytes=utf8(canonicalJson(item)).length+1;
+      insist(bytes+200<=partBytes,'row_part_too_large:'+table.name);
+      if(size+bytes>partBytes)break;items.push(item);size+=bytes;cursor.row++;
+    }
+    return items;
+  }
+  const entries=[...manifest.objects,...manifest.payloads];
+  if(cursor.binary===entries.length)return null;
+  const entry=entries[cursor.binary],length=Math.min(entry.size-cursor.offset,Math.max(1,Math.floor((partBytes-200-utf8(canonicalJson({bucket:entry.bucket??null,key:entry.key??null,name:entry.name??null})).length)*3/4)));
+  let bytes,item;
+  if(cursor.binary<manifest.objects.length) {
+    const object=await buckets[entry.bucket].get(entry.key,length?{range:{offset:cursor.offset,length}}:undefined);
+    insist(object && object.etag===entry.etag,'object_changed:'+entry.bucket+':'+entry.key);
+    bytes=new Uint8Array(await object.arrayBuffer());insist(bytes.length===length,'object_changed:'+entry.key);
+    item={kind:'object',bucket:entry.bucket,key:entry.key,offset:cursor.offset,data:base64(bytes)};
+  }else{
+    bytes=await payloadRange(db,row.id,cursor.binary-manifest.objects.length,cursor.offset,length);
+    item={kind:'payload',name:entry.name,offset:cursor.offset,data:base64(bytes)};
+  }
+  const hash=new StreamHash();if(cursor.hash){hash.h=Uint32Array.from(cursor.hash.h);hash.block=Uint8Array.from(cursor.hash.block);hash.size=cursor.hash.size;hash.used=cursor.hash.used;}
+  hash.update(bytes);cursor.offset+=length;
+  if(cursor.offset===entry.size) {
+    const digest=hash.digest();if(entry.sha256)insist(entry.sha256===digest,'payload_changed');entry.sha256=digest;delete entry.etag;
+    cursor.binary++;cursor.offset=0;delete cursor.hash;
+  }else cursor.hash={h:[...hash.h],block:[...hash.block],size:hash.size,used:hash.used};
+  return [item];
+}
+export async function advanceExport(context,id,{expectedPart,maxParts=1}={}) {
+  insist(Number.isSafeInteger(maxParts) && maxParts>0 && maxParts<=100,'advance_limit');
+  for(let n=0;n<maxParts;n++) {
+    const row=await job(context.db,id,'export');
+    if(row.phase==='complete')return exportStatus(context,id);
+    insist(row.phase==='building','export_phase',409);
+    if(expectedPart!==undefined && row.completed!==expectedPart)return exportStatus(context,id);
+    const before=row.completed,items=await nextItems(context,row);
+    if(items) {
+      const part=await makePart(before,items);await checkPart(part,before,row.metadata.partBytes);row.completed++;
+      try{await context.db.batch([guard(context.db,'(SELECT completed FROM rer_portability_jobs WHERE id=?)=?',id,before),...partStatements(context.db,id,part),save(context.db,row),context.db.prepare('DELETE FROM rer_portability_guard')]);}
+      catch(e){if((await job(context.db,id)).completed===before)throw e;}
+      if(expectedPart!==undefined)expectedPart++;
+    }else{
+      // References are checked again against the archived, normalized snapshot.
+      await checkReferences(context,row);
+      row.metadata.manifest.partCount=row.completed;
+      const verifier=new ContentVerifier(row.metadata.manifest),archive=new ArchiveVerifier(row.metadata.manifest);
+      for await(const part of parts(context.db,id,row.completed)){archive.add(part);for(const item of part.items)verifier.add(item);}
+      row.metadata.contentHash=verifier.digest();row.metadata.archiveHash=archive.digest();row.phase='complete';await save(context.db,row).run();
+      // Finished snapshots are kept until abort so erasure can invalidate them.
+      return exportStatus(context,id);
+    }
+  }
+  return exportStatus(context,id);
+}
+export async function exportStatus(context,id) {
+  const row=await job(context.db,id,'export');return {id,kind:'export',phase:row.phase,completed:row.completed,total:row.phase==='complete'?row.completed:null,contentHash:row.metadata.contentHash??null};
+}
+export async function downloadExport(context,id) {
+  const row=await job(context.db,id,'export');insist(row.phase==='complete','export_incomplete',409);
+  return stream((async function*(){
+    yield utf8(canonicalJson(row.metadata.manifest)+'\n');
+    for(let i=0;i<row.completed;i++)yield utf8(canonicalJson(await readPart(context.db,id,i))+'\n');
+    yield utf8(canonicalJson({kind:'closing',partCount:row.completed,contentHash:row.metadata.contentHash,archiveHash:row.metadata.archiveHash})+'\n');
+  })());
+}
+export async function abortExport(context,id) {
+  const row=await job(context.db,id,'export');await cleanCopies(context.db,row);row.phase='aborted';await save(context.db,row).run();return exportStatus(context,id);
+}

package/modules/portability/files.mjs

@@ -0,0 +1,65 @@
+import { open, mkdir, link, unlink } from 'node:fs/promises';
+import { constants } from 'node:fs';
+import { dirname } from 'node:path';
+import { Readable } from 'node:stream';
+import { randomUUID } from 'node:crypto';
+import { readArchive } from './index.mjs';
+export async function* fileRecords(path,reader=readArchive) {
+ const file = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW);
+ try { yield* reader(Readable.toWeb(file.createReadStream({ autoClose: false }))); }
+ finally { await file.close(); }
+}
+function receipt(manifest, closing, transient) {
+ return {
+  contentHash: closing.contentHash,
+  archiveHash: closing.archiveHash,
+  tables: manifest.tables.map(({ name, count }) => ({ name, count })),
+  objects: manifest.objects.length, payloads: manifest.payloads.length,
+  exclusions: { tables: manifest.excludedTables, columns: manifest.excludedColumns, configuration: true, transient },
+  requiredConfiguration: manifest.requiredConfiguration,
+ };
+}
+export async function exportArchive(service, path, transient) {
+ let id, temporary, file;
+ try {
+  await mkdir(dirname(path), { recursive: true, mode: 0o700 });
+  temporary = path + '.' + randomUUID() + '.partial';
+  file = await open(temporary, 'wx', 0o600);
+  id = await service.exports.create();
+  while ((await service.exports.advance(id, { maxParts: 16 })).phase === 'building') {}
+  const reader = (await service.exports.download(id)).getReader();
+  try {
+   for (;;) {
+    const { done, value } = await reader.read(); if (done) break;
+    let offset = 0;
+    while (offset < value.length) offset += (await file.write(value, offset)).bytesWritten;
+   }
+  } finally { await reader.cancel(); reader.releaseLock(); }
+  await file.sync(); await file.close(); file = undefined;
+  let manifest, closing;
+  for await (const record of fileRecords(temporary)) { if (record.kind === 'manifest') manifest = record; if (record.kind === 'closing') closing = record; }
+  await link(temporary, path);
+  return receipt(manifest, closing, transient);
+ } finally {
+  await file?.close();
+  if (temporary) await unlink(temporary).catch(error => { if (error.code !== 'ENOENT') throw error; });
+  if (id) await service.exports.abort(id);
+ }
+}
+export async function importArchive(service, path, transient, journal) {
+ let id, manifest, closing, complete = false;
+ try {
+  // Validate both complete archives before creating any restore state.
+  let later;
+  for (const file of [path, journal].filter(Boolean)) for await (const record of fileRecords(file)) {
+   if (file === journal && record.kind === 'manifest') later = { sourceIdentity: record.sourceIdentity, epoch: record.epoch, journal: record.journal };
+  }
+  for await (const record of fileRecords(path)) {
+   if (record.kind === 'manifest') { manifest = record; id = await service.imports.create(manifest); }
+   else if (record.kind === 'part') await service.imports.uploadPart(id, record.index, record);
+   else closing = record;
+  }
+  const status = await service.imports.finalize(id, { contentHash: closing.contentHash, archiveHash: closing.archiveHash, ...(later ? { laterJournal: later } : {}) }); complete = true;
+  return { ...receipt(manifest, closing, transient), ...(status.bridged ? { bridged: status.bridged } : {}) };
+ } finally { if (id && !complete) await service.imports.abort(id); }
+}