@runeditrun/rer 0.0.0-stage → 0.1.1

package/modules/portability/format.mjs

@@ -0,0 +1,122 @@
+import { archiveLines } from './lines.mjs';
+import { canonicalJson, sha256, StreamHash, unbase64, utf8 } from './hash.mjs';
+import { insist } from './storage.mjs';
+import { validateRow } from './catalogue.mjs';
+export const FORMAT='runeditrun-portability';
+export const MAX_LINE_BYTES=24*1024*1024;
+export const MAX_PART_BYTES=16*1024*1024;
+export async function makePart(index,items) {
+  const content={kind:'part',index,items}; return {...content,sha256:await sha256(canonicalJson(content))};
+}
+export async function checkPart(part,index,partBytes=MAX_PART_BYTES) {
+  insist(part?.kind==='part' && part.index===index && Array.isArray(part.items) && part.items.length>0 && Object.keys(part).sort().join(',')==='index,items,kind,sha256','part_order');
+  insist(Number.isSafeInteger(partBytes) && partBytes>0 && partBytes<=MAX_PART_BYTES,'invalid_part_limit');
+  insist(utf8(canonicalJson(part)).length<=partBytes,'part_byte_limit',413);
+  insist(part.sha256===await sha256(canonicalJson({kind:part.kind,index:part.index,items:part.items})),'part_checksum');
+  insist(part.items.filter(x=>x.kind==='row').length<=1000,'part_row_limit');
+}
+export async function checkManifest(m) {
+  insist(m?.kind==='manifest' && m.format===FORMAT,'archive_format');
+  // Version 1 never authenticated restoration identity or erasure metadata.
+  // Relabelling it cannot recover the missing binding.
+  insist(m.version!==1,'legacy_archive_unverifiable',409);
+  insist(m.version===2,'archive_format');
+  await checkManifestFields(m);
+}
+async function checkLegacyManifest(m) {
+  insist(m?.kind==='manifest' && m.format===FORMAT && m.version===1,'legacy_v1_required',409);
+  await checkManifestFields(m);
+  insist(m.journal.every(x=>typeof x==='string' && /^[a-f0-9]{64}$/.test(x)),'invalid_journal');
+}
+async function checkManifestFields(m) {
+  insist(typeof m.product==='string' && typeof m.sourceIdentity==='string' && typeof m.exportedAt==='string','manifest_identity');
+  insist(Number.isSafeInteger(m.partCount) && m.partCount>=0 && Number.isSafeInteger(m.epoch) && m.epoch>=0,'manifest_counts');
+  insist(Array.isArray(m.tables) && Array.isArray(m.objects) && Array.isArray(m.payloads) && Array.isArray(m.journal) && Array.isArray(m.requiredConfiguration),'manifest_inventory');
+  // Applied product migrations, recorded by archives since additive restore.
+  insist(m.migrations===undefined || (Array.isArray(m.migrations) && m.migrations.every(x=>typeof x==='string')),'manifest_inventory');
+  insist(m.schema===await sha256(canonicalJson(m.tables.map(({name,sql})=>({name,sql})))),'schema_checksum');
+  insist(m.tables.every((t,i)=>typeof t.name==='string' && typeof t.sql==='string' && Array.isArray(t.columns) && Array.isArray(t.dependencies) && Number.isSafeInteger(t.count) && t.count>=0 && (!i || m.tables[i-1].name<t.name)),'table_inventory');
+  for(const collection of [m.objects,m.payloads]) {
+    const keys=new Set();
+    for(const entry of collection) {
+      const key=entry.bucket===undefined?entry.name:canonicalJson([entry.bucket,entry.key]);
+      insist(typeof key==='string' && !keys.has(key) && Number.isSafeInteger(entry.size) && entry.size>=0 && /^[a-f0-9]{64}$/.test(entry.sha256),'binary_inventory'); keys.add(key);
+    }
+  }
+}
+// Bind the entire manifest, including future policy-visible fields, and every
+// ordered part digest. Part checksums cover all row/object/payload fields.
+// contentHash remains independent of export time, identity and part boundaries.
+export class ArchiveVerifier {
+  constructor(manifest) {
+    this.hash=new StreamHash().update('{"manifest":'+canonicalJson(manifest)+',"parts":[');
+    this.count=0;
+  }
+  add(part) {
+    if(this.count++)this.hash.update(',');
+    this.hash.update(canonicalJson(part.sha256));
+  }
+  digest() { return this.hash.update(']}').digest(); }
+}
+export class ContentVerifier {
+  constructor(manifest) {
+    this.manifest=manifest; this.hash=new StreamHash(); this.table=0; this.row=0; this.binary=0; this.offset=0; this.binaryHash=new StreamHash();
+    this.entries=[...manifest.objects.map(x=>({...x,kind:'object'})),...manifest.payloads.map(x=>({...x,kind:'payload'}))];
+    const objects=manifest.objects.map(({bucket,key,size,sha256,httpMetadata,customMetadata})=>({bucket,key,size,sha256,metadata:{httpMetadata,customMetadata}}));
+    this.hash.update('{"objects":'+canonicalJson(objects)+',"payloads":'+canonicalJson(manifest.payloads)+',"requiredConfiguration":'+canonicalJson(manifest.requiredConfiguration)+',"schema":'+canonicalJson(manifest.schema)+',"tables":[');
+    this.open=false; this.rows=0;
+  }
+  emptyTables() {
+    while(this.table<this.manifest.tables.length) {
+      const t=this.manifest.tables[this.table];
+      if(!this.open) {if(this.table)this.hash.update(',');this.hash.update('{"name":'+canonicalJson(t.name)+',"rows":[');this.open=true;}
+      if(this.row<t.count) break;
+      this.hash.update(']}');this.table++;this.row=0;this.open=false;
+    }
+  }
+  add(item) {
+    this.emptyTables();
+    if(item.kind==='row') {
+      const t=this.manifest.tables[this.table];
+      insist(t && item.table===t.name && item.index===this.row,'row_order');
+      validateRow(t,item.row,this.manifest.excludedColumns[t.name]??[]);
+      if(t.rowidAlias)insist(Number.isSafeInteger(item.rowid),'rowid_inventory');
+      if(this.row)this.hash.update(',');this.hash.update(canonicalJson(item.row));this.row++;this.rows++;return;
+    }
+    insist(this.table===this.manifest.tables.length,'rows_missing');
+    const entry=this.entries[this.binary];
+    insist(entry && item.kind===entry.kind && (entry.kind==='object'?item.bucket===entry.bucket && item.key===entry.key:item.name===entry.name) && item.offset===this.offset,'binary_order');
+    const bytes=unbase64(item.data); insist(this.offset+bytes.length<=entry.size && (bytes.length>0 || entry.size===0),'binary_size');
+    this.binaryHash.update(bytes);this.offset+=bytes.length;
+    if(this.offset===entry.size) {insist(this.binaryHash.digest()===entry.sha256,'binary_checksum');this.binary++;this.offset=0;this.binaryHash=new StreamHash();}
+  }
+  digest() {
+    this.emptyTables();insist(this.table===this.manifest.tables.length && this.binary===this.entries.length,'archive_incomplete');
+    return this.hash.update(']}').digest();
+  }
+}
+export async function* readArchive(source,{maxLineBytes=MAX_LINE_BYTES,partBytes=MAX_PART_BYTES}={}) {
+  yield* verifiedRecords(source,maxLineBytes,partBytes,checkManifest,true);
+}
+// Deliberately separate from the normal reader: only the acknowledged local
+// recovery path uses this. It cannot establish the missing manifest binding.
+export async function* readLegacyArchive(source,{maxLineBytes=MAX_LINE_BYTES,partBytes=MAX_PART_BYTES}={}) {
+  yield* verifiedRecords(source,maxLineBytes,partBytes,checkLegacyManifest,false);
+}
+async function* verifiedRecords(source,maxLineBytes,partBytes,validateManifest,requireBinding) {
+  let manifest,verifier,archive,index=0,closed=false;
+  for await(const line of archiveLines(source,maxLineBytes)) {
+    insist(!closed,'data_after_closing');
+    let record;try{record=JSON.parse(line);}catch{throw new Error('archive_json');}
+    if(!manifest) {await validateManifest(record);manifest=record;verifier=new ContentVerifier(manifest);archive=new ArchiveVerifier(manifest);yield record;continue;}
+    if(record.kind==='closing') {
+      insist(index===manifest.partCount && record.partCount===index,'part_count');
+      insist(record.contentHash===verifier.digest(),'content_checksum');
+      if(requireBinding)insist(record.archiveHash===archive.digest(),'archive_checksum',409);closed=true;yield record;continue;
+    }
+    insist(index<manifest.partCount,'extra_part');await checkPart(record,index++,partBytes);
+    archive.add(record);
+    for(const item of record.items)verifier.add(item);yield record;
+  }
+  insist(closed,'closing_missing');
+}

package/modules/portability/generated.mjs

@@ -0,0 +1,60 @@
+import { quote, insist } from './storage.mjs';
+
+// SQLite does not expose generated expressions through a pragma. Read their
+// original SQL, retaining quoted tokens and balanced expressions verbatim.
+// Copies deliberately omit constraints: FK cycles and normalized duplicate
+// keys must reach the existing atomic destination validation, not fail staging.
+export function tokens(sql) {
+ const result=[];
+ for(let i=0;i<sql.length;) {
+  const start=i,c=sql[i];
+  if(/[ \t\n\v\f\r]/.test(c)){i++;continue;}
+  if(sql.startsWith('--',i)){while(i<sql.length&&sql[i]!=='\n')i++;continue;}
+  if(sql.startsWith('/*',i)){const end=sql.indexOf('*/',i+2);insist(end>=0,'generated_sql');i=end+2;continue;}
+  if(['"',"'",'`','['].includes(c)) {
+   const end=c==='['?']':c;i++;
+   while(i<sql.length){if(sql[i++]===end){if(c!=='['&&sql[i]===end){i++;continue;}break;}}
+   result.push({text:sql.slice(start,i),start,end:i,quoted:true});continue;
+  }
+  if(/[(),]/.test(c)){i++;result.push({text:c,start,end:i});continue;}
+  while(i<sql.length&&!/[ \t\n\v\f\r(),"'`\[]/.test(sql[i])&&!sql.startsWith('--',i)&&!sql.startsWith('/*',i))i++;
+  result.push({text:sql.slice(start,i),start,end:i});
+ }
+ return result;
+}
+const word=(token,value)=>token&&!token.quoted&&folded(token.text)===value.toLowerCase();
+export const identifier=token=>{
+ if(!token.quoted)return token.text;
+ const value=token.text.slice(1,-1),delimiter=token.text[0];
+ // Bracket quoting has no doubled-delimiter escape; opening brackets are literal.
+ return delimiter==='['?value:value.replaceAll(delimiter+delimiter,delimiter);
+};
+export const folded=value=>value.replace(/[A-Z]/g,letter=>letter.toLowerCase());
+function definitions(sql) {
+ const list=tokens(sql),start=list.findIndex(token=>token.text==='(');let depth=0,part=[],result=[];
+ for(const token of list.slice(start+1)) {
+  if(token.text==='(')depth++;
+  if(token.text===')'){if(!depth){result.push(part);return result;}depth--;}
+  if(token.text===','&&!depth){result.push(part);part=[];}else part.push({...token,depth});
+ }
+ insist(false,'generated_sql');
+}
+export function generatedCopySQL(table,copy) {
+ const declarations=definitions(table.sql);
+ const columns=table.columns.map(column=>{
+  const definition=declarations.find(part=>part[0]&&folded(identifier(part[0]))===folded(column.name));
+  insist(definition,'generated_sql:'+table.name);
+  const collate=definition.findLastIndex((token,i)=>i>0&&token.depth===0&&word(token,'COLLATE'));
+  const collation=collate<0?'':' COLLATE '+definition[collate+1].text;
+  let expression='';
+  if(column.generated) {
+   const as=definition.findIndex((token,i)=>i>0&&token.depth===0&&word(token,'AS')&&definition[i+1]?.text==='(');
+   insist(as>=0,'generated_sql:'+table.name+':'+column.name);
+   const close=definition.findIndex((token,i)=>i>as+1&&token.text===')'&&token.depth===0);
+   insist(close>=0,'generated_sql:'+table.name+':'+column.name);
+   expression=' AS '+table.sql.slice(definition[as+1].start,definition[close].end)+' '+column.generated.toUpperCase();
+  }
+  return quote(column.name)+(column.type?' '+quote(column.type):'')+collation+expression;
+ });
+ return `CREATE TABLE ${quote(copy)} (${columns.join(',')})`;
+}

package/modules/portability/hash.mjs

@@ -0,0 +1,79 @@
+export const utf8 = value => new TextEncoder().encode(value);
+export const canonicalJson = value => {
+  if (value instanceof Date) return JSON.stringify(value.toISOString());
+  if (value === null || typeof value !== 'object') {
+    if (value === undefined || (typeof value === 'number' && !Number.isFinite(value))) throw new Error('invalid_json');
+    return JSON.stringify(value);
+  }
+  if (Array.isArray(value)) return '[' + value.map(canonicalJson).join(',') + ']';
+  return '{' + Object.keys(value).filter(key => value[key] !== undefined).sort().map(key => JSON.stringify(key) + ':' + canonicalJson(value[key])).join(',') + '}';
+};
+export async function sha256(value) {
+  const bytes = typeof value === 'string' ? utf8(value) : value;
+  return [...new Uint8Array(await crypto.subtle.digest('SHA-256', bytes))].map(x => x.toString(16).padStart(2, '0')).join('');
+}
+export function base64(bytes) {
+  let result = '';
+  for (let i = 0; i < bytes.length; i += 8192) result += String.fromCharCode(...bytes.subarray(i, i + 8192));
+  return btoa(result);
+}
+export function unbase64(value) {
+  if (typeof value !== 'string' || !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(value)) throw new Error('invalid_base64');
+  return Uint8Array.from(atob(value), c => c.charCodeAt(0));
+}
+
+// Incremental SHA-256 for semantic JSON and objects. SubtleCrypto has no streaming
+// digest API; part digests use Web Crypto, while this keeps memory independent of
+// the number of rows. FIPS 180-4 compression, checked against SubtleCrypto in tests.
+const K = Uint32Array.from([
+  0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5,0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5,
+  0xd807aa98,0x12835b01,0x243185be,0x550c7dc3,0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174,
+  0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc,0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da,
+  0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7,0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967,
+  0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13,0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85,
+  0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3,0xd192e819,0xd6990624,0xf40e3585,0x106aa070,
+  0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5,0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3,
+  0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208,0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2,
+]);
+const rotate = (x,n) => (x >>> n) | (x << (32-n));
+export class StreamHash {
+  constructor() {
+    this.h = Uint32Array.from([0x6a09e667,0xbb67ae85,0x3c6ef372,0xa54ff53a,0x510e527f,0x9b05688c,0x1f83d9ab,0x5be0cd19]);
+    this.block = new Uint8Array(64); this.used = 0; this.size = 0;
+  }
+  update(input) {
+    const bytes = typeof input === 'string' ? utf8(input) : input;
+    this.size += bytes.length;
+    let offset = 0;
+    while (offset < bytes.length) {
+      const n = Math.min(64-this.used, bytes.length-offset);
+      this.block.set(bytes.subarray(offset,offset+n),this.used); this.used += n; offset += n;
+      if (this.used === 64) { this.compress(); this.used = 0; }
+    }
+    return this;
+  }
+  compress() {
+    const w = new Uint32Array(64), view = new DataView(this.block.buffer);
+    for (let i=0;i<16;i++) w[i] = view.getUint32(i*4);
+    for (let i=16;i<64;i++) {
+      const a=w[i-15], b=w[i-2];
+      w[i]=w[i-16]+(rotate(a,7)^rotate(a,18)^(a>>>3))+w[i-7]+(rotate(b,17)^rotate(b,19)^(b>>>10));
+    }
+    let [a,b,c,d,e,f,g,h]=this.h;
+    for (let i=0;i<64;i++) {
+      const t1=(h+(rotate(e,6)^rotate(e,11)^rotate(e,25))+((e&f)^(~e&g))+K[i]+w[i])>>>0;
+      const t2=((rotate(a,2)^rotate(a,13)^rotate(a,22))+((a&b)^(a&c)^(b&c)))>>>0;
+      h=g;g=f;f=e;e=(d+t1)>>>0;d=c;c=b;b=a;a=(t1+t2)>>>0;
+    }
+    [a,b,c,d,e,f,g,h].forEach((v,i)=>{this.h[i]=(this.h[i]+v)>>>0;});
+  }
+  digest() {
+    const size=this.size;
+    this.update(Uint8Array.of(128));
+    while(this.used!==56) this.update(Uint8Array.of(0));
+    const tail=new Uint8Array(8),view=new DataView(tail.buffer);
+    view.setUint32(0,Math.floor(size/0x20000000)); view.setUint32(4,(size*8)>>>0);
+    this.update(tail);
+    return [...this.h].map(x=>x.toString(16).padStart(8,'0')).join('');
+  }
+}

package/modules/portability/http.mjs

@@ -0,0 +1,42 @@
+import { createPortability } from './index.mjs';
+import { insist, PortabilityError } from './storage.mjs';
+import { MAX_LINE_BYTES } from './format.mjs';
+export async function boundedJson(request,limit) {
+  const reader=request.body?.getReader();if(!reader)return {};
+  const chunks=[];let size=0;
+  try {for(;;){const {value,done}=await reader.read();if(done)break;size+=value.length;insist(size<=limit,'body_too_large',413);chunks.push(value);}}
+  finally{await reader.cancel();reader.releaseLock();}
+  const bytes=new Uint8Array(size);let offset=0;for(const c of chunks){bytes.set(c,offset);offset+=c.length;}
+  try{return JSON.parse(new TextDecoder('utf-8',{fatal:true}).decode(bytes));}catch{throw new PortabilityError('invalid_json');}
+}
+const json=(value,status=200)=>new Response(JSON.stringify(value),{status,headers:{'Content-Type':'application/json','Cache-Control':'no-store'}});
+export async function createPortabilityHandler({db,buckets,policy,appOrigin,basePath='/api/portability',authorize,maxBodyBytes=MAX_LINE_BYTES}) {
+  insist(typeof authorize==='function' && new URL(appOrigin).origin===appOrigin,'invalid_handler_configuration');
+  insist(/^\/[A-Za-z0-9/_-]+$/.test(basePath) && !basePath.endsWith('/') && Number.isSafeInteger(maxBodyBytes) && maxBodyBytes>0,'invalid_handler_configuration');
+  const service=await createPortability({db,buckets,policy});
+  return async request=>{
+    const url=new URL(request.url);
+    if(url.pathname!==basePath && !url.pathname.startsWith(basePath+'/'))return null;
+    const route=url.pathname.slice(basePath.length+1).split('/'),[collection,id,action,index]=route;
+    const operation=collection==='exports'?'export':collection==='imports'?'import':'unknown';
+    try {
+      const actor=await authorize(request,{operation,jobId:id??null});insist(typeof actor==='string' && actor.length>0,'unauthorized',401);
+      if(!['GET','HEAD'].includes(request.method))insist(request.headers.get('Origin')===appOrigin,'origin_refused',403);
+      if(id)insist(await service.actor(id)===actor,'job_not_found',404);
+      insist(['exports','imports'].includes(collection) && route.length<=4,'route_not_found',404);
+      const api=service[collection];
+      if(request.method==='POST' && !id) {
+        const body=await boundedJson(request,maxBodyBytes);
+        if(collection==='imports'){insist(body.confirmed===true,'confirmation_required');return json({id:await api.create(body.manifest,actor)},201);}
+        return json({id:await api.create(actor)},201);
+      }
+      if(request.method==='GET' && id && !action)return json(await api.status(id));
+      if(request.method==='POST' && id && action==='abort' && !index)return json(await api.abort(id));
+      if(collection==='exports' && request.method==='POST' && id && action==='advance' && !index)return json(await api.advance(id,await boundedJson(request,maxBodyBytes)));
+      if(collection==='exports' && request.method==='GET' && id && action==='download' && !index)return new Response(await api.download(id),{headers:{'Content-Type':'application/x-ndjson','Cache-Control':'no-store'}});
+      if(collection==='imports' && request.method==='PUT' && id && action==='parts' && /^(0|[1-9]\d*)$/.test(index??''))return json(await api.uploadPart(id,Number(index),await boundedJson(request,maxBodyBytes)));
+      if(collection==='imports' && request.method==='POST' && id && action==='finalize' && !index)return json(await api.finalize(id,await boundedJson(request,maxBodyBytes)));
+      throw new PortabilityError('route_not_found',404);
+    }catch(e){return json({error:e instanceof PortabilityError?e.code:'operation_failed'},e instanceof PortabilityError?e.status:500);}
+  };
+}

package/modules/portability/import.mjs

@@ -0,0 +1,185 @@
+import { all, statement, state, guard, job, save, partStatements, parts, readPart, cleanCopies, quote, stream, insist, run } from './storage.mjs';
+import { refuseProviderData } from './provider-data.mjs';
+import { refuseDevData } from './dev-data.mjs';
+import { catalogue, validateRow, bindCell, writableColumns } from './catalogue.mjs';
+import { canonicalJson, unbase64 } from './hash.mjs';
+import { checkManifest, checkPart, ContentVerifier, ArchiveVerifier } from './format.mjs';
+import { fence, mergeJournal, tombstone } from './journal.mjs';
+import { writeObject } from './payloads.mjs';
+import { createCopy,sidecar,readCopy,restoreCopy } from './copies.mjs';
+import { excludedDependentGuards } from './restore-safety.mjs';
+import { additiveBridge, sameSchemaMigrations } from './bridge.mjs';
+import { legacyEmptyGuards, legacyAuditStatements, requireLegacyEmpty } from './legacy-safety.mjs';
+export async function createImport(context,manifest,actor='direct',legacyRecovery) {
+  const {db,config,buckets}=context;await checkManifest(manifest);await state(db);
+  insist(manifest.product===config.product,'wrong_product');
+  const cat=await catalogue(db,config),upgrade=config.acceptedSchemas[manifest.schema];
+  const same=manifest.schema===cat.identity;
+  const bridge=same || upgrade?undefined:await additiveBridge(db,manifest,cat,context.migrations);
+  if(same && !upgrade)await sameSchemaMigrations(db,manifest,cat,context.migrations);
+  if(!bridge)insist(canonicalJson(manifest.tables.map(x=>x.name))===canonicalJson(cat.tables.map(x=>x.name)),'table_inventory');
+  if(!upgrade && !bridge)for(const table of manifest.tables) {
+    const target=cat.tables.find(t=>t.name===table.name);
+    insist(canonicalJson(table.columns)===canonicalJson(target.columns) && canonicalJson(table.dependencies)===canonicalJson(target.dependencies),'column_inventory');
+  }
+  // Bridged tables stage and restore only the archived columns; the destination
+  // supplies defaults for later columns and leaves later tables empty.
+  const targets=cat.tables.map(t=>bridge?.columns[t.name]?{...t,columns:manifest.tables.find(x=>x.name===t.name).columns,added:bridge.columns[t.name]}:t);
+  insist(canonicalJson(manifest.excludedColumns)===canonicalJson(config.secretColumns),'secret_inventory');
+  insist(canonicalJson(manifest.requiredConfiguration)===canonicalJson(config.requiredConfiguration),'configuration_inventory');
+  for(const o of manifest.objects) insist(buckets[o.bucket] && config.buckets[o.bucket] && o.key.startsWith(config.buckets[o.bucket].prefix??''),'bucket_inventory');
+  insist(canonicalJson(manifest.payloads.map(x=>x.name))===canonicalJson(Object.keys(config.extraPayloads).sort()),'payload_inventory');
+  const id=crypto.randomUUID(),copies=Object.fromEntries(targets.map((t,i)=>[t.name,'rer_portability_stage_'+id.replaceAll('-','')+'_'+i]));
+  const row={id,kind:'import',actor,phase:'staging',completed:0,metadata:{manifest,copies,targets,triggers:cat.triggers,...(bridge?{bridge}:{}),...(legacyRecovery?{legacyRecovery}: {})}};
+  try {
+    await db.batch([guard(db,'(SELECT import_job FROM rer_portability_state WHERE id=1) IS NULL'),
+      statement(db,'UPDATE rer_portability_state SET import_job=? WHERE id=1',id),
+      ...targets.flatMap(t=>createCopy(db,t,copies[t.name])),
+      statement(db,'INSERT INTO rer_portability_jobs(id,kind,actor,phase,metadata) VALUES(?,?,?,?,?)',id,'import',actor,row.phase,canonicalJson(row.metadata)),db.prepare('DELETE FROM rer_portability_guard')]);
+  }catch(e){if((await state(db)).import_job)throw new (await import('./storage.mjs')).PortabilityError('import_running',409);throw e;}
+  return id;
+}
+export async function uploadPart(context,id,index,part) {
+  const {db,config}=context,row=await job(db,id,'import');await checkPart(part,index,config.partBytes);
+  const previous=await statement(db,'SELECT sha256 FROM rer_portability_parts WHERE job=? AND idx=?',id,index).first();
+  if(previous){insist(previous.sha256===part.sha256,'part_conflict',409);return importStatus(context,id);}
+  insist(row.phase==='staging','import_phase',409);insist(index===row.completed && index<row.metadata.manifest.partCount,'part_order',409);
+  const inserts=[],groups=new Map(),manifest=row.metadata.manifest,upgrade=config.acceptedSchemas[manifest.schema];
+  for(const item of part.items) {
+    if(item.kind==='row') {
+      const table=manifest.tables.find(x=>x.name===item.table);insist(table && Number.isSafeInteger(item.index) && item.index>=0 && item.index<table.count,'row_inventory');
+      validateRow(table,item.row,manifest.excludedColumns[table.name]??[]);
+      const target=row.metadata.targets.find(x=>x.name===table.name);
+      const data=upgrade?upgrade.upgradeRow(table.name,structuredClone(item.row)):item.row;
+      insist(!(data instanceof Promise),'upgrade_row_must_be_pure');validateRow(target,data,config.secretColumns[table.name]??[]);
+      const columns=writableColumns(target);
+      const values=columns.map(c=>bindCell(config.secretColumns[table.name]?.includes(c.name)?null:data[c.name]));
+      const pk=table.columns.filter(c=>c.pk).sort((a,b)=>a.pk-b.pk);
+      if(!pk.length)insist(Number.isSafeInteger(item.rowid),'rowid_inventory');
+      const key=pk.length?Object.fromEntries(pk.map(c=>[c.name,item.row[c.name]])):{rowid:item.rowid};
+      const hash=await tombstone(manifest.sourceIdentity,table.name,key);
+      if(!groups.has(table.name))groups.set(table.name,{columns:columns.map(c=>quote(c.name)).join(','),rows:[],metadata:[]});
+      groups.get(table.name).rows.push(values);
+      groups.get(table.name).metadata.push([item.index,item.rowid??null,hash]);
+    }else{
+      insist(['object','payload'].includes(item.kind) && Number.isSafeInteger(item.offset) && item.offset>=0,'binary_inventory');
+      const entry=item.kind==='object'?manifest.objects.find(x=>x.bucket===item.bucket && x.key===item.key):manifest.payloads.find(x=>x.name===item.name);
+      insist(entry && item.offset+unbase64(item.data).length<=entry.size,'binary_inventory');
+    }
+  }
+  row.metadata.stagedCounts=Object.assign(Object.create(null),row.metadata.stagedCounts);
+  for(const [table,group] of groups) {
+    const width=group.rows[0].length,count=Math.min(30,Math.floor(100/width));
+    insist(count>0,'unsupported_restore_width:'+table);
+    const copy=row.metadata.copies[table],first=(row.metadata.stagedCounts[table]??0)+1;
+    for(let start=0;start<group.rows.length;start+=count) {
+      const chunk=group.rows.slice(start,start+count);
+      inserts.push(statement(db,`INSERT INTO ${quote(copy)} (${group.columns}) VALUES${chunk.map(()=> '('+Array(width).fill('?').join(',')+')').join(',')}`,...chunk.flat()));
+      inserts.push(statement(db,`INSERT INTO ${quote(sidecar(copy))} VALUES${chunk.map((_,i)=>`(?,${first+start+i},?,?)`).join(',')}`,...group.metadata.slice(start,start+count).flat()));
+    }
+    row.metadata.stagedCounts[table]=first-1+group.rows.length;
+  }
+  const before=row.completed;row.completed++;
+  try{await db.batch([guard(db,'(SELECT completed FROM rer_portability_jobs WHERE id=?)=?',id,before),...inserts,...partStatements(db,id,part),save(db,row),db.prepare('DELETE FROM rer_portability_guard')]);}
+  catch(e){const duplicate=await statement(db,'SELECT sha256 FROM rer_portability_parts WHERE job=? AND idx=?',id,index).first();if(!duplicate || duplicate.sha256!==part.sha256)throw e;}
+  return importStatus(context,id);
+}
+const binaryKey=entry=>canonicalJson(entry.bucket===undefined?['payload',entry.name]:['object',entry.bucket,entry.key]);
+function binaryStream(context,id,entry,indexes) {
+  const selected=indexes.get(binaryKey(entry));insist(selected,'binary_inventory');
+  return stream((async function*(){
+    for(const index of selected)for(const item of (await readPart(context.db,id,index)).items) {
+      if(entry.bucket===undefined?item.kind==='payload' && item.name===entry.name:item.kind==='object' && item.bucket===entry.bucket && item.key===entry.key)yield unbase64(item.data);
+    }
+  })());
+}
+async function bucketsEmpty(context,row) {
+  const owned=await all(context.db,'SELECT bucket,key FROM rer_portability_owned WHERE job=?',row.id);
+  for(const [name,spec] of Object.entries(context.config.buckets)) {
+    let cursor;
+    do {const page=await context.buckets[name].list({prefix:spec.prefix??'',cursor});
+      insist(page.objects.every(o=>owned.some(x=>x.bucket===name && x.key===o.key)),'bucket_not_empty:'+name,409);
+      cursor=page.truncated?page.cursor:undefined;
+    }while(cursor);
+  }
+}
+function emptyGuards(context,row) {
+  return row.metadata.targets.filter(t=>!context.config.replaceableTables.includes(t.name)).map(t=>guard(context.db,`NOT EXISTS(SELECT 1 FROM ${quote(t.name)})`));
+}
+async function removeOwned(context,id) {
+  const owned=await all(context.db,'SELECT bucket,key FROM rer_portability_owned WHERE job=?',id);
+  for(const o of owned)await context.buckets[o.bucket].delete(o.key);
+  await run(context.db,'DELETE FROM rer_portability_owned WHERE job=?',id);
+}
+export async function finalizeImport(context,id,{contentHash,archiveHash,laterJournal}={}) {
+  const {db,config}=context,row=await job(db,id,'import');
+  if(row.phase==='complete')return importStatus(context,id);
+  insist(['staging','applying'].includes(row.phase),'import_phase',409);
+  const manifest=row.metadata.manifest;insist(row.completed===manifest.partCount,'parts_missing');
+  const verifier=new ContentVerifier(manifest),archive=new ArchiveVerifier(manifest),binaryIndexes=new Map();let index=0;
+  // Rebuild from the fully verified parts on every finalize, including resume.
+  // Keep only part positions: object reads must not rescan unrelated bytes.
+  for await(const part of parts(db,id,row.completed)){
+    await checkPart(part,index,config.partBytes);archive.add(part);
+    for(const item of part.items){
+      verifier.add(item);
+      if(item.kind==='object'||item.kind==='payload'){
+        const key=binaryKey(item),selected=binaryIndexes.get(key)??[];
+        if(selected.at(-1)!==index)selected.push(index);
+        binaryIndexes.set(key,selected);
+      }
+    }
+    index++;
+  }
+  const digest=verifier.digest();if(contentHash!==undefined)insist(contentHash===digest,'content_checksum');
+  insist(archiveHash===archive.digest(),'archive_checksum',409);
+  const checked=await fence(db,manifest,row.metadata.copies,laterJournal);
+  if(row.metadata.legacyRecovery)await requireLegacyEmpty(context,row.metadata.legacyRecovery.emptyTables);
+  for(const table of row.metadata.targets) if(!config.replaceableTables.includes(table.name))insist(!(await statement(db,`SELECT 1 FROM ${quote(table.name)} LIMIT 1`).first()),'target_not_empty:'+table.name,409);
+  const dependentGuards=await excludedDependentGuards(db,row.metadata.targets,config.replaceableTables);
+  await bucketsEmpty(context,row);
+  await refuseDevData(db,row.metadata.copies,row.metadata.targets);
+  await refuseProviderData(db,row.metadata.copies,row.metadata.targets);
+  if(config.validateStaged) {
+    const read=async table=>{insist(row.metadata.copies[table],'staged_table');const target=row.metadata.targets.find(t=>t.name===table);return (await readCopy(db,target,row.metadata.copies[table],0,row.metadata.manifest.tables.find(t=>t.name===table)?.count??0)).map(x=>x.row);};
+    insist(await config.validateStaged({db,tables:{...row.metadata.copies},read,manifest})!==false,'staged_validation');
+  }
+  row.phase='applying';
+  await db.batch([guard(db,'(SELECT import_job FROM rer_portability_state WHERE id=1)=?',id),db.prepare('UPDATE rer_portability_state SET applying=1 WHERE id=1'),save(db,row),db.prepare('DELETE FROM rer_portability_guard')]);
+  try {
+    for(const object of manifest.objects) {
+      await run(db,'INSERT OR IGNORE INTO rer_portability_owned VALUES(?,?,?)',id,object.bucket,object.key);
+      await writeObject(context.buckets[object.bucket],object,binaryStream(context,id,object,binaryIndexes));
+      const stored=await context.buckets[object.bucket].get(object.key);insist(stored,'object_write_failed');
+      // Read back independently, including multipart results, before the commit.
+      const hash=new (await import('./hash.mjs')).StreamHash();const reader=stored.body.getReader();
+      for(;;){const {value,done}=await reader.read();if(done)break;hash.update(value);}reader.releaseLock();
+      insist(hash.digest()===object.sha256,'object_write_checksum');
+    }
+    for(const payload of manifest.payloads)await config.extraPayloads[payload.name].consume(binaryStream(context,id,payload,binaryIndexes),payload);
+    row.phase='complete';row.metadata.contentHash=digest;
+    await db.batch([db.prepare('PRAGMA defer_foreign_keys=ON'),...emptyGuards(context,row),...dependentGuards,...legacyEmptyGuards(db,row),
+      guard(db,'NOT EXISTS(SELECT 1 FROM rer_portability_epochs WHERE source=? AND epoch>?) AND NOT EXISTS(SELECT 1 FROM rer_portability_state WHERE source_identity=? AND epoch>?)',manifest.sourceIdentity,checked.epoch,manifest.sourceIdentity,checked.epoch),
+      ...row.metadata.targets.map(t=>guard(db,`NOT EXISTS(SELECT 1 FROM ${quote(sidecar(row.metadata.copies[t.name]))} s JOIN rer_portability_journal j ON j.source=? AND j.tombstone=s.rer_portability_tombstone)`,manifest.sourceIdentity)),
+      ...row.metadata.triggers.map(t=>db.prepare(`DROP TRIGGER ${quote(t.name)}`)),
+      ...config.replaceableTables.map(t=>db.prepare(`DELETE FROM ${quote(t)}`)),
+      ...row.metadata.targets.flatMap(t=>restoreCopy(db,t,row.metadata.copies[t.name])),
+      ...row.metadata.triggers.map(t=>db.prepare(t.sql)),...mergeJournal(db,manifest,checked),...legacyAuditStatements(db,row),save(db,row),
+      db.prepare('UPDATE rer_portability_state SET import_job=NULL,applying=0 WHERE id=1'),db.prepare('DELETE FROM rer_portability_guard')]);
+  }catch(e){
+    // D1 may have committed even when its acknowledgement failed.
+    if((await job(db,id)).phase==='complete')return importStatus(context,id);
+    await removeOwned(context,id);row.phase='staging';await db.batch([save(db,row),db.prepare('UPDATE rer_portability_state SET applying=0 WHERE id=1')]);throw e;
+  }
+  return importStatus(context,id);
+}
+export async function importStatus(context,id) {
+  const row=await job(context.db,id,'import');return {id,kind:'import',phase:row.phase,completed:row.completed,total:row.metadata.manifest.partCount,
+    contentHash:row.metadata.contentHash??null,requiredConfiguration:row.phase==='complete'?row.metadata.manifest.requiredConfiguration:[],...(row.metadata.bridge?{bridged:row.metadata.bridge}:{})};
+}
+export async function abortImport(context,id) {
+  const row=await job(context.db,id,'import');insist(row.phase!=='complete','import_complete',409);
+  await removeOwned(context,id);await cleanCopies(context.db,row);row.phase='aborted';
+  await context.db.batch([save(context.db,row),statement(context.db,'UPDATE rer_portability_state SET import_job=NULL,applying=0 WHERE id=1 AND import_job=?',id)]);
+  return importStatus(context,id);
+}

package/modules/portability/index.d.mts

@@ -0,0 +1,58 @@
+export type Cell = null | string | number | { blob: string };
+export type Row = Record<string, Cell>;
+export interface Column { name: string; type: string; notnull: number; pk: number; generated?: 'virtual'|'stored' }
+export interface Table { name: string; sql: string; columns: Column[]; dependencies: string[]; count: number; rowidAlias: string|null }
+export interface ObjectDescriptor { bucket: string; key: string; size: number; sha256: string; httpMetadata: Omit<R2HTTPMetadata,'cacheExpiry'> & {cacheExpiry?:string}; customMetadata: Record<string,string> }
+export interface PayloadDescriptor { name: string; size: number; sha256: string }
+export interface Manifest {
+  kind: 'manifest'; format: 'runeditrun-portability'; version: 2; product: string; exportedAt: string; sourceIdentity: string;
+  schema: string; tables: Table[]; objects: ObjectDescriptor[]; payloads: PayloadDescriptor[];
+  excludedTables: string[]; excludedColumns: Record<string,string[]>; requiredConfiguration: string[]; partCount: number; epoch: number; journal: string[];
+  /** Applied product migrations; absent from archives made before additive restore. */
+  migrations?: string[];
+}
+export type Item = {kind:'row';table:string;index:number;row:Row;rowid?:number} | {kind:'object';bucket:string;key:string;offset:number;data:string} | {kind:'payload';name:string;offset:number;data:string};
+export interface Part {kind:'part';index:number;items:Item[];sha256:string}
+export interface Closing {kind:'closing';partCount:number;contentHash:string;archiveHash:string}
+export interface LaterJournal {sourceIdentity:string;epoch:number;journal:string[]}
+export interface Staged {
+  db: D1Database; tables: Record<string,string>; manifest: Manifest;
+  read(table:string): Promise<Record<string,unknown>[]>;
+}
+export interface ArchivePolicy {
+  product: string; excludeTables?: string[]; includeTables?: string[]; derivedTables?: string[];
+  secretColumns?: Record<string,string[]>; exportRow?: (table:string,row:Row)=>Row;
+  replaceableTables?: string[]; requiredConfiguration?: string[]; validateStaged?: (staged:Staged)=>boolean|void|Promise<boolean|void>;
+  acceptedSchemas?: Record<string,{upgradeRow:(table:string,row:Row)=>Row}>;
+  buckets?: Record<string,{prefix?:string;objectReferences?:{table:string;keyColumn:string}}>;
+  extraPayloads?: Record<string,{produce:()=>Uint8Array|ReadableStream<Uint8Array>|Promise<Uint8Array|ReadableStream<Uint8Array>>;consume:(source:ReadableStream<Uint8Array>,descriptor:PayloadDescriptor)=>void|Promise<void>}>;
+  partBytes?: number;
+}
+export interface Status {id:string;kind:'export'|'import';phase:'building'|'staging'|'applying'|'complete'|'aborted';completed:number;total:number|null;contentHash:string|null;requiredConfiguration?:string[];bridged?:Bridge}
+/** Later additive migrations an older archive was restored across. */
+export interface Bridge {fromSchema:string;toSchema:string;migrations:string[];tables:string[];columns:Record<string,string[]>}
+/** A runner's replay of the destination's applied migrations: the archive schema identity after a chosen set, and each migration's statements. */
+export interface MigrationReplay {names:string[];statements(name:string):string[];schemaAfter(names:string[]):Promise<string>}
+export interface PortabilityOptions {db:D1Database;buckets?:Record<string,R2Bucket>;policy:ArchivePolicy;migrations?:()=>Promise<MigrationReplay>}
+export interface Portability {
+  exports: {create(actor?:string):Promise<string>;advance(id:string,options?:{expectedPart?:number;maxParts?:number}):Promise<Status>;status(id:string):Promise<Status>;download(id:string):Promise<ReadableStream<Uint8Array>>;abort(id:string):Promise<Status>};
+  imports: {create(manifest:Manifest,actor?:string):Promise<string>;uploadPart(id:string,index:number,part:Part):Promise<Status>;finalize(id:string,options:{archiveHash:string;contentHash?:string;laterJournal?:LaterJournal}):Promise<Status>;status(id:string):Promise<Status>;abort(id:string):Promise<Status>};
+  status(id:string):Promise<Status>;actor(id:string):Promise<string>;
+  erasureStatements(table:string,primaryKey:Record<string,Cell>):Promise<D1PreparedStatement[]>;
+  redactionStatements():Promise<D1PreparedStatement[]>;
+  invalidateTransfersStatements():Promise<D1PreparedStatement[]>;
+}
+export function createPortability(options:PortabilityOptions):Promise<Portability>;
+export interface HandlerOptions extends PortabilityOptions {
+  appOrigin:string;basePath?:string;maxBodyBytes?:number;
+  authorize:(request:Request,context:{operation:'export'|'import'|'unknown';jobId:string|null})=>string|null|Promise<string|null>;
+}
+export function createPortabilityHandler(options:HandlerOptions):Promise<(request:Request)=>Promise<Response|null>>;
+export function archiveLines(source:ReadableStream<Uint8Array>,maxLineBytes:number):AsyncGenerator<string>;
+export function readArchive(source:ReadableStream<Uint8Array>,options?:{maxLineBytes?:number;partBytes?:number}):AsyncGenerator<Manifest|Part|Closing>;
+export function makePart(index:number,items:Item[]):Promise<Part>;
+export function canonicalJson(value:unknown):string;
+export function sha256(value:string|Uint8Array):Promise<string>;
+export const MAX_LINE_BYTES:number;
+export const MAX_PART_BYTES:number;
+export class PortabilityError extends Error {code:string;status:number;constructor(code:string,status?:number)}

package/modules/portability/index.mjs

@@ -0,0 +1,22 @@
+import { policy, catalogue } from './catalogue.mjs';
+import { insist, job } from './storage.mjs';
+import { createExport, advanceExport, exportStatus, downloadExport, abortExport } from './export.mjs';
+import { createImport, uploadPart, finalizeImport, importStatus, abortImport } from './import.mjs';
+import { erasureStatements, redactionStatements, invalidateTransfersStatements } from './journal.mjs';
+export { archiveLines } from './lines.mjs';
+export { readArchive, makePart, MAX_LINE_BYTES, MAX_PART_BYTES } from './format.mjs';
+export { canonicalJson, sha256 } from './hash.mjs';
+export { PortabilityError } from './storage.mjs';
+export { createPortabilityHandler } from './http.mjs';
+export async function createPortability({db,buckets={},policy:input,migrations}) {
+  const config=policy(input);await catalogue(db,config);
+  for(const name of Object.keys(config.buckets))insist(buckets[name] && typeof buckets[name].get==='function','missing_bucket:'+name);
+  const context={db,buckets,config,migrations};
+  return {
+    exports:{create:actor=>createExport(context,actor),advance:(id,options)=>advanceExport(context,id,options),status:id=>exportStatus(context,id),download:id=>downloadExport(context,id),abort:id=>abortExport(context,id)},
+    imports:{create:(manifest,actor)=>createImport(context,manifest,actor),uploadPart:(id,index,part)=>uploadPart(context,id,index,part),finalize:(id,options)=>finalizeImport(context,id,options),status:id=>importStatus(context,id),abort:id=>abortImport(context,id)},
+    status:async id=>(await job(db,id)).kind==='export'?exportStatus(context,id):importStatus(context,id),
+    actor:async id=>(await job(db,id)).actor,
+    erasureStatements:(table,key)=>erasureStatements(db,table,key),redactionStatements:()=>redactionStatements(db),invalidateTransfersStatements:()=>invalidateTransfersStatements(db),
+  };
+}

package/modules/portability/journal.mjs

@@ -0,0 +1,49 @@
+import { all, statement, state, insist, quote } from './storage.mjs';
+import { canonicalJson, sha256 } from './hash.mjs';
+import { sidecar } from './copies.mjs';
+export async function tombstone(source,table,key) { return sha256(canonicalJson([source,table,key])); }
+export async function erasureStatements(db,table,primaryKey) {
+  const s=await state(db), hash=await tombstone(s.source_identity,table,primaryKey);
+  return [statement(db,'INSERT OR IGNORE INTO rer_portability_journal VALUES(?,?)',s.source_identity,hash),
+    db.prepare('UPDATE rer_portability_state SET epoch=epoch+1 WHERE id=1'),
+    statement(db,'INSERT INTO rer_portability_epochs SELECT source_identity,epoch FROM rer_portability_state WHERE id=1 ON CONFLICT(source) DO UPDATE SET epoch=MAX(epoch,excluded.epoch)')];
+}
+/** Redaction keeps a value-free row portable. Fence the source checkpoint instead
+ * of banning that retained primary key from every future archive. */
+export async function redactionStatements(db) {
+  return erasureStatements(db,'rer_portability_state',{id:1});
+}
+export async function invalidateTransfersStatements(db) {
+  const tables=await all(db,"SELECT name FROM sqlite_master WHERE type='table' AND (name GLOB 'rer_portability_snap_*' OR name GLOB 'rer_portability_stage_*')");
+  return [...tables.map(x=>db.prepare(`DROP TABLE ${quote(x.name)}`)),db.prepare('DELETE FROM rer_portability_segments'),db.prepare('DELETE FROM rer_portability_parts'),
+    db.prepare('DELETE FROM rer_portability_jobs'),db.prepare('UPDATE rer_portability_state SET import_job=NULL,applying=0 WHERE id=1')];
+}
+export async function journal(db,source) {
+  return (await all(db,'SELECT tombstone FROM rer_portability_journal WHERE source=? ORDER BY tombstone',source)).map(x=>x.tombstone);
+}
+export async function fence(db,manifest,copies,later) {
+  const recorded=await statement(db,'SELECT epoch FROM rer_portability_epochs WHERE source=?',manifest.sourceIdentity).first();
+  const s=await state(db);
+  insist(manifest.epoch>=Math.max(recorded?.epoch??0,s.source_identity===manifest.sourceIdentity?s.epoch:0),'erasure_epoch');
+  if(later) insist(later.sourceIdentity===manifest.sourceIdentity && Number.isSafeInteger(later.epoch) && later.epoch>=manifest.epoch && Array.isArray(later.journal),'later_journal');
+  const hashes=new Set([...manifest.journal,...await journal(db,manifest.sourceIdentity),...(later?.journal??[])]);
+  insist([...hashes].every(x=>/^[a-f0-9]{64}$/.test(x)),'invalid_journal');
+  // A later trusted redaction checkpoint refuses the entire older archive; an
+  // archive made after redaction still carries its anonymous attribution rows.
+  const checkpoint=await tombstone(manifest.sourceIdentity,'rer_portability_state',{id:1});
+  if(later && hashes.has(checkpoint)) insist(manifest.epoch>=later.epoch,'erasure_epoch');
+  for(const table of manifest.tables) {
+    for(let offset=0;offset<table.count;offset+=500) {
+      const rows=await all(db,`SELECT rer_portability_tombstone FROM ${quote(sidecar(copies[table.name]))} ORDER BY rer_portability_row_order LIMIT 500 OFFSET ?`,offset);
+      for(const row of rows) {
+        insist(!hashes.has(row.rer_portability_tombstone),'erased_row:'+table.name);
+      }
+    }
+  }
+  return {hashes:[...hashes],epoch:later?.epoch??manifest.epoch};
+}
+export function mergeJournal(db,manifest,checked) {
+  return [...checked.hashes.map(h=>statement(db,'INSERT OR IGNORE INTO rer_portability_journal VALUES(?,?)',manifest.sourceIdentity,h)),
+    statement(db,'INSERT INTO rer_portability_epochs VALUES(?,?) ON CONFLICT(source) DO UPDATE SET epoch=MAX(epoch,excluded.epoch)',manifest.sourceIdentity,checked.epoch),
+    statement(db,'UPDATE rer_portability_state SET source_identity=?,epoch=? WHERE id=1',manifest.sourceIdentity,checked.epoch)];
+}