@runeditrun/rer 0.2.67 → 0.2.69

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,5 +1,7 @@
1
1
  import { all, statement, state, guard, job, save, partQueries, parts, readPart, cleanCopies, quote, stream, insist, run } from './storage.mjs';
2
2
  import { stagePart } from './import-staging.mjs';
3
+ import { normalizeStaged } from './import-safety.mjs';
4
+ import { ensureImportLeases, createImportLease, importLeaseStatus, importRunning, withImportLease } from './import-lease.mjs';
3
5
  import { refuseProviderData } from './provider-data.mjs';
4
6
  import { refuseDevData } from './dev-data.mjs';
5
7
  import { catalogue, validateRow, bindCell, writableColumns } from './catalogue.mjs';
@@ -11,18 +13,21 @@ import { writeObject } from './payloads.mjs';
11
13
  import { createCopy,sidecar,readCopy,restoreCopy } from './copies.mjs';
12
14
  import { excludedDependentGuards, migrationSeeded, seededCondition } from './restore-safety.mjs';
13
15
  import { additiveBridge, sameSchemaMigrations, appliedMigrations } from './bridge.mjs';
16
+ import { checkArchivedLedgers, archivedHistory } from './ledgers.mjs';
14
17
  import { legacyEmptyGuards, legacyAuditStatements, requireLegacyEmpty } from './legacy-safety.mjs';
15
- export async function createImport(context,manifest,actor='direct',legacyRecovery) {
16
- const {db,config,buckets}=context;await checkManifest(manifest);
18
+ // ledgers: the rows of each migration ledger table the archive carries, as the
19
+ // caller read them from it ({table:rows}); uploadPart binds them to its parts.
20
+ export async function createImport(context,manifest,actor='direct',legacyRecovery,ledgers) {
21
+ const {db,config,buckets}=context;await checkManifest(manifest);checkArchivedLedgers(manifest,ledgers);
17
22
  insist(manifest.product===config.product,'wrong_product');
18
23
  // Explicit schema upgrades can reconcile rows, but cannot attest an interrupted deployment.
19
24
  await appliedMigrations(db);await state(db);
20
25
  const original=manifest;manifest=await restorableManifest(original);
21
26
  const skippedTables=original.tables.filter(table=>platformTable(table.name)).map(table=>table.name);
22
27
  const cat=await catalogue(db,config),upgrade=config.acceptedSchemas[original.schema];
23
- const same=manifest.schema===cat.identity;
24
- const bridge=same || upgrade?undefined:await additiveBridge(db,manifest,cat,context.migrations);
25
- if(same && !upgrade)await sameSchemaMigrations(db,manifest,cat,context.migrations);
28
+ const same=manifest.schema===cat.identity,history=()=>archivedHistory(original,ledgers);
29
+ const bridge=same || upgrade?undefined:await additiveBridge(db,manifest,cat,context.migrations,history);
30
+ if(same && !upgrade)await sameSchemaMigrations(db,manifest,cat,context.migrations,history);
26
31
  if(!bridge)insist(canonicalJson(manifest.tables.map(x=>x.name))===canonicalJson(cat.tables.map(x=>x.name)),'table_inventory');
27
32
  if(!upgrade && !bridge)for(const table of manifest.tables) {
28
33
  const target=cat.tables.find(t=>t.name===table.name);
@@ -35,26 +40,36 @@ export async function createImport(context,manifest,actor='direct',legacyRecover
35
40
  insist(canonicalJson(manifest.requiredConfiguration)===canonicalJson(config.requiredConfiguration),'configuration_inventory');
36
41
  for(const o of manifest.objects) insist(buckets[o.bucket] && config.buckets[o.bucket] && o.key.startsWith(config.buckets[o.bucket].prefix??''),'bucket_inventory');
37
42
  insist(canonicalJson(manifest.payloads.map(x=>x.name))===canonicalJson(Object.keys(config.extraPayloads).sort()),'payload_inventory');
43
+ await ensureImportLeases(db);
44
+ const previous=(await state(db)).import_job;
45
+ if(previous) {
46
+ if(!(await importLeaseStatus(db,await job(db,previous,'import'))).stale)throw await importRunning(db,previous);
47
+ await abortImport(context,previous,{staleOnly:true});
48
+ }
38
49
  const id=crypto.randomUUID(),copies=Object.fromEntries(targets.map((t,i)=>[t.name,'rer_portability_stage_'+id.replaceAll('-','')+'_'+i]));
39
- const row={id,kind:'import',actor,phase:'staging',completed:0,metadata:{manifest:original,...(skippedTables.length?{restoreManifest:manifest,skippedTables}:{}),copies,targets,triggers:cat.triggers,...(bridge?{bridge}:{}),...(legacyRecovery?{legacyRecovery}: {})}};
50
+ const row={id,kind:'import',actor,phase:'staging',completed:0,metadata:{manifest:original,...(skippedTables.length?{restoreManifest:manifest,skippedTables}:{}),copies,targets,triggers:cat.triggers,...(bridge?{bridge}:{}),...(legacyRecovery?{legacyRecovery}: {}),...(ledgers?{ledgers}:{})}};
40
51
  try {
41
52
  await db.batch([guard(db,'(SELECT import_job FROM rer_portability_state WHERE id=1) IS NULL'),
42
53
  statement(db,'UPDATE rer_portability_state SET import_job=? WHERE id=1',id),
43
54
  ...targets.flatMap(t=>createCopy(db,t,copies[t.name])),
44
- statement(db,'INSERT INTO rer_portability_jobs(id,kind,actor,phase,metadata) VALUES(?,?,?,?,?)',id,'import',actor,row.phase,canonicalJson(row.metadata)),db.prepare('DELETE FROM rer_portability_guard')]);
45
- }catch(e){if((await state(db)).import_job)throw new (await import('./storage.mjs')).PortabilityError('import_running',409);throw e;}
55
+ statement(db,'INSERT INTO rer_portability_jobs(id,kind,actor,phase,metadata) VALUES(?,?,?,?,?)',id,'import',actor,row.phase,canonicalJson(row.metadata)),createImportLease(db,id),db.prepare('DELETE FROM rer_portability_guard')]);
56
+ }catch(e){const running=(await state(db)).import_job;if(running)throw await importRunning(db,running);throw e;}
46
57
  return id;
47
58
  }
48
59
  export async function uploadPart(context,id,index,part) {
60
+ return withImportLease(context,id,leased=>uploadLeasedPart(leased,id,index,part));
61
+ }
62
+ async function uploadLeasedPart(context,id,index,part) {
49
63
  const {db,config}=context,row=await job(db,id,'import');await checkPart(part,index,config.partBytes);
50
64
  const previous=await statement(db,'SELECT sha256 FROM rer_portability_parts WHERE job=? AND idx=?',id,index).first();
51
- if(previous){insist(previous.sha256===part.sha256,'part_conflict',409);return importStatus(context,id);}
65
+ if(previous){insist(previous.sha256===part.sha256,'part_conflict',409);return importStatus(context,id,{includeLease:false});}
52
66
  insist(row.phase==='staging','import_phase',409);insist(index===row.completed && index<row.metadata.manifest.partCount,'part_order',409);
53
67
  const inserts=[],groups=new Map(),manifest=row.metadata.manifest,upgrade=config.acceptedSchemas[manifest.schema];
54
68
  for(const item of part.items) {
55
69
  if(item.kind==='row') {
56
70
  const table=manifest.tables.find(x=>x.name===item.table);insist(table && Number.isSafeInteger(item.index) && item.index>=0 && item.index<table.count,'row_inventory');
57
71
  validateRow(table,item.row,manifest.excludedColumns[table.name]??[]);
72
+ if(row.metadata.ledgers?.[table.name])insist(canonicalJson(item.row)===canonicalJson(row.metadata.ledgers[table.name][item.index]),'ledger_inventory');
58
73
  if(platformTable(table.name))continue;
59
74
  const target=row.metadata.targets.find(x=>x.name===table.name);
60
75
  const data=upgrade?upgrade.upgradeRow(table.name,structuredClone(item.row)):item.row;
@@ -88,7 +103,7 @@ export async function uploadPart(context,id,index,part) {
88
103
  }
89
104
  try{await stagePart(context,row,part,[...inserts,...partQueries(id,part).slice(1)],stagedCounts);}
90
105
  catch(e){const duplicate=await statement(db,'SELECT sha256 FROM rer_portability_parts WHERE job=? AND idx=?',id,index).first();if(!duplicate || duplicate.sha256!==part.sha256)throw e;}
91
- return importStatus(context,id);
106
+ return importStatus(context,id,{includeLease:false});
92
107
  }
93
108
  const binaryKey=entry=>canonicalJson(entry.bucket===undefined?['payload',entry.name]:['object',entry.bucket,entry.key]);
94
109
  function binaryStream(context,id,entry,indexes) {
@@ -129,7 +144,11 @@ async function removeOwned(context,id) {
129
144
  for(const o of owned)await context.buckets[o.bucket].delete(o.key);
130
145
  await run(context.db,'DELETE FROM rer_portability_owned WHERE job=?',id);
131
146
  }
132
- export async function finalizeImport(context,id,{contentHash,archiveHash,laterJournal}={}) {
147
+ export async function finalizeImport(context,id,options={}) {
148
+ if((await job(context.db,id,'import')).phase==='complete')return importStatus(context,id);
149
+ return withImportLease(context,id,leased=>finalizeLeasedImport(leased,id,options));
150
+ }
151
+ async function finalizeLeasedImport(context,id,{contentHash,archiveHash,laterJournal}={}) {
133
152
  const {db,config}=context,row=await job(db,id,'import');
134
153
  if(row.phase==='complete')return importStatus(context,id);
135
154
  insist(['staging','applying'].includes(row.phase),'import_phase',409);
@@ -158,6 +177,7 @@ export async function finalizeImport(context,id,{contentHash,archiveHash,laterJo
158
177
  await bucketsEmpty(context,row);
159
178
  await refuseDevData(db,row.metadata.copies,row.metadata.targets);
160
179
  await refuseProviderData(db,row.metadata.copies,row.metadata.targets);
180
+ await normalizeStaged(context,row);
161
181
  if(config.validateStaged) {
162
182
  const read=async table=>{insist(row.metadata.copies[table],'staged_table');const target=row.metadata.targets.find(t=>t.name===table);return (await readCopy(db,target,row.metadata.copies[table],0,row.metadata.manifest.tables.find(t=>t.name===table)?.count??0)).map(x=>x.row);};
163
183
  insist(await config.validateStaged({db,tables:{...row.metadata.copies},read,manifest})!==false,'staged_validation');
@@ -187,16 +207,27 @@ export async function finalizeImport(context,id,{contentHash,archiveHash,laterJo
187
207
  }catch(e){
188
208
  // D1 may have committed even when its acknowledgement failed.
189
209
  if((await job(db,id)).phase==='complete')return importStatus(context,id);
210
+ if(!await context.importLease.owned())throw new (await import('./storage.mjs')).PortabilityError('import_lease_expired',409);
190
211
  await removeOwned(context,id);row.phase='staging';await db.batch([save(db,row),db.prepare('UPDATE rer_portability_state SET applying=0 WHERE id=1')]);throw e;
191
212
  }
192
213
  return importStatus(context,id);
193
214
  }
194
- export async function importStatus(context,id) {
195
- const row=await job(context.db,id,'import');return {id,kind:'import',phase:row.phase,completed:row.completed,total:row.metadata.manifest.partCount,
196
- contentHash:row.metadata.contentHash??null,requiredConfiguration:row.phase==='complete'?row.metadata.manifest.requiredConfiguration:[],...(row.metadata.bridge?{bridged:row.metadata.bridge}:{}),...(row.metadata.skippedTables?{skippedTables:row.metadata.skippedTables}:{})};
215
+ // Upload receipts stay idempotent; the status door exposes the moving lease.
216
+ export async function importStatus(context,id,{includeLease=true}={}) {
217
+ const row=await job(context.db,id,'import');return {id,kind:'import',phase:row.phase,completed:row.completed,total:row.metadata.manifest.partCount,...(includeLease?await importLeaseStatus(context.db,row):{}),
218
+ contentHash:row.metadata.contentHash??null,requiredConfiguration:row.phase==='complete'?row.metadata.manifest.requiredConfiguration:[],...(row.metadata.bridge?{bridged:row.metadata.bridge}:{}),...(row.metadata.skippedTables?{skippedTables:row.metadata.skippedTables}:{}),...(row.metadata.safetyChanges?{safetyChanges:row.metadata.safetyChanges}:{})};
197
219
  }
198
- export async function abortImport(context,id) {
220
+ export async function activeImport(context,actor='direct') {
221
+ const id=(await state(context.db)).import_job;
222
+ return id && (await job(context.db,id,'import')).actor===actor?importStatus(context,id):null;
223
+ }
224
+ export async function abortImport(context,id,options={}) {
199
225
  const row=await job(context.db,id,'import');insist(row.phase!=='complete','import_complete',409);
226
+ if(row.phase==='aborted')return importStatus(context,id);
227
+ return withImportLease(context,id,leased=>abortLeasedImport(leased,row),{abort:true,...options});
228
+ }
229
+ async function abortLeasedImport(context,row) {
230
+ const id=row.id;
200
231
  await removeOwned(context,id);await cleanCopies(context.db,row);row.phase='aborted';
201
232
  await context.db.batch([save(context.db,row),statement(context.db,'UPDATE rer_portability_state SET import_job=NULL,applying=0 WHERE id=1 AND import_job=?',id)]);
202
233
  return importStatus(context,id);
@@ -207,5 +238,5 @@ export async function abortImport(context,id) {
207
238
  export async function abortAbandonedImport(context) {
208
239
  const id=(await state(context.db)).import_job;
209
240
  if(!id)return null;
210
- await abortImport(context,id);return id;
241
+ await abortImport(context,id,{soleOperator:true});return id;
211
242
  }
@@ -22,21 +22,27 @@ export interface Staged {
22
22
  export interface ArchivePolicy {
23
23
  product: string; excludeTables?: string[]; includeTables?: string[]; derivedTables?: string[];
24
24
  secretColumns?: Record<string,string[]>; exportRow?: (table:string,row:Row)=>Row;
25
+ /** Nonempty finite safe-value sets for writable non-key columns; requires importRow. */
26
+ importSafety?: Record<string,Record<string,Cell[]>>;
27
+ /** Pure, synchronous, deterministic and idempotent; called only for importSafety tables before validateStaged. */
28
+ importRow?: (table:string,row:Row)=>Row;
25
29
  replaceableTables?: string[]; requiredConfiguration?: string[]; validateStaged?: (staged:Staged)=>boolean|void|Promise<boolean|void>;
26
30
  acceptedSchemas?: Record<string,{upgradeRow:(table:string,row:Row)=>Row}>;
27
31
  buckets?: Record<string,{prefix?:string;objectReferences?:{table:string;keyColumn:string}}>;
28
32
  extraPayloads?: Record<string,{produce:()=>Uint8Array|ReadableStream<Uint8Array>|Promise<Uint8Array|ReadableStream<Uint8Array>>;consume:(source:ReadableStream<Uint8Array>,descriptor:PayloadDescriptor)=>void|Promise<void>}>;
29
33
  partBytes?: number;
30
34
  }
31
- export interface Status {id:string;kind:'export'|'import';phase:'building'|'staging'|'applying'|'complete'|'aborted';completed:number;total:number|null;/** Export only: bytes of the parts built so far. */bytes?:number;contentHash:string|null;requiredConfiguration?:string[];bridged?:Bridge;/** Platform bookkeeping ignored in an older archive. */skippedTables?:string[]}
35
+ /** Actual changed rows grouped by table, column and safe replacement value; never original values. */
36
+ export interface SafetyChange {table:string;column:string;value:Cell;rows:number}
37
+ export interface Status {id:string;kind:'export'|'import';phase:'building'|'staging'|'applying'|'complete'|'aborted';completed:number;total:number|null;/** Import only: inactivity deadline and whether it has passed. */abortableAt?:string|null;stale?:boolean;/** Export only: bytes of the parts built so far. */bytes?:number;contentHash:string|null;requiredConfiguration?:string[];bridged?:Bridge;/** Platform bookkeeping ignored in an older archive. */skippedTables?:string[];/** Durable summary of bounded import normalization. */safetyChanges?:SafetyChange[]}
32
38
  /** Later additive migrations an older archive was restored across. */
33
39
  export interface Bridge {fromSchema:string;toSchema:string;migrations:string[];tables:string[];columns:Record<string,string[]>}
34
40
  /** A runner's replay of the destination's applied migrations: the archive schema identity after a chosen set, and each migration's statements. */
35
- export interface MigrationReplay {names:string[];statements(name:string):string[];schemaAfter(names:string[]):Promise<string>}
41
+ export interface MigrationReplay {names:string[];statements(name:string):string[];/** The migration file's SHA-256 hex, where the runner read the file. */sha256?(name:string):string|undefined;schemaAfter(names:string[]):Promise<string>}
36
42
  export interface PortabilityOptions {db:D1Database;buckets?:Record<string,R2Bucket>;policy:ArchivePolicy;migrations?:()=>Promise<MigrationReplay>}
37
43
  export interface Portability {
38
44
  exports: {create(actor?:string):Promise<string>;advance(id:string,options?:{expectedPart?:number;maxParts?:number}):Promise<Status>;status(id:string):Promise<Status>;download(id:string):Promise<ReadableStream<Uint8Array>>;abort(id:string):Promise<Status>};
39
- imports: {create(manifest:Manifest,actor?:string):Promise<string>;uploadPart(id:string,index:number,part:Part):Promise<Status>;finalize(id:string,options:{archiveHash:string;contentHash?:string;laterJournal?:LaterJournal}):Promise<Status>;status(id:string):Promise<Status>;abort(id:string):Promise<Status>;/** Aborts the import left staged by a crashed restore and returns its id; call only as the destination's sole operator. */abortAbandoned():Promise<string|null>};
45
+ imports: {active(actor?:string):Promise<Status|null>;/** ledgers: the rows of each migration ledger table an old archive carries, as read from it; they place it in the destination's history and must equal its parts. */create(manifest:Manifest,actor?:string,options?:{ledgers?:Record<string,Row[]>}):Promise<string>;uploadPart(id:string,index:number,part:Part):Promise<Status>;finalize(id:string,options:{archiveHash:string;contentHash?:string;laterJournal?:LaterJournal}):Promise<Status>;status(id:string):Promise<Status>;abort(id:string):Promise<Status>;/** Aborts the import left staged by a crashed restore and returns its id; call only as the destination's sole operator. */abortAbandoned():Promise<string|null>};
40
46
  status(id:string):Promise<Status>;actor(id:string):Promise<string>;
41
47
  erasureStatements(table:string,primaryKey:Record<string,Cell>):Promise<D1PreparedStatement[]>;
42
48
  redactionStatements():Promise<D1PreparedStatement[]>;
@@ -59,4 +65,4 @@ export const excludedTables:readonly string[];
59
65
  /** True when no archive under this policy carries `table`. */
60
66
  export function excludedTable(table:string,policy?:Pick<ArchivePolicy,'excludeTables'|'includeTables'>):boolean;
61
67
  export const MAX_PART_BYTES:number;
62
- export class PortabilityError extends Error {code:string;status:number;constructor(code:string,status?:number)}
68
+ export class PortabilityError extends Error {code:string;status:number;jobId?:string;abortableAt?:string|null;constructor(code:string,status?:number)}
@@ -1,8 +1,9 @@
1
+ import { ensureImportLeases } from './import-lease.mjs';
1
2
  import { policy, catalogue } from './catalogue.mjs';
2
3
  import { wireDatabase } from '../d1-transport/index.mjs';
3
4
  import { insist, job } from './storage.mjs';
4
5
  import { createExport, advanceExport, exportStatus, downloadExport, abortExport } from './export.mjs';
5
- import { createImport, uploadPart, finalizeImport, importStatus, abortImport, abortAbandonedImport } from './import.mjs';
6
+ import { createImport, uploadPart, finalizeImport, importStatus, activeImport, abortImport, abortAbandonedImport } from './import.mjs';
6
7
  import { erasureStatements, redactionStatements, invalidateTransfersStatements } from './journal.mjs';
7
8
  export { archiveLines } from './lines.mjs';
8
9
  export { excludedTables, excludedTable } from './catalogue.mjs';
@@ -14,10 +15,11 @@ export async function createPortability({db,buckets={},policy:input,migrations})
14
15
  db=wireDatabase(db);
15
16
  const config=policy(input);await catalogue(db,config);
16
17
  for(const name of Object.keys(config.buckets))insist(buckets[name] && typeof buckets[name].get==='function','missing_bucket:'+name);
18
+ await ensureImportLeases(db);
17
19
  const context={db,buckets,config,migrations};
18
20
  return {
19
21
  exports:{create:actor=>createExport(context,actor),advance:(id,options)=>advanceExport(context,id,options),status:id=>exportStatus(context,id),download:id=>downloadExport(context,id),abort:id=>abortExport(context,id)},
20
- imports:{create:(manifest,actor)=>createImport(context,manifest,actor),uploadPart:(id,index,part)=>uploadPart(context,id,index,part),finalize:(id,options)=>finalizeImport(context,id,options),status:id=>importStatus(context,id),abort:id=>abortImport(context,id),abortAbandoned:()=>abortAbandonedImport(context)},
22
+ imports:{active:actor=>activeImport(context,actor),create:(manifest,actor,options={})=>createImport(context,manifest,actor,undefined,options.ledgers),uploadPart:(id,index,part)=>uploadPart(context,id,index,part),finalize:(id,options)=>finalizeImport(context,id,options),status:id=>importStatus(context,id),abort:id=>abortImport(context,id),abortAbandoned:()=>abortAbandonedImport(context)},
21
23
  status:async id=>(await job(db,id)).kind==='export'?exportStatus(context,id):importStatus(context,id),
22
24
  actor:async id=>(await job(db,id)).actor,
23
25
  erasureStatements:(table,key)=>erasureStatements(db,table,key),redactionStatements:()=>redactionStatements(db),invalidateTransfersStatements:()=>invalidateTransfersStatements(db),
@@ -0,0 +1,83 @@
1
+ import { all, insist } from './storage.mjs';
2
+ import { identifier } from './bookkeeping.mjs';
3
+ import { canonicalJson } from './hash.mjs';
4
+ import { validateRow } from './catalogue.mjs';
5
+
6
+ // Migration ledgers, by the runner that writes them: Base deploy and container
7
+ // runners (_rer_deploy_migrations), Wrangler and the Base deploy kit
8
+ // (d1_migrations, tracked or plain) and RER Platform (rer_migrations).
9
+ export const ledgerNames=['_rer_deploy_migrations','d1_migrations','rer_migrations'];
10
+ // Platform's ledger exactly as its runner creates it
11
+ // (platform src/integrations/cloudflare-migrations.ts:33).
12
+ const platformShape=canonicalJson([{name:'name',type:'TEXT',notnull:0,pk:1},{name:'sha256',type:'TEXT',notnull:1,pk:0},{name:'applied_at',type:'TEXT',notnull:1,pk:0}]);
13
+ const shape=columns=>canonicalJson(columns.map(({name,type,notnull,pk})=>({name,type,notnull,pk})));
14
+ // Platform records a file applied in part as <name>#<start>-<end> until the
15
+ // file completes (platform src/integrations/migration-batch.ts:31): its effects
16
+ // are uncertain, like a tracked ledger's 'applying' row.
17
+ const partial=name=>name.includes('#');
18
+
19
+ /** The destination's applied product migrations, [{name,sha256?}] in the ledger's own order, or undefined without a ledger. */
20
+ export async function destinationLedger(db) {
21
+ const found=(await all(db,`SELECT name FROM sqlite_master WHERE type='table' AND name IN (${ledgerNames.map(n=>`'${n}'`).join(',')})`)).map(x=>x.name);
22
+ if(found.includes('_rer_deploy_migrations'))return (await all(db,"SELECT * FROM _rer_deploy_migrations WHERE state='applied' ORDER BY name")).map(entry);
23
+ if(found.includes('d1_migrations')) {
24
+ const columns=(await all(db,'PRAGMA table_info(d1_migrations)')).map(x=>x.name);
25
+ const tracked=columns.includes('state') || columns.includes('sha256');
26
+ if(tracked)insist(columns.includes('state') && columns.includes('sha256'),'schema_mismatch:migration_history',409);
27
+ const rows=await all(db,`SELECT name${tracked?',state,sha256':''} FROM d1_migrations ORDER BY id`);
28
+ if(tracked)insist(rows.every(row=>row.state==='applied'),'schema_mismatch:uncertain_migrations',409);
29
+ return rows.map(entry);
30
+ }
31
+ if(!found.includes('rer_migrations'))return;
32
+ // Platform applies pending files in its plan's order and inserts each ledger
33
+ // row in the same request as the file's SQL (migration-batch.ts:35-38,43-64;
34
+ // cloudflare-migrations.ts:34-44), so rowid order is application order.
35
+ insist(shape(await all(db,'PRAGMA table_info(rer_migrations)'))===platformShape,'schema_mismatch:migration_history',409);
36
+ const rows=await all(db,'SELECT name,sha256 FROM rer_migrations ORDER BY rowid');
37
+ insist(!rows.some(row=>partial(row.name)),'schema_mismatch:uncertain_migrations',409);
38
+ return rows.map(entry);
39
+ }
40
+ const entry=row=>({name:row.name,...(typeof row.sha256==='string'?{sha256:row.sha256}:{})});
41
+
42
+ /** The ledger tables an archive carries (old archives carried rer_migrations before it became bookkeeping). */
43
+ export const archivedLedgerTables=manifest=>manifest.tables.filter(t=>ledgerNames.includes(identifier(t.name)));
44
+
45
+ /** Collects the rows of an archive's ledger tables while its records are read: add(record) each, then ledgers() is {table:rows}, or undefined without one. */
46
+ export function ledgerReader() {
47
+ let ledgers;
48
+ return {
49
+ add(record) {
50
+ if(record.kind==='manifest') {const tables=archivedLedgerTables(record);if(tables.length)ledgers=Object.fromEntries(tables.map(t=>[t.name,[]]));}
51
+ if(record.kind==='part' && ledgers)for(const item of record.items)if(item.kind==='row' && ledgers[item.table])ledgers[item.table].push(item.row);
52
+ },
53
+ ledgers:()=>ledgers,
54
+ };
55
+ }
56
+ /** Checks rows a caller read from the archive for its ledger tables ({table:rows}), so uploadPart can bind each to the authenticated parts. */
57
+ export function checkArchivedLedgers(manifest,ledgers) {
58
+ if(ledgers===undefined)return;
59
+ const tables=archivedLedgerTables(manifest);
60
+ insist(ledgers && typeof ledgers==='object' && Object.keys(ledgers).every(name=>tables.some(t=>t.name===name)),'ledger_inventory');
61
+ for(const [name,rows] of Object.entries(ledgers)) {
62
+ const table=tables.find(t=>t.name===name);
63
+ insist(Array.isArray(rows) && rows.length===table.count,'ledger_inventory');
64
+ for(const row of rows)validateRow(table,row,manifest.excludedColumns[name]??[]);
65
+ }
66
+ }
67
+
68
+ /** The applied migrations an archive's own ledger records, [{name,sha256?}]; undefined when it carries no ledger. */
69
+ export function archivedHistory(manifest,ledgers) {
70
+ const tables=archivedLedgerTables(manifest);
71
+ if(!tables.length)return;
72
+ insist(tables.length===1 && ledgers?.[tables[0].name],'schema_mismatch:migration_history');
73
+ const [table]=tables,rows=ledgers[table.name],columns=table.columns.map(c=>c.name);
74
+ if(identifier(table.name)==='rer_migrations') {
75
+ insist(shape(table.columns)===platformShape,'schema_mismatch:migration_history');
76
+ insist(!rows.some(row=>partial(row.name)),'schema_mismatch:uncertain_migrations');
77
+ }else{
78
+ insist(columns.includes('name') && columns.includes('state')===columns.includes('sha256'),'schema_mismatch:migration_history');
79
+ insist(rows.every(row=>row.state===undefined || row.state==='applied'),'schema_mismatch:uncertain_migrations');
80
+ }
81
+ insist(rows.every(row=>typeof row.name==='string') && new Set(rows.map(row=>row.name)).size===rows.length,'schema_mismatch:migration_history');
82
+ return rows.map(entry);
83
+ }
@@ -6,13 +6,16 @@ import { readLegacyArchive, ArchiveVerifier } from './format.mjs';
6
6
  import { fileRecords, exportArchive } from './files.mjs';
7
7
  import { ensureLegacyAudit, legacyTargetState, requireLegacyEmpty } from './legacy-safety.mjs';
8
8
  import { insist, PortabilityError } from './storage.mjs';
9
+ import { ledgerReader } from './ledgers.mjs';
9
10
 
10
11
  export async function recoverLegacyArchive({db,buckets,policy:input,migrations},path,output,transient,acknowledged) {
11
12
  insist(acknowledged===true,'legacy_acknowledgement_required',409);
12
13
  insist(typeof output==='string' && output!==path,'legacy_output_required',409);
13
14
  insist(!(await lstat(output).catch(error=>{if(error.code==='ENOENT')return null;throw error;})),'legacy_output_exists',409);
14
- // Whole-file verification precedes even staging/audit schema writes.
15
- for await(const _record of fileRecords(path,readLegacyArchive)){}
15
+ // Whole-file verification precedes even staging/audit schema writes. It also
16
+ // reads the archive's own migration ledger, if any, for placement (bridge.mjs).
17
+ const ledger=ledgerReader();
18
+ for await(const record of fileRecords(path,readLegacyArchive))ledger.add(record);
16
19
  const context={db,buckets,config:policy(input),migrations},target=await legacyTargetState(db,migrations);
17
20
  await requireLegacyEmpty(context,target);
18
21
  let id,manifest,closing,integrity,status,complete=false;
@@ -26,7 +29,7 @@ export async function recoverLegacyArchive({db,buckets,policy:input,migrations},
26
29
  event:'legacy_v1_unverified_restore',acknowledgedUnverifiedMetadata:true,
27
30
  sourceIdentity:record.sourceIdentity,exportedAt:record.exportedAt,
28
31
  recoveryOutput:output,...target,
29
- });
32
+ },ledger.ledgers());
30
33
  // A refused schema writes nothing, not even the audit table.
31
34
  await ensureLegacyAudit(db);
32
35
  }else if(record.kind==='part') {integrity.add(record);await uploadPart(context,id,record.index,record);}
@@ -38,6 +41,6 @@ export async function recoverLegacyArchive({db,buckets,policy:input,migrations},
38
41
  // A failure here must be reported; the durable restore audit remains intact.
39
42
  try {
40
43
  const recovered=await exportArchive(await createPortability({db,buckets,policy:input}),output,transient);
41
- return {...recovered,legacyRecovery:true,unverifiedMetadata:true,restoredContentHash:closing.contentHash,...(status.bridged?{bridged:status.bridged}:{})};
44
+ return {...recovered,legacyRecovery:true,unverifiedMetadata:true,restoredContentHash:closing.contentHash,...(status.bridged?{bridged:status.bridged}:{}),...(status.safetyChanges?{safetyChanges:status.safetyChanges}:{})};
42
45
  }catch{throw new PortabilityError('legacy_restored_export_failed',500);}
43
46
  }
@@ -4,10 +4,11 @@ import { policy, schemaIdentity } from './catalogue.mjs';
4
4
  // Replays a chosen set of the destination's migrations, in its applied order,
5
5
  // into a scratch database and returns the resulting archive schema identity.
6
6
  // Node runners only: restore uses it to place an older archive within the
7
- // destination's history (see bridge.mjs). plan: [{name,statements}].
7
+ // destination's history (see bridge.mjs). plan: [{name,statements,sha256?}],
8
+ // sha256 being the migration file's hash where the runner read the file.
8
9
  export function migrationReplay(plan,input) {
9
10
  const config=policy(input);
10
- return {names:plan.map(migration=>migration.name),statements:name=>plan.find(m=>m.name===name).statements,async schemaAfter(names) {
11
+ return {names:plan.map(migration=>migration.name),statements:name=>plan.find(m=>m.name===name).statements,sha256:name=>plan.find(m=>m.name===name).sha256,async schemaAfter(names) {
11
12
  const scratch=new DatabaseSync(':memory:');
12
13
  const db={prepare:sql=>{const query=scratch.prepare(sql);let values=[];return {bind(...args){values=args;return this;},async all(){return {results:query.all(...values)};}};}};
13
14
  try {
@@ -3,18 +3,92 @@ import { tokens, identifier, folded } from './generated.mjs';
3
3
  // Whether one migration statement can change rows a restore carries. Only
4
4
  // CREATE TABLE (not AS SELECT), CREATE INDEX and ALTER TABLE ... ADD COLUMN
5
5
  // are additive: SQLite refuses ADD COLUMN with a non-constant default, so a
6
- // restored row takes the same value the migration gave existing rows. Any
7
- // other statement (INSERT, UPDATE, DELETE, triggers, table rebuilds) changes
8
- // data unless it names none of the restorable tables, as when it only touches
9
- // operator-auth or other excluded tables.
10
- export function changesData(sql,restorable) {
11
- const list=tokens(sql),words=list.map(t=>t.quoted?undefined:folded(t.text));
12
- const kind=words.slice(0,4).filter(w=>!['temp','temporary','unique'].includes(w)).slice(0,2).join(' ');
6
+ // restored row takes the same value the migration gave existing rows.
7
+ // CREATE TRIGGER and DROP TRIGGER are schema-only whatever the body does:
8
+ // neither runs the body, so no row changes when the migration runs, and a
9
+ // restore inserts rows with the destination's triggers dropped and recreates
10
+ // them afterwards (import.mjs finalizeImport), exactly as the live upgrade left
11
+ // the rows that predate the trigger. That holds only for a statement bounded
12
+ // as exactly one trigger statement; any other (INSERT, UPDATE, DELETE, table
13
+ // rebuilds, a trigger glued to a further statement) changes data unless it
14
+ // names none of `reached` (reachingTables), as when it only touches
15
+ // operator-auth or other excluded tables that no trigger links to restorable ones.
16
+ // `empty` lists the restorable tables the archive holds no rows of: restore
17
+ // leaves them empty, and an UPDATE or DELETE of an empty table changes nothing
18
+ // (no row fires a trigger either), so such a statement changes data only if it
19
+ // names another reached table (one it could write, or read into its writes).
20
+ // Any other statement naming an empty table still does: an INSERT would leave
21
+ // rows the restore lacks.
22
+ export function changesData(sql,reached,empty=[]) {
23
+ const list=tokens(sql),words=list.map(t=>t.quoted?undefined:folded(t.text)),kind=kindOf(words);
13
24
  if(kind==='create table')return topLevelWords(list,words).includes('as');
14
25
  if(kind==='create index')return false;
15
26
  if(kind==='alter table' && words.find((w,i)=>i>2 && ['add','rename','drop'].includes(w))==='add')return false;
16
- const names=new Set(restorable.map(folded));
17
- return list.some(t=>t.quoted ? !t.text.startsWith("'") && names.has(folded(identifier(t))) : t.text.split('.').some(part=>names.has(folded(part))));
27
+ if(kind==='create trigger' && createdTrigger(list))return false;
28
+ if(kind==='drop trigger' && droppedTrigger(list))return false;
29
+ const skipped=new Set(['update','delete'].includes(words[0])?empty.map(folded):[]);
30
+ return names(list,new Set(reached.map(folded).filter(name=>!skipped.has(name))));
31
+ }
32
+ // The restorable tables and every table whose writes reach one through a
33
+ // trigger any of `statements` creates: a write to such a table during a later
34
+ // migration changes restorable rows the archive lacks. A trigger counts when
35
+ // its statement names a reached table anywhere; one this parser cannot bound
36
+ // adds every name it holds.
37
+ export function reachingTables(statements,restorable) {
38
+ const reached=new Set(restorable.map(folded));
39
+ const triggers=statements.map(sql=>tokens(sql)).filter(list=>kindOf(list.map(t=>t.quoted?undefined:folded(t.text)))==='create trigger')
40
+ .map(list=>({list,on:createdTrigger(list)?.on}));
41
+ for(let grew=true;grew;) {
42
+ grew=false;
43
+ for(const {list,on} of triggers)if(names(list,reached))for(const name of on===undefined?identifiers(list):[on])if(!reached.has(name)){reached.add(name);grew=true;}
44
+ }
45
+ return [...reached];
46
+ }
47
+ const kindOf=words=>words.slice(0,4).filter(w=>!['temp','temporary','unique'].includes(w)).slice(0,2).join(' ');
48
+ const identifiers=list=>list.flatMap(t=>t.quoted?(t.text.startsWith("'")?[]:[folded(identifier(t))]):folded(t.text).split(/[^a-z0-9_$\u0080-\uffff]+/).filter(Boolean));
49
+ const names=(list,reached)=>identifiers(list).some(name=>reached.has(name));
50
+ // Tokens with each unquoted ';' split out, and trailing ones dropped.
51
+ function atoms(list) {
52
+ const result=list.flatMap(t=>t.quoted?[t]:t.text.split(/(;)/).filter(Boolean).map(text=>({text})));
53
+ while(result.length && semicolon(result.at(-1)))result.pop();
54
+ return result;
55
+ }
56
+ const semicolon=a=>!a.quoted && a.text===';';
57
+ const word=(a,value)=>a && !a.quoted && folded(a.text)===value;
58
+ // {on} for CREATE [TEMP] TRIGGER ... ON table ... BEGIN statement; ... END and
59
+ // nothing after it. The body ends at the first END that starts a statement;
60
+ // CASE ... END inside a statement does not. Anything else is unbounded.
61
+ function createdTrigger(list) {
62
+ const parts=atoms(list);let depth=0,begin=-1,on;
63
+ for(let i=0;i<parts.length && begin<0;i++) {
64
+ const a=parts[i];
65
+ if(a.quoted)continue;
66
+ if(a.text===';')return;
67
+ if(a.text==='(')depth++;
68
+ else if(a.text===')'){if(--depth<0)return;}
69
+ else if(!depth && on===undefined && word(a,'on')) {
70
+ const name=[];
71
+ for(let j=i+1;j<parts.length && !['for','when','begin'].some(w=>word(parts[j],w));j++)name.push(parts[j]);
72
+ const last=name.at(-1);
73
+ if(!last || name.length>3)return;
74
+ on=folded(last.quoted?identifier(last):last.text.split('.').at(-1));
75
+ if(!on)return;
76
+ }else if(!depth && word(a,'begin'))begin=i;
77
+ }
78
+ if(begin<0 || on===undefined)return;
79
+ let start=true,statements=0;
80
+ for(let i=begin+1;i<parts.length;i++) {
81
+ const a=parts[i];
82
+ if(start && word(a,'end'))return i===parts.length-1 && statements && !depth?{on}:undefined;
83
+ if(semicolon(a)){if(start || depth)return;start=true;statements++;continue;}
84
+ if(!a.quoted){if(a.text==='(')depth++;else if(a.text===')' && --depth<0)return;}
85
+ start=false;
86
+ }
87
+ }
88
+ // DROP TRIGGER [IF EXISTS] [schema.]name, and nothing after it.
89
+ function droppedTrigger(list) {
90
+ const parts=atoms(list),from=word(parts[2],'if') && word(parts[3],'exists')?4:2,name=parts.slice(from);
91
+ return name.length>=1 && name.length<=3 && name.every(a=>a.quoted?!a.text.startsWith("'"):!/[;(),]/.test(a.text));
18
92
  }
19
93
  function topLevelWords(list,words) {
20
94
  let depth=0;const result=[];
@@ -17,6 +17,8 @@ The `rer` bin and `node bin/rer.mjs` expose the same command. Node 22+, Python 3
17
17
  Git, pnpm and tar are required, along with authenticated `gh` for GitHub sources.
18
18
  Use a full-history, clean seed clone with its declared package-manager version;
19
19
  `--filter=blob:none` is supported, while shallow history is refused. Before
20
+ release work begins, tracked changes and untracked files in the source checkout
21
+ are refused; ignored local files do not become release inputs. Before
20
22
  reading source files, destination validation uses the publication scanner's
21
23
  single batched hydration fetch. An unavailable blob reports its object ID and
22
24
  committed path with `fetch the full history` guidance, separately from identity
@@ -97,9 +99,14 @@ path needs the seed's browser prerequisites and any resources its verification
97
99
  requires. Routine CI and default verification do not invoke the release-only
98
100
  script.
99
101
 
100
- Both paths then scan source history with publish-check. When the checkout's
101
- origin is a confirmed public repository, the scanner uses that source
102
- repository's advertised default branch and published tags as its public base.
102
+ Both paths then run publish-check in a fresh disposable clone checked out at
103
+ the selected source commit. Its working tree contains exactly that commit's
104
+ committed files, isolated from the caller's ignored or untracked files and from
105
+ installation, build and verification outputs. The clone retains all source refs
106
+ and full history: committed secrets and historical findings remain subject to
107
+ the publication checks. When the source checkout's origin is a confirmed public
108
+ repository, the scanner uses that source repository's advertised default branch
109
+ and published tags as its public base.
103
110
  Already-public findings are classified pre-existing; unpublished content still
104
111
  blocks. Other sources retain complete local-ref history inspection.
105
112
  A passing step's receipt retains exit status and numeric final summaries from
@@ -212,9 +219,10 @@ Public export commit and annotated-tag messages contain the version and optional
212
219
  `RELEASE.json.publicReleaseNotes` only. Private source commit identities stay in
213
220
  the private release receipt. The clean export's final annotated tag exists
214
221
  before publication inspection, and the checked repository is copied unchanged
215
- into `public-git/`. Ordinary releases inspect their source history and stage the
216
- exact final annotation in a disposable checkout for inspection before creating
217
- the source tag. Release notes pass the same content rules as the public tree.
222
+ into `public-git/`. Ordinary releases stage the exact final annotation in the
223
+ fresh selected-commit checkout and inspect it with the source history before
224
+ creating the source tag. Release notes pass the same content rules as the
225
+ public tree.
218
226
 
219
227
  ## Releasing a seed that is already public
220
228
 
@@ -2,7 +2,7 @@ import { execFileSync } from 'node:child_process';
2
2
  import { resolve } from 'node:path';
3
3
  import { writeFileSync } from 'node:fs';
4
4
  import { git } from './source.mjs';
5
- import { annotate } from './export.mjs';
5
+ import { annotate, sourceTree } from './export.mjs';
6
6
  import { runningVersion } from '../../cli/version.mjs';
7
7
 
8
8
  export function validatePublicationDestination(source, project, repository) {
@@ -36,26 +36,32 @@ export function publicationScan(repo, ref, envFile, project, repository, publicB
36
36
  }
37
37
 
38
38
  // This is the ordinary release gate, also usable without installing/building a
39
- // seed. The scanner sees the original checkout and every source ref; only the
40
- // new annotation is read from the clean candidate, never verification output.
39
+ // seed. Materialize raw committed bytes, without checkout filters or export
40
+ // attributes, and mirror every source ref for the existing history policy.
41
+ // Neither the caller's local files nor verification output enters this tree.
41
42
  export function sourcePublicationScan(source, temporary, envFile, project, repository, publicBase, tagged, adopted) {
42
43
  const started = performance.now();
44
+ const candidate = sourceTree(source, resolve(temporary, 'publication'));
45
+ git(source.repo, 'clone', '--mirror', '--no-hardlinks', '--', source.repo, resolve(candidate, '.git'));
46
+ git(candidate, 'config', 'core.bare', 'false');
47
+ git(candidate, 'update-ref', '--no-deref', 'HEAD', source.commit);
48
+ git(candidate, 'read-tree', source.commit);
43
49
  let annotationFile;
44
50
  if (adopted) {
45
51
  annotationFile = resolve(temporary, 'annotation.tag');
46
52
  writeFileSync(annotationFile, git(adopted.repo, 'cat-file', 'tag', adopted.object), { mode: 0o600 });
47
53
  } else if (tagged) {
48
- const candidate = resolve(temporary, 'annotation');
49
- git(source.repo, 'clone', '--no-hardlinks', '--no-checkout', '--', source.repo, candidate);
50
- git(candidate, 'checkout', '--detach', source.commit);
51
54
  const staged = git(candidate, 'tag', '--list', source.tag).toString().trim();
52
55
  const name = staged ? `${source.tag}-publication-candidate` : source.tag;
53
56
  annotate(candidate, source, source.commit, name);
54
57
  annotationFile = resolve(temporary, 'annotation.tag');
55
58
  writeFileSync(annotationFile, git(candidate, 'cat-file', 'tag', `refs/tags/${name}`), { mode: 0o600 });
59
+ // Inspect the new tag once via annotationFile, leaving the mirrored source
60
+ // refs unchanged for the full-history scan.
61
+ git(candidate, 'tag', '-d', name);
56
62
  }
57
63
  const stagingSeconds = (performance.now() - started) / 1000;
58
- const report = publicationScan(source.repo, source.commit, envFile, project, repository, publicBase, annotationFile);
64
+ const report = publicationScan(candidate, source.commit, envFile, project, repository, publicBase, annotationFile);
59
65
  Object.assign(report.invocation, { stagingSeconds, totalSeconds: (performance.now() - started) / 1000 });
60
66
  return report;
61
67
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@runeditrun/rer",
3
- "version": "0.2.67",
3
+ "version": "0.2.69",
4
4
  "description": "The rer command: install, set up, run and launch Run Edit Run checkouts.",
5
5
  "license": "MIT",
6
6
  "type": "module",