@runeditrun/rer 0.2.90 → 0.2.93

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. package/cli/README.md +1 -1
  2. package/cli/agent-door.mjs +15 -4
  3. package/cli/agent-initializer.mjs +27 -0
  4. package/cli/agent-journey-fixtures.mjs +8 -4
  5. package/cli/agent-verifier.mjs +5 -1
  6. package/cli/deploy-command.mjs +19 -8
  7. package/cli/deploy.mjs +1 -1
  8. package/cli/ledger-refusal.mjs +1 -1
  9. package/cli/portability-migrations.mjs +9 -5
  10. package/cli/portability-worker.mjs +4 -2
  11. package/foundation/runtime/contract-docs.mjs +26 -12
  12. package/global/CHANGELOG.md +47 -0
  13. package/modules/agent-door/conformance/audit.mjs +82 -7
  14. package/modules/agent-door/conformance/crypto.mjs +14 -2
  15. package/modules/agent-door/conformance/fixtures.mjs +22 -3
  16. package/modules/agent-door/conformance/index.mjs +9 -2
  17. package/modules/agent-door/conformance/journey.mjs +26 -2
  18. package/modules/agent-door/conformance/negative.mjs +29 -1
  19. package/modules/agent-door/conformance/policy-phases.mjs +43 -1
  20. package/modules/agent-door/conformance/protocol.mjs +44 -8
  21. package/modules/agent-door/conformance/sponsors.mjs +12 -1
  22. package/modules/agent-door/conformance/step-hooks.mjs +3 -1
  23. package/modules/agent-door/conformance/surface.mjs +10 -0
  24. package/modules/deploy-kit/owner-cli.mjs +9 -1
  25. package/modules/erasure/README.md +4 -2
  26. package/modules/operator-auth/identity/oidc/README.md +2 -1
  27. package/modules/portability/README.md +52 -2
  28. package/modules/portability/backfills.mjs +96 -3
  29. package/modules/portability/bridge.mjs +3 -1
  30. package/modules/portability/format.mjs +1 -0
  31. package/modules/portability/import.mjs +58 -3
  32. package/modules/portability/index.d.mts +2 -0
  33. package/modules/portability/index.mjs +16 -5
  34. package/modules/portability/library-history.mjs +152 -0
  35. package/modules/portability/migration-history.mjs +3 -2
  36. package/modules/portability/snapshot-objects.mjs +16 -0
  37. package/onboarding/adapter.mjs +3 -1
  38. package/package.json +1 -1
@@ -1,9 +1,11 @@
1
1
  import { statement, quote, insist } from './storage.mjs';
2
- import { tokens, folded } from './generated.mjs';
2
+ import { tokens, folded, identifier } from './generated.mjs';
3
3
  import { changesData, dataStatement, reachingTables, triggerBodies, schemaChange, statementNames } from './migration-statements.mjs';
4
4
  import { splitStatements } from './sql-text.mjs';
5
5
  import { preservingRebuilds } from './rebuilds.mjs';
6
6
  import { replayViews, viewReads } from './replay-views.mjs';
7
+ import { sqliteTokens } from './sqlite-tokens.mjs';
8
+ import { schemaSQL } from './schema-identity.mjs';
7
9
  import { transientTables } from './transient-tables.mjs';
8
10
 
9
11
  // The data statements of later migrations, judged by the tables they write
@@ -31,13 +33,24 @@ import { transientTables } from './transient-tables.mjs';
31
33
  // statements after it is judged as the statements it holds.
32
34
  // Returns the replays in migration order and the proven CHECK rebuilds, or refuses with the first migration
33
35
  // (in `unapplied` order) holding a statement that cannot be admitted.
34
- export function laterData({applied,unapplied,replay,catalogue,archived,empty,tables,secrets,manifest}) {
36
+ export function laterData({applied,unapplied,replay,catalogue,archived,empty,tables,secrets,manifest,replayEmpty=[]}) {
35
37
  const split=name=>replay.statements(name).flatMap(splitStatements),fold=list=>list.map(folded);
36
38
  const statements=applied.flatMap(split);
37
39
  const restorable=fold(catalogue),carried=fold(archived);
38
40
  const reached=reachingTables(statements,carried),writable=reachingTables(statements,restorable);
39
41
  const later=unapplied.flatMap(migration=>split(migration).map((sql,index)=>({migration,index,sql,data:dataStatement(sql)})));
40
42
  const {admitted,tables:rebuilt}=preservingRebuilds(later,manifest,tables,statements),views=replayViews(statements);
43
+ const restored=restoreCommitStatements(applied,split,later);
44
+ restored.forEach(s=>admitted.add(s));
45
+ // Renaming an uncarried table is metadata-only. Later row statements still
46
+ // face their own checks; carried table definitions still face the bridge.
47
+ for(const s of later) {
48
+ const list=sqliteTokens(s.sql),words=list.map(t=>folded(t.value));
49
+ if(words.length===6 && words[0]==='alter' && words[1]==='table' && words[3]==='rename' && words[4]==='to' && !carried.includes(words[2]) && !carried.includes(words[5]))admitted.add(s);
50
+ }
51
+ const absentEmpty=emptyAbsentTables(later,statements,carried,replayEmpty,admitted);
52
+ // An empty absent source contributes no row to a direct INSERT SELECT.
53
+ for(const s of later)if(s.data && emptySelect(s.sql,absentEmpty))admitted.add(s);
41
54
  const {missing,plans:transients}=transientTables(statements,later,fold(tables),carried);
42
55
  const current=fold(tables),known=[...current,...missing],outside=later.filter(s=>s.data && !writable.includes(s.data.target) && !transients.has(s.data.target)),tainted=new Set();
43
56
  // Tables an admitted write fills from archived rows (directly, through its
@@ -61,7 +74,7 @@ export function laterData({applied,unapplied,replay,catalogue,archived,empty,tab
61
74
  // Dependencies are checked below before any of the plan is selected.
62
75
  const available=[...transients.keys(),...(!carried.includes(table)?[...writes.keys()].filter(name=>!carried.includes(name)):[])];
63
76
  const allowUpdate=!transient && !carried.includes(table) && list.some(s=>['insert','replace'].includes(s.data.verb));
64
- const options={table,allowUpdate,statements,restorable,writable,carried:[...carried,...available],tables:known,secrets,views,manifest};
77
+ const options={table,allowUpdate,statements,restorable,writable,carried:[...carried,...absentEmpty,...available],tables:known,secrets,views,manifest};
65
78
  // Carried target rules must use the actual archive, not reconstructed reads.
66
79
  const bad=carried.includes(table) && list.length>1?list:list.filter(s=>!replayable(s,{...options,targetCarried:carried.includes(table)}));
67
80
  // Writes confined to tables created after the archive retain their
@@ -141,3 +154,83 @@ export function replayStatements(db,row) {
141
154
  statement(db,`UPDATE rer_portability_jobs SET metadata=json_set(metadata,'$.bridge.replayed[${i}].rows',${r.schema?'0':'changes()'}) WHERE id=?`,row.id)])];
142
155
  }
143
156
  const autoincrement=sql=>tokens(sql).some(t=>!t.quoted && folded(t.text)==='autoincrement');
157
+
158
+ // Only a surviving destination-owned trigger on the actual restore transition
159
+ // can cover DML. The complete body must recur contiguously, in the same file
160
+ // and order, preserving literals and identifiers. Nothing is replayed twice.
161
+ function restoreCommitStatements(applied,split,later) {
162
+ const live=new Map();
163
+ for(const migration of applied)for(const sql of split(migration)) {
164
+ const list=tokens(sql),words=list.map(t=>folded(t.text));
165
+ if(!['create','drop'].includes(words[0]) || words[1]!=='trigger')continue;
166
+ const at=words[2]==='if'?(words[0]==='create'?5:4):2,name=folded(identifier(list[at]));
167
+ if(words[0]==='drop'){live.delete(name);continue;}
168
+ if(words[2]==='if' && live.has(name))continue;
169
+ live.set(name,{migration,sql,list,at});
170
+ }
171
+ const admitted=new Set();
172
+ const header='AFTER UPDATE OF import_job ON rer_portability_state WHEN OLD.import_job IS NOT NULL AND NEW.import_job IS NULL AND OLD.applying=1';
173
+ for(const {migration,sql,list,at} of live.values()) {
174
+ const begin=list.findIndex(t=>!t.quoted && folded(t.text)==='begin');
175
+ if(begin<0 || schemaChange(sql)!=='trigger' || schemaSQL(sql.slice(list[at].end,list[begin].start))!==schemaSQL(header))continue;
176
+ const end=list.at(-1);if(!/^end;?$/i.test(end.text))continue;
177
+ const body=splitStatements(sql.slice(list[begin].end,end.start));
178
+ if(!body.length || body.some(sql=>!dataStatement(sql)))continue;
179
+ const file=later.filter(s=>s.migration===migration),images=body.map(schemaSQL);
180
+ for(let i=0;i+body.length<=file.length;i++)if(images.every((image,k)=>file[i+k].data && schemaSQL(file[i+k].sql)===image))file.slice(i,i+body.length).forEach(s=>admitted.add(s));
181
+ }
182
+ return admitted;
183
+ }
184
+ // Prove an absent table stays empty rather than assuming every newly created
185
+ // table is empty. Full replay supplies the empty baseline; every later INSERT
186
+ // and every possible trigger writer must also preserve it. A local empty
187
+ // create/copy/drop/rename rebuild carries no row, including through its DROP.
188
+ function emptyAbsentTables(later,statements,carried,replayEmpty,admitted) {
189
+ const empty=new Set(replayEmpty.map(folded).filter(name=>!carried.includes(name)));
190
+ const rebuilds=[];
191
+ for(let i=0;i+3<later.length;i++) {
192
+ const block=later.slice(i,i+4),a=tokens(block[0].sql),b=tokens(block[1].sql),c=tokens(block[2].sql),d=tokens(block[3].sql);
193
+ const text=list=>list.map(t=>folded(identifier(t)));
194
+ const [aa,bb,cc,dd]=[a,b,c,d].map(text);
195
+ if(block.some(s=>s.migration!==block[0].migration) || aa[0]!=='create' || aa[1]!=='table' || a[3]?.text!=='(' || bb.length!==7 || bb[0]!=='insert' || bb[1]!=='into' || bb[2]!==aa[2] || bb[3]!=='select' || bb[4]!=='*' || bb[5]!=='from' || cc.join(' ')!==`drop table ${bb[6]}` || dd.join(' ')!==`alter table ${aa[2]} rename to ${bb[6]}` || !empty.has(bb[6]) || carried.includes(aa[2]))continue;
196
+ if(later.some(s=>!block.includes(s) && statementNames(s.sql).includes(aa[2])))continue;
197
+ rebuilds.push({block,source:bb[6]});
198
+ }
199
+ const copyStatements=new Set(rebuilds.flatMap(r=>r.block)),triggerTargets=new Set();
200
+ for(const sql of statements) {
201
+ const list=tokens(sql);if(folded(list[0]?.text??'')!=='create' || folded(list[1]?.text??'')!=='trigger')continue;
202
+ const on=list.findIndex(t=>!t.quoted && folded(t.text)==='on');
203
+ const begin=list.findIndex(t=>!t.quoted && folded(t.text)==='begin');
204
+ if(on<0 || begin<0 || schemaChange(sql)!=='trigger'){statementNames(sql).forEach(name=>triggerTargets.add(name));continue;}
205
+ for(const body of splitStatements(sql.slice(list[begin].end,list.at(-1).start))) {
206
+ const data=dataStatement(body);
207
+ if(data)triggerTargets.add(data.target);
208
+ else if(!/^\s*SELECT\b/i.test(body))statementNames(body).forEach(name=>triggerTargets.add(name));
209
+ }
210
+ }
211
+ for(let changed=true;changed;) {
212
+ changed=false;
213
+ for(const name of empty) {
214
+ const writes=later.filter(s=>s.data && s.data.target===name && !copyStatements.has(s) && ['insert','replace'].includes(s.data.verb));
215
+ const triggered=triggerTargets.has(name);
216
+ if(triggered || writes.some(s=>!emptySelect(s.sql,empty))) {empty.delete(name);changed=true;}
217
+ }
218
+ }
219
+ for(const {block,source} of rebuilds)if(empty.has(source))block.forEach(s=>admitted.add(s));
220
+ return [...empty];
221
+ }
222
+ function emptySelect(sql,empty) {
223
+ const list=sqliteTokens(sql),word=t=>['keyword','identifier'].includes(t?.type)?folded(t.value):undefined;
224
+ let depth=0,select=-1,from=-1;
225
+ for(let i=0;i<list.length;i++) {
226
+ const t=list[i],w=word(t);
227
+ if(!depth && ['union','except','intersect','right','full','returning'].includes(w))return false;
228
+ if(!depth && w==='select' && select<0)select=i;
229
+ if(!depth && w==='from' && select>=0 && from<0)from=i;
230
+ // An aggregate over an empty input can still produce one row. Permit
231
+ // only known scalar expressions and EXISTS before the outer FROM.
232
+ if(select>=0 && from<0 && list[i+1]?.text==='(' && w && !['coalesce','exists','in'].includes(w))return false;
233
+ if(t.text==='(')depth++;if(t.text===')')depth--;
234
+ }
235
+ return from>select && [...empty].includes(folded(list[from+1]?.value??''));
236
+ }
@@ -66,7 +66,9 @@ async function placement(db,manifest,cat,replay,ledger,history,legacyRecovery,un
66
66
  const catalogue=cat.tables.map(t=>t.name),archivedTables=catalogue.filter(name=>manifest.tables.some(t=>t.name===name));
67
67
  const empty=archivedTables.filter(name=>!manifest.tables.find(t=>t.name===name).count);
68
68
  const tables=(await all(db,"SELECT name FROM sqlite_master WHERE type IN ('table','view')")).map(t=>t.name);
69
- const {replays,rebuilt}=laterData({applied,unapplied,replay,catalogue,archived:archivedTables,empty,tables,secrets:manifest.excludedColumns??{},manifest:manifest.tables});
69
+ const baseline=await replay.rowImages?.(cat.tables.filter(t=>!archivedTables.includes(t.name)).map(t=>({name:t.name,columns:t.columns.map(c=>c.name)})),0);
70
+ const replayEmpty=baseline?[...baseline].filter(([,rows])=>!rows.length).map(([name])=>name):[];
71
+ const {replays,rebuilt}=laterData({applied,unapplied,replay,catalogue,archived:archivedTables,empty,tables,secrets:manifest.excludedColumns??{},manifest:manifest.tables,replayEmpty});
70
72
  return {applied,earlier,replays,rebuilt};
71
73
  }
72
74
  // Old v2 writers recorded tables and applied migration names, but no behavioral
@@ -36,6 +36,7 @@ async function checkManifestFields(m) {
36
36
  insist(typeof m.product==='string' && typeof m.sourceIdentity==='string' && typeof m.exportedAt==='string','manifest_identity');
37
37
  insist(Number.isSafeInteger(m.partCount) && m.partCount>=0 && Number.isSafeInteger(m.epoch) && m.epoch>=0,'manifest_counts');
38
38
  insist(Array.isArray(m.tables) && Array.isArray(m.objects) && Array.isArray(m.payloads) && Array.isArray(m.journal) && Array.isArray(m.requiredConfiguration),'manifest_inventory');
39
+ insist(m.schemaObjects===undefined || m.excludedTables===undefined || (Array.isArray(m.excludedTables) && m.excludedTables.every(name=>typeof name==='string') && new Set(m.excludedTables).size===m.excludedTables.length),'manifest_inventory');
39
40
  // Applied product migrations, recorded by archives since additive restore.
40
41
  insist(m.migrations===undefined || (Array.isArray(m.migrations) && m.migrations.every(x=>typeof x==='string')),'manifest_inventory');
41
42
  // The ledger an export from a RER Platform installation records, by migration identity.
@@ -1,3 +1,4 @@
1
+ import { libraryMigrations } from './library-history.mjs';
1
2
  import { all, statement, state, guard, job, save, partQueries, parts, readPart, dropCopyStatements, cleanCopyStatements, quote, stream, insist, run } from './storage.mjs';
2
3
  import { stagePart } from './import-staging.mjs';
3
4
  import { normalizeStaged } from './import-safety.mjs';
@@ -7,6 +8,7 @@ import { refusePolicyApprovals } from './policy-data.mjs';
7
8
  import { refuseDevData } from './dev-data.mjs';
8
9
  import { catalogue, validateRow, bindCell, writableColumns } from './catalogue.mjs';
9
10
  import { platformTable, restorableManifest } from './bookkeeping.mjs';
11
+ import { withoutAbsentExcludedObjects } from './snapshot-objects.mjs';
10
12
  import { canonicalJson, unbase64 } from './hash.mjs';
11
13
  import { checkManifest, checkPart, ContentVerifier, ArchiveVerifier } from './format.mjs';
12
14
  import { fence, mergeJournal, tombstone } from './journal.mjs';
@@ -15,13 +17,13 @@ import { createCopy,sidecar,readCopy,restoreCopy } from './copies.mjs';
15
17
  import { excludedDependentGuards, migrationSeeded, seededCondition } from './restore-safety.mjs';
16
18
  import { additiveBridge, sameSchemaMigrations, appliedMigrations, verifiedLegacySchema } from './bridge.mjs';
17
19
  import { replayStatements } from './backfills.mjs';
18
- import { checkArchivedLedgers, archivedHistory } from './ledgers.mjs';
20
+ import { checkArchivedLedgers, archivedHistory, archivedLedgerTables, ledgerReader, matchMigrationLedger } from './ledgers.mjs';
19
21
  import { legacyEmptyGuards, legacyAuditStatements, requireLegacyEmpty } from './legacy-safety.mjs';
20
22
  import { checkReferences } from './references.mjs';
21
23
  import { rebuildDerivedStatements } from './derived.mjs';
22
24
  // ledgers: the rows of each migration ledger table the archive carries, as the
23
25
  // caller read them from it ({table:rows}); uploadPart binds them to its parts.
24
- export async function createImport(context,manifest,actor='direct',legacyRecovery,ledgers,acknowledgeUnverifiedSchemaObjects=false) {
26
+ export async function createImport(context,manifest,actor='direct',legacyRecovery,ledgers,acknowledgeUnverifiedSchemaObjects=false,id=crypto.randomUUID()) {
25
27
  const {db,config,buckets}=context;await checkManifest(manifest);checkArchivedLedgers(manifest,ledgers);
26
28
  insist(manifest.product===config.product,'wrong_product');
27
29
  // Explicit schema upgrades can reconcile rows, but cannot attest an interrupted deployment.
@@ -29,6 +31,7 @@ export async function createImport(context,manifest,actor='direct',legacyRecover
29
31
  const original=manifest;manifest=await restorableManifest(original);
30
32
  const skippedTables=original.tables.filter(table=>platformTable(table.name)).map(table=>table.name);
31
33
  const cat=await catalogue(db,config),upgrade=config.acceptedSchemas[original.schema];
34
+ manifest=await withoutAbsentExcludedObjects(manifest,cat,config);
32
35
  const legacy=original.version===2 && manifest.schemaObjects===undefined;
33
36
  // An explicit acknowledgment keeps its existing recovery semantics. Without
34
37
  // it, legacy v2 needs positive replay evidence for its recorded schema and
@@ -70,7 +73,7 @@ export async function createImport(context,manifest,actor='direct',legacyRecover
70
73
  if(!(await importLeaseStatus(db,await job(db,previous,'import'))).stale)throw await importRunning(db,previous);
71
74
  await abortImport(context,previous,{staleOnly:true});
72
75
  }
73
- const id=crypto.randomUUID(),copies=Object.fromEntries(targets.map((t,i)=>[t.name,'rer_portability_stage_'+id.replaceAll('-','')+'_'+i]));
76
+ const copies=Object.fromEntries(targets.map((t,i)=>[t.name,'rer_portability_stage_'+id.replaceAll('-','')+'_'+i]));
74
77
  const row={id,kind:'import',actor,phase:'staging',completed:0,metadata:{manifest:original,...(unverified?{schemaUnverified:{reason:'acknowledged recovery of a version 2 archive without schemaObjects',sourceIdentity:original.sourceIdentity,exportedAt:original.exportedAt,acknowledgedAt:new Date().toISOString()}}:{}),...(skippedTables.length?{restoreManifest:manifest,skippedTables}:{}),copies,targets,triggers:cat.triggers,...(bridge.fromSchema?{bridge}:{}),...(replays?{replays}:{}),...(legacyRecovery?{legacyRecovery}: {}),...(ledgers?{ledgers}:{})}};
75
78
  try {
76
79
  await db.batch([guard(db,'(SELECT import_job FROM rer_portability_state WHERE id=1) IS NULL'),
@@ -273,3 +276,55 @@ export async function abortAbandonedImport(context) {
273
276
  if(!id)return null;
274
277
  await abortImport(context,id,{soleOperator:true});return id;
275
278
  }
279
+
280
+ // A ledger carried in parts cannot authorize a durable job at create time.
281
+ // Collect this legacy upload in the service instance, authenticate it in full,
282
+ // then enter the ordinary durable import path. A lost collector is re-uploaded;
283
+ // it owns no destination data, lock, staging table or job to recover.
284
+ export function libraryImports(context,direct,migrationRoot) {
285
+ const pending=new Map();
286
+ const status=row=>({id:row.id,kind:'import',phase:row.phase,completed:row.parts.length,total:row.manifest.partCount,contentHash:null,requiredConfiguration:[]});
287
+ return {
288
+ ...direct,
289
+ async create(manifest,actor='direct',options={}) {
290
+ if(manifest?.version!==2 || manifest.schemaObjects!==undefined || options.acknowledgeUnverifiedSchemaObjects || options.ledgers)return direct.create(manifest,actor,options);
291
+ await checkManifest(manifest);
292
+ if(!archivedLedgerTables(manifest).length)return direct.create(manifest,actor,options);
293
+ insist(manifest.product===context.config.product,'wrong_product');
294
+ await appliedMigrations(context.db);
295
+ insist(!(await statement(context.db,'SELECT import_job FROM rer_portability_state WHERE id=1').first())?.import_job && ![...pending.values()].some(row=>row.phase==='staging'),'import_running',409);
296
+ context.migrations??=await libraryMigrations(context.db,context.inputPolicy,migrationRoot);
297
+ const id=crypto.randomUUID();pending.set(id,{id,manifest:structuredClone(manifest),actor,parts:[],phase:'staging'});return id;
298
+ },
299
+ async uploadPart(id,index,part) {
300
+ const row=pending.get(id);if(!row)return direct.uploadPart(id,index,part);
301
+ insist(row.phase==='staging','import_phase',409);await checkPart(part,index,context.config.partBytes);
302
+ if(row.parts[index]){insist(row.parts[index].sha256===part.sha256,'part_conflict',409);return status(row);}
303
+ insist(index===row.parts.length && index<row.manifest.partCount,'part_order',409);
304
+ row.parts.push(structuredClone(part));return status(row);
305
+ },
306
+ async finalize(id,options={}) {
307
+ const row=pending.get(id);if(!row)return direct.finalize(id,options);
308
+ const {manifest,parts}=row;insist(row.phase==='staging','import_phase',409);insist(parts.length===manifest.partCount,'parts_missing');
309
+ const content=new ContentVerifier(manifest),archive=new ArchiveVerifier(manifest),reader=ledgerReader();reader.add(manifest);
310
+ for(const part of parts){archive.add(part);reader.add(part);for(const item of part.items)content.add(item);}
311
+ const digest=content.digest();
312
+ if(options.contentHash!==undefined)insist(digest===options.contentHash,'content_checksum');
313
+ insist(options.archiveHash===archive.digest(),'archive_checksum',409);
314
+ const ledgers=reader.ledgers(),history=archivedHistory(manifest,ledgers);
315
+ insist(history?.length && history.every(entry=>entry.sha256),'schema_mismatch:legacy_schema',409);
316
+ const replay=await context.migrations(),files=replay.names.map(name=>({name,sha256:replay.sha256?.(name),aliases:replay.aliases?.(name)}));
317
+ const recorded=matchMigrationLedger(history,files,replay);
318
+ insist(canonicalJson(recorded)===canonicalJson(replay.names.slice(0,recorded.length)),'schema_mismatch:migration_history',409);
319
+ const actual=await direct.create(manifest,row.actor,{ledgers,id});
320
+ pending.delete(id);
321
+ for(const part of parts)await direct.uploadPart(actual,part.index,part);
322
+ return direct.finalize(actual,options);
323
+ },
324
+ async status(id){return pending.has(id)?status(pending.get(id)):direct.status(id);},
325
+ async active(actor='direct'){const row=[...pending.values()].find(row=>row.actor===actor && row.phase==='staging');return row?status(row):direct.active(actor);},
326
+ async abort(id){const row=pending.get(id);if(!row)return direct.abort(id);row.phase='aborted';row.parts=[];return status(row);},
327
+ async lookup(id){return pending.has(id)?status(pending.get(id)):direct.lookup(id);},
328
+ async actor(id){return pending.get(id)?.actor??direct.actor(id);},
329
+ };
330
+ }
@@ -48,6 +48,8 @@ export interface MigrationReplay {names:string[];/** Seed-declared data-only Pla
48
48
  /** Export job creation only; no record data, object keys or file paths. */
49
49
  export interface ExportEvent {id:string;/** Opaque caller id; the direct/CLI sentinel is normalized to null. */actor:string|null;scope:'installation'}
50
50
  export interface PortabilityOptions {
51
+ /** Node checkout root for automatic legacy evidence; defaults to cwd. */
52
+ migrationRoot?:string;
51
53
  db:D1Database;buckets?:Record<string,R2Bucket>;policy:ArchivePolicy;migrations?:()=>Promise<MigrationReplay>;
52
54
  /** Seed-owned audit SQL, appended to the snapshot/job batch. Throwing or SQL failure refuses creation. */
53
55
  onExport?:(event:ExportEvent)=>[sql:string,args:unknown[]][];
@@ -3,7 +3,7 @@ import { policy, catalogue } from './catalogue.mjs';
3
3
  import { wireDatabase } from '../d1-transport/index.mjs';
4
4
  import { insist, job } from './storage.mjs';
5
5
  import { createExport, advanceExport, exportStatus, downloadExport, abortExport } from './export.mjs';
6
- import { createImport, uploadPart, finalizeImport, importStatus, activeImport, abortImport, abortAbandonedImport } from './import.mjs';
6
+ import { libraryImports, createImport, uploadPart, finalizeImport, importStatus, activeImport, abortImport, abortAbandonedImport } from './import.mjs';
7
7
  import { erasureStatements, redactionStatements, invalidateTransfersStatements } from './journal.mjs';
8
8
  export { archiveLines } from './lines.mjs';
9
9
  export { excludedTables, excludedTable } from './catalogue.mjs';
@@ -11,18 +11,29 @@ export { readArchive, makePart, MAX_LINE_BYTES, MAX_PART_BYTES } from './format.
11
11
  export { canonicalJson, sha256 } from './hash.mjs';
12
12
  export { PortabilityError } from './storage.mjs';
13
13
  export { createPortabilityHandler } from './http.mjs';
14
- export async function createPortability({db,buckets={},policy:input,migrations,onExport}) {
14
+ export async function createPortability({db,buckets={},policy:input,migrations,onExport,migrationRoot}) {
15
15
  insist(onExport===undefined || typeof onExport==='function','invalid_on_export');
16
16
  db=wireDatabase(db);
17
17
  const config=policy(input);await catalogue(db,config);
18
18
  for(const name of Object.keys(config.buckets))insist(buckets[name] && typeof buckets[name].get==='function','missing_bucket:'+name);
19
19
  await ensureImportLeases(db);
20
20
  const context={db,buckets,config,migrations,onExport};
21
+ const importContext={...context,inputPolicy:input};
22
+ const {lookup,actor,...imports}=libraryImports(importContext,{
23
+ active:actor=>activeImport(importContext,actor),
24
+ create:(manifest,actor,options={})=>createImport(importContext,manifest,actor,undefined,options.ledgers,options.acknowledgeUnverifiedSchemaObjects,options.id),
25
+ uploadPart:(id,index,part)=>uploadPart(importContext,id,index,part),
26
+ finalize:(id,options)=>finalizeImport(importContext,id,options),
27
+ status:id=>importStatus(importContext,id),abort:id=>abortImport(importContext,id),
28
+ abortAbandoned:()=>abortAbandonedImport(importContext),
29
+ lookup:async id=>(await job(db,id)).kind==='export'?exportStatus(context,id):importStatus(importContext,id),
30
+ actor:id=>job(db,id).then(row=>row.actor),
31
+ },migrationRoot);
21
32
  return {
22
33
  exports:{create:actor=>createExport(context,actor),advance:(id,options)=>advanceExport(context,id,options),status:id=>exportStatus(context,id),download:id=>downloadExport(context,id),abort:id=>abortExport(context,id)},
23
- imports:{active:actor=>activeImport(context,actor),create:(manifest,actor,options={})=>createImport(context,manifest,actor,undefined,options.ledgers,options.acknowledgeUnverifiedSchemaObjects),uploadPart:(id,index,part)=>uploadPart(context,id,index,part),finalize:(id,options)=>finalizeImport(context,id,options),status:id=>importStatus(context,id),abort:id=>abortImport(context,id),abortAbandoned:()=>abortAbandonedImport(context)},
24
- status:async id=>(await job(db,id)).kind==='export'?exportStatus(context,id):importStatus(context,id),
25
- actor:async id=>(await job(db,id)).actor,
34
+ imports,
35
+ status:lookup,
36
+ actor,
26
37
  erasureStatements:(table,key)=>erasureStatements(db,table,key),redactionStatements:()=>redactionStatements(db),invalidateTransfersStatements:()=>invalidateTransfersStatements(db),
27
38
  };
28
39
  }
@@ -0,0 +1,152 @@
1
+ import { catalogue, policy } from './catalogue.mjs';
2
+ import { sha256 } from './hash.mjs';
3
+ import { destinationLedger, matchMigrationLedger } from './ledgers.mjs';
4
+ import { splitStatements } from './sql-text.mjs';
5
+ import { insist, PortabilityError } from './storage.mjs';
6
+ import { migrationReplay } from './migration-history.mjs';
7
+
8
+ // Node library callers have the same checkout evidence as the runner. Resolve
9
+ // only its single declared D1 migration directory, never an archive-supplied path.
10
+ // Variable imports keep Node builtins out of a Worker's module graph.
11
+ const nodeModule = name => import(name);
12
+ export async function libraryMigrations(db,config,root) {
13
+ try{return await checkoutMigrations(db,config,root);}
14
+ catch(error){if(error instanceof PortabilityError)throw error;throw new PortabilityError('schema_mismatch:legacy_evidence',409);}
15
+ }
16
+ async function checkoutMigrations(db,config,root) {
17
+ insist(typeof process!=='undefined' && process.versions?.node,'schema_mismatch:legacy_evidence',409);
18
+ const [{readFile,readdir},{resolve,sep}]=await Promise.all([nodeModule('node:fs/promises'),nodeModule('node:path')]);
19
+ root=resolve(root??process.cwd());
20
+ const declaration=parseJsonc(await readFile(resolve(root,'wrangler.jsonc'),'utf8'));
21
+ insist(declaration.d1_databases?.length===1,'schema_mismatch:legacy_evidence',409);
22
+ const directory=resolve(root,declaration.d1_databases[0].migrations_dir??'migrations');
23
+ insist(directory===root || directory.startsWith(root+sep),'schema_mismatch:legacy_evidence',409);
24
+ const files=await Promise.all((await readdir(directory)).filter(name=>name.endsWith('.sql')).sort().map(async name=>{
25
+ const bytes=await readFile(resolve(directory,name));
26
+ return {name,sha256:await sha256(bytes),statements:splitStatements(bytes.toString('utf8'))};
27
+ }));
28
+ const ledger=await destinationLedger(db)??[];
29
+ insist(ledger.length && ledger.every(row=>row.sha256),'schema_mismatch:legacy_evidence',409);
30
+ const applied=matchMigrationLedger(ledger,files);
31
+ const replay=migrationReplay(applied.map(name=>files.find(file=>file.name===name)),config);
32
+ insist(await replay.schemaAfter(replay.names)===(await catalogue(db,policy(config))).identity,'schema_mismatch');
33
+ return async()=>replay;
34
+ }
35
+
36
+ // This copied runtime cannot depend on the authoring CLI. Keep the strict
37
+ // JSONC reader local so checkout configuration has the same syntax rules.
38
+ function location(text, position) {
39
+ const lines = text.slice(0, position).split('\n');
40
+ return `position ${position} (line ${lines.length} column ${lines.at(-1).length + 1})`;
41
+ }
42
+
43
+ // Validate JSONC before JSON.parse so diagnostics do not depend on the Node
44
+ // version's error wording. An explicit stack also handles deeply nested input.
45
+ // Only comments and commas accepted by the grammar are masked; offsets stay in
46
+ // the original source and comment removal can never join separate tokens.
47
+ function parseJsonc(text, onProperty) {
48
+ const result = text.split(''), stack = [{ kind: 'root', state: 'value', path: [] }];
49
+ let index = 0;
50
+ const fail = (message, position = index) => {
51
+ throw new SyntaxError(`${position === text.length ? 'Unexpected end of JSON input' : message} at ${location(text, position)}`);
52
+ };
53
+ const unterminated = (kind, start) => {
54
+ throw new SyntaxError(`Unterminated ${kind} at ${location(text, start)}; close the ${kind}.`);
55
+ };
56
+ const blank = start => {
57
+ for (let i = start; i < index; i++) if (text[i] !== '\n' && text[i] !== '\r') result[i] = ' ';
58
+ };
59
+ const space = () => {
60
+ while (index < text.length) {
61
+ if (' \t\r\n'.includes(text[index])) { index++; continue; }
62
+ const start = index;
63
+ if (text[index] === '/' && text[index + 1] === '/') {
64
+ index += 2;
65
+ while (index < text.length && text[index] !== '\n' && text[index] !== '\r') index++;
66
+ } else if (text[index] === '/' && text[index + 1] === '*') {
67
+ const end = text.indexOf('*/', index + 2);
68
+ if (end === -1) unterminated('block comment', start);
69
+ index = end + 2;
70
+ } else break;
71
+ blank(start);
72
+ }
73
+ };
74
+ const string = () => {
75
+ const start = index++;
76
+ let end = index;
77
+ while (end < text.length && text[end] !== '"') end = Math.min(text.length, end + (text[end] === '\\' ? 2 : 1));
78
+ if (end === text.length) unterminated('string', start);
79
+ while (index < end) {
80
+ if (text.charCodeAt(index) < 32) fail('Unescaped control character in string');
81
+ if (text[index++] !== '\\') continue;
82
+ if (text[index] === 'u') {
83
+ index++;
84
+ for (let digit = 0; digit < 4; digit++, index++) {
85
+ if (index >= end || !/[0-9a-fA-F]/.test(text[index])) fail('Invalid Unicode escape');
86
+ }
87
+ } else {
88
+ if (!'"\\/bfnrt'.includes(text[index])) fail('Invalid string escape');
89
+ index++;
90
+ }
91
+ }
92
+ index++;
93
+ };
94
+ const digit = () => index < text.length && text[index] >= '0' && text[index] <= '9';
95
+ const digits = () => {
96
+ if (!digit()) fail('Expected a digit');
97
+ do { index++; } while (digit());
98
+ };
99
+ const number = () => {
100
+ if (text[index] === '-') index++;
101
+ if (text[index] === '0') index++;
102
+ else digits();
103
+ if (text[index] === '.') { index++; digits(); }
104
+ if (text[index] === 'e' || text[index] === 'E') {
105
+ index++;
106
+ if (text[index] === '+' || text[index] === '-') index++;
107
+ digits();
108
+ }
109
+ };
110
+ while (stack.length) {
111
+ space();
112
+ const frame = stack.at(-1), char = text[index], closing = frame.kind === 'object' ? '}' : ']';
113
+ if (frame.state === 'end') {
114
+ if (index < text.length) fail('Unexpected token after JSON value');
115
+ stack.pop();
116
+ } else if ((frame.state === 'keyOrEnd' || frame.state === 'valueOrEnd' || frame.state === 'comma') && char === closing) {
117
+ index++; stack.pop();
118
+ } else if (frame.state === 'keyOrEnd') {
119
+ if (char !== '"') fail('Expected a property name');
120
+ const start = index; string();
121
+ frame.key = onProperty ? JSON.parse(text.slice(start, index)) : undefined;
122
+ onProperty?.([...frame.path, frame.key], start);
123
+ frame.state = 'colon';
124
+ } else if (frame.state === 'colon') {
125
+ if (char !== ':') fail("Expected ':' after property name");
126
+ index++; frame.state = 'value';
127
+ } else if (frame.state === 'comma') {
128
+ if (char !== ',') fail(`Expected ',' or '${closing}'`);
129
+ const comma = index++;
130
+ space();
131
+ if (text[index] === closing) result[comma] = ' ';
132
+ frame.state = frame.kind === 'object' ? 'keyOrEnd' : 'valueOrEnd';
133
+ } else {
134
+ const path = !onProperty ? undefined : frame.kind === 'root' ? [] : [...frame.path, frame.kind === 'array' ? frame.next++ : frame.key];
135
+ frame.state = frame.kind === 'root' ? 'end' : 'comma';
136
+ if (char === '{' || char === '[') {
137
+ index++;
138
+ stack.push({ kind: char === '{' ? 'object' : 'array', state: char === '{' ? 'keyOrEnd' : 'valueOrEnd', path, next: 0 });
139
+ } else if (char === '"') string();
140
+ else if (char === '-' || digit()) number();
141
+ else {
142
+ const literal = { t: 'true', f: 'false', n: 'null' }[char];
143
+ if (!literal) fail('Unexpected token in JSON value');
144
+ for (const expected of literal) {
145
+ if (text[index] !== expected) fail(`Expected '${literal}'`);
146
+ index++;
147
+ }
148
+ }
149
+ }
150
+ }
151
+ return JSON.parse(result.join(''));
152
+ }
@@ -1,4 +1,3 @@
1
- import { DatabaseSync } from 'node:sqlite';
2
1
  import { policy, schemaIdentity } from './catalogue.mjs';
3
2
 
4
3
  // Replays a chosen set of the destination's migrations, in its applied order,
@@ -7,8 +6,9 @@ import { policy, schemaIdentity } from './catalogue.mjs';
7
6
  // destination's history (see bridge.mjs). plan: [{name,statements,sha256?}],
8
7
  // sha256 being the migration file's hash where the runner read the file.
9
8
  export function migrationReplay(plan,input,{dataOnly=[]}={}) {
10
- const config=policy(input);
9
+ const config=policy(input),sqlite='node:sqlite';
11
10
  return {names:plan.map(migration=>migration.name),dataOnly,aliases:name=>plan.find(m=>m.name===name).aliases,statements:name=>plan.find(m=>m.name===name).statements,sha256:name=>plan.find(m=>m.name===name).sha256,async schemaAfter(names,options) {
11
+ const {DatabaseSync}=await import(sqlite);
12
12
  const scratch=new DatabaseSync(':memory:');
13
13
  const db={prepare:sql=>{const query=scratch.prepare(sql);let values=[];return {bind(...args){values=args;return this;},async all(){return {results:query.all(...values)};}};}};
14
14
  try {
@@ -19,6 +19,7 @@ export function migrationReplay(plan,input,{dataOnly=[]}={}) {
19
19
  // The rows the whole replay leaves in each requested table, as sorted images (legacy-safety.mjs).
20
20
  // A table the replay lacks, or one holding more than `limit` rows, has no entry.
21
21
  async rowImages(requests,limit) {
22
+ const {DatabaseSync}=await import(sqlite);
22
23
  const scratch=new DatabaseSync(':memory:'),found=new Map();
23
24
  try {
24
25
  for(const migration of plan)for(const sql of migration.statements)scratch.exec(sql);
@@ -0,0 +1,16 @@
1
+ import { excludedTable } from './catalogue.mjs';
2
+ import { identifier } from './bookkeeping.mjs';
3
+ import { schemaIdentityOf } from './schema-identity.mjs';
4
+
5
+ // Project only the archive's identity. Live and replay catalogues still attest
6
+ // every object they contain; the original manifest remains the digest input.
7
+ export async function withoutAbsentExcludedObjects(manifest,cat,config) {
8
+ if(manifest.schemaObjects===undefined)return manifest;
9
+ const restorable=new Set([...manifest.tables,...cat.tables].map(table=>identifier(table.name)));
10
+ const present=new Set([...cat.excludedTables,...cat.tables.map(table=>table.name)].map(identifier));
11
+ const schemaObjects=manifest.schemaObjects.filter(object=>!(
12
+ manifest.excludedTables?.includes(object.table) && !restorable.has(identifier(object.table))
13
+ && !present.has(identifier(object.table)) && excludedTable(object.table,config)));
14
+ if(schemaObjects.length===manifest.schemaObjects.length)return manifest;
15
+ return {...manifest,schemaObjects,schema:await schemaIdentityOf(manifest.tables,schemaObjects)};
16
+ }
@@ -216,7 +216,9 @@ async function start(context, hooks) {
216
216
  throw installationFailure('command_failed', { step: 'start', diagnostic: error });
217
217
  }
218
218
  let announcement,detectedMismatch;
219
- const mismatch=port=>({fatal:detectedMismatch=`the application listened on port ${port} but the CLI expected ${expectedPort}`});
219
+ // Keep context after the number: conservative redaction treats a trailing
220
+ // credential prefix as private even when it happens to be an ordinary port.
221
+ const mismatch=port=>({fatal:detectedMismatch=`the application listened on port ${port} but the CLI expected ${expectedPort} for the runtime listener`});
220
222
  const announced=message=>{if(message?.type==='rer-listening')announcement=message;};
221
223
  if(runtimePortHandshake)child.on('message',announced);
222
224
  context.app = child;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@runeditrun/rer",
3
- "version": "0.2.90",
3
+ "version": "0.2.93",
4
4
  "description": "The rer command: install, set up, run and launch Run Edit Run checkouts.",
5
5
  "license": "MIT",
6
6
  "type": "module",