@runeditrun/rer 0.2.122 → 0.2.124

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. package/cli/README.md +8 -0
  2. package/cli/bounded.mjs +3 -3
  3. package/cli/business/agent-output.mjs +4 -2
  4. package/cli/business/agent-verbs.mjs +10 -4
  5. package/cli/business/api.mjs +12 -9
  6. package/cli/business/carry.mjs +3 -2
  7. package/cli/business/choices.mjs +4 -1
  8. package/cli/business/connect.mjs +2 -1
  9. package/cli/business/local-commands.mjs +4 -1
  10. package/cli/business/mcp-bridge.mjs +7 -2
  11. package/cli/business/questions.mjs +5 -4
  12. package/cli/business/refusal.mjs +47 -0
  13. package/cli/business/run-grant.mjs +15 -16
  14. package/cli/business/setup-command.mjs +7 -2
  15. package/cli/business/supervisor.mjs +4 -1
  16. package/cli/deploy.mjs +2 -1
  17. package/cli/dev-identity.mjs +7 -0
  18. package/cli/errors.mjs +45 -9
  19. package/cli/export.mjs +1 -0
  20. package/cli/inspect.mjs +2 -0
  21. package/cli/main.mjs +17 -3
  22. package/cli/managed-server.mjs +8 -4
  23. package/cli/portability-empty.mjs +4 -2
  24. package/cli/portability-evidence.mjs +9 -1
  25. package/cli/portability-migrations.mjs +30 -8
  26. package/cli/portability-owners.mjs +54 -0
  27. package/cli/portability-worker.mjs +13 -5
  28. package/cli/portability.mjs +6 -4
  29. package/cli/render-warmup.mjs +31 -0
  30. package/cli/setup-input.mjs +14 -2
  31. package/cli/site-private.mjs +9 -1
  32. package/cli/site.mjs +5 -5
  33. package/edition-authoring/cli.mjs +3 -0
  34. package/edition-authoring/project-hashes.mjs +58 -0
  35. package/global/CHANGELOG.md +45 -0
  36. package/global/launch/dev.mjs +2 -2
  37. package/global/launch/render.mjs +12 -1
  38. package/global/launch/state.mjs +11 -4
  39. package/global/launch/worker.mjs +22 -5
  40. package/modules/deploy-kit/owner-cli.mjs +5 -2
  41. package/modules/deploy-kit/owner-remove.mjs +23 -5
  42. package/modules/job-queue/identity.mjs +11 -6
  43. package/modules/job-queue/policy.mjs +16 -2
  44. package/modules/operator-auth/README.md +6 -0
  45. package/modules/operator-auth/accounts/oa.mjs +1 -1
  46. package/modules/operator-auth/dev/index.mjs +32 -14
  47. package/modules/operator-auth/dev/persona-entry.mjs +38 -0
  48. package/modules/operator-auth/dev/persona-grant.mjs +16 -0
  49. package/modules/operator-auth/dev-admission.d.mts +2 -2
  50. package/modules/operator-auth/dev-admission.mjs +2 -2
  51. package/modules/operator-auth/dev-gate-script.mjs +28 -0
  52. package/modules/operator-auth/dev-gate.mjs +7 -10
  53. package/modules/portability/README.md +7 -2
  54. package/modules/portability/bridge.mjs +25 -23
  55. package/modules/portability/checkout-history.mjs +7 -6
  56. package/modules/portability/export.mjs +6 -3
  57. package/modules/portability/format.mjs +2 -2
  58. package/modules/portability/http-errors.mjs +1 -1
  59. package/modules/portability/http.mjs +2 -2
  60. package/modules/portability/import.mjs +13 -11
  61. package/modules/portability/index.d.mts +4 -2
  62. package/modules/portability/index.mjs +2 -2
  63. package/modules/portability/ledgers.mjs +16 -3
  64. package/modules/portability/legacy-files.mjs +3 -3
  65. package/modules/portability/migration-history.mjs +4 -3
  66. package/modules/portability/owner-history.mjs +62 -0
  67. package/modules/portability/worker-history.d.mts +1 -0
  68. package/modules/portability/worker-history.mjs +4 -4
  69. package/modules/release/README.md +7 -0
  70. package/modules/release/produce.mjs +3 -0
  71. package/onboarding/adapter.mjs +9 -9
  72. package/onboarding/commands.mjs +15 -5
  73. package/onboarding/operator-installer.mjs +2 -2
  74. package/onboarding/operator-settings.mjs +10 -8
  75. package/onboarding/redaction.mjs +1 -1
  76. package/package.json +1 -1
@@ -2,16 +2,18 @@ import { all, quote, insist } from './storage.mjs';
2
2
  import { canonicalJson } from './hash.mjs';
3
3
  import { tokens, identifier, folded } from './generated.mjs';
4
4
  import { laterData } from './backfills.mjs';
5
- import { destinationLedger, matchMigrationLedger } from './ledgers.mjs';
5
+ import { matchMigrationLedger } from './ledgers.mjs';
6
6
  import { checkRebuild } from './rebuilds.mjs';
7
7
  import { schemaSQL } from './schema-identity.mjs';
8
8
 
9
- // Applied product migrations in the ledger's own order (ledgers.mjs). A
10
- // state-bearing ledger may hold an unacknowledged migration whose effects are
11
- // unknown; neither an archive nor a restore can attest its history until that
12
- // is resolved.
13
- export async function appliedMigrations(db) {
14
- return (await destinationLedger(db))?.map(m=>m.name);
9
+ function recordedMigrations(manifest, archived, files, replay) {
10
+ const entries=manifest.migrations?.map(name=>archived?.find(entry=>entry.name===name)??{name})??archived;
11
+ if(manifest.migrationLedger?.origin!=='d1_migrations')return entries && matchMigrationLedger(entries,files,replay);
12
+ // Native owner identities come from the checkout's declared directories.
13
+ // A matching digest under another name cannot stand in for a changed file.
14
+ insist(entries?.every(entry=>files.some(file=>file.name===entry.name && file.sha256===entry.sha256)),
15
+ 'schema_mismatch:owner_migration_history',409);
16
+ return entries.map(entry=>entry.name);
15
17
  }
16
18
  // Where an archive sits in the destination's applied migrations: those its
17
19
  // rows passed through (earlier) and those applied after it (unapplied).
@@ -30,8 +32,7 @@ async function placement(db,manifest,cat,replay,ledger,history,legacyRecovery,un
30
32
  if(canonicalJson(replay.names)!==canonicalJson(applied))applied=matchMigrationLedger(ledger,files,replay);
31
33
  insist(canonicalJson(replay.names)===canonicalJson(applied) && await replay.schemaAfter(applied)===cat.identity,'schema_mismatch');
32
34
  const archived=history?.();
33
- const historyEntries=manifest.migrations?.map(name=>archived?.find(entry=>entry.name===name) ?? {name}) ?? archived;
34
- const recorded=historyEntries && matchMigrationLedger(historyEntries,files,replay);
35
+ const recorded=recordedMigrations(manifest,archived,files,replay);
35
36
  // Only explicitly acknowledged v1 recovery may use the old projection for
36
37
  // archive placement. The live destination check above remains complete, and
37
38
  // current v2 imports use their authenticated object inventory. Verified legacy
@@ -40,8 +41,9 @@ async function placement(db,manifest,cat,replay,ledger,history,legacyRecovery,un
40
41
  if(unverified && archived && manifest.migrations)insist(canonicalJson(matchMigrationLedger(archived,files,replay))===canonicalJson(recorded),'schema_mismatch:migration_history',409);
41
42
  let earlier,unapplied;
42
43
  if(recorded) {
43
- if(unverified)insist(recorded.every((name,i)=>applied.indexOf(name)>(i?applied.indexOf(recorded[i-1]):-1)),'schema_mismatch:migration_history',409);
44
- earlier=applied.filter(name=>recorded.includes(name));
44
+ const mismatch=manifest.migrationLedger?.origin==='d1_migrations'?'schema_mismatch:owner_migration_history':'schema_mismatch:migration_history';
45
+ insist(recorded.every((name,i)=>applied.indexOf(name)>(i?applied.indexOf(recorded[i-1]):-1)),mismatch,409);
46
+ earlier=recorded;
45
47
  insist(await schemaAfter(earlier)===manifest.schema,'schema_mismatch');
46
48
  unapplied=applied.filter(name=>!earlier.includes(name));
47
49
  }else{
@@ -76,15 +78,14 @@ async function placement(db,manifest,cat,replay,ledger,history,legacyRecovery,un
76
78
  // Verify the archive's historical projection and the complete live replay, then
77
79
  // return the recorded schema's complete identity for ordinary bridge placement.
78
80
  // Never guess an unrecorded prefix or install source SQL to fill missing evidence.
79
- export async function verifiedLegacySchema(db,manifest,cat,migrations,history) {
81
+ export async function verifiedLegacySchema(ledger,manifest,cat,migrations,history) {
80
82
  if(!migrations)return false;
81
- const replay=await migrations(),ledger=await destinationLedger(db)??[];
83
+ const replay=await migrations();
82
84
  try {
83
85
  const files=replay.names.map(name=>({name,sha256:replay.sha256?.(name),aliases:replay.aliases?.(name)}));
84
- const archived=history(),entries=manifest.migrations?.map(name=>archived?.find(entry=>entry.name===name)??{name})??archived;
85
- if(!entries?.length)return false;
86
- const applied=matchMigrationLedger(ledger,files,replay),recorded=matchMigrationLedger(entries,files,replay);
87
- if(!recorded.length || canonicalJson(applied)!==canonicalJson(replay.names)
86
+ const archived=history(),recorded=recordedMigrations(manifest,archived,files,replay);
87
+ const applied=matchMigrationLedger(ledger,files,replay);
88
+ if(!recorded?.length || canonicalJson(applied)!==canonicalJson(replay.names)
88
89
  || !recorded.every((name,i)=>applied.indexOf(name)>(i?applied.indexOf(recorded[i-1]):-1)))return false;
89
90
  if(archived && manifest.migrations && canonicalJson(matchMigrationLedger(archived,files,replay))!==canonicalJson(recorded))return false;
90
91
  if(await replay.schemaAfter(applied)!==cat.identity
@@ -99,8 +100,9 @@ export async function verifiedLegacySchema(db,manifest,cat,migrations,history) {
99
100
  // migrations the destination's ledger has outgrown is refused. An archive that
100
101
  // records a migration the destination never applied is newer than it, even
101
102
  // when that migration left the schema as it was (an index, a data change).
102
- export async function sameSchemaMigrations(db,manifest,cat,migrations,history) {
103
- const ledger=await destinationLedger(db),applied=ledger?.map(m=>m.name);
103
+ export async function sameSchemaMigrations(db,ledger,manifest,cat,migrations,history) {
104
+ const applied=ledger?.map(m=>m.name);
105
+ if(manifest.migrationLedger?.origin==='d1_migrations')insist(migrations && applied,'schema_mismatch:owner_migration_history',409);
104
106
  if(!applied)return;
105
107
  if(!migrations){
106
108
  if(manifest.migrations)matchMigrationLedger(manifest.migrations.map(name=>({name})),ledger);
@@ -108,17 +110,17 @@ export async function sameSchemaMigrations(db,manifest,cat,migrations,history) {
108
110
  }
109
111
  // A data-only migration leaves the schema unchanged but still needs its
110
112
  // carried-table replay plan retained for finalize.
111
- const bridge=await additiveBridge(db,manifest,cat,migrations,history);
113
+ const bridge=await additiveBridge(db,ledger,manifest,cat,migrations,history);
112
114
  return bridge.replays?.length?bridge:undefined;
113
115
  }
114
116
  // An archive made before later migrations restores only when those migrations
115
117
  // are additive, or a proven row-preserving CHECK rebuild: new tables, new
116
118
  // columns that take a default, or later schema objects established by replay.
117
119
  // Anything else stays a schema_mismatch refusal.
118
- export async function additiveBridge(db,manifest,cat,migrations,history,legacyRecovery,unverified=false) {
120
+ export async function additiveBridge(db,ledger,manifest,cat,migrations,history,legacyRecovery,unverified=false) {
119
121
  insist(migrations,'schema_mismatch');
120
- const replay=await migrations(),ledger=await destinationLedger(db)??[];
121
- const {applied,earlier,replays,rebuilt}=await placement(db,manifest,cat,replay,ledger,history,legacyRecovery,unverified);
122
+ const replay=await migrations();
123
+ const {applied,earlier,replays,rebuilt}=await placement(db,manifest,cat,replay,ledger??[],history,legacyRecovery,unverified);
122
124
  const columns={};
123
125
  for(const table of manifest.tables) {
124
126
  const target=cat.tables.find(t=>t.name===table.name),label=':'+table.name;
@@ -1,20 +1,21 @@
1
1
  import { catalogue, policy } from './catalogue.mjs';
2
2
  import { readLedger, matchMigrationLedger } from './ledgers.mjs';
3
+ import { nativeMigrationHistory, ownerMigrationPolicy } from './owner-history.mjs';
3
4
  import { insist } from './storage.mjs';
4
5
 
5
6
  // Both evidence adapters bind the seed's files to live history and the complete
6
7
  // schema. Only an explicitly bundled Worker may bind plain Wrangler history:
7
8
  // literal names must select an ordered checkout prefix, never aliases or gaps.
8
- export async function checkoutHistory(db,input,files,replayFor,{nativeWrangler=false}={}) {
9
- const history=await readLedger(db),ledger=history?.entries??[];
9
+ export async function checkoutHistory(db,input,files,replayFor,{nativeWrangler=false,ownerSources=[]}={}) {
10
+ const history=await (ownerSources.length?nativeMigrationHistory(db,ownerSources):readLedger(db)),ledger=history?.entries??[];
10
11
  insist(ledger.length,'schema_mismatch:legacy_evidence',409);
11
12
  if(!ledger.every(row=>row.sha256)) {
12
13
  insist(nativeWrangler && history.table==='d1_migrations'
13
- && ledger.every((row,i)=>row.sha256===undefined && row.name===files[i]?.name),
14
+ && ledger.every((row,i)=>row.name===files[i]?.name && (history.owners || row.sha256===undefined)),
14
15
  'schema_mismatch:legacy_evidence',409);
15
16
  }
16
17
  const applied=matchMigrationLedger(ledger,files);
17
- const replay=await replayFor(applied);
18
- insist(await replay.schemaAfter(replay.names)===(await catalogue(db,policy(input))).identity,'schema_mismatch');
19
- return replay;
18
+ const config=policy(ownerMigrationPolicy(input,ownerSources)),replay=await replayFor(applied,config);
19
+ insist(await replay.schemaAfter(replay.names)===(await catalogue(db,config)).identity,'schema_mismatch');
20
+ return ownerSources.length?{...replay,ownerSources,history}:replay;
20
21
  }
@@ -7,13 +7,15 @@ import { createCopy,readCopy,normalizeCopy } from './copies.mjs';
7
7
  import { canonicalJson, utf8, base64, StreamHash } from './hash.mjs';
8
8
  import { FORMAT, makePart, checkPart, ContentVerifier, ArchiveVerifier } from './format.mjs';
9
9
  import { journal } from './journal.mjs';
10
- import { readLedger, platformOrigin } from './ledgers.mjs';
10
+ import { platformOrigin, nativeOrigin } from './ledgers.mjs';
11
+ import { transferHistory } from './owner-history.mjs';
11
12
  import { capturePayload, payloadRange } from './payloads.mjs';
12
13
  import { dropProviderData } from './provider-data.mjs';
13
14
  import { dropDevData } from './dev-data.mjs';
14
15
  import { revokePolicyApprovals, refusePolicyApprovals } from './policy-data.mjs';
15
16
  export async function createExport(context,actor='direct') {
16
- const {db,config,buckets}=context,s=await state(db);insist(!s.applying,'import_applying',409);
17
+ const {db,buckets}=context,s=await state(db);insist(!s.applying,'import_applying',409);
18
+ const {config,ledger,migrations:replay}=await transferHistory(context);
17
19
  const cat=await catalogue(db,config),id=crypto.randomUUID(),objects=[];
18
20
  for(const [binding,spec] of Object.entries(config.buckets).sort()) {
19
21
  let cursor;
@@ -25,7 +27,8 @@ export async function createExport(context,actor='direct') {
25
27
  }
26
28
  objects.sort((a,b)=>a.bucket<b.bucket?-1:a.bucket>b.bucket?1:a.key<b.key?-1:a.key>b.key?1:0);
27
29
  const copies=Object.fromEntries(cat.tables.map((t,i)=>[t.name,'rer_portability_snap_'+id.replaceAll('-','')+'_'+i]));
28
- const ledger=await readLedger(db),origin=await platformOrigin(ledger,context.migrations),migrations=origin?origin.entries.map(entry=>entry.name):ledger?.entries.map(entry=>entry.name);
30
+ const origin=await (ledger?.owners?nativeOrigin(ledger,replay,cat.identity):platformOrigin(ledger,replay));
31
+ const migrations=origin?origin.entries.map(entry=>entry.name):ledger?.entries.map(entry=>entry.name);
29
32
  const manifest={kind:'manifest',format:FORMAT,version:2,product:config.product,exportedAt:new Date().toISOString(),sourceIdentity:s.source_identity,schema:cat.identity,tables:cat.tables,schemaObjects:cat.schemaObjects,objects,payloads:[],excludedTables:cat.excludedTables,excludedColumns:config.secretColumns,requiredConfiguration:config.requiredConfiguration,partCount:0,epoch:s.epoch,journal:await journal(db,s.source_identity),...(migrations?{migrations}:{}),...(origin?{migrationLedger:origin}:{})};
30
33
  const row={id,kind:'export',actor,phase:'building',completed:0,metadata:{manifest,copies,cursor:{table:0,row:0,binary:0,offset:0},partBytes:config.partBytes}};
31
34
  const audit=context.onExport?context.onExport({id,actor:actor==='direct'?null:actor,scope:'installation'}):[];
@@ -39,9 +39,9 @@ async function checkManifestFields(m) {
39
39
  insist(m.schemaObjects===undefined || m.excludedTables===undefined || (Array.isArray(m.excludedTables) && m.excludedTables.every(name=>typeof name==='string') && new Set(m.excludedTables).size===m.excludedTables.length),'manifest_inventory');
40
40
  // Applied product migrations, recorded by archives since additive restore.
41
41
  insist(m.migrations===undefined || (Array.isArray(m.migrations) && m.migrations.every(x=>typeof x==='string')),'manifest_inventory');
42
- // The ledger an export from a RER Platform installation records, by migration identity.
42
+ // A hash-bearing ledger from Platform or from a native core and owner history.
43
43
  const ledger=m.migrationLedger;
44
- insist(ledger===undefined || (ledger && typeof ledger==='object' && Object.keys(ledger).sort().join()==='entries,origin' && ledger.origin==='rer_migrations' && Array.isArray(ledger.entries)
44
+ insist(ledger===undefined || (ledger && typeof ledger==='object' && Object.keys(ledger).sort().join()==='entries,origin' && ['rer_migrations','d1_migrations'].includes(ledger.origin) && Array.isArray(ledger.entries)
45
45
  && ledger.entries.every(e=>e && typeof e==='object' && Object.keys(e).sort().join()==='name,sha256' && typeof e.name==='string' && /^[a-f0-9]{64}$/.test(e.sha256))
46
46
  && canonicalJson(m.migrations)===canonicalJson(ledger.entries.map(e=>e.name)) && new Set(m.migrations).size===m.migrations.length),'manifest_inventory');
47
47
  insist(m.tables.every((t,i)=>typeof t.name==='string' && typeof t.sql==='string' && Array.isArray(t.columns) && Array.isArray(t.dependencies) && Number.isSafeInteger(t.count) && t.count>=0 && (!i || m.tables[i-1].name<t.name)),'table_inventory');
@@ -27,7 +27,7 @@ const publicCodes = new Set(`
27
27
  upgrade_row_must_be_pure virtual_table wrong_product
28
28
  unauthenticated forbidden owner_required authority_changed step_up_required rate_limited operation_failed
29
29
  `.trim().split(/\s+/));
30
- const publicCode = code => typeof code === 'string' && publicCodes.has(code.split(':')[0]) ? code.split(':')[0] : 'operation_failed';
30
+ const publicCode = code => code === 'schema_mismatch:owner_migration_history' ? code : typeof code === 'string' && publicCodes.has(code.split(':')[0]) ? code.split(':')[0] : 'operation_failed';
31
31
 
32
32
  // Public errors use the existing {error:code} vocabulary. Native messages can
33
33
  // contain SQL, row values or credentials; inspect causes only to choose status.
@@ -11,10 +11,10 @@ export async function boundedJson(request,limit) {
11
11
  try{return JSON.parse(new TextDecoder('utf-8',{fatal:true}).decode(bytes));}catch{throw new PortabilityError('invalid_json');}
12
12
  }
13
13
  const json=(value,status=200)=>new Response(JSON.stringify(value),{status,headers:{'Content-Type':'application/json','Cache-Control':'no-store'}});
14
- export async function createPortabilityHandler({db,buckets,policy,migrations,appOrigin,basePath='/api/portability',authorize,maxBodyBytes=MAX_LINE_BYTES}) {
14
+ export async function createPortabilityHandler({db,buckets,policy,migrations,migrationHistory,appOrigin,basePath='/api/portability',authorize,maxBodyBytes=MAX_LINE_BYTES}) {
15
15
  insist(typeof authorize==='function' && new URL(appOrigin).origin===appOrigin,'invalid_handler_configuration');
16
16
  insist(/^\/[A-Za-z0-9/_-]+$/.test(basePath) && !basePath.endsWith('/') && Number.isSafeInteger(maxBodyBytes) && maxBodyBytes>0,'invalid_handler_configuration');
17
- const service=await createPortability({db,buckets,policy,migrations});
17
+ const service=await createPortability({db,buckets,policy,migrations,migrationHistory});
18
18
  return async request=>{
19
19
  const url=new URL(request.url);
20
20
  if(url.pathname!==basePath && !url.pathname.startsWith(basePath+'/'))return null;
@@ -16,25 +16,27 @@ import { checkEpoch, fence, mergeJournal, tombstone } from './journal.mjs';
16
16
  import { writeObject } from './payloads.mjs';
17
17
  import { createCopy,sidecar,readCopy,restoreCopy } from './copies.mjs';
18
18
  import { excludedDependentGuards, migrationSeeded, seededCondition } from './restore-safety.mjs';
19
- import { additiveBridge, sameSchemaMigrations, appliedMigrations, verifiedLegacySchema } from './bridge.mjs';
19
+ import { additiveBridge, sameSchemaMigrations, verifiedLegacySchema } from './bridge.mjs';
20
20
  import { replayStatements } from './backfills.mjs';
21
- import { checkArchivedLedgers, archivedHistory, archivedLedgerTables, ledgerReader, matchMigrationLedger } from './ledgers.mjs';
21
+ import { checkArchivedLedgers, archivedHistory, archivedLedgerTables, ledgerReader, matchMigrationLedger, readLedger } from './ledgers.mjs';
22
+ import { transferHistory } from './owner-history.mjs';
22
23
  import { legacyEmptyGuards, legacyAuditStatements, requireLegacyEmpty } from './legacy-safety.mjs';
23
24
  import { checkReferences } from './references.mjs';
24
25
  import { rebuildDerivedStatements } from './derived.mjs';
26
+ const acceptedUpgrade=(config,manifest)=>manifest.migrationLedger?.origin==='d1_migrations'?undefined:config.acceptedSchemas[manifest.schema];
25
27
  // ledgers: the rows of each migration ledger table the archive carries, as the
26
28
  // caller read them from it ({table:rows}); uploadPart binds them to its parts.
27
29
  export async function createImport(context,manifest,actor='direct',legacyRecovery,ledgers,acknowledgeUnverifiedSchemaObjects=false,id=crypto.randomUUID()) {
28
- const {db,config,buckets}=context;await checkManifest(manifest);checkArchivedLedgers(manifest,ledgers);
29
- insist(manifest.product===config.product,'wrong_product');
30
+ const {db,buckets}=context;await checkManifest(manifest);checkArchivedLedgers(manifest,ledgers);
31
+ insist(manifest.product===context.config.product,'wrong_product');
30
32
  // An already-stale backup cannot become restorable. Refuse before schema
31
33
  // inventory, staging copies and part uploads; fence checks again at finalize.
32
34
  await checkEpoch(db,manifest);
33
35
  // Explicit schema upgrades can reconcile rows, but cannot attest an interrupted deployment.
34
- await appliedMigrations(db);
36
+ const {config,ledger:historyLedger,migrations}=await transferHistory(context),ledger=historyLedger?.entries;
35
37
  const original=manifest;manifest=await restorableManifest(original,{canonical:true});
36
38
  const skippedTables=original.tables.filter(table=>platformTable(table.name)).map(table=>table.name);
37
- const cat=await catalogue(db,config),upgrade=config.acceptedSchemas[original.schema];
39
+ const cat=await catalogue(db,config),upgrade=acceptedUpgrade(config,original);
38
40
  manifest=await withoutAbsentExcludedObjects(manifest,cat,config);
39
41
  const legacy=original.version===2 && manifest.schemaObjects===undefined;
40
42
  // An explicit acknowledgment keeps its existing recovery semantics. Without
@@ -43,7 +45,7 @@ export async function createImport(context,manifest,actor='direct',legacyRecover
43
45
  const unverified=legacy && acknowledgeUnverifiedSchemaObjects===true;
44
46
  const verifyingLegacy=legacy && !unverified && !legacyRecovery;
45
47
  const history=()=>archivedHistory(original,ledgers,{allowOlderShape:unverified || verifyingLegacy});
46
- const verified=(verifyingLegacy || (!legacy && manifest.schema!==cat.identity)) && await verifiedLegacySchema(db,manifest,cat,context.migrations,history);
48
+ const verified=(verifyingLegacy || (!legacy && manifest.schema!==cat.identity)) && await verifiedLegacySchema(ledger??[],manifest,cat,migrations,history);
47
49
  insist(!legacy || legacyRecovery || unverified || verified,'schema_mismatch:legacy_schema',409);
48
50
  if(verified)manifest={...manifest,schema:verified};
49
51
  if(unverified)for(const table of manifest.tables) {
@@ -53,7 +55,7 @@ export async function createImport(context,manifest,actor='direct',legacyRecover
53
55
  insist(table.dependencies.every(d=>target.dependencies.includes(d)),'schema_mismatch:dependencies:'+table.name,409);
54
56
  }
55
57
  const same=manifest.schema===cat.identity;
56
- const {replays,...bridge}=unverified?await additiveBridge(db,manifest,cat,context.migrations,history,legacyRecovery,true):upgrade?{}:same?(await sameSchemaMigrations(db,manifest,cat,context.migrations,history)??{}):await additiveBridge(db,manifest,cat,context.migrations,history,legacyRecovery);
58
+ const {replays,...bridge}=unverified?await additiveBridge(db,ledger,manifest,cat,migrations,history,legacyRecovery,true):upgrade?{}:same?(await sameSchemaMigrations(db,ledger,manifest,cat,migrations,history)??{}):await additiveBridge(db,ledger,manifest,cat,migrations,history,legacyRecovery);
57
59
  // Finalize checks object references on the staged rows; a later migration
58
60
  // replayed into a reference table would publish rows that check never saw.
59
61
  const referenced=replays?.find(r=>Object.values(config.buckets).some(spec=>spec.objectReferences?.table===r.table));
@@ -70,7 +72,7 @@ export async function createImport(context,manifest,actor='direct',legacyRecover
70
72
  insist(canonicalJson(manifest.requiredConfiguration)===canonicalJson(config.requiredConfiguration),'configuration_inventory');
71
73
  for(const o of manifest.objects) insist(buckets[o.bucket] && config.buckets[o.bucket] && o.key.startsWith(config.buckets[o.bucket].prefix??''),'bucket_inventory');
72
74
  insist(canonicalJson(manifest.payloads.map(x=>x.name))===canonicalJson(Object.keys(config.extraPayloads).sort()),'payload_inventory');
73
- if(unverified)await destinationState(context,{metadata:{targets:cat.tables,manifest:original}});
75
+ if(unverified)await destinationState({...context,config,migrations},{metadata:{targets:cat.tables,manifest:original}});
74
76
  await ensureImportLeases(db);
75
77
  const previous=(await state(db)).import_job;
76
78
  if(previous) {
@@ -96,7 +98,7 @@ async function uploadLeasedPart(context,id,index,part) {
96
98
  const previous=await statement(db,'SELECT sha256 FROM rer_portability_parts WHERE job=? AND idx=?',id,index).first();
97
99
  if(previous){insist(previous.sha256===part.sha256,'part_conflict',409);return importStatus(context,id,{includeLease:false});}
98
100
  insist(row.phase==='staging','import_phase',409);insist(index===row.completed && index<row.metadata.manifest.partCount,'part_order',409);
99
- const inserts=[],groups=new Map(),manifest=row.metadata.manifest,upgrade=config.acceptedSchemas[manifest.schema];
101
+ const inserts=[],groups=new Map(),manifest=row.metadata.manifest,upgrade=acceptedUpgrade(config,manifest);
100
102
  for(const item of part.items) {
101
103
  if(item.kind==='row') {
102
104
  const table=manifest.tables.find(x=>x.name===item.table);insist(table && Number.isSafeInteger(item.index) && item.index>=0 && item.index<table.count,'row_inventory');
@@ -303,7 +305,7 @@ export function libraryImports(context,direct,migrationRoot) {
303
305
  await checkManifest(manifest);
304
306
  if(!archivedLedgerTables(manifest).length)return direct.create(manifest,actor,options);
305
307
  insist(manifest.product===context.config.product,'wrong_product');
306
- await appliedMigrations(context.db);
308
+ await (context.migrationHistory?.()??readLedger(context.db));
307
309
  insist(!(await statement(context.db,'SELECT import_job FROM rer_portability_state WHERE id=1').first())?.import_job && ![...pending.values()].some(row=>row.phase==='staging'),'import_running',409);
308
310
  context.migrations??=await libraryMigrations(context.db,context.inputPolicy,migrationRoot);
309
311
  const id=crypto.randomUUID();pending.set(id,{id,manifest:structuredClone(manifest),actor,parts:[],phase:'staging'});return id;
@@ -13,8 +13,8 @@ export interface Manifest {
13
13
  excludedTables: string[]; excludedColumns: Record<string,string[]>; requiredConfiguration: string[]; partCount: number; epoch: number; journal: string[];
14
14
  /** Applied product migrations; absent from archives made before additive restore. */
15
15
  migrations?: string[];
16
- /** Present when RER Platform's runner migrated the source: each applied migration by native file name and SHA-256, in application order. */
17
- migrationLedger?: {origin:'rer_migrations';entries:{name:string;sha256:string}[]};
16
+ /** Hash-bound application order from RER Platform or the local core and owner ledgers. */
17
+ migrationLedger?: {origin:'rer_migrations'|'d1_migrations';entries:{name:string;sha256:string}[]};
18
18
  }
19
19
  export type Item = {kind:'row';table:string;index:number;row:Row;rowid?:number} | {kind:'object';bucket:string;key:string;offset:number;data:string} | {kind:'payload';name:string;offset:number;data:string};
20
20
  export interface Part {kind:'part';index:number;items:Item[];sha256:string}
@@ -51,6 +51,8 @@ export interface PortabilityOptions {
51
51
  /** Node checkout root for automatic legacy evidence; defaults to cwd. */
52
52
  migrationRoot?:string;
53
53
  db:D1Database;buckets?:Record<string,R2Bucket>;policy:ArchivePolicy;migrations?:()=>Promise<MigrationReplay>;
54
+ /** Runtime-owned history for databases with separate owner migration ledgers. */
55
+ migrationHistory?:()=>Promise<{table:string;entries:{name:string;sha256?:string}[];owners?:boolean}|undefined>;
54
56
  /** Seed-owned audit SQL, appended to the snapshot/job batch. Throwing or SQL failure refuses creation. */
55
57
  onExport?:(event:ExportEvent)=>[sql:string,args:unknown[]][];
56
58
  }
@@ -12,13 +12,13 @@ export { readArchive, makePart, MAX_LINE_BYTES, MAX_PART_BYTES } from './format.
12
12
  export { canonicalJson, sha256 } from './hash.mjs';
13
13
  export { PortabilityError } from './storage.mjs';
14
14
  export { createPortabilityHandler } from './http.mjs';
15
- export async function createPortability({db,buckets={},policy:input,migrations,onExport,migrationRoot}) {
15
+ export async function createPortability({db,buckets={},policy:input,migrations,migrationHistory,onExport,migrationRoot}) {
16
16
  insist(onExport===undefined || typeof onExport==='function','invalid_on_export');
17
17
  db=wireDatabase(db);
18
18
  const config=policy(input);await ensureObjectInventory(db);await catalogue(db,config);
19
19
  for(const name of Object.keys(config.buckets))insist(buckets[name] && typeof buckets[name].get==='function','missing_bucket:'+name);
20
20
  await ensureImportLeases(db);
21
- const context={db,buckets,config,migrations,onExport};
21
+ const context={db,buckets,config,migrations,migrationHistory,onExport};
22
22
  const importContext={...context,inputPolicy:input};
23
23
  const {lookup,actor,...imports}=libraryImports(importContext,{
24
24
  active:actor=>activeImport(importContext,actor),
@@ -76,8 +76,7 @@ export function matchMigrationLedger(entries,files,{dataOnly=[]}={}) {
76
76
  * {origin:'rer_migrations',entries:[{name,sha256}]} in application order, each
77
77
  * entry named by the checkout file with that hash (the native identity) where
78
78
  * the exporter can read the checkout's files. Declared data-only entries and
79
- * hosted exporters, which have no files, keep the ledger's own names. Native
80
- * sources record nothing here, so their archives are unchanged.
79
+ * hosted exporters, which have no files, keep the ledger's own names.
81
80
  */
82
81
  export async function platformOrigin(ledger,migrations) {
83
82
  if(ledger?.table!=='rer_migrations')return;
@@ -88,6 +87,20 @@ export async function platformOrigin(ledger,migrations) {
88
87
  return {origin:'rer_migrations',entries};
89
88
  }
90
89
 
90
+ // Separate native owner ledgers are also bookkeeping. Bind the merged applied
91
+ // history to the checkout's files without moving any of those tables' rows.
92
+ export async function nativeOrigin(ledger,migrations,schema) {
93
+ const replay=await migrations?.();
94
+ insist(replay && canonicalJson(replay.names)===canonicalJson(ledger.entries.map(row=>row.name)),'schema_mismatch:migration_history',409);
95
+ const entries=replay.names.map(name=>{
96
+ const sha256=replay.sha256?.(name);
97
+ insist(typeof sha256==='string' && /^[a-f0-9]{64}$/.test(sha256),'schema_mismatch:migration_history',409);
98
+ return {name,sha256};
99
+ });
100
+ insist(await replay.schemaAfter(replay.names)===schema,'schema_mismatch',409);
101
+ return {origin:'d1_migrations',entries};
102
+ }
103
+
91
104
  /** The ledger tables an archive carries (old archives carried rer_migrations before it became bookkeeping). */
92
105
  export const archivedLedgerTables=manifest=>manifest.tables.filter(t=>ledgerNames.includes(identifier(t.name)));
93
106
 
@@ -117,7 +130,7 @@ export function checkArchivedLedgers(manifest,ledgers) {
117
130
  /** The applied migrations an archive's own ledger records, [{name,sha256?}]; undefined when it carries no ledger. */
118
131
  export function archivedHistory(manifest,ledgers,{allowOlderShape=false}={}) {
119
132
  const tables=archivedLedgerTables(manifest);
120
- // A Platform-origin export records its ledger in the manifest, never as a table.
133
+ // Hash-bearing exports record their ledger in the manifest.
121
134
  if(manifest.migrationLedger)insist(!tables.length,'schema_mismatch:migration_history');
122
135
  if(!tables.length)return manifest.migrationLedger?.entries.map(entry);
123
136
  insist(tables.length===1 && ledgers?.[tables[0].name],'schema_mismatch:migration_history');
@@ -8,7 +8,7 @@ import { ensureLegacyAudit, legacyTargetState, requireLegacyEmpty } from './lega
8
8
  import { insist, PortabilityError } from './storage.mjs';
9
9
  import { ledgerReader } from './ledgers.mjs';
10
10
 
11
- export async function recoverLegacyArchive({db,buckets,policy:input,migrations},path,output,transient,acknowledged) {
11
+ export async function recoverLegacyArchive({db,buckets,policy:input,migrations,migrationHistory},path,output,transient,acknowledged) {
12
12
  insist(acknowledged===true,'legacy_acknowledgement_required',409);
13
13
  insist(typeof output==='string' && output!==path,'legacy_output_required',409);
14
14
  insist(!(await lstat(output).catch(error=>{if(error.code==='ENOENT')return null;throw error;})),'legacy_output_exists',409);
@@ -16,7 +16,7 @@ export async function recoverLegacyArchive({db,buckets,policy:input,migrations},
16
16
  // reads the archive's own migration ledger, if any, for placement (bridge.mjs).
17
17
  const ledger=ledgerReader();
18
18
  for await(const record of fileRecords(path,readLegacyArchive))ledger.add(record);
19
- const context={db,buckets,config:policy(input),migrations},target=await legacyTargetState(db,migrations);
19
+ const context={db,buckets,config:policy(input),migrations,migrationHistory},target=await legacyTargetState(db,migrations);
20
20
  await requireLegacyEmpty(context,target);
21
21
  let id,manifest,closing,integrity,status,complete=false;
22
22
  try {
@@ -40,7 +40,7 @@ export async function recoverLegacyArchive({db,buckets,policy:input,migrations},
40
40
  // A fresh export binds the recovered installation from this point forward.
41
41
  // A failure here must be reported; the durable restore audit remains intact.
42
42
  try {
43
- const recovered=await exportArchive(await createPortability({db,buckets,policy:input}),output,transient);
43
+ const recovered=await exportArchive(await createPortability({db,buckets,policy:input,migrations,migrationHistory}),output,transient);
44
44
  return {...recovered,legacyRecovery:true,unverifiedMetadata:true,restoredContentHash:closing.contentHash,...(status.bridged?{bridged:status.bridged}:{}),...(status.safetyChanges?{safetyChanges:status.safetyChanges}:{})};
45
45
  }catch{throw new PortabilityError('legacy_restored_export_failed',500);}
46
46
  }
@@ -1,18 +1,19 @@
1
1
  import { policy, schemaIdentity } from './catalogue.mjs';
2
2
 
3
- // Replays a chosen set of the destination's migrations, in its applied order,
3
+ // Replays a chosen sequence of the destination's migrations, in recorded order,
4
4
  // into a scratch database and returns the resulting archive schema identity.
5
5
  // Node runners only: restore uses it to place an older archive within the
6
6
  // destination's history (see bridge.mjs). plan: [{name,statements,sha256?}],
7
7
  // sha256 being the migration file's hash where the runner read the file.
8
8
  export function migrationReplay(plan,input,{dataOnly=[]}={}) {
9
9
  const config=policy(input),sqlite='node:sqlite';
10
- return {names:plan.map(migration=>migration.name),dataOnly,aliases:name=>plan.find(m=>m.name===name).aliases,statements:name=>plan.find(m=>m.name===name).statements,sha256:name=>plan.find(m=>m.name===name).sha256,async schemaAfter(names,options) {
10
+ const files=new Map(plan.map(migration=>[migration.name,migration]));
11
+ return {names:plan.map(migration=>migration.name),dataOnly,aliases:name=>files.get(name).aliases,statements:name=>files.get(name).statements,sha256:name=>files.get(name).sha256,async schemaAfter(names,options) {
11
12
  const {DatabaseSync}=await import(sqlite);
12
13
  const scratch=new DatabaseSync(':memory:');
13
14
  const db={prepare:sql=>{const query=scratch.prepare(sql);let values=[];return {bind(...args){values=args;return this;},async all(){return {results:query.all(...values)};}};}};
14
15
  try {
15
- for(const migration of plan.filter(m=>names.includes(m.name)))for(const sql of migration.statements)scratch.exec(sql);
16
+ for(const name of names)for(const sql of files.get(name).statements)scratch.exec(sql);
16
17
  return await schemaIdentity(db,config,options);
17
18
  }finally{scratch.close();}
18
19
  },
@@ -0,0 +1,62 @@
1
+ import { ledgerNames, readLedger } from './ledgers.mjs';
2
+ import { all, insist, quote } from './storage.mjs';
3
+
4
+ export function ownerMigrationPolicy(policy, sources) {
5
+ if (!sources.length) return policy;
6
+ const tables = sources.map(source => source.table);
7
+ insist(!policy.includeTables?.some(name => tables.includes(name.toLowerCase())), 'migration_ledger:incompatible', 409);
8
+ return { ...policy, excludeTables: [...new Set([...(policy.excludeTables ?? []), ...tables])] };
9
+ }
10
+
11
+ const ownerColumns = columns => columns.length === 3 && columns[0].name === 'id' && columns[0].pk === 1
12
+ && columns[1].name === 'name' && columns[2].name === 'applied_at';
13
+ const timestamp = /^\d{4}-\d\d-\d\d \d\d:\d\d:\d\d$/;
14
+ async function nativeRows(db, table, owner = false) {
15
+ const columns = await all(db, `PRAGMA table_info(${quote(table)})`);
16
+ if (!columns.length) return [];
17
+ insist(owner ? ownerColumns(columns) : ownerColumns(columns.slice(0, 3)), 'migration_ledger:incompatible', 409);
18
+ const rows = await all(db, `SELECT name, applied_at FROM ${quote(table)} ORDER BY id`);
19
+ insist(new Set(rows.map(row => row.name)).size === rows.length
20
+ && rows.every((row, i) => typeof row.name === 'string' && timestamp.test(row.applied_at)
21
+ && (!i || rows[i - 1].applied_at <= row.applied_at)), 'migration_ledger:incompatible', 409);
22
+ return rows;
23
+ }
24
+
25
+ export async function nativeMigrationHistory(db, sources) {
26
+ const ledger = await readLedger(db);
27
+ const owners = (await Promise.all(sources.map(async ({ table, prefix }) =>
28
+ (await nativeRows(db, table, true)).map(row => ({ ...row, name: prefix + row.name, table }))))).flat();
29
+ if (!owners.length) return ledger;
30
+ insist(ledger?.table === 'd1_migrations', 'migration_ledger:multiple', 409);
31
+ const core = (await nativeRows(db, ledger.table)).map((row, i) => ({ ...ledger.entries[i], applied_at: row.applied_at, table: ledger.table }));
32
+ const rows = [...core, ...owners].sort((a, b) => a.applied_at.localeCompare(b.applied_at));
33
+ // IDs order each ledger. Across ledgers equal timestamps carry no order.
34
+ insist(rows.every((row, i) => !i || row.applied_at !== rows[i - 1].applied_at || row.table === rows[i - 1].table), 'migration_ledger:uncertain', 409);
35
+ const entries = rows.map(({ name, sha256 }) => ({ name, ...(sha256 ? { sha256 } : {}) }));
36
+ insist(new Set(entries.map(row => row.name)).size === entries.length, 'migration_ledger:ambiguous', 409);
37
+ return { table: ledger.table, entries, owners: true };
38
+ }
39
+
40
+ // The existing service callback is also used for core-only replay. Consult it
41
+ // early only when native owner bookkeeping may exist. The replay supplies the
42
+ // declared table identities; a similar product table never becomes bookkeeping.
43
+ async function hasOwnerLedger(db) {
44
+ const candidates = await all(db, "SELECT name FROM sqlite_master WHERE type='table' AND name GLOB '[a-z]*_*_migrations'");
45
+ for (const { name } of candidates) {
46
+ if (!ledgerNames.includes(name) && ownerColumns(await all(db, `PRAGMA table_info(${quote(name)})`))) return true;
47
+ }
48
+ return false;
49
+ }
50
+
51
+ export async function transferHistory({ db, config, migrations, migrationHistory }) {
52
+ if (migrationHistory) return { config, migrations, ledger: await migrationHistory() };
53
+ const ledger = await readLedger(db);
54
+ if (migrations && ledger?.table === 'd1_migrations' && await hasOwnerLedger(db)) {
55
+ const replay = await migrations();
56
+ if (replay?.ownerSources) return {
57
+ config: ownerMigrationPolicy(config, replay.ownerSources), migrations: async () => replay, ledger: replay.history,
58
+ };
59
+ return { config, migrations: async () => replay, ledger };
60
+ }
61
+ return { config, migrations, ledger };
62
+ }
@@ -2,6 +2,7 @@ import type { ArchivePolicy, MigrationReplay } from './index.mjs';
2
2
  export interface MigrationEvidence {
3
3
  version:1;
4
4
  files:{name:string;sha256:string;statements:string[]}[];
5
+ ownerSources?:{prefix:string;table:string}[];
5
6
  objects:{name:string;sql:string;type:string;tbl_name:string}[];
6
7
  prefixes:{objects:number[];shadows:string[];seeds:{name:string;columns:string[];hashes:string[]}[]}[];
7
8
  }
@@ -1,4 +1,4 @@
1
- import { policy, inventorySchemaIdentity } from './catalogue.mjs';
1
+ import { inventorySchemaIdentity } from './catalogue.mjs';
2
2
  import { canonicalJson } from './hash.mjs';
3
3
  import { checkoutHistory } from './checkout-history.mjs';
4
4
  import { insist, PortabilityError } from './storage.mjs';
@@ -8,8 +8,8 @@ import { insist, PortabilityError } from './storage.mjs';
8
8
  export async function workerMigrations(db,input,evidence) {
9
9
  try {
10
10
  insist(evidence?.version===1 && evidence.prefixes.length===evidence.files.length+1,'schema_mismatch:legacy_evidence',409);
11
- const config=policy(input),files=evidence.files;
12
- return await checkoutHistory(db,input,files,names=>{
11
+ const {files,ownerSources}=evidence;
12
+ return await checkoutHistory(db,input,files,(names,config)=>{
13
13
  const prefix=selected=>{
14
14
  insist(selected.length<=names.length && canonicalJson(selected)===canonicalJson(files.slice(0,selected.length).map(f=>f.name)),'schema_mismatch:legacy_evidence',409);
15
15
  const found=evidence.prefixes[selected.length];
@@ -24,6 +24,6 @@ export async function workerMigrations(db,input,evidence) {
24
24
  async rowImages(requests){return new Map(requests.filter(r=>final.seeds.some(s=>s.name===r.name && !s.hashes.length)).map(r=>[r.name,[]]));},
25
25
  async rowHashes(requests,limit){return new Map(requests.flatMap(r=>{const found=final.seeds.find(s=>s.name===r.name && s.hashes.length<=limit && canonicalJson(s.columns)===canonicalJson(r.columns));return found?[[r.name,found.hashes]]:[];}));},
26
26
  };
27
- },{nativeWrangler:true});
27
+ },{nativeWrangler:true,ownerSources});
28
28
  }catch(error){if(error instanceof PortabilityError)throw error;throw new PortabilityError('schema_mismatch:legacy_evidence',409);}
29
29
  }
@@ -69,6 +69,13 @@ URLs; identity comparison ignores case and a terminal `.git` in URLs. Missing,
69
69
  malformed, partial or inconsistent identities fail closed. The selected
70
70
  committed tree supplies the declarations.
71
71
 
72
+ Every object in `rer-project.json` that declares a `path` and a `sha256` or
73
+ `digest` must name a regular file in the project tree. Release compares each
74
+ declaration with the raw file bytes at the selected commit before staging
75
+ output or tags. Use a hexadecimal SHA-256, with an optional `sha256:` prefix
76
+ for `digest`. The same check uses checkout files in `rer verify`, `rer check`
77
+ and the edition source check.
78
+
72
79
  Temporary release directories are removed recursively, including incidental
73
80
  Finder metadata. A cleanup failure keeps a completed release successful and
74
81
  records a `temporary-cleanup-failed` warning in stderr and the release receipt;
@@ -15,6 +15,7 @@ import { listingContent } from './listing.mjs';
15
15
  import { previousListing } from './previous-listing.mjs';
16
16
  import { cleanupTemporary } from './cleanup.mjs';
17
17
  import { parseTimeoutSeconds } from '../../scripts/publish-check/timeout.mjs';
18
+ import { projectHashProblems, projectManifest } from '../../edition-authoring/project-hashes.mjs';
18
19
 
19
20
  // Failure output belongs to the private receipt, not the site preview.
20
21
  const withoutOutput = gates => JSON.parse(JSON.stringify(gates, (key, value) => key === 'output' ? undefined : value));
@@ -30,6 +31,8 @@ export async function release({ repo, ref = 'HEAD', output, project, repository,
30
31
  // metadata, placeholder inspection or release export reads their contents.
31
32
  validatePublicationDestination({ repo, commit: ref }, project, repository);
32
33
  const source = inspectSource(repo, ref);
34
+ const hashProblems = projectHashProblems(source.entries.has(projectManifest) ? source.bytes(projectManifest) : undefined, source.bytes);
35
+ if (hashProblems.length) throw Error(hashProblems.join('\n'));
33
36
  // A release refuses a seed still wearing Base's placeholder mark unless the
34
37
  // caller allows it explicitly; the receipt records that allowance.
35
38
  const files = [...source.entries].filter(([, entry]) => entry.type === 'blob' && ['100644', '100755'].includes(entry.mode)).map(([path]) => path);