@runeditrun/rer 0.2.69 → 0.2.71

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/cli/README.md +2 -0
  2. package/cli/deploy-claim-session.mjs +25 -0
  3. package/cli/deploy-command.mjs +8 -3
  4. package/cli/deploy-observation.mjs +7 -2
  5. package/cli/deploy-owner.mjs +5 -4
  6. package/cli/deploy-product.mjs +30 -0
  7. package/cli/deploy.mjs +3 -1
  8. package/cli/portability-worker.mjs +7 -4
  9. package/global/CHANGELOG.md +17 -0
  10. package/modules/deploy-kit/README.md +2 -0
  11. package/modules/deploy-kit/owner.mjs +1 -1
  12. package/modules/operator-auth/README.md +31 -10
  13. package/modules/operator-auth/SCHEDULED-POLICY.md +137 -0
  14. package/modules/operator-auth/accounts/admin.mjs +2 -10
  15. package/modules/operator-auth/accounts/schema/last-signin.sql +11 -0
  16. package/modules/operator-auth/accounts/schema/operator-auth.sql +25 -1
  17. package/modules/operator-auth/accounts/schema/scheduled-policy-hardening.sql +8 -0
  18. package/modules/operator-auth/accounts/schema/scheduled-policy.sql +18 -0
  19. package/modules/operator-auth/accounts/status.mjs +18 -0
  20. package/modules/operator-auth/accounts/store.mjs +3 -0
  21. package/modules/operator-auth/branding.mjs +5 -2
  22. package/modules/operator-auth/conformance/README.md +7 -3
  23. package/modules/operator-auth/conformance/index.d.mts +2 -2
  24. package/modules/operator-auth/conformance/index.mjs +6 -4
  25. package/modules/operator-auth/conformance/presentation.mjs +8 -3
  26. package/modules/operator-auth/dev-gate.mjs +5 -3
  27. package/modules/operator-auth/index.d.mts +6 -2
  28. package/modules/operator-auth/page.mjs +5 -3
  29. package/modules/operator-auth/routes.mjs +6 -1
  30. package/modules/operator-auth/scheduled/definition.mjs +38 -0
  31. package/modules/operator-auth/scheduled/expiry.mjs +17 -0
  32. package/modules/operator-auth/scheduled/fence.mjs +28 -0
  33. package/modules/operator-auth/scheduled/history.mjs +30 -0
  34. package/modules/operator-auth/scheduled/index.d.mts +21 -0
  35. package/modules/operator-auth/scheduled/index.mjs +83 -0
  36. package/modules/operator-auth/scheduled/runner.mjs +68 -0
  37. package/modules/operator-auth/service.mjs +2 -0
  38. package/modules/portability/README.md +57 -4
  39. package/modules/portability/backfills.mjs +94 -0
  40. package/modules/portability/bridge.mjs +18 -12
  41. package/modules/portability/export.mjs +5 -1
  42. package/modules/portability/import.mjs +11 -7
  43. package/modules/portability/index.d.mts +1 -1
  44. package/modules/portability/migration-statements.mjs +105 -11
  45. package/modules/portability/policy-data.mjs +10 -0
  46. package/modules/portability/sql-text.mjs +95 -0
  47. package/onboarding/adapter.mjs +5 -2
  48. package/onboarding/process-diagnostics.mjs +7 -2
  49. package/package.json +1 -1
@@ -1,5 +1,6 @@
1
1
  // Composed identity over Base's OA accounts, or an explicitly paired native
2
2
  // accounts/proof adapter. Native composition never constructs the OA store.
3
+ import { scheduledPolicies } from './scheduled/index.mjs';
3
4
  import { fencedCredentials } from './accounts/credentials.mjs';
4
5
  import { providerControl } from './accounts/provider-control.mjs';
5
6
  import { linkedProviders } from '../dependencies/auth.mjs';
@@ -76,6 +77,7 @@ export function createOperatorAuth(options) {
76
77
  const external = legacyAdmission({ admission, config });
77
78
  return Object.freeze({
78
79
  ...admin,
80
+ ...(options.scheduledPolicies ? { scheduledPolicies: scheduledPolicies({ definitions: options.scheduledPolicies, store, config, clock, authenticate, requireFreshProof, rolePolicy }) } : {}),
79
81
  ...providerControl({ store, accounts, authenticate, config, clock }),
80
82
  ...fencedCredentials({store,authenticate,requireFreshProof,config,clock,hash}),
81
83
  ...ownerSignIn({ config, store, clock, sessions, limit }),
@@ -132,6 +132,12 @@ that generation and the content hash, while retained domain and identity rows
132
132
  remain equal. Re-linking keeps the installation sector and person bindings;
133
133
  restored provider sign-in remains off until the owner explicitly enables it.
134
134
 
135
+ Included standing policy approvals retain their provenance and run references,
136
+ but export revokes them in the snapshot. The source approval stays unchanged.
137
+ Restore refuses an archive containing a live approval, even with valid archive
138
+ checksums. The destination owner must approve its policies again. Included
139
+ operators retain their durable `last_signin_at` when sessions are scrubbed.
140
+
135
141
  SQLite VIRTUAL and STORED generated columns retain their visible schema metadata
136
142
  and original table SQL. Only writable base cells appear in archived rows or
137
143
  restore inserts; the destination's migrated schema computes generated values and
@@ -286,7 +292,25 @@ contain only `CREATE TABLE` (not `AS SELECT`), `CREATE INDEX` and
286
292
  restorable tables. A later migration with any other statement that names a
287
293
  restorable table, such as an `INSERT`, `UPDATE` or `DELETE` that purges or
288
294
  backfills rows or a trigger, is refused with `schema_mismatch`; statements that
289
- touch only excluded tables are allowed. An archived table the archive holds no
295
+ touch only excluded tables are allowed. A data statement (`INSERT`, `REPLACE`,
296
+ `UPDATE` or `DELETE`, including `INSERT ... SELECT` and a leading `WITH`) is
297
+ judged by the table it writes: when that table is outside the restorable
298
+ catalogue (excluded, derived or platform) and no trigger or foreign key action
299
+ (`ON DELETE`/`ON UPDATE CASCADE` or `SET`) carries its writes into a catalogue
300
+ table, it changes no restored row and is allowed whatever it reads. Such a table
301
+ then holds live rows the restore lacks, so a later statement that copies it into
302
+ a catalogue table is refused. A statement this parser cannot bound to one table
303
+ keeps the rule above. Each chunk a runner's splitter hands over is first split
304
+ again where SQLite itself ends statements (`sql-text.mjs`, sqlite3_complete's
305
+ state machine): a trigger ends only at `; END ;`, so `CASE ... END||x` in its
306
+ body, quoted or commented semicolons never end it, and Wrangler's splitter
307
+ gluing such triggers to the statements after them changes nothing. `rer
308
+ import` replays the installation's migrations with that splitter too. A
309
+ statement names a table only through an identifier that can be one: SQL
310
+ keywords outside table position, function names (also table-valued ones such as
311
+ `pragma_index_list(...)`), the column of a qualified `t.column`, string and blob
312
+ literals, numbers and comments never count. Text Base cannot read to a
313
+ statement boundary, such as an unclosed block comment, is refused. An archived table the archive holds no
290
314
  rows of restores empty, so an `UPDATE` or `DELETE` of it changes nothing and is
291
315
  allowed unless it also names a table the archive holds rows of (which it could
292
316
  read into its writes); an `INSERT` into it, or any other statement naming it,
@@ -294,6 +318,28 @@ is still refused. For archives without a migration record,
294
318
  every migration after the earliest prefix that gives the archive's schema is
295
319
  checked, since the archive may predate any of them.
296
320
 
321
+ A backfill of a catalogue table the archive does not carry (one a later
322
+ migration adds, restored empty) is **replayed** instead of refused: an
323
+ `INSERT ... SELECT` (or `INSERT ... VALUES`) into that table. Finalize empties
324
+ the table (resetting its `AUTOINCREMENT` counter, as the live upgrade created
325
+ it) and runs each of its later writes in migration order, exactly as written,
326
+ after every archived row is inserted and while the destination's triggers are
327
+ still dropped, in the same atomic D1 transaction as the rest of the restore. The
328
+ table then holds what the live upgrade gave it. Time functions such as
329
+ `strftime(..., 'now')` take the restore's time, as they took the upgrade's; an
330
+ `INSERT ... SELECT` without `ORDER BY` assigns new keys in the order SQLite
331
+ scans the restored rows. Rows the restore normalizes (import safety) are read
332
+ as restored. The restore is refused with
333
+ `schema_mismatch:migration_data:<migration>` instead when the archive carries
334
+ the table (even with no rows), when a write to it is not a plain `INSERT` or
335
+ `REPLACE`, reads any table other than archived ones and itself, selects `*` or
336
+ a secret column, when a trigger on it or a foreign key action from it reaches a
337
+ catalogue table, when any other later statement writes it (directly or through
338
+ a trigger) or changes its columns other than by `ADD COLUMN` (allowed only when
339
+ every replay names its columns), or when the statement cannot be bounded.
340
+ A failed or interrupted apply leaves no replayed row; finalizing again replays
341
+ from the start.
342
+
297
343
  The same check runs when the archive's schema identity equals the destination's:
298
344
  a later migration that only purges or backfills rows leaves the schema unchanged,
299
345
  yet the archive's rows never passed through it. Such an archive is refused with
@@ -313,9 +359,13 @@ before any staging or audit write. Without a replayer (the HTTP handler, direct
313
359
  bindings), only identical schemas and `acceptedSchemas` restore.
314
360
 
315
361
  The import status and CLI receipt carry `bridged`: `migrations` (the later
316
- migrations bridged), `tables` (later tables left empty), `columns` (later columns
317
- per table that took defaults), `fromSchema` and `toSchema`. The import job keeps it
318
- and v1 recovery adds it to its audit record. `archiveHash`, `contentHash`,
362
+ migrations bridged), `tables` (later tables, left empty unless replayed),
363
+ `columns` (later columns per table that took defaults), `fromSchema` and
364
+ `toSchema`, and with any replay `replayed`: one `{migration, statement, table,
365
+ rows}` per replayed statement in the order run (`statement` is its index, from
366
+ 0, in the migration's split statements, `rows` the rows it wrote, recorded in the same
367
+ transaction). The import job keeps it as the restore's journal of replays and
368
+ v1 recovery adds it to its audit record. `archiveHash`, `contentHash`,
319
369
  the erasure fence and v1's own checks are unchanged.
320
370
 
321
371
  ## Backup-only v1 recovery
@@ -581,11 +631,14 @@ modules/portability/storage.mjs
581
631
  modules/portability/copies.mjs
582
632
  modules/portability/dev-data.mjs
583
633
  modules/portability/provider-data.mjs
634
+ modules/portability/policy-data.mjs
584
635
  modules/portability/generated.mjs
585
636
  modules/portability/restore-safety.mjs
586
637
  modules/portability/bridge.mjs
587
638
  modules/portability/ledgers.mjs
588
639
  modules/portability/migration-statements.mjs
640
+ modules/portability/sql-text.mjs
641
+ modules/portability/backfills.mjs
589
642
  modules/portability/legacy-safety.mjs
590
643
  modules/portability/export.mjs
591
644
  modules/portability/import.mjs
@@ -0,0 +1,94 @@
1
+ import { statement, quote, insist } from './storage.mjs';
2
+ import { tokens, folded } from './generated.mjs';
3
+ import { changesData, dataStatement, reachingTables, triggerBodies, schemaChange, statementNames } from './migration-statements.mjs';
4
+ import { splitStatements } from './sql-text.mjs';
5
+
6
+ // The data statements of later migrations, judged by the tables they write
7
+ // (bridge.mjs placement). Restored rows never pass through a later migration.
8
+ // - A bounded INSERT, REPLACE, UPDATE or DELETE whose target lies outside the
9
+ // restorable catalogue (an excluded, derived or platform table) and whose
10
+ // writes reach no catalogue table through a trigger or foreign key action
11
+ // (reachingTables) changes no restored row, whatever it reads: admitted.
12
+ // - One whose target is a catalogue table the archive does not carry (restored
13
+ // empty) and that today's rule refuses (it reads archived rows: a backfill)
14
+ // is replayed at finalize. Every later write to that table must then be a
15
+ // replayable INSERT: they run in migration order, exactly as written, after
16
+ // the archived rows are inserted and with the destination's triggers still
17
+ // dropped, so the table holds what the live upgrade gave it. A replay may read
18
+ // only archived tables (complete by then) and the table itself, by named
19
+ // columns that are not secret; no trigger on the table may write a catalogue
20
+ // table; no later statement may write it another way (directly or through a
21
+ // trigger) or change its columns other than by ADD COLUMN.
22
+ // - Everything else keeps today's rule (changesData), with the tables admitted
23
+ // writes fill from archived rows counted as archived: copying one into a
24
+ // catalogue table is refused, since the restore does not hold its live rows.
25
+ // Each chunk the runner's splitter handed over is split again at SQLite's own
26
+ // statement boundaries first (sql-text.mjs), so a trigger glued to the
27
+ // statements after it is judged as the statements it holds.
28
+ // Returns the replays in migration order, or refuses with the first migration
29
+ // (in `unapplied` order) holding a statement that cannot be admitted.
30
+ export function laterData({applied,unapplied,replay,catalogue,archived,empty,tables,secrets}) {
31
+ const split=name=>replay.statements(name).flatMap(splitStatements),fold=list=>list.map(folded);
32
+ const statements=applied.flatMap(split);
33
+ const restorable=fold(catalogue),carried=fold(archived),absent=restorable.filter(name=>!carried.includes(name));
34
+ const reached=reachingTables(statements,carried),writable=reachingTables(statements,restorable);
35
+ const later=unapplied.flatMap(migration=>split(migration).map((sql,index)=>({migration,index,sql,data:dataStatement(sql)})));
36
+ const known=fold(tables),outside=later.filter(s=>s.data && !writable.includes(s.data.target)),tainted=new Set();
37
+ // Tables an admitted write fills from archived rows (directly, through its
38
+ // target's triggers, or from another such table) hold rows the restore lacks.
39
+ const taint=name=>{if(known.includes(name) && !tainted.has(name)){tainted.add(name);triggerBodies(statements,name).forEach(taint);return true;}};
40
+ for(let grew=true;grew;)grew=outside.filter(s=>s.data.names.some(name=>carried.includes(name) || tainted.has(name))).map(s=>taint(s.data.target)).some(Boolean);
41
+ const refused=new Set(),writes=new Map();
42
+ for(const s of later) {
43
+ if(outside.includes(s))continue;
44
+ s.today=changesData(s.sql,[...reached,...tainted],empty);
45
+ if(s.data && absent.includes(s.data.target)){if(!writes.has(s.data.target))writes.set(s.data.target,[]);writes.get(s.data.target).push(s);}
46
+ else if(s.today)refused.add(s);
47
+ }
48
+ const replays=[];
49
+ for(const [table,list] of writes) {
50
+ if(!list.some(s=>s.today))continue;
51
+ const bad=list.filter(s=>!replayable(s,{table,statements,restorable,writable,carried,tables:known,secrets}));
52
+ const reach=reachingTables(statements,[table]),other=later.filter(s=>!list.includes(s) && writesAnother(s,table,reach,list));
53
+ if(bad.length || other.length){[...bad,...other].forEach(s=>refused.add(s));continue;}
54
+ replays.push(...list);
55
+ }
56
+ const first=later.find(s=>refused.has(s));
57
+ insist(!first,'schema_mismatch:migration_data:'+first?.migration);
58
+ const named=Object.fromEntries(catalogue.map(name=>[folded(name),name]));
59
+ return later.filter(s=>replays.includes(s)).map(s=>({migration:s.migration,statement:s.index,table:named[s.data.target],sql:s.sql}));
60
+ }
61
+ function replayable({data},{table,statements,restorable,writable,carried,tables,secrets}) {
62
+ if(!['insert','replace'].includes(data.verb) || data.star)return false;
63
+ const reads=data.names.filter(name=>tables.includes(name) && name!==table);
64
+ if(!reads.every(name=>carried.includes(name)))return false;
65
+ if(reads.some(name=>Object.entries(secrets).some(([t,columns])=>folded(t)===name && columns.some(c=>data.words.includes(folded(c))))))return false;
66
+ if(triggerBodies(statements,table).some(name=>writable.includes(name)))return false;
67
+ // Foreign key actions: a row the replay deletes or rekeys (REPLACE, upsert) must reach no other catalogue table.
68
+ return !reachingTables(statements,restorable.filter(name=>name!==table)).includes(table);
69
+ }
70
+ // Whether a later statement other than the table's replays writes it, through a
71
+ // trigger as well, or changes its columns other than by ADD COLUMN.
72
+ function writesAnother(s,table,reach,replays) {
73
+ if(s.data)return reach.includes(s.data.target);
74
+ if(!statementNames(s.sql).includes(table))return false;
75
+ const shape=schemaChange(s.sql);
76
+ if(shape==='add column')return !replays.every(r=>r.data.columns);
77
+ return !['create table','create index','drop index','create view','drop view','trigger'].includes(shape);
78
+ }
79
+ // Finalize statements for the job's replays, to run after the archived rows are
80
+ // inserted and before the triggers are recreated: each replayed table is
81
+ // emptied (and its AUTOINCREMENT counter reset, as the live upgrade created it)
82
+ // and the statements run in migration order. Each records the rows it wrote in
83
+ // the job's bridge record (`replayed[i].rows`), in the same transaction.
84
+ export function replayStatements(db,row) {
85
+ const replays=row.metadata.replays??[];
86
+ const reset=[...new Set(replays.map(r=>r.table))].flatMap(name=>{
87
+ const target=row.metadata.targets.find(t=>t.name===name);
88
+ insist(target,'schema_mismatch:replay_table:'+name);
89
+ const sequenced=tokens(target.sql).some(t=>!t.quoted && folded(t.text)==='autoincrement');
90
+ return [statement(db,`DELETE FROM ${quote(name)}`),...(sequenced?[statement(db,'DELETE FROM sqlite_sequence WHERE name=?',name)]:[])];
91
+ });
92
+ return [...reset,...replays.flatMap((r,i)=>[db.prepare(r.sql),
93
+ statement(db,`UPDATE rer_portability_jobs SET metadata=json_set(metadata,'$.bridge.replayed[${i}].rows',changes()) WHERE id=?`,row.id)])];
94
+ }
@@ -1,7 +1,7 @@
1
1
  import { all, quote, insist } from './storage.mjs';
2
2
  import { canonicalJson } from './hash.mjs';
3
3
  import { tokens, identifier, folded } from './generated.mjs';
4
- import { changesData, reachingTables } from './migration-statements.mjs';
4
+ import { laterData } from './backfills.mjs';
5
5
  import { destinationLedger } from './ledgers.mjs';
6
6
 
7
7
  // Applied product migrations in the ledger's own order (ledgers.mjs). A
@@ -18,8 +18,9 @@ export async function appliedMigrations(db) {
18
18
  // migrations it records or, for archives that predate that record, of those
19
19
  // its own migration ledger records or, without one, of an earlier prefix.
20
20
  // Restored rows never pass through a later migration, so one that writes or
21
- // rewrites restorable data is refused, whether or not it changes the schema.
22
- async function placement(manifest,cat,replay,ledger,history) {
21
+ // rewrites restorable data is refused, whether or not it changes the schema,
22
+ // unless its statement is replayed at finalize (backfills.mjs).
23
+ async function placement(db,manifest,cat,replay,ledger,history) {
23
24
  const applied=ledger.map(m=>m.name);
24
25
  insist(canonicalJson(replay.names)===canonicalJson(applied) && await replay.schemaAfter(applied)===cat.identity,'schema_mismatch');
25
26
  const recorded=manifest.migrations ?? archivedApplied(history?.(),ledger,replay);
@@ -44,12 +45,14 @@ async function placement(manifest,cat,replay,ledger,history) {
44
45
  // such tables (a newly added module's own) change no restored row, unless a
45
46
  // trigger from any applied migration carries their writes into one that is
46
47
  // restorable. Trigger DDL itself changes no row, nor does an UPDATE or DELETE
47
- // of a table the archive holds no rows of (migration-statements.mjs).
48
- const restorable=cat.tables.map(t=>t.name).filter(name=>manifest.tables.some(t=>t.name===name));
49
- const reached=reachingTables(applied.flatMap(name=>replay.statements(name)),restorable);
50
- const empty=restorable.filter(name=>!manifest.tables.find(t=>t.name===name).count);
51
- for(const name of unapplied)insist(!replay.statements(name).some(sql=>changesData(sql,reached,empty)),'schema_mismatch:migration_data:'+name);
52
- return {earlier};
48
+ // of a table the archive holds no rows of (migration-statements.mjs). A data
49
+ // statement that writes only tables outside the catalogue is admitted
50
+ // whatever it reads; a backfill of a table the archive lacks is replayed.
51
+ const catalogue=cat.tables.map(t=>t.name),archived=catalogue.filter(name=>manifest.tables.some(t=>t.name===name));
52
+ const empty=archived.filter(name=>!manifest.tables.find(t=>t.name===name).count);
53
+ const tables=(await all(db,"SELECT name FROM sqlite_master WHERE type IN ('table','view')")).map(t=>t.name);
54
+ const replays=laterData({applied,unapplied,replay,catalogue,archived,empty,tables,secrets:manifest.excludedColumns??{}});
55
+ return {earlier,replays};
53
56
  }
54
57
  // An archive without a migration record that carries its source's migration
55
58
  // ledger (an old archive from before that ledger became bookkeeping) records
@@ -77,7 +80,8 @@ export async function sameSchemaMigrations(db,manifest,cat,migrations,history) {
77
80
  if(!applied)return;
78
81
  insist(!manifest.migrations || manifest.migrations.every(name=>applied.includes(name)),'schema_mismatch:migrations');
79
82
  if(!migrations){insist(!manifest.migrations || applied.every(name=>manifest.migrations.includes(name)),'schema_mismatch:migrations');return;}
80
- await placement(manifest,cat,await migrations(),ledger,history);
83
+ // Same schema: every catalogue table is archived, so nothing is replayed.
84
+ await placement(db,manifest,cat,await migrations(),ledger,history);
81
85
  }
82
86
  // An archive made before later migrations restores only when those migrations
83
87
  // are additive: new tables, new columns that take a default, or indexes (which
@@ -86,7 +90,7 @@ export async function additiveBridge(db,manifest,cat,migrations,history) {
86
90
  insist(migrations,'schema_mismatch');
87
91
  const replay=await migrations(),ledger=await destinationLedger(db)??[],applied=ledger.map(m=>m.name);
88
92
  insist(!manifest.migrations || manifest.migrations.every(name=>applied.includes(name)),'schema_mismatch:migrations');
89
- const {earlier}=await placement(manifest,cat,replay,ledger,history);
93
+ const {earlier,replays}=await placement(db,manifest,cat,replay,ledger,history);
90
94
  const columns={};
91
95
  for(const table of manifest.tables) {
92
96
  const target=cat.tables.find(t=>t.name===table.name),label=':'+table.name;
@@ -99,8 +103,10 @@ export async function additiveBridge(db,manifest,cat,migrations,history) {
99
103
  insist(appended(table.sql,target.sql,added.map(c=>c.name)),'schema_mismatch:definition'+label);
100
104
  columns[table.name]=added.map(c=>c.name);
101
105
  }
106
+ // replays: the statements finalize runs (backfills.mjs); replayed: the job's record of them.
102
107
  return {fromSchema:manifest.schema,toSchema:cat.identity,migrations:applied.filter(name=>!earlier.includes(name)),
103
- tables:cat.tables.filter(t=>!manifest.tables.some(x=>x.name===t.name)).map(t=>t.name),columns};
108
+ tables:cat.tables.filter(t=>!manifest.tables.some(x=>x.name===t.name)).map(t=>t.name),columns,
109
+ ...(replays.length?{replays,replayed:replays.map(({migration,statement,table})=>({migration,statement,table,rows:null}))}:{})};
104
110
  }
105
111
  // The destination definition is the archived one with only the added column
106
112
  // definitions inserted, as ALTER TABLE ADD COLUMN writes it: no other column,
@@ -8,6 +8,7 @@ import { appliedMigrations } from './bridge.mjs';
8
8
  import { capturePayload, payloadRange } from './payloads.mjs';
9
9
  import { dropProviderData } from './provider-data.mjs';
10
10
  import { dropDevData } from './dev-data.mjs';
11
+ import { revokePolicyApprovals, refusePolicyApprovals } from './policy-data.mjs';
11
12
  export async function createExport(context,actor='direct') {
12
13
  const {db,config,buckets}=context,s=await state(db);insist(!s.applying,'import_applying',409);
13
14
  const cat=await catalogue(db,config),id=crypto.randomUUID(),objects=[];
@@ -28,7 +29,8 @@ export async function createExport(context,actor='direct') {
28
29
  ...cat.tables.flatMap(t=>createCopy(db,t,copies[t.name],true)),statement(db,'INSERT INTO rer_portability_jobs(id,kind,actor,phase,metadata) VALUES(?,?,?,?,?)',id,'export',actor,row.phase,canonicalJson(row.metadata)),db.prepare('DELETE FROM rer_portability_guard')]);
29
30
  try {
30
31
  await dropDevData(db,cat.tables,copies);
31
- await dropProviderData(db,cat.tables,copies);
32
+ await dropProviderData(db,cat.tables,copies);
33
+ await revokePolicyApprovals(db,copies);
32
34
  for(const t of manifest.tables) t.count=(await statement(db,`SELECT COUNT(*) AS n FROM ${quote(copies[t.name])}`).first()).n;
33
35
  for(const [name,producer] of Object.entries(config.extraPayloads).sort()) {
34
36
  const descriptor=await capturePayload(db,id,manifest.payloads.length,await producer.produce());manifest.payloads.push({name,...descriptor});
@@ -52,6 +54,8 @@ async function nextItems(context,row) {
52
54
  const table=manifest.tables[cursor.table];
53
55
  if(!row.metadata.normalized?.includes(table.name)) {
54
56
  await normalizeCopy(context,table,copies[table.name]);
57
+ // A product row transform cannot re-enable a delegation after scrubbing.
58
+ if(table.name==='oa_policy_approvals')await refusePolicyApprovals(db,copies);
55
59
  row.metadata.normalized??=[];row.metadata.normalized.push(table.name);await save(db,row).run();
56
60
  }
57
61
  const records=await readCopy(db,table,copies[table.name],cursor.row,Math.min(1000,Math.ceil(partBytes/128)),true);
@@ -3,6 +3,7 @@ import { stagePart } from './import-staging.mjs';
3
3
  import { normalizeStaged } from './import-safety.mjs';
4
4
  import { ensureImportLeases, createImportLease, importLeaseStatus, importRunning, withImportLease } from './import-lease.mjs';
5
5
  import { refuseProviderData } from './provider-data.mjs';
6
+ import { refusePolicyApprovals } from './policy-data.mjs';
6
7
  import { refuseDevData } from './dev-data.mjs';
7
8
  import { catalogue, validateRow, bindCell, writableColumns } from './catalogue.mjs';
8
9
  import { platformTable, restorableManifest } from './bookkeeping.mjs';
@@ -13,6 +14,7 @@ import { writeObject } from './payloads.mjs';
13
14
  import { createCopy,sidecar,readCopy,restoreCopy } from './copies.mjs';
14
15
  import { excludedDependentGuards, migrationSeeded, seededCondition } from './restore-safety.mjs';
15
16
  import { additiveBridge, sameSchemaMigrations, appliedMigrations } from './bridge.mjs';
17
+ import { replayStatements } from './backfills.mjs';
16
18
  import { checkArchivedLedgers, archivedHistory } from './ledgers.mjs';
17
19
  import { legacyEmptyGuards, legacyAuditStatements, requireLegacyEmpty } from './legacy-safety.mjs';
18
20
  // ledgers: the rows of each migration ledger table the archive carries, as the
@@ -26,16 +28,16 @@ export async function createImport(context,manifest,actor='direct',legacyRecover
26
28
  const skippedTables=original.tables.filter(table=>platformTable(table.name)).map(table=>table.name);
27
29
  const cat=await catalogue(db,config),upgrade=config.acceptedSchemas[original.schema];
28
30
  const same=manifest.schema===cat.identity,history=()=>archivedHistory(original,ledgers);
29
- const bridge=same || upgrade?undefined:await additiveBridge(db,manifest,cat,context.migrations,history);
31
+ const {replays,...bridge}=same || upgrade?{}:await additiveBridge(db,manifest,cat,context.migrations,history);
30
32
  if(same && !upgrade)await sameSchemaMigrations(db,manifest,cat,context.migrations,history);
31
- if(!bridge)insist(canonicalJson(manifest.tables.map(x=>x.name))===canonicalJson(cat.tables.map(x=>x.name)),'table_inventory');
32
- if(!upgrade && !bridge)for(const table of manifest.tables) {
33
+ if(!bridge.fromSchema)insist(canonicalJson(manifest.tables.map(x=>x.name))===canonicalJson(cat.tables.map(x=>x.name)),'table_inventory');
34
+ if(!upgrade && !bridge.fromSchema)for(const table of manifest.tables) {
33
35
  const target=cat.tables.find(t=>t.name===table.name);
34
36
  insist(canonicalJson(table.columns)===canonicalJson(target.columns) && canonicalJson(table.dependencies)===canonicalJson(target.dependencies),'column_inventory');
35
37
  }
36
38
  // Bridged tables stage and restore only the archived columns; the destination
37
39
  // supplies defaults for later columns and leaves later tables empty.
38
- const targets=cat.tables.map(t=>bridge?.columns[t.name]?{...t,columns:manifest.tables.find(x=>x.name===t.name).columns,added:bridge.columns[t.name]}:t);
40
+ const targets=cat.tables.map(t=>bridge.columns?.[t.name]?{...t,columns:manifest.tables.find(x=>x.name===t.name).columns,added:bridge.columns[t.name]}:t);
39
41
  insist(canonicalJson(manifest.excludedColumns)===canonicalJson(config.secretColumns),'secret_inventory');
40
42
  insist(canonicalJson(manifest.requiredConfiguration)===canonicalJson(config.requiredConfiguration),'configuration_inventory');
41
43
  for(const o of manifest.objects) insist(buckets[o.bucket] && config.buckets[o.bucket] && o.key.startsWith(config.buckets[o.bucket].prefix??''),'bucket_inventory');
@@ -47,7 +49,7 @@ export async function createImport(context,manifest,actor='direct',legacyRecover
47
49
  await abortImport(context,previous,{staleOnly:true});
48
50
  }
49
51
  const id=crypto.randomUUID(),copies=Object.fromEntries(targets.map((t,i)=>[t.name,'rer_portability_stage_'+id.replaceAll('-','')+'_'+i]));
50
- const row={id,kind:'import',actor,phase:'staging',completed:0,metadata:{manifest:original,...(skippedTables.length?{restoreManifest:manifest,skippedTables}:{}),copies,targets,triggers:cat.triggers,...(bridge?{bridge}:{}),...(legacyRecovery?{legacyRecovery}: {}),...(ledgers?{ledgers}:{})}};
52
+ const row={id,kind:'import',actor,phase:'staging',completed:0,metadata:{manifest:original,...(skippedTables.length?{restoreManifest:manifest,skippedTables}:{}),copies,targets,triggers:cat.triggers,...(bridge.fromSchema?{bridge}:{}),...(replays?{replays}:{}),...(legacyRecovery?{legacyRecovery}: {}),...(ledgers?{ledgers}:{})}};
51
53
  try {
52
54
  await db.batch([guard(db,'(SELECT import_job FROM rer_portability_state WHERE id=1) IS NULL'),
53
55
  statement(db,'UPDATE rer_portability_state SET import_job=? WHERE id=1',id),
@@ -177,7 +179,9 @@ async function finalizeLeasedImport(context,id,{contentHash,archiveHash,laterJou
177
179
  await bucketsEmpty(context,row);
178
180
  await refuseDevData(db,row.metadata.copies,row.metadata.targets);
179
181
  await refuseProviderData(db,row.metadata.copies,row.metadata.targets);
182
+ await refusePolicyApprovals(db,row.metadata.copies);
180
183
  await normalizeStaged(context,row);
184
+ await refusePolicyApprovals(db,row.metadata.copies);
181
185
  if(config.validateStaged) {
182
186
  const read=async table=>{insist(row.metadata.copies[table],'staged_table');const target=row.metadata.targets.find(t=>t.name===table);return (await readCopy(db,target,row.metadata.copies[table],0,row.metadata.manifest.tables.find(t=>t.name===table)?.count??0)).map(x=>x.row);};
183
187
  insist(await config.validateStaged({db,tables:{...row.metadata.copies},read,manifest})!==false,'staged_validation');
@@ -201,8 +205,8 @@ async function finalizeLeasedImport(context,id,{contentHash,archiveHash,laterJou
201
205
  ...row.metadata.targets.map(t=>guard(db,`NOT EXISTS(SELECT 1 FROM ${quote(sidecar(row.metadata.copies[t.name]))} s JOIN rer_portability_journal j ON j.source=? AND j.tombstone=s.rer_portability_tombstone)`,manifest.sourceIdentity)),
202
206
  ...row.metadata.triggers.map(t=>db.prepare(`DROP TRIGGER ${quote(t.name)}`)),
203
207
  ...replaced.map(t=>db.prepare(`DELETE FROM ${quote(t)}`)),
204
- ...row.metadata.targets.flatMap(t=>restoreCopy(db,t,row.metadata.copies[t.name])),
205
- ...row.metadata.triggers.map(t=>db.prepare(t.sql)),...mergeJournal(db,manifest,checked),...legacyAuditStatements(db,row),save(db,row),
208
+ ...row.metadata.targets.flatMap(t=>restoreCopy(db,t,row.metadata.copies[t.name])),save(db,row),...replayStatements(db,row),
209
+ ...row.metadata.triggers.map(t=>db.prepare(t.sql)),...mergeJournal(db,manifest,checked),...legacyAuditStatements(db,row),
206
210
  db.prepare('UPDATE rer_portability_state SET import_job=NULL,applying=0 WHERE id=1'),db.prepare('DELETE FROM rer_portability_guard')]);
207
211
  }catch(e){
208
212
  // D1 may have committed even when its acknowledgement failed.
@@ -36,7 +36,7 @@ export interface ArchivePolicy {
36
36
  export interface SafetyChange {table:string;column:string;value:Cell;rows:number}
37
37
  export interface Status {id:string;kind:'export'|'import';phase:'building'|'staging'|'applying'|'complete'|'aborted';completed:number;total:number|null;/** Import only: inactivity deadline and whether it has passed. */abortableAt?:string|null;stale?:boolean;/** Export only: bytes of the parts built so far. */bytes?:number;contentHash:string|null;requiredConfiguration?:string[];bridged?:Bridge;/** Platform bookkeeping ignored in an older archive. */skippedTables?:string[];/** Durable summary of bounded import normalization. */safetyChanges?:SafetyChange[]}
38
38
  /** Later additive migrations an older archive was restored across. */
39
- export interface Bridge {fromSchema:string;toSchema:string;migrations:string[];tables:string[];columns:Record<string,string[]>}
39
+ export interface Bridge {fromSchema:string;toSchema:string;migrations:string[];tables:string[];columns:Record<string,string[]>;/** Later backfills of tables the archive lacks, replayed at finalize in this order; rows is null until the apply commits. */replayed?:{migration:string;statement:number;table:string;rows:number|null}[]}
40
40
  /** A runner's replay of the destination's applied migrations: the archive schema identity after a chosen set, and each migration's statements. */
41
41
  export interface MigrationReplay {names:string[];statements(name:string):string[];/** The migration file's SHA-256 hex, where the runner read the file. */sha256?(name:string):string|undefined;schemaAfter(names:string[]):Promise<string>}
42
42
  export interface PortabilityOptions {db:D1Database;buckets?:Record<string,R2Bucket>;policy:ArchivePolicy;migrations?:()=>Promise<MigrationReplay>}
@@ -1,4 +1,5 @@
1
1
  import { tokens, identifier, folded } from './generated.mjs';
2
+ import { tableNames, splitStatements } from './sql-text.mjs';
2
3
 
3
4
  // Whether one migration statement can change rows a restore carries. Only
4
5
  // CREATE TABLE (not AS SELECT), CREATE INDEX and ALTER TABLE ... ADD COLUMN
@@ -18,8 +19,12 @@ import { tokens, identifier, folded } from './generated.mjs';
18
19
  // (no row fires a trigger either), so such a statement changes data only if it
19
20
  // names another reached table (one it could write, or read into its writes).
20
21
  // Any other statement naming an empty table still does: an INSERT would leave
21
- // rows the restore lacks.
22
+ // rows the restore lacks. A chunk holding several statements (sql-text.mjs
23
+ // splitStatements) changes data when any of them does.
22
24
  export function changesData(sql,reached,empty=[]) {
25
+ return splitStatements(sql).some(one=>statementChanges(one,reached,empty));
26
+ }
27
+ function statementChanges(sql,reached,empty) {
23
28
  const list=tokens(sql),words=list.map(t=>t.quoted?undefined:folded(t.text)),kind=kindOf(words);
24
29
  if(kind==='create table')return topLevelWords(list,words).includes('as');
25
30
  if(kind==='create index')return false;
@@ -30,23 +35,112 @@ export function changesData(sql,reached,empty=[]) {
30
35
  return names(list,new Set(reached.map(folded).filter(name=>!skipped.has(name))));
31
36
  }
32
37
  // The restorable tables and every table whose writes reach one through a
33
- // trigger any of `statements` creates: a write to such a table during a later
34
- // migration changes restorable rows the archive lacks. A trigger counts when
35
- // its statement names a reached table anywhere; one this parser cannot bound
36
- // adds every name it holds.
38
+ // trigger any of `statements` creates, or through a foreign key action (ON
39
+ // DELETE/UPDATE CASCADE or SET) declared by a reached table: a write to such a
40
+ // table during a later migration changes restorable rows the archive lacks. A
41
+ // trigger counts when its statement names a reached table anywhere; one this
42
+ // parser cannot bound adds every table name it holds (sql-text.mjs tableNames:
43
+ // never a keyword, function, qualified column or literal).
37
44
  export function reachingTables(statements,restorable) {
38
- const reached=new Set(restorable.map(folded));
39
- const triggers=statements.map(sql=>tokens(sql)).filter(list=>kindOf(list.map(t=>t.quoted?undefined:folded(t.text)))==='create trigger')
40
- .map(list=>({list,on:createdTrigger(list)?.on}));
45
+ const reached=new Set(restorable.map(folded)),parsed=statements.flatMap(splitStatements).map(sql=>tokens(sql)).map(list=>({list,words:list.map(t=>t.quoted?undefined:folded(t.text))}));
46
+ const triggers=parsed.filter(({words})=>kindOf(words)==='create trigger').map(({list})=>({list,on:createdTrigger(list)?.on}));
47
+ const actions=parsed.filter(({words})=>['create table','alter table'].includes(kindOf(words)) && words.some((w,i)=>w==='cascade' || (w==='set' && ['delete','update'].includes(words[i-1])))).map(({list})=>referencing(list));
41
48
  for(let grew=true;grew;) {
42
49
  grew=false;
43
- for(const {list,on} of triggers)if(names(list,reached))for(const name of on===undefined?identifiers(list):[on])if(!reached.has(name)){reached.add(name);grew=true;}
50
+ const add=name=>{if(!reached.has(name)){reached.add(name);grew=true;}};
51
+ for(const {list,on} of triggers)if(names(list,reached))(on===undefined?tableNames(list):[on]).forEach(add);
52
+ for(const {table,parents} of actions)if(table===undefined?parents.some(name=>reached.has(name)):reached.has(table))parents.forEach(add);
44
53
  }
45
54
  return [...reached];
46
55
  }
56
+ // The table a CREATE TABLE or ALTER TABLE statement declares and the tables its
57
+ // REFERENCES clauses name (table undefined when it cannot be read).
58
+ function referencing(list) {
59
+ const parts=atoms(list),at=word(parts[2],'if') && word(parts[3],'not') && word(parts[4],'exists')?5:2;
60
+ return {table:tableName(parts.slice(at,at+1)),parents:parts.flatMap((a,i)=>word(a,'references') && parts[i+1]?[tableName([parts[i+1]])].filter(Boolean):[])};
61
+ }
62
+ // The last part of a [schema.]table name read from one to three atoms.
63
+ function tableName(name) {
64
+ const last=name.at(-1);
65
+ if(!last || name.length>3 || name.some(a=>!a.quoted && !/^[A-Za-z0-9_$.\u0080-\uffff]+$/.test(a.text)))return;
66
+ return folded(last.quoted?identifier(last):last.text.split('.').at(-1))||undefined;
67
+ }
68
+ // The one table a bounded INSERT, REPLACE, UPDATE or DELETE statement writes
69
+ // directly, after an optional WITH clause, with what a replay must know of it:
70
+ // {verb, target, conflict (an OR clause's resolution, 'replace' for REPLACE),
71
+ // columns (an explicit insert column list), upsert (ON CONFLICT ... DO UPDATE),
72
+ // star (a * or t.* result column), names (the table names it holds, tableNames),
73
+ // words (every identifier it holds, columns and keywords too)}. Undefined
74
+ // for any other statement or one this parser cannot bound. Triggers on the
75
+ // target and foreign key actions are the only further writes such a statement
76
+ // makes; reachingTables accounts for both.
77
+ export function dataStatement(sql) {
78
+ const parts=atoms(tokens(sql));
79
+ if(parts.some(semicolon))return;
80
+ let i=0;
81
+ if(word(parts[0],'with')) {
82
+ i=word(parts[1],'recursive')?2:1;
83
+ for(;;) {
84
+ i++;// the table name
85
+ if(parts[i]?.text==='(')i=closing(parts,i);
86
+ if(!word(parts[i],'as'))return;
87
+ i++;
88
+ if(word(parts[i],'not'))i++;
89
+ if(word(parts[i],'materialized'))i++;
90
+ if(parts[i]?.text!=='(')return;
91
+ i=closing(parts,i);
92
+ if(parts[i]?.text!==',')break;
93
+ i++;
94
+ }
95
+ }
96
+ const verb=['insert','replace','update','delete'].find(v=>word(parts[i],v));
97
+ if(!verb)return;
98
+ let conflict=verb==='replace'?'replace':undefined;
99
+ i++;
100
+ if(['insert','update'].includes(verb) && word(parts[i],'or')){conflict=parts[i+1] && !parts[i+1].quoted?folded(parts[i+1].text):undefined;if(!conflict)return;i+=2;}
101
+ if(verb!=='update'){if(!word(parts[i],verb==='delete'?'from':'into'))return;i++;}
102
+ const name=[];
103
+ const joined=a=>!name.length || (!a.quoted && a.text.startsWith('.')) || (!name.at(-1).quoted && name.at(-1).text.endsWith('.'));
104
+ for(;i<parts.length && !['(',',',')'].includes(parts[i].text) && joined(parts[i]);i++)name.push(parts[i]);
105
+ const target=tableName(name);
106
+ if(!target)return;
107
+ if(word(parts[i],'as'))i+=2;
108
+ const star=parts.some((a,k)=>!a.quoted && /(^|\.)\*$/.test(a.text) && !(parts[k-1]?.text==='(' && parts[k+1]?.text===')'));
109
+ const upsert=parts.some((a,k)=>word(a,'do') && word(parts[k+1],'update'));
110
+ return {verb,target,conflict,columns:verb==='insert' || verb==='replace'?parts[i]?.text==='(':undefined,upsert,star,names:tableNames(parts),words:identifiers(parts)};
111
+ }
112
+ // The index after the parenthesized group opening at `start`, or the end when unbalanced.
113
+ function closing(parts,start) {
114
+ let depth=0;
115
+ for(let i=start;i<parts.length;i++){if(parts[i].quoted)continue;if(parts[i].text==='(')depth++;else if(parts[i].text===')' && !--depth)return i+1;}
116
+ return parts.length;
117
+ }
118
+ // The table names in the bodies of triggers on `table` that `statements`
119
+ // create, and every one in a trigger this parser cannot bound that names the table.
120
+ export function triggerBodies(statements,table) {
121
+ const result=[];
122
+ for(const list of statements.flatMap(splitStatements).map(sql=>tokens(sql))) {
123
+ if(kindOf(list.map(t=>t.quoted?undefined:folded(t.text)))!=='create trigger')continue;
124
+ const created=createdTrigger(list);
125
+ if(!created){if(tableNames(list).includes(table))result.push(...tableNames(list));}
126
+ else if(created.on===table)result.push(...tableNames(atoms(list).slice(created.begin+1)));
127
+ }
128
+ return result;
129
+ }
130
+ // What a statement that changes only the schema does: 'create table' (not AS
131
+ // SELECT), 'create index', 'drop index', 'create view', 'drop view', 'add
132
+ // column' or 'trigger' (a bounded CREATE or DROP TRIGGER); undefined otherwise.
133
+ export function schemaChange(sql) {
134
+ const list=tokens(sql),words=list.map(t=>t.quoted?undefined:folded(t.text)),kind=kindOf(words);
135
+ if(kind==='create table')return topLevelWords(list,words).includes('as')?undefined:kind;
136
+ if(['create index','drop index','create view','drop view'].includes(kind))return kind;
137
+ if(kind==='alter table' && words.find((w,i)=>i>2 && ['add','rename','drop'].includes(w))==='add')return 'add column';
138
+ if(kind==='create trigger' && createdTrigger(list) || kind==='drop trigger' && droppedTrigger(list))return 'trigger';
139
+ }
140
+ export const statementNames=sql=>tableNames(tokens(sql));
47
141
  const kindOf=words=>words.slice(0,4).filter(w=>!['temp','temporary','unique'].includes(w)).slice(0,2).join(' ');
48
142
  const identifiers=list=>list.flatMap(t=>t.quoted?(t.text.startsWith("'")?[]:[folded(identifier(t))]):folded(t.text).split(/[^a-z0-9_$\u0080-\uffff]+/).filter(Boolean));
49
- const names=(list,reached)=>identifiers(list).some(name=>reached.has(name));
143
+ const names=(list,reached)=>tableNames(list).some(name=>reached.has(name));
50
144
  // Tokens with each unquoted ';' split out, and trailing ones dropped.
51
145
  function atoms(list) {
52
146
  const result=list.flatMap(t=>t.quoted?[t]:t.text.split(/(;)/).filter(Boolean).map(text=>({text})));
@@ -79,7 +173,7 @@ function createdTrigger(list) {
79
173
  let start=true,statements=0;
80
174
  for(let i=begin+1;i<parts.length;i++) {
81
175
  const a=parts[i];
82
- if(start && word(a,'end'))return i===parts.length-1 && statements && !depth?{on}:undefined;
176
+ if(start && word(a,'end'))return i===parts.length-1 && statements && !depth?{on,begin}:undefined;
83
177
  if(semicolon(a)){if(start || depth)return;start=true;statements++;continue;}
84
178
  if(!a.quoted){if(a.text==='(')depth++;else if(a.text===')' && --depth<0)return;}
85
179
  start=false;
@@ -0,0 +1,10 @@
1
+ import { all, quote, insist } from './storage.mjs';
2
+
3
+ // Keep approval provenance and run references, but never transfer a live delegation.
4
+ export async function revokePolicyApprovals(db,copies) {
5
+ if(copies.oa_policy_approvals)await db.prepare(`UPDATE ${quote(copies.oa_policy_approvals)} SET revoked_at=MAX(approved_at,CAST(strftime('%s','now') AS INTEGER)*1000) WHERE revoked_at IS NULL`).run();
6
+ }
7
+
8
+ export async function refusePolicyApprovals(db,copies) {
9
+ if(copies.oa_policy_approvals)insist(!(await all(db,`SELECT 1 FROM ${quote(copies.oa_policy_approvals)} WHERE revoked_at IS NULL LIMIT 1`)).length,'live_policy_approval',409);
10
+ }