@runeditrun/rer 0.2.69 → 0.2.70
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cli/README.md +2 -0
- package/cli/deploy-command.mjs +5 -1
- package/cli/deploy-product.mjs +30 -0
- package/cli/deploy.mjs +3 -1
- package/global/CHANGELOG.md +9 -0
- package/modules/deploy-kit/README.md +2 -0
- package/modules/deploy-kit/owner.mjs +1 -1
- package/modules/operator-auth/README.md +31 -10
- package/modules/operator-auth/SCHEDULED-POLICY.md +130 -0
- package/modules/operator-auth/accounts/admin.mjs +2 -10
- package/modules/operator-auth/accounts/schema/last-signin.sql +11 -0
- package/modules/operator-auth/accounts/schema/operator-auth.sql +25 -1
- package/modules/operator-auth/accounts/schema/scheduled-policy-hardening.sql +8 -0
- package/modules/operator-auth/accounts/schema/scheduled-policy.sql +18 -0
- package/modules/operator-auth/accounts/status.mjs +18 -0
- package/modules/operator-auth/accounts/store.mjs +3 -0
- package/modules/operator-auth/branding.mjs +5 -2
- package/modules/operator-auth/conformance/README.md +7 -3
- package/modules/operator-auth/conformance/index.d.mts +2 -2
- package/modules/operator-auth/conformance/index.mjs +6 -4
- package/modules/operator-auth/conformance/presentation.mjs +8 -3
- package/modules/operator-auth/dev-gate.mjs +5 -3
- package/modules/operator-auth/index.d.mts +6 -2
- package/modules/operator-auth/page.mjs +5 -3
- package/modules/operator-auth/routes.mjs +6 -1
- package/modules/operator-auth/scheduled/definition.mjs +37 -0
- package/modules/operator-auth/scheduled/fence.mjs +27 -0
- package/modules/operator-auth/scheduled/history.mjs +30 -0
- package/modules/operator-auth/scheduled/index.d.mts +21 -0
- package/modules/operator-auth/scheduled/index.mjs +83 -0
- package/modules/operator-auth/scheduled/runner.mjs +65 -0
- package/modules/operator-auth/service.mjs +2 -0
- package/modules/portability/README.md +46 -4
- package/modules/portability/backfills.mjs +89 -0
- package/modules/portability/bridge.mjs +18 -12
- package/modules/portability/export.mjs +5 -1
- package/modules/portability/import.mjs +11 -7
- package/modules/portability/index.d.mts +1 -1
- package/modules/portability/migration-statements.mjs +96 -9
- package/modules/portability/policy-data.mjs +10 -0
- package/onboarding/adapter.mjs +5 -2
- package/onboarding/process-diagnostics.mjs +7 -2
- package/package.json +1 -1
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
import { statement, quote, insist } from './storage.mjs';
|
|
2
|
+
import { tokens, folded } from './generated.mjs';
|
|
3
|
+
import { changesData, dataStatement, reachingTables, triggerBodies, schemaChange, statementNames } from './migration-statements.mjs';
|
|
4
|
+
|
|
5
|
+
// The data statements of later migrations, judged by the tables they write
|
|
6
|
+
// (bridge.mjs placement). Restored rows never pass through a later migration.
|
|
7
|
+
// - A bounded INSERT, REPLACE, UPDATE or DELETE whose target lies outside the
|
|
8
|
+
// restorable catalogue (an excluded, derived or platform table) and whose
|
|
9
|
+
// writes reach no catalogue table through a trigger or foreign key action
|
|
10
|
+
// (reachingTables) changes no restored row, whatever it reads: admitted.
|
|
11
|
+
// - One whose target is a catalogue table the archive does not carry (restored
|
|
12
|
+
// empty) and that today's rule refuses (it reads archived rows: a backfill)
|
|
13
|
+
// is replayed at finalize. Every later write to that table must then be a
|
|
14
|
+
// replayable INSERT: they run in migration order, exactly as written, after
|
|
15
|
+
// the archived rows are inserted and with the destination's triggers still
|
|
16
|
+
// dropped, so the table holds what the live upgrade gave it. A replay may read
|
|
17
|
+
// only archived tables (complete by then) and the table itself, by named
|
|
18
|
+
// columns that are not secret; no trigger on the table may write a catalogue
|
|
19
|
+
// table; no later statement may write it another way (directly or through a
|
|
20
|
+
// trigger) or change its columns other than by ADD COLUMN.
|
|
21
|
+
// - Everything else keeps today's rule (changesData), with the tables admitted
|
|
22
|
+
// writes fill from archived rows counted as archived: copying one into a
|
|
23
|
+
// catalogue table is refused, since the restore does not hold its live rows.
|
|
24
|
+
// Returns the replays in migration order, or refuses with the first migration
|
|
25
|
+
// (in `unapplied` order) holding a statement that cannot be admitted.
|
|
26
|
+
export function laterData({applied,unapplied,replay,catalogue,archived,empty,tables,secrets}) {
|
|
27
|
+
const statements=applied.flatMap(name=>replay.statements(name)),fold=list=>list.map(folded);
|
|
28
|
+
const restorable=fold(catalogue),carried=fold(archived),absent=restorable.filter(name=>!carried.includes(name));
|
|
29
|
+
const reached=reachingTables(statements,carried),writable=reachingTables(statements,restorable);
|
|
30
|
+
const later=unapplied.flatMap(migration=>replay.statements(migration).map((sql,index)=>({migration,index,sql,data:dataStatement(sql)})));
|
|
31
|
+
const known=fold(tables),outside=later.filter(s=>s.data && !writable.includes(s.data.target)),tainted=new Set();
|
|
32
|
+
// Tables an admitted write fills from archived rows (directly, through its
|
|
33
|
+
// target's triggers, or from another such table) hold rows the restore lacks.
|
|
34
|
+
const taint=name=>{if(known.includes(name) && !tainted.has(name)){tainted.add(name);triggerBodies(statements,name).forEach(taint);return true;}};
|
|
35
|
+
for(let grew=true;grew;)grew=outside.filter(s=>s.data.names.some(name=>carried.includes(name) || tainted.has(name))).map(s=>taint(s.data.target)).some(Boolean);
|
|
36
|
+
const refused=new Set(),writes=new Map();
|
|
37
|
+
for(const s of later) {
|
|
38
|
+
if(outside.includes(s))continue;
|
|
39
|
+
s.today=changesData(s.sql,[...reached,...tainted],empty);
|
|
40
|
+
if(s.data && absent.includes(s.data.target)){if(!writes.has(s.data.target))writes.set(s.data.target,[]);writes.get(s.data.target).push(s);}
|
|
41
|
+
else if(s.today)refused.add(s);
|
|
42
|
+
}
|
|
43
|
+
const replays=[];
|
|
44
|
+
for(const [table,list] of writes) {
|
|
45
|
+
if(!list.some(s=>s.today))continue;
|
|
46
|
+
const bad=list.filter(s=>!replayable(s,{table,statements,restorable,writable,carried,tables:known,secrets}));
|
|
47
|
+
const reach=reachingTables(statements,[table]),other=later.filter(s=>!list.includes(s) && writesAnother(s,table,reach,list));
|
|
48
|
+
if(bad.length || other.length){[...bad,...other].forEach(s=>refused.add(s));continue;}
|
|
49
|
+
replays.push(...list);
|
|
50
|
+
}
|
|
51
|
+
const first=later.find(s=>refused.has(s));
|
|
52
|
+
insist(!first,'schema_mismatch:migration_data:'+first?.migration);
|
|
53
|
+
const named=Object.fromEntries(catalogue.map(name=>[folded(name),name]));
|
|
54
|
+
return later.filter(s=>replays.includes(s)).map(s=>({migration:s.migration,statement:s.index,table:named[s.data.target],sql:s.sql}));
|
|
55
|
+
}
|
|
56
|
+
function replayable({data},{table,statements,restorable,writable,carried,tables,secrets}) {
|
|
57
|
+
if(!['insert','replace'].includes(data.verb) || data.star)return false;
|
|
58
|
+
const reads=data.names.filter(name=>tables.includes(name) && name!==table);
|
|
59
|
+
if(!reads.every(name=>carried.includes(name)))return false;
|
|
60
|
+
if(reads.some(name=>Object.entries(secrets).some(([t,columns])=>folded(t)===name && columns.some(c=>data.names.includes(folded(c))))))return false;
|
|
61
|
+
if(triggerBodies(statements,table).some(name=>writable.includes(name)))return false;
|
|
62
|
+
// Foreign key actions: a row the replay deletes or rekeys (REPLACE, upsert) must reach no other catalogue table.
|
|
63
|
+
return !reachingTables(statements,restorable.filter(name=>name!==table)).includes(table);
|
|
64
|
+
}
|
|
65
|
+
// Whether a later statement other than the table's replays writes it, through a
|
|
66
|
+
// trigger as well, or changes its columns other than by ADD COLUMN.
|
|
67
|
+
function writesAnother(s,table,reach,replays) {
|
|
68
|
+
if(s.data)return reach.includes(s.data.target);
|
|
69
|
+
if(!statementNames(s.sql).includes(table))return false;
|
|
70
|
+
const shape=schemaChange(s.sql);
|
|
71
|
+
if(shape==='add column')return !replays.every(r=>r.data.columns);
|
|
72
|
+
return !['create table','create index','drop index','create view','drop view','trigger'].includes(shape);
|
|
73
|
+
}
|
|
74
|
+
// Finalize statements for the job's replays, to run after the archived rows are
|
|
75
|
+
// inserted and before the triggers are recreated: each replayed table is
|
|
76
|
+
// emptied (and its AUTOINCREMENT counter reset, as the live upgrade created it)
|
|
77
|
+
// and the statements run in migration order. Each records the rows it wrote in
|
|
78
|
+
// the job's bridge record (`replayed[i].rows`), in the same transaction.
|
|
79
|
+
export function replayStatements(db,row) {
|
|
80
|
+
const replays=row.metadata.replays??[];
|
|
81
|
+
const reset=[...new Set(replays.map(r=>r.table))].flatMap(name=>{
|
|
82
|
+
const target=row.metadata.targets.find(t=>t.name===name);
|
|
83
|
+
insist(target,'schema_mismatch:replay_table:'+name);
|
|
84
|
+
const sequenced=tokens(target.sql).some(t=>!t.quoted && folded(t.text)==='autoincrement');
|
|
85
|
+
return [statement(db,`DELETE FROM ${quote(name)}`),...(sequenced?[statement(db,'DELETE FROM sqlite_sequence WHERE name=?',name)]:[])];
|
|
86
|
+
});
|
|
87
|
+
return [...reset,...replays.flatMap((r,i)=>[db.prepare(r.sql),
|
|
88
|
+
statement(db,`UPDATE rer_portability_jobs SET metadata=json_set(metadata,'$.bridge.replayed[${i}].rows',changes()) WHERE id=?`,row.id)])];
|
|
89
|
+
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { all, quote, insist } from './storage.mjs';
|
|
2
2
|
import { canonicalJson } from './hash.mjs';
|
|
3
3
|
import { tokens, identifier, folded } from './generated.mjs';
|
|
4
|
-
import {
|
|
4
|
+
import { laterData } from './backfills.mjs';
|
|
5
5
|
import { destinationLedger } from './ledgers.mjs';
|
|
6
6
|
|
|
7
7
|
// Applied product migrations in the ledger's own order (ledgers.mjs). A
|
|
@@ -18,8 +18,9 @@ export async function appliedMigrations(db) {
|
|
|
18
18
|
// migrations it records or, for archives that predate that record, of those
|
|
19
19
|
// its own migration ledger records or, without one, of an earlier prefix.
|
|
20
20
|
// Restored rows never pass through a later migration, so one that writes or
|
|
21
|
-
// rewrites restorable data is refused, whether or not it changes the schema
|
|
22
|
-
|
|
21
|
+
// rewrites restorable data is refused, whether or not it changes the schema,
|
|
22
|
+
// unless its statement is replayed at finalize (backfills.mjs).
|
|
23
|
+
async function placement(db,manifest,cat,replay,ledger,history) {
|
|
23
24
|
const applied=ledger.map(m=>m.name);
|
|
24
25
|
insist(canonicalJson(replay.names)===canonicalJson(applied) && await replay.schemaAfter(applied)===cat.identity,'schema_mismatch');
|
|
25
26
|
const recorded=manifest.migrations ?? archivedApplied(history?.(),ledger,replay);
|
|
@@ -44,12 +45,14 @@ async function placement(manifest,cat,replay,ledger,history) {
|
|
|
44
45
|
// such tables (a newly added module's own) change no restored row, unless a
|
|
45
46
|
// trigger from any applied migration carries their writes into one that is
|
|
46
47
|
// restorable. Trigger DDL itself changes no row, nor does an UPDATE or DELETE
|
|
47
|
-
// of a table the archive holds no rows of (migration-statements.mjs).
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
const
|
|
51
|
-
|
|
52
|
-
|
|
48
|
+
// of a table the archive holds no rows of (migration-statements.mjs). A data
|
|
49
|
+
// statement that writes only tables outside the catalogue is admitted
|
|
50
|
+
// whatever it reads; a backfill of a table the archive lacks is replayed.
|
|
51
|
+
const catalogue=cat.tables.map(t=>t.name),archived=catalogue.filter(name=>manifest.tables.some(t=>t.name===name));
|
|
52
|
+
const empty=archived.filter(name=>!manifest.tables.find(t=>t.name===name).count);
|
|
53
|
+
const tables=(await all(db,"SELECT name FROM sqlite_master WHERE type IN ('table','view')")).map(t=>t.name);
|
|
54
|
+
const replays=laterData({applied,unapplied,replay,catalogue,archived,empty,tables,secrets:manifest.excludedColumns??{}});
|
|
55
|
+
return {earlier,replays};
|
|
53
56
|
}
|
|
54
57
|
// An archive without a migration record that carries its source's migration
|
|
55
58
|
// ledger (an old archive from before that ledger became bookkeeping) records
|
|
@@ -77,7 +80,8 @@ export async function sameSchemaMigrations(db,manifest,cat,migrations,history) {
|
|
|
77
80
|
if(!applied)return;
|
|
78
81
|
insist(!manifest.migrations || manifest.migrations.every(name=>applied.includes(name)),'schema_mismatch:migrations');
|
|
79
82
|
if(!migrations){insist(!manifest.migrations || applied.every(name=>manifest.migrations.includes(name)),'schema_mismatch:migrations');return;}
|
|
80
|
-
|
|
83
|
+
// Same schema: every catalogue table is archived, so nothing is replayed.
|
|
84
|
+
await placement(db,manifest,cat,await migrations(),ledger,history);
|
|
81
85
|
}
|
|
82
86
|
// An archive made before later migrations restores only when those migrations
|
|
83
87
|
// are additive: new tables, new columns that take a default, or indexes (which
|
|
@@ -86,7 +90,7 @@ export async function additiveBridge(db,manifest,cat,migrations,history) {
|
|
|
86
90
|
insist(migrations,'schema_mismatch');
|
|
87
91
|
const replay=await migrations(),ledger=await destinationLedger(db)??[],applied=ledger.map(m=>m.name);
|
|
88
92
|
insist(!manifest.migrations || manifest.migrations.every(name=>applied.includes(name)),'schema_mismatch:migrations');
|
|
89
|
-
const {earlier}=await placement(manifest,cat,replay,ledger,history);
|
|
93
|
+
const {earlier,replays}=await placement(db,manifest,cat,replay,ledger,history);
|
|
90
94
|
const columns={};
|
|
91
95
|
for(const table of manifest.tables) {
|
|
92
96
|
const target=cat.tables.find(t=>t.name===table.name),label=':'+table.name;
|
|
@@ -99,8 +103,10 @@ export async function additiveBridge(db,manifest,cat,migrations,history) {
|
|
|
99
103
|
insist(appended(table.sql,target.sql,added.map(c=>c.name)),'schema_mismatch:definition'+label);
|
|
100
104
|
columns[table.name]=added.map(c=>c.name);
|
|
101
105
|
}
|
|
106
|
+
// replays: the statements finalize runs (backfills.mjs); replayed: the job's record of them.
|
|
102
107
|
return {fromSchema:manifest.schema,toSchema:cat.identity,migrations:applied.filter(name=>!earlier.includes(name)),
|
|
103
|
-
tables:cat.tables.filter(t=>!manifest.tables.some(x=>x.name===t.name)).map(t=>t.name),columns
|
|
108
|
+
tables:cat.tables.filter(t=>!manifest.tables.some(x=>x.name===t.name)).map(t=>t.name),columns,
|
|
109
|
+
...(replays.length?{replays,replayed:replays.map(({migration,statement,table})=>({migration,statement,table,rows:null}))}:{})};
|
|
104
110
|
}
|
|
105
111
|
// The destination definition is the archived one with only the added column
|
|
106
112
|
// definitions inserted, as ALTER TABLE ADD COLUMN writes it: no other column,
|
|
@@ -8,6 +8,7 @@ import { appliedMigrations } from './bridge.mjs';
|
|
|
8
8
|
import { capturePayload, payloadRange } from './payloads.mjs';
|
|
9
9
|
import { dropProviderData } from './provider-data.mjs';
|
|
10
10
|
import { dropDevData } from './dev-data.mjs';
|
|
11
|
+
import { revokePolicyApprovals, refusePolicyApprovals } from './policy-data.mjs';
|
|
11
12
|
export async function createExport(context,actor='direct') {
|
|
12
13
|
const {db,config,buckets}=context,s=await state(db);insist(!s.applying,'import_applying',409);
|
|
13
14
|
const cat=await catalogue(db,config),id=crypto.randomUUID(),objects=[];
|
|
@@ -28,7 +29,8 @@ export async function createExport(context,actor='direct') {
|
|
|
28
29
|
...cat.tables.flatMap(t=>createCopy(db,t,copies[t.name],true)),statement(db,'INSERT INTO rer_portability_jobs(id,kind,actor,phase,metadata) VALUES(?,?,?,?,?)',id,'export',actor,row.phase,canonicalJson(row.metadata)),db.prepare('DELETE FROM rer_portability_guard')]);
|
|
29
30
|
try {
|
|
30
31
|
await dropDevData(db,cat.tables,copies);
|
|
31
|
-
|
|
32
|
+
await dropProviderData(db,cat.tables,copies);
|
|
33
|
+
await revokePolicyApprovals(db,copies);
|
|
32
34
|
for(const t of manifest.tables) t.count=(await statement(db,`SELECT COUNT(*) AS n FROM ${quote(copies[t.name])}`).first()).n;
|
|
33
35
|
for(const [name,producer] of Object.entries(config.extraPayloads).sort()) {
|
|
34
36
|
const descriptor=await capturePayload(db,id,manifest.payloads.length,await producer.produce());manifest.payloads.push({name,...descriptor});
|
|
@@ -52,6 +54,8 @@ async function nextItems(context,row) {
|
|
|
52
54
|
const table=manifest.tables[cursor.table];
|
|
53
55
|
if(!row.metadata.normalized?.includes(table.name)) {
|
|
54
56
|
await normalizeCopy(context,table,copies[table.name]);
|
|
57
|
+
// A product row transform cannot re-enable a delegation after scrubbing.
|
|
58
|
+
if(table.name==='oa_policy_approvals')await refusePolicyApprovals(db,copies);
|
|
55
59
|
row.metadata.normalized??=[];row.metadata.normalized.push(table.name);await save(db,row).run();
|
|
56
60
|
}
|
|
57
61
|
const records=await readCopy(db,table,copies[table.name],cursor.row,Math.min(1000,Math.ceil(partBytes/128)),true);
|
|
@@ -3,6 +3,7 @@ import { stagePart } from './import-staging.mjs';
|
|
|
3
3
|
import { normalizeStaged } from './import-safety.mjs';
|
|
4
4
|
import { ensureImportLeases, createImportLease, importLeaseStatus, importRunning, withImportLease } from './import-lease.mjs';
|
|
5
5
|
import { refuseProviderData } from './provider-data.mjs';
|
|
6
|
+
import { refusePolicyApprovals } from './policy-data.mjs';
|
|
6
7
|
import { refuseDevData } from './dev-data.mjs';
|
|
7
8
|
import { catalogue, validateRow, bindCell, writableColumns } from './catalogue.mjs';
|
|
8
9
|
import { platformTable, restorableManifest } from './bookkeeping.mjs';
|
|
@@ -13,6 +14,7 @@ import { writeObject } from './payloads.mjs';
|
|
|
13
14
|
import { createCopy,sidecar,readCopy,restoreCopy } from './copies.mjs';
|
|
14
15
|
import { excludedDependentGuards, migrationSeeded, seededCondition } from './restore-safety.mjs';
|
|
15
16
|
import { additiveBridge, sameSchemaMigrations, appliedMigrations } from './bridge.mjs';
|
|
17
|
+
import { replayStatements } from './backfills.mjs';
|
|
16
18
|
import { checkArchivedLedgers, archivedHistory } from './ledgers.mjs';
|
|
17
19
|
import { legacyEmptyGuards, legacyAuditStatements, requireLegacyEmpty } from './legacy-safety.mjs';
|
|
18
20
|
// ledgers: the rows of each migration ledger table the archive carries, as the
|
|
@@ -26,16 +28,16 @@ export async function createImport(context,manifest,actor='direct',legacyRecover
|
|
|
26
28
|
const skippedTables=original.tables.filter(table=>platformTable(table.name)).map(table=>table.name);
|
|
27
29
|
const cat=await catalogue(db,config),upgrade=config.acceptedSchemas[original.schema];
|
|
28
30
|
const same=manifest.schema===cat.identity,history=()=>archivedHistory(original,ledgers);
|
|
29
|
-
const bridge=same || upgrade?
|
|
31
|
+
const {replays,...bridge}=same || upgrade?{}:await additiveBridge(db,manifest,cat,context.migrations,history);
|
|
30
32
|
if(same && !upgrade)await sameSchemaMigrations(db,manifest,cat,context.migrations,history);
|
|
31
|
-
if(!bridge)insist(canonicalJson(manifest.tables.map(x=>x.name))===canonicalJson(cat.tables.map(x=>x.name)),'table_inventory');
|
|
32
|
-
if(!upgrade && !bridge)for(const table of manifest.tables) {
|
|
33
|
+
if(!bridge.fromSchema)insist(canonicalJson(manifest.tables.map(x=>x.name))===canonicalJson(cat.tables.map(x=>x.name)),'table_inventory');
|
|
34
|
+
if(!upgrade && !bridge.fromSchema)for(const table of manifest.tables) {
|
|
33
35
|
const target=cat.tables.find(t=>t.name===table.name);
|
|
34
36
|
insist(canonicalJson(table.columns)===canonicalJson(target.columns) && canonicalJson(table.dependencies)===canonicalJson(target.dependencies),'column_inventory');
|
|
35
37
|
}
|
|
36
38
|
// Bridged tables stage and restore only the archived columns; the destination
|
|
37
39
|
// supplies defaults for later columns and leaves later tables empty.
|
|
38
|
-
const targets=cat.tables.map(t=>bridge?.
|
|
40
|
+
const targets=cat.tables.map(t=>bridge.columns?.[t.name]?{...t,columns:manifest.tables.find(x=>x.name===t.name).columns,added:bridge.columns[t.name]}:t);
|
|
39
41
|
insist(canonicalJson(manifest.excludedColumns)===canonicalJson(config.secretColumns),'secret_inventory');
|
|
40
42
|
insist(canonicalJson(manifest.requiredConfiguration)===canonicalJson(config.requiredConfiguration),'configuration_inventory');
|
|
41
43
|
for(const o of manifest.objects) insist(buckets[o.bucket] && config.buckets[o.bucket] && o.key.startsWith(config.buckets[o.bucket].prefix??''),'bucket_inventory');
|
|
@@ -47,7 +49,7 @@ export async function createImport(context,manifest,actor='direct',legacyRecover
|
|
|
47
49
|
await abortImport(context,previous,{staleOnly:true});
|
|
48
50
|
}
|
|
49
51
|
const id=crypto.randomUUID(),copies=Object.fromEntries(targets.map((t,i)=>[t.name,'rer_portability_stage_'+id.replaceAll('-','')+'_'+i]));
|
|
50
|
-
const row={id,kind:'import',actor,phase:'staging',completed:0,metadata:{manifest:original,...(skippedTables.length?{restoreManifest:manifest,skippedTables}:{}),copies,targets,triggers:cat.triggers,...(bridge?{bridge}:{}),...(legacyRecovery?{legacyRecovery}: {}),...(ledgers?{ledgers}:{})}};
|
|
52
|
+
const row={id,kind:'import',actor,phase:'staging',completed:0,metadata:{manifest:original,...(skippedTables.length?{restoreManifest:manifest,skippedTables}:{}),copies,targets,triggers:cat.triggers,...(bridge.fromSchema?{bridge}:{}),...(replays?{replays}:{}),...(legacyRecovery?{legacyRecovery}: {}),...(ledgers?{ledgers}:{})}};
|
|
51
53
|
try {
|
|
52
54
|
await db.batch([guard(db,'(SELECT import_job FROM rer_portability_state WHERE id=1) IS NULL'),
|
|
53
55
|
statement(db,'UPDATE rer_portability_state SET import_job=? WHERE id=1',id),
|
|
@@ -177,7 +179,9 @@ async function finalizeLeasedImport(context,id,{contentHash,archiveHash,laterJou
|
|
|
177
179
|
await bucketsEmpty(context,row);
|
|
178
180
|
await refuseDevData(db,row.metadata.copies,row.metadata.targets);
|
|
179
181
|
await refuseProviderData(db,row.metadata.copies,row.metadata.targets);
|
|
182
|
+
await refusePolicyApprovals(db,row.metadata.copies);
|
|
180
183
|
await normalizeStaged(context,row);
|
|
184
|
+
await refusePolicyApprovals(db,row.metadata.copies);
|
|
181
185
|
if(config.validateStaged) {
|
|
182
186
|
const read=async table=>{insist(row.metadata.copies[table],'staged_table');const target=row.metadata.targets.find(t=>t.name===table);return (await readCopy(db,target,row.metadata.copies[table],0,row.metadata.manifest.tables.find(t=>t.name===table)?.count??0)).map(x=>x.row);};
|
|
183
187
|
insist(await config.validateStaged({db,tables:{...row.metadata.copies},read,manifest})!==false,'staged_validation');
|
|
@@ -201,8 +205,8 @@ async function finalizeLeasedImport(context,id,{contentHash,archiveHash,laterJou
|
|
|
201
205
|
...row.metadata.targets.map(t=>guard(db,`NOT EXISTS(SELECT 1 FROM ${quote(sidecar(row.metadata.copies[t.name]))} s JOIN rer_portability_journal j ON j.source=? AND j.tombstone=s.rer_portability_tombstone)`,manifest.sourceIdentity)),
|
|
202
206
|
...row.metadata.triggers.map(t=>db.prepare(`DROP TRIGGER ${quote(t.name)}`)),
|
|
203
207
|
...replaced.map(t=>db.prepare(`DELETE FROM ${quote(t)}`)),
|
|
204
|
-
...row.metadata.targets.flatMap(t=>restoreCopy(db,t,row.metadata.copies[t.name])),
|
|
205
|
-
...row.metadata.triggers.map(t=>db.prepare(t.sql)),...mergeJournal(db,manifest,checked),...legacyAuditStatements(db,row),
|
|
208
|
+
...row.metadata.targets.flatMap(t=>restoreCopy(db,t,row.metadata.copies[t.name])),save(db,row),...replayStatements(db,row),
|
|
209
|
+
...row.metadata.triggers.map(t=>db.prepare(t.sql)),...mergeJournal(db,manifest,checked),...legacyAuditStatements(db,row),
|
|
206
210
|
db.prepare('UPDATE rer_portability_state SET import_job=NULL,applying=0 WHERE id=1'),db.prepare('DELETE FROM rer_portability_guard')]);
|
|
207
211
|
}catch(e){
|
|
208
212
|
// D1 may have committed even when its acknowledgement failed.
|
|
@@ -36,7 +36,7 @@ export interface ArchivePolicy {
|
|
|
36
36
|
export interface SafetyChange {table:string;column:string;value:Cell;rows:number}
|
|
37
37
|
export interface Status {id:string;kind:'export'|'import';phase:'building'|'staging'|'applying'|'complete'|'aborted';completed:number;total:number|null;/** Import only: inactivity deadline and whether it has passed. */abortableAt?:string|null;stale?:boolean;/** Export only: bytes of the parts built so far. */bytes?:number;contentHash:string|null;requiredConfiguration?:string[];bridged?:Bridge;/** Platform bookkeeping ignored in an older archive. */skippedTables?:string[];/** Durable summary of bounded import normalization. */safetyChanges?:SafetyChange[]}
|
|
38
38
|
/** Later additive migrations an older archive was restored across. */
|
|
39
|
-
export interface Bridge {fromSchema:string;toSchema:string;migrations:string[];tables:string[];columns:Record<string,string[]
|
|
39
|
+
export interface Bridge {fromSchema:string;toSchema:string;migrations:string[];tables:string[];columns:Record<string,string[]>;/** Later backfills of tables the archive lacks, replayed at finalize in this order; rows is null until the apply commits. */replayed?:{migration:string;statement:number;table:string;rows:number|null}[]}
|
|
40
40
|
/** A runner's replay of the destination's applied migrations: the archive schema identity after a chosen set, and each migration's statements. */
|
|
41
41
|
export interface MigrationReplay {names:string[];statements(name:string):string[];/** The migration file's SHA-256 hex, where the runner read the file. */sha256?(name:string):string|undefined;schemaAfter(names:string[]):Promise<string>}
|
|
42
42
|
export interface PortabilityOptions {db:D1Database;buckets?:Record<string,R2Bucket>;policy:ArchivePolicy;migrations?:()=>Promise<MigrationReplay>}
|
|
@@ -30,20 +30,107 @@ export function changesData(sql,reached,empty=[]) {
|
|
|
30
30
|
return names(list,new Set(reached.map(folded).filter(name=>!skipped.has(name))));
|
|
31
31
|
}
|
|
32
32
|
// The restorable tables and every table whose writes reach one through a
|
|
33
|
-
// trigger any of `statements` creates
|
|
34
|
-
//
|
|
35
|
-
//
|
|
36
|
-
//
|
|
33
|
+
// trigger any of `statements` creates, or through a foreign key action (ON
|
|
34
|
+
// DELETE/UPDATE CASCADE or SET) declared by a reached table: a write to such a
|
|
35
|
+
// table during a later migration changes restorable rows the archive lacks. A
|
|
36
|
+
// trigger counts when its statement names a reached table anywhere; one this
|
|
37
|
+
// parser cannot bound adds every name it holds.
|
|
37
38
|
export function reachingTables(statements,restorable) {
|
|
38
|
-
const reached=new Set(restorable.map(folded));
|
|
39
|
-
const triggers=
|
|
40
|
-
|
|
39
|
+
const reached=new Set(restorable.map(folded)),parsed=statements.map(sql=>tokens(sql)).map(list=>({list,words:list.map(t=>t.quoted?undefined:folded(t.text))}));
|
|
40
|
+
const triggers=parsed.filter(({words})=>kindOf(words)==='create trigger').map(({list})=>({list,on:createdTrigger(list)?.on}));
|
|
41
|
+
const actions=parsed.filter(({words})=>['create table','alter table'].includes(kindOf(words)) && words.some((w,i)=>w==='cascade' || (w==='set' && ['delete','update'].includes(words[i-1])))).map(({list})=>referencing(list));
|
|
41
42
|
for(let grew=true;grew;) {
|
|
42
43
|
grew=false;
|
|
43
|
-
|
|
44
|
+
const add=name=>{if(!reached.has(name)){reached.add(name);grew=true;}};
|
|
45
|
+
for(const {list,on} of triggers)if(names(list,reached))(on===undefined?identifiers(list):[on]).forEach(add);
|
|
46
|
+
for(const {table,parents} of actions)if(table===undefined?parents.some(name=>reached.has(name)):reached.has(table))parents.forEach(add);
|
|
44
47
|
}
|
|
45
48
|
return [...reached];
|
|
46
49
|
}
|
|
50
|
+
// The table a CREATE TABLE or ALTER TABLE statement declares and the tables its
|
|
51
|
+
// REFERENCES clauses name (table undefined when it cannot be read).
|
|
52
|
+
function referencing(list) {
|
|
53
|
+
const parts=atoms(list),at=word(parts[2],'if') && word(parts[3],'not') && word(parts[4],'exists')?5:2;
|
|
54
|
+
return {table:tableName(parts.slice(at,at+1)),parents:parts.flatMap((a,i)=>word(a,'references') && parts[i+1]?[tableName([parts[i+1]])].filter(Boolean):[])};
|
|
55
|
+
}
|
|
56
|
+
// The last part of a [schema.]table name read from one to three atoms.
|
|
57
|
+
function tableName(name) {
|
|
58
|
+
const last=name.at(-1);
|
|
59
|
+
if(!last || name.length>3 || name.some(a=>!a.quoted && !/^[A-Za-z0-9_$.\u0080-\uffff]+$/.test(a.text)))return;
|
|
60
|
+
return folded(last.quoted?identifier(last):last.text.split('.').at(-1))||undefined;
|
|
61
|
+
}
|
|
62
|
+
// The one table a bounded INSERT, REPLACE, UPDATE or DELETE statement writes
|
|
63
|
+
// directly, after an optional WITH clause, with what a replay must know of it:
|
|
64
|
+
// {verb, target, conflict (an OR clause's resolution, 'replace' for REPLACE),
|
|
65
|
+
// columns (an explicit insert column list), upsert (ON CONFLICT ... DO UPDATE),
|
|
66
|
+
// star (a * or t.* result column), names (every identifier it holds)}. Undefined
|
|
67
|
+
// for any other statement or one this parser cannot bound. Triggers on the
|
|
68
|
+
// target and foreign key actions are the only further writes such a statement
|
|
69
|
+
// makes; reachingTables accounts for both.
|
|
70
|
+
export function dataStatement(sql) {
|
|
71
|
+
const parts=atoms(tokens(sql));
|
|
72
|
+
if(parts.some(semicolon))return;
|
|
73
|
+
let i=0;
|
|
74
|
+
if(word(parts[0],'with')) {
|
|
75
|
+
i=word(parts[1],'recursive')?2:1;
|
|
76
|
+
for(;;) {
|
|
77
|
+
i++;// the table name
|
|
78
|
+
if(parts[i]?.text==='(')i=closing(parts,i);
|
|
79
|
+
if(!word(parts[i],'as'))return;
|
|
80
|
+
i++;
|
|
81
|
+
if(word(parts[i],'not'))i++;
|
|
82
|
+
if(word(parts[i],'materialized'))i++;
|
|
83
|
+
if(parts[i]?.text!=='(')return;
|
|
84
|
+
i=closing(parts,i);
|
|
85
|
+
if(parts[i]?.text!==',')break;
|
|
86
|
+
i++;
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
const verb=['insert','replace','update','delete'].find(v=>word(parts[i],v));
|
|
90
|
+
if(!verb)return;
|
|
91
|
+
let conflict=verb==='replace'?'replace':undefined;
|
|
92
|
+
i++;
|
|
93
|
+
if(['insert','update'].includes(verb) && word(parts[i],'or')){conflict=parts[i+1] && !parts[i+1].quoted?folded(parts[i+1].text):undefined;if(!conflict)return;i+=2;}
|
|
94
|
+
if(verb!=='update'){if(!word(parts[i],verb==='delete'?'from':'into'))return;i++;}
|
|
95
|
+
const name=[];
|
|
96
|
+
const joined=a=>!name.length || (!a.quoted && a.text.startsWith('.')) || (!name.at(-1).quoted && name.at(-1).text.endsWith('.'));
|
|
97
|
+
for(;i<parts.length && !['(',',',')'].includes(parts[i].text) && joined(parts[i]);i++)name.push(parts[i]);
|
|
98
|
+
const target=tableName(name);
|
|
99
|
+
if(!target)return;
|
|
100
|
+
if(word(parts[i],'as'))i+=2;
|
|
101
|
+
const star=parts.some((a,k)=>!a.quoted && /(^|\.)\*$/.test(a.text) && !(parts[k-1]?.text==='(' && parts[k+1]?.text===')'));
|
|
102
|
+
const upsert=parts.some((a,k)=>word(a,'do') && word(parts[k+1],'update'));
|
|
103
|
+
return {verb,target,conflict,columns:verb==='insert' || verb==='replace'?parts[i]?.text==='(':undefined,upsert,star,names:identifiers(parts)};
|
|
104
|
+
}
|
|
105
|
+
// The index after the parenthesized group opening at `start`, or the end when unbalanced.
|
|
106
|
+
function closing(parts,start) {
|
|
107
|
+
let depth=0;
|
|
108
|
+
for(let i=start;i<parts.length;i++){if(parts[i].quoted)continue;if(parts[i].text==='(')depth++;else if(parts[i].text===')' && !--depth)return i+1;}
|
|
109
|
+
return parts.length;
|
|
110
|
+
}
|
|
111
|
+
// The names in the bodies of triggers on `table` that `statements` create, and
|
|
112
|
+
// every name in a trigger this parser cannot bound that names the table.
|
|
113
|
+
export function triggerBodies(statements,table) {
|
|
114
|
+
const result=[];
|
|
115
|
+
for(const list of statements.map(sql=>tokens(sql))) {
|
|
116
|
+
if(kindOf(list.map(t=>t.quoted?undefined:folded(t.text)))!=='create trigger')continue;
|
|
117
|
+
const created=createdTrigger(list);
|
|
118
|
+
if(!created){if(identifiers(list).includes(table))result.push(...identifiers(list));}
|
|
119
|
+
else if(created.on===table)result.push(...identifiers(atoms(list).slice(created.begin+1)));
|
|
120
|
+
}
|
|
121
|
+
return result;
|
|
122
|
+
}
|
|
123
|
+
// What a statement that changes only the schema does: 'create table' (not AS
|
|
124
|
+
// SELECT), 'create index', 'drop index', 'create view', 'drop view', 'add
|
|
125
|
+
// column' or 'trigger' (a bounded CREATE or DROP TRIGGER); undefined otherwise.
|
|
126
|
+
export function schemaChange(sql) {
|
|
127
|
+
const list=tokens(sql),words=list.map(t=>t.quoted?undefined:folded(t.text)),kind=kindOf(words);
|
|
128
|
+
if(kind==='create table')return topLevelWords(list,words).includes('as')?undefined:kind;
|
|
129
|
+
if(['create index','drop index','create view','drop view'].includes(kind))return kind;
|
|
130
|
+
if(kind==='alter table' && words.find((w,i)=>i>2 && ['add','rename','drop'].includes(w))==='add')return 'add column';
|
|
131
|
+
if(kind==='create trigger' && createdTrigger(list) || kind==='drop trigger' && droppedTrigger(list))return 'trigger';
|
|
132
|
+
}
|
|
133
|
+
export const statementNames=sql=>identifiers(tokens(sql));
|
|
47
134
|
const kindOf=words=>words.slice(0,4).filter(w=>!['temp','temporary','unique'].includes(w)).slice(0,2).join(' ');
|
|
48
135
|
const identifiers=list=>list.flatMap(t=>t.quoted?(t.text.startsWith("'")?[]:[folded(identifier(t))]):folded(t.text).split(/[^a-z0-9_$\u0080-\uffff]+/).filter(Boolean));
|
|
49
136
|
const names=(list,reached)=>identifiers(list).some(name=>reached.has(name));
|
|
@@ -79,7 +166,7 @@ function createdTrigger(list) {
|
|
|
79
166
|
let start=true,statements=0;
|
|
80
167
|
for(let i=begin+1;i<parts.length;i++) {
|
|
81
168
|
const a=parts[i];
|
|
82
|
-
if(start && word(a,'end'))return i===parts.length-1 && statements && !depth?{on}:undefined;
|
|
169
|
+
if(start && word(a,'end'))return i===parts.length-1 && statements && !depth?{on,begin}:undefined;
|
|
83
170
|
if(semicolon(a)){if(start || depth)return;start=true;statements++;continue;}
|
|
84
171
|
if(!a.quoted){if(a.text==='(')depth++;else if(a.text===')' && --depth<0)return;}
|
|
85
172
|
start=false;
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { all, quote, insist } from './storage.mjs';
|
|
2
|
+
|
|
3
|
+
// Keep approval provenance and run references, but never transfer a live delegation.
|
|
4
|
+
export async function revokePolicyApprovals(db,copies) {
|
|
5
|
+
if(copies.oa_policy_approvals)await db.prepare(`UPDATE ${quote(copies.oa_policy_approvals)} SET revoked_at=MAX(approved_at,CAST(strftime('%s','now') AS INTEGER)*1000) WHERE revoked_at IS NULL`).run();
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
export async function refusePolicyApprovals(db,copies) {
|
|
9
|
+
if(copies.oa_policy_approvals)insist(!(await all(db,`SELECT 1 FROM ${quote(copies.oa_policy_approvals)} WHERE revoked_at IS NULL LIMIT 1`)).length,'live_policy_approval',409);
|
|
10
|
+
}
|
package/onboarding/adapter.mjs
CHANGED
|
@@ -238,8 +238,11 @@ async function start(context, hooks) {
|
|
|
238
238
|
// its /health would pass. Claim and sign in only against the process this
|
|
239
239
|
// setup started.
|
|
240
240
|
let ownership;
|
|
241
|
-
try { ownership=await listenerOwnership(child.pid,expectedPort,context.runtimePort?'127.0.0.1':new URL(url).hostname,inspectionContext({port:expectedPort,verb:context.verb??'setup'})); }
|
|
242
|
-
catch(error){
|
|
241
|
+
try { ownership=await listenerOwnership(child.pid,expectedPort,context.runtimePort?'127.0.0.1':new URL(url).hostname,inspectionContext({port:expectedPort,verb:context.verb??'setup',signal})); }
|
|
242
|
+
catch(error){
|
|
243
|
+
if (signal?.aborted) throw error;
|
|
244
|
+
throw installationFailure('command_failed',{step:'start',diagnostic:error});
|
|
245
|
+
}
|
|
243
246
|
if (ownership.status !== 'owned') {
|
|
244
247
|
const message = (store.redact??(value=>value))(ownership.status==='other'
|
|
245
248
|
? `Another process answered on port ${expectedPort}; the application did not bind it. Listener PID ${ownership.pid}. Nothing was sent. Stop that process or choose another port, then run setup again.`
|
|
@@ -25,8 +25,10 @@ const run=async(command,args,context=inspectionContext(),{maxBuffer=1024*1024,no
|
|
|
25
25
|
const remaining=Math.floor(context.deadline-performance.now());
|
|
26
26
|
if(remaining<=0)break;
|
|
27
27
|
const timeout=attempt===0?Math.min(3000,remaining):remaining;
|
|
28
|
-
try{return (await exec(command,args,{timeout,maxBuffer})).stdout;}
|
|
28
|
+
try{return (await exec(command,args,{timeout,maxBuffer,signal:context.signal})).stdout;}
|
|
29
29
|
catch(error){
|
|
30
|
+
// A cancelled caller stops inspecting: the abort is never an ownership verdict.
|
|
31
|
+
if(context.signal?.aborted)throw context.signal.reason;
|
|
30
32
|
// A selected state file positively observed before a timeout still has a
|
|
31
33
|
// holder. Partial output can never establish the absence of a holder.
|
|
32
34
|
if(typeof error.stdout==='string'&&positiveResult?.(error.stdout))return error.stdout;
|
|
@@ -166,6 +168,9 @@ export async function listenerPorts(pid,context=inspectionContext()){
|
|
|
166
168
|
export async function listenerOwnership(pid,port,host,context=inspectionContext()){
|
|
167
169
|
context.port=port;
|
|
168
170
|
do {
|
|
171
|
+
// A cancelled launch has stopped its app, so its listener never appears:
|
|
172
|
+
// end now rather than polling ps and lsof until the inspection budget.
|
|
173
|
+
context.signal?.throwIfAborted();
|
|
169
174
|
const owned=await descendants(pid,context);
|
|
170
175
|
const ownRows=await listeners([...owned],context);
|
|
171
176
|
const own=ownRows.find(row=>owned.has(row.pid)&&row.sockets.some(socket=>serves(socket,port,host)===true));
|
|
@@ -180,7 +185,7 @@ export async function listenerOwnership(pid,port,host,context=inspectionContext(
|
|
|
180
185
|
if(row)return {status:owned.has(row.pid)?'owned':'other',pid:row.pid};
|
|
181
186
|
const remaining=context.deadline-performance.now();
|
|
182
187
|
if(remaining<=0)return {status:'none'};
|
|
183
|
-
await delay(Math.min(100,remaining));
|
|
188
|
+
await delay(Math.min(100,remaining),undefined,{signal:context.signal});
|
|
184
189
|
}while(performance.now()<context.deadline);
|
|
185
190
|
return {status:'none'};
|
|
186
191
|
}
|