@runeditrun/rer 0.2.91 → 0.2.93
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cli/README.md +1 -1
- package/cli/deploy-command.mjs +19 -8
- package/cli/deploy.mjs +1 -1
- package/cli/ledger-refusal.mjs +1 -1
- package/cli/portability-migrations.mjs +9 -5
- package/cli/portability-worker.mjs +4 -2
- package/foundation/runtime/contract-docs.mjs +26 -12
- package/global/CHANGELOG.md +27 -0
- package/modules/agent-door/conformance/crypto.mjs +14 -2
- package/modules/agent-door/conformance/journey.mjs +26 -2
- package/modules/agent-door/conformance/negative.mjs +4 -1
- package/modules/agent-door/conformance/policy-phases.mjs +43 -1
- package/modules/agent-door/conformance/protocol.mjs +42 -7
- package/modules/agent-door/conformance/sponsors.mjs +4 -0
- package/modules/agent-door/conformance/surface.mjs +10 -0
- package/modules/deploy-kit/owner-cli.mjs +9 -1
- package/modules/erasure/README.md +4 -2
- package/modules/operator-auth/identity/oidc/README.md +2 -1
- package/modules/portability/README.md +52 -2
- package/modules/portability/backfills.mjs +96 -3
- package/modules/portability/bridge.mjs +3 -1
- package/modules/portability/format.mjs +1 -0
- package/modules/portability/import.mjs +58 -3
- package/modules/portability/index.d.mts +2 -0
- package/modules/portability/index.mjs +16 -5
- package/modules/portability/library-history.mjs +152 -0
- package/modules/portability/migration-history.mjs +3 -2
- package/modules/portability/snapshot-objects.mjs +16 -0
- package/onboarding/adapter.mjs +3 -1
- package/package.json +1 -1
|
@@ -4,6 +4,7 @@ import { execFileSync } from 'node:child_process';
|
|
|
4
4
|
import { resolve, join } from 'node:path';
|
|
5
5
|
import { pathToFileURL } from 'node:url';
|
|
6
6
|
import { parseArgs } from 'node:util';
|
|
7
|
+
import { AsyncLocalStorage } from 'node:async_hooks';
|
|
7
8
|
import { assert, hexId } from './validation.mjs';
|
|
8
9
|
import { regularBytes } from './artifact.mjs';
|
|
9
10
|
import { packageBuild } from './prepare.mjs';
|
|
@@ -21,6 +22,11 @@ import { publicOrigin } from './public-origin.mjs';
|
|
|
21
22
|
export const cronNotice = 'This Worker has a Cron Trigger. Cloudflare may take up to about two hours to start its first scheduled run on some accounts.';
|
|
22
23
|
const usage = 'usage: owner-cli.mjs deploy|dry-run|local-plan|health|remove|inspect|record-owner|signin|rekey --declaration <file> --environment <name> [--product <dir>] [--state <file>] [--outcome claimed|kept] | secret ... --set NAME [--set NAME]|--unset NAME [--unset NAME]';
|
|
23
24
|
|
|
25
|
+
// The public offline command also contains legacy hooks that call kit dry-run.
|
|
26
|
+
// Keep their plan local without changing direct kit callers or real deploys.
|
|
27
|
+
const localPlans = new AsyncLocalStorage();
|
|
28
|
+
export const withLocalDeployPlan = run => localPlans.run(true, run);
|
|
29
|
+
|
|
24
30
|
/** The owner's committed source and existing local build. Run the seed's build first. */
|
|
25
31
|
async function ownerArtifact(root, product) {
|
|
26
32
|
const git = (...args) => { try { return execFileSync('git',args,{cwd:root,encoding:'utf8',stdio:['ignore','pipe','pipe']}).trim(); } catch { throw new Error('owner-source-not-a-git-checkout'); } };
|
|
@@ -34,8 +40,10 @@ export async function main(argv = process.argv.slice(2), env = process.env, {fet
|
|
|
34
40
|
let parsed;
|
|
35
41
|
try { parsed = parseArgs({args:argv,allowPositionals:true,strict:true,options:{declaration:{type:'string'},environment:{type:'string'},product:{type:'string',default:''},state:{type:'string'},set:{type:'string',multiple:true},unset:{type:'string',multiple:true},outcome:{type:'string'}}}); }
|
|
36
42
|
catch { throw new Error('usage'); }
|
|
37
|
-
const {positionals:[
|
|
43
|
+
const {positionals:[requestedCommand,...rest],values} = parsed;
|
|
44
|
+
const command = localPlans.getStore() && requestedCommand === 'dry-run' ? 'local-plan' : requestedCommand;
|
|
38
45
|
if (!['deploy','dry-run','local-plan','health','secret','remove','inspect','record-owner','signin','rekey'].includes(command) || rest.length || !values.declaration || !values.environment) throw new Error('usage');
|
|
46
|
+
if (localPlans.getStore() && command !== 'local-plan') throw new Error('offline-deploy-refuses-provider-operation');
|
|
39
47
|
if ((command === 'secret') !== (values.set !== undefined || values.unset !== undefined) || values.set !== undefined && values.unset !== undefined) throw new Error('usage');
|
|
40
48
|
if (values.outcome !== undefined && command !== 'record-owner' || command === 'record-owner' && !['claimed','kept'].includes(values.outcome)) throw new Error('usage');
|
|
41
49
|
await refuseLocalModels(resolve(cwd));
|
|
@@ -25,12 +25,13 @@ retained exceptions and effects outside D1 and R2. Clauses are in
|
|
|
25
25
|
|
|
26
26
|
```js
|
|
27
27
|
import { defineErasure, createErasure } from './vendor/base/modules/erasure/index.mjs';
|
|
28
|
+
const fileColumn = 'storage_key';
|
|
28
29
|
export const person = defineErasure({
|
|
29
30
|
entity: 'person',
|
|
30
31
|
root: { table: 'people' },
|
|
31
32
|
edges: [
|
|
32
33
|
{ table: 'person_notes', from: 'people', on: { person_id: 'id' } },
|
|
33
|
-
{ table: 'person_files', from: 'people', on: { person_id: 'id' }, blob: { bucket: 'FILES', keyColumn:
|
|
34
|
+
{ table: 'person_files', from: 'people', on: { person_id: 'id' }, blob: { bucket: 'FILES', keyColumn: fileColumn } },
|
|
34
35
|
{ table: 'signals', from: 'people', where: 'instr(payload_json,?)>0', params: ['email'] },
|
|
35
36
|
],
|
|
36
37
|
redact: [{ table: 'messages', from: 'people', on: { author_id: 'id' }, set: { body: '' }, reason: 'transcript order is kept' }],
|
|
@@ -122,8 +123,9 @@ the declaration erases both together.
|
|
|
122
123
|
```js
|
|
123
124
|
import { rateEdges, rateWrite } from './vendor/base/modules/erasure/index.mjs';
|
|
124
125
|
const root = { table: 'people', column: 'id' };
|
|
126
|
+
const counterColumn = 'key';
|
|
125
127
|
// in the declaration
|
|
126
|
-
edges: [...rateEdges({ entity: 'person', root, counters: [{ table: 'rate_limits', keyColumn:
|
|
128
|
+
edges: [...rateEdges({ entity: 'person', root, counters: [{ table: 'rate_limits', keyColumn: counterColumn }] })],
|
|
127
129
|
// at every write whose key identifies a known person
|
|
128
130
|
const [counted] = await rateWrite(db, { entity: 'person', root, subject: person.id, rateTable: 'rate_limits', rateKey: key },
|
|
129
131
|
[db.prepare('INSERT INTO rate_limits(key,window,count) VALUES(?,?,1) ON CONFLICT(key) DO UPDATE SET count=count+1 RETURNING count').bind(key, window)]);
|
|
@@ -10,6 +10,7 @@ again as its next migration.
|
|
|
10
10
|
|
|
11
11
|
```js
|
|
12
12
|
import { createOperatorAuth, createAuthHandler, createSignInHandler } from './modules/operator-auth/index.mjs';
|
|
13
|
+
const clientAuthMethod = 'client_secret_basic';
|
|
13
14
|
const auth = createOperatorAuth({
|
|
14
15
|
db: env.DB, clientIp, mailSender,
|
|
15
16
|
// The runtime's fetch; on workerd use redirect 'manual' and refuse every 3xx.
|
|
@@ -22,7 +23,7 @@ const auth = createOperatorAuth({
|
|
|
22
23
|
providers: [
|
|
23
24
|
// The app's private key (a Worker secret) authenticates it: no shared secret exists.
|
|
24
25
|
{ id: 'auth', preset: 'auth', issuer: env.IDENTITY_PROVIDER, clientId: env.IDENTITY_CLIENT_ID, privateKey: JSON.parse(env.IDENTITY_PRIVATE_KEY) },
|
|
25
|
-
{ id: 'google', preset: 'google', clientId: env.GOOGLE_CLIENT_ID, tokenAuthMethod:
|
|
26
|
+
{ id: 'google', preset: 'google', clientId: env.GOOGLE_CLIENT_ID, tokenAuthMethod: clientAuthMethod, clientSecret: env.GOOGLE_CLIENT_SECRET },
|
|
26
27
|
{ id: 'corp', preset: 'okta', issuer: 'https://corp.okta.com', clientId: env.OKTA_CLIENT_ID, privateKey: JSON.parse(env.OKTA_PRIVATE_KEY),
|
|
27
28
|
subjectEmails: { '00u1abcd': 'operator@corp.example' } },
|
|
28
29
|
],
|
|
@@ -249,7 +249,15 @@ provider operation.
|
|
|
249
249
|
| Unknown hash/name, ambiguous hash/alias or conflicting name/hash | `migration_ledger:unknown`, `:ambiguous` or `:conflict`; use the matching checkout and declaration. |
|
|
250
250
|
|
|
251
251
|
Refusals throw `PortabilityError` with status 409 and a constant `code`;
|
|
252
|
-
ledger values never enter these diagnostics.
|
|
252
|
+
ledger values never enter these diagnostics. `migration` names a pending
|
|
253
|
+
checkout file, and `identity` may name a checkout
|
|
254
|
+
file, a declared alias or the known `rer_migrations` table. The worker appends
|
|
255
|
+
these safe identities to its public refusal token, retaining
|
|
256
|
+
`migration_ledger:unapplied:<file>`, `migration_ledger:multiple:rer_migrations`
|
|
257
|
+
and named `unknown`, `ambiguous` or `conflict` tokens only when the name exactly
|
|
258
|
+
equals a checkout filename or declared alias. Historical
|
|
259
|
+
`migration_ledger:unknown:initial-page.sql` stays category-only without that
|
|
260
|
+
provenance. Propagate failures and stop before
|
|
253
261
|
native apply. IO and provider failures also propagate. A successful check
|
|
254
262
|
creates no ledger, changes no row and does not attest unrecorded historical
|
|
255
263
|
bytes in a plain Wrangler ledger. It never silently skips pending migrations.
|
|
@@ -367,7 +375,7 @@ extra-payload recovery responsibilities and transfer erasure behavior still appl
|
|
|
367
375
|
The UTF-8 JSON-lines archive has a manifest, indexed checksummed parts and a
|
|
368
376
|
closing record. `readArchive(stream)` uses the shared bounded, fatal UTF-8 line
|
|
369
377
|
reader. The default maximum line size is 24 MiB; callers can set a smaller bound.
|
|
370
|
-
|
|
378
|
+
Current-archive paths do not materialize the whole archive; the legacy ledger collector described below buffers its initial upload. The manifest opens the archive and holds the final part count and payload hashes, so an export builds all its parts in the job before any byte is written; `exportArchive` and `importArchive` take an optional `progress` callback (`progress.mjs` turns its events into throttled lines with a time-left estimate). Readers enforce a 16 MiB
|
|
371
379
|
canonical UTF-8 part limit; `readArchive(stream,{partBytes})` can lower it.
|
|
372
380
|
Import uploads enforce the policy's part limit before any staging write.
|
|
373
381
|
A part has at most 1,000 rows;
|
|
@@ -400,6 +408,14 @@ Comments and harmless inter-token whitespace therefore compare equally, while
|
|
|
400
408
|
`x '31'` and `x'31'`, separated minus signs, numbers and combined operators stay
|
|
401
409
|
distinct. No regular expression rewrites SQL.
|
|
402
410
|
|
|
411
|
+
Restore may omit an archived schema object from its compatibility comparison
|
|
412
|
+
only when its table is in the archive's excluded inventory, the destination
|
|
413
|
+
policy also excludes it, neither side restores it, and the destination lacks
|
|
414
|
+
it. This permits transient export snapshots without installing source SQL.
|
|
415
|
+
Objects on restorable tables and excluded tables present at the destination
|
|
416
|
+
remain verified against the unchanged live and migration-replay catalogues;
|
|
417
|
+
the original manifest and parts still supply the archive digest.
|
|
418
|
+
|
|
403
419
|
Quoted identifiers (double quotes, backticks, brackets, or a single-quoted string
|
|
404
420
|
used as a name), strings, blobs, numbers, variables and ordinary identifiers keep
|
|
405
421
|
their exact spelling. Only unambiguous reserved keywords fold ASCII case.
|
|
@@ -548,6 +564,37 @@ archive's tables restore with their archived columns, later columns take their
|
|
|
548
564
|
defaults, and later tables stay empty unless their eligible seed or backfill is
|
|
549
565
|
replayed below. This applies to normal v2 import and to backup-only v1 recovery.
|
|
550
566
|
|
|
567
|
+
The Node in-process library door also verifies old v2 archives carrying a
|
|
568
|
+
hash-bearing migration ledger without caller-supplied replay or ledger rows.
|
|
569
|
+
`createPortability({db,policy})` reads the checkout's single D1 migration directory
|
|
570
|
+
from `wrangler.jsonc` (or use `migrationRoot` to name the checkout), matches its
|
|
571
|
+
files to the destination ledger, and collects the original parts in memory until
|
|
572
|
+
finalize authenticates their hashes and closing binding. The archived ledger must
|
|
573
|
+
be a hash-matched prefix; recorded replay must reproduce the archived tables and
|
|
574
|
+
full replay the complete destination identity before any durable import job is
|
|
575
|
+
created. Keep the same service instance through this initial upload; restarting
|
|
576
|
+
it requires uploading again. Acknowledged recovery and runner-supplied evidence
|
|
577
|
+
retain their existing paths.
|
|
578
|
+
|
|
579
|
+
Worker evidence remains pending: a Worker has neither checkout files nor Node
|
|
580
|
+
SQLite, and `db` plus `policy` cannot reveal migration SQL that is absent from the
|
|
581
|
+
live schema. The required refresh/lock integration should generate a seed-owned
|
|
582
|
+
module containing ordered file hashes, SQL statements, complete sqlite_master
|
|
583
|
+
inventories and seeded row images for every migration prefix. Bind that artifact
|
|
584
|
+
to the destination's hashed ledger, apply the runtime policy to each inventory,
|
|
585
|
+
and verify both historical and complete live identities before admitting the
|
|
586
|
+
same collector. Do not import archive SQL or substitute the live schema for a
|
|
587
|
+
historical replay. Until that artifact is wired into seed builds, this automatic
|
|
588
|
+
library path refuses in Workers with `schema_mismatch:legacy_evidence`.
|
|
589
|
+
|
|
590
|
+
Later data statements also qualify when a surviving trigger from that same
|
|
591
|
+
migration repeats the complete statement block, in order, on Base's restore
|
|
592
|
+
commit transition. Those statements run through that destination-owned trigger,
|
|
593
|
+
not a second finalize replay.
|
|
594
|
+
Absent tables count as empty only with an empty full replay and proof that every
|
|
595
|
+
later insert and possible trigger writer preserves emptiness; a bounded empty
|
|
596
|
+
copy/drop/rename rebuild and a SELECT from such an empty source carry no row.
|
|
597
|
+
|
|
551
598
|
The Base CLI runners (local Workers and container) pass `createPortability` a
|
|
552
599
|
`migrations` replayer (`migration-history.mjs`, Node only) built from the
|
|
553
600
|
destination's applied migration ledger and migration files, with each file's
|
|
@@ -1018,6 +1065,7 @@ modules/portability/schema-identity.mjs
|
|
|
1018
1065
|
modules/portability/schema-legacy.mjs
|
|
1019
1066
|
modules/portability/sqlite-tokens.mjs
|
|
1020
1067
|
modules/portability/bookkeeping.mjs
|
|
1068
|
+
modules/portability/snapshot-objects.mjs
|
|
1021
1069
|
modules/portability/storage.mjs
|
|
1022
1070
|
modules/portability/copies.mjs
|
|
1023
1071
|
modules/portability/derived.mjs
|
|
@@ -1027,6 +1075,8 @@ modules/portability/policy-data.mjs
|
|
|
1027
1075
|
modules/portability/generated.mjs
|
|
1028
1076
|
modules/portability/restore-safety.mjs
|
|
1029
1077
|
modules/portability/bridge.mjs
|
|
1078
|
+
modules/portability/library-history.mjs
|
|
1079
|
+
modules/portability/migration-history.mjs
|
|
1030
1080
|
modules/portability/ledgers.mjs
|
|
1031
1081
|
modules/portability/migration-statements.mjs
|
|
1032
1082
|
modules/portability/sql-text.mjs
|
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
import { statement, quote, insist } from './storage.mjs';
|
|
2
|
-
import { tokens, folded } from './generated.mjs';
|
|
2
|
+
import { tokens, folded, identifier } from './generated.mjs';
|
|
3
3
|
import { changesData, dataStatement, reachingTables, triggerBodies, schemaChange, statementNames } from './migration-statements.mjs';
|
|
4
4
|
import { splitStatements } from './sql-text.mjs';
|
|
5
5
|
import { preservingRebuilds } from './rebuilds.mjs';
|
|
6
6
|
import { replayViews, viewReads } from './replay-views.mjs';
|
|
7
|
+
import { sqliteTokens } from './sqlite-tokens.mjs';
|
|
8
|
+
import { schemaSQL } from './schema-identity.mjs';
|
|
7
9
|
import { transientTables } from './transient-tables.mjs';
|
|
8
10
|
|
|
9
11
|
// The data statements of later migrations, judged by the tables they write
|
|
@@ -31,13 +33,24 @@ import { transientTables } from './transient-tables.mjs';
|
|
|
31
33
|
// statements after it is judged as the statements it holds.
|
|
32
34
|
// Returns the replays in migration order and the proven CHECK rebuilds, or refuses with the first migration
|
|
33
35
|
// (in `unapplied` order) holding a statement that cannot be admitted.
|
|
34
|
-
export function laterData({applied,unapplied,replay,catalogue,archived,empty,tables,secrets,manifest}) {
|
|
36
|
+
export function laterData({applied,unapplied,replay,catalogue,archived,empty,tables,secrets,manifest,replayEmpty=[]}) {
|
|
35
37
|
const split=name=>replay.statements(name).flatMap(splitStatements),fold=list=>list.map(folded);
|
|
36
38
|
const statements=applied.flatMap(split);
|
|
37
39
|
const restorable=fold(catalogue),carried=fold(archived);
|
|
38
40
|
const reached=reachingTables(statements,carried),writable=reachingTables(statements,restorable);
|
|
39
41
|
const later=unapplied.flatMap(migration=>split(migration).map((sql,index)=>({migration,index,sql,data:dataStatement(sql)})));
|
|
40
42
|
const {admitted,tables:rebuilt}=preservingRebuilds(later,manifest,tables,statements),views=replayViews(statements);
|
|
43
|
+
const restored=restoreCommitStatements(applied,split,later);
|
|
44
|
+
restored.forEach(s=>admitted.add(s));
|
|
45
|
+
// Renaming an uncarried table is metadata-only. Later row statements still
|
|
46
|
+
// face their own checks; carried table definitions still face the bridge.
|
|
47
|
+
for(const s of later) {
|
|
48
|
+
const list=sqliteTokens(s.sql),words=list.map(t=>folded(t.value));
|
|
49
|
+
if(words.length===6 && words[0]==='alter' && words[1]==='table' && words[3]==='rename' && words[4]==='to' && !carried.includes(words[2]) && !carried.includes(words[5]))admitted.add(s);
|
|
50
|
+
}
|
|
51
|
+
const absentEmpty=emptyAbsentTables(later,statements,carried,replayEmpty,admitted);
|
|
52
|
+
// An empty absent source contributes no row to a direct INSERT SELECT.
|
|
53
|
+
for(const s of later)if(s.data && emptySelect(s.sql,absentEmpty))admitted.add(s);
|
|
41
54
|
const {missing,plans:transients}=transientTables(statements,later,fold(tables),carried);
|
|
42
55
|
const current=fold(tables),known=[...current,...missing],outside=later.filter(s=>s.data && !writable.includes(s.data.target) && !transients.has(s.data.target)),tainted=new Set();
|
|
43
56
|
// Tables an admitted write fills from archived rows (directly, through its
|
|
@@ -61,7 +74,7 @@ export function laterData({applied,unapplied,replay,catalogue,archived,empty,tab
|
|
|
61
74
|
// Dependencies are checked below before any of the plan is selected.
|
|
62
75
|
const available=[...transients.keys(),...(!carried.includes(table)?[...writes.keys()].filter(name=>!carried.includes(name)):[])];
|
|
63
76
|
const allowUpdate=!transient && !carried.includes(table) && list.some(s=>['insert','replace'].includes(s.data.verb));
|
|
64
|
-
const options={table,allowUpdate,statements,restorable,writable,carried:[...carried,...available],tables:known,secrets,views,manifest};
|
|
77
|
+
const options={table,allowUpdate,statements,restorable,writable,carried:[...carried,...absentEmpty,...available],tables:known,secrets,views,manifest};
|
|
65
78
|
// Carried target rules must use the actual archive, not reconstructed reads.
|
|
66
79
|
const bad=carried.includes(table) && list.length>1?list:list.filter(s=>!replayable(s,{...options,targetCarried:carried.includes(table)}));
|
|
67
80
|
// Writes confined to tables created after the archive retain their
|
|
@@ -141,3 +154,83 @@ export function replayStatements(db,row) {
|
|
|
141
154
|
statement(db,`UPDATE rer_portability_jobs SET metadata=json_set(metadata,'$.bridge.replayed[${i}].rows',${r.schema?'0':'changes()'}) WHERE id=?`,row.id)])];
|
|
142
155
|
}
|
|
143
156
|
const autoincrement=sql=>tokens(sql).some(t=>!t.quoted && folded(t.text)==='autoincrement');
|
|
157
|
+
|
|
158
|
+
// Only a surviving destination-owned trigger on the actual restore transition
|
|
159
|
+
// can cover DML. The complete body must recur contiguously, in the same file
|
|
160
|
+
// and order, preserving literals and identifiers. Nothing is replayed twice.
|
|
161
|
+
function restoreCommitStatements(applied,split,later) {
|
|
162
|
+
const live=new Map();
|
|
163
|
+
for(const migration of applied)for(const sql of split(migration)) {
|
|
164
|
+
const list=tokens(sql),words=list.map(t=>folded(t.text));
|
|
165
|
+
if(!['create','drop'].includes(words[0]) || words[1]!=='trigger')continue;
|
|
166
|
+
const at=words[2]==='if'?(words[0]==='create'?5:4):2,name=folded(identifier(list[at]));
|
|
167
|
+
if(words[0]==='drop'){live.delete(name);continue;}
|
|
168
|
+
if(words[2]==='if' && live.has(name))continue;
|
|
169
|
+
live.set(name,{migration,sql,list,at});
|
|
170
|
+
}
|
|
171
|
+
const admitted=new Set();
|
|
172
|
+
const header='AFTER UPDATE OF import_job ON rer_portability_state WHEN OLD.import_job IS NOT NULL AND NEW.import_job IS NULL AND OLD.applying=1';
|
|
173
|
+
for(const {migration,sql,list,at} of live.values()) {
|
|
174
|
+
const begin=list.findIndex(t=>!t.quoted && folded(t.text)==='begin');
|
|
175
|
+
if(begin<0 || schemaChange(sql)!=='trigger' || schemaSQL(sql.slice(list[at].end,list[begin].start))!==schemaSQL(header))continue;
|
|
176
|
+
const end=list.at(-1);if(!/^end;?$/i.test(end.text))continue;
|
|
177
|
+
const body=splitStatements(sql.slice(list[begin].end,end.start));
|
|
178
|
+
if(!body.length || body.some(sql=>!dataStatement(sql)))continue;
|
|
179
|
+
const file=later.filter(s=>s.migration===migration),images=body.map(schemaSQL);
|
|
180
|
+
for(let i=0;i+body.length<=file.length;i++)if(images.every((image,k)=>file[i+k].data && schemaSQL(file[i+k].sql)===image))file.slice(i,i+body.length).forEach(s=>admitted.add(s));
|
|
181
|
+
}
|
|
182
|
+
return admitted;
|
|
183
|
+
}
|
|
184
|
+
// Prove an absent table stays empty rather than assuming every newly created
|
|
185
|
+
// table is empty. Full replay supplies the empty baseline; every later INSERT
|
|
186
|
+
// and every possible trigger writer must also preserve it. A local empty
|
|
187
|
+
// create/copy/drop/rename rebuild carries no row, including through its DROP.
|
|
188
|
+
function emptyAbsentTables(later,statements,carried,replayEmpty,admitted) {
|
|
189
|
+
const empty=new Set(replayEmpty.map(folded).filter(name=>!carried.includes(name)));
|
|
190
|
+
const rebuilds=[];
|
|
191
|
+
for(let i=0;i+3<later.length;i++) {
|
|
192
|
+
const block=later.slice(i,i+4),a=tokens(block[0].sql),b=tokens(block[1].sql),c=tokens(block[2].sql),d=tokens(block[3].sql);
|
|
193
|
+
const text=list=>list.map(t=>folded(identifier(t)));
|
|
194
|
+
const [aa,bb,cc,dd]=[a,b,c,d].map(text);
|
|
195
|
+
if(block.some(s=>s.migration!==block[0].migration) || aa[0]!=='create' || aa[1]!=='table' || a[3]?.text!=='(' || bb.length!==7 || bb[0]!=='insert' || bb[1]!=='into' || bb[2]!==aa[2] || bb[3]!=='select' || bb[4]!=='*' || bb[5]!=='from' || cc.join(' ')!==`drop table ${bb[6]}` || dd.join(' ')!==`alter table ${aa[2]} rename to ${bb[6]}` || !empty.has(bb[6]) || carried.includes(aa[2]))continue;
|
|
196
|
+
if(later.some(s=>!block.includes(s) && statementNames(s.sql).includes(aa[2])))continue;
|
|
197
|
+
rebuilds.push({block,source:bb[6]});
|
|
198
|
+
}
|
|
199
|
+
const copyStatements=new Set(rebuilds.flatMap(r=>r.block)),triggerTargets=new Set();
|
|
200
|
+
for(const sql of statements) {
|
|
201
|
+
const list=tokens(sql);if(folded(list[0]?.text??'')!=='create' || folded(list[1]?.text??'')!=='trigger')continue;
|
|
202
|
+
const on=list.findIndex(t=>!t.quoted && folded(t.text)==='on');
|
|
203
|
+
const begin=list.findIndex(t=>!t.quoted && folded(t.text)==='begin');
|
|
204
|
+
if(on<0 || begin<0 || schemaChange(sql)!=='trigger'){statementNames(sql).forEach(name=>triggerTargets.add(name));continue;}
|
|
205
|
+
for(const body of splitStatements(sql.slice(list[begin].end,list.at(-1).start))) {
|
|
206
|
+
const data=dataStatement(body);
|
|
207
|
+
if(data)triggerTargets.add(data.target);
|
|
208
|
+
else if(!/^\s*SELECT\b/i.test(body))statementNames(body).forEach(name=>triggerTargets.add(name));
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
for(let changed=true;changed;) {
|
|
212
|
+
changed=false;
|
|
213
|
+
for(const name of empty) {
|
|
214
|
+
const writes=later.filter(s=>s.data && s.data.target===name && !copyStatements.has(s) && ['insert','replace'].includes(s.data.verb));
|
|
215
|
+
const triggered=triggerTargets.has(name);
|
|
216
|
+
if(triggered || writes.some(s=>!emptySelect(s.sql,empty))) {empty.delete(name);changed=true;}
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
for(const {block,source} of rebuilds)if(empty.has(source))block.forEach(s=>admitted.add(s));
|
|
220
|
+
return [...empty];
|
|
221
|
+
}
|
|
222
|
+
function emptySelect(sql,empty) {
|
|
223
|
+
const list=sqliteTokens(sql),word=t=>['keyword','identifier'].includes(t?.type)?folded(t.value):undefined;
|
|
224
|
+
let depth=0,select=-1,from=-1;
|
|
225
|
+
for(let i=0;i<list.length;i++) {
|
|
226
|
+
const t=list[i],w=word(t);
|
|
227
|
+
if(!depth && ['union','except','intersect','right','full','returning'].includes(w))return false;
|
|
228
|
+
if(!depth && w==='select' && select<0)select=i;
|
|
229
|
+
if(!depth && w==='from' && select>=0 && from<0)from=i;
|
|
230
|
+
// An aggregate over an empty input can still produce one row. Permit
|
|
231
|
+
// only known scalar expressions and EXISTS before the outer FROM.
|
|
232
|
+
if(select>=0 && from<0 && list[i+1]?.text==='(' && w && !['coalesce','exists','in'].includes(w))return false;
|
|
233
|
+
if(t.text==='(')depth++;if(t.text===')')depth--;
|
|
234
|
+
}
|
|
235
|
+
return from>select && [...empty].includes(folded(list[from+1]?.value??''));
|
|
236
|
+
}
|
|
@@ -66,7 +66,9 @@ async function placement(db,manifest,cat,replay,ledger,history,legacyRecovery,un
|
|
|
66
66
|
const catalogue=cat.tables.map(t=>t.name),archivedTables=catalogue.filter(name=>manifest.tables.some(t=>t.name===name));
|
|
67
67
|
const empty=archivedTables.filter(name=>!manifest.tables.find(t=>t.name===name).count);
|
|
68
68
|
const tables=(await all(db,"SELECT name FROM sqlite_master WHERE type IN ('table','view')")).map(t=>t.name);
|
|
69
|
-
const
|
|
69
|
+
const baseline=await replay.rowImages?.(cat.tables.filter(t=>!archivedTables.includes(t.name)).map(t=>({name:t.name,columns:t.columns.map(c=>c.name)})),0);
|
|
70
|
+
const replayEmpty=baseline?[...baseline].filter(([,rows])=>!rows.length).map(([name])=>name):[];
|
|
71
|
+
const {replays,rebuilt}=laterData({applied,unapplied,replay,catalogue,archived:archivedTables,empty,tables,secrets:manifest.excludedColumns??{},manifest:manifest.tables,replayEmpty});
|
|
70
72
|
return {applied,earlier,replays,rebuilt};
|
|
71
73
|
}
|
|
72
74
|
// Old v2 writers recorded tables and applied migration names, but no behavioral
|
|
@@ -36,6 +36,7 @@ async function checkManifestFields(m) {
|
|
|
36
36
|
insist(typeof m.product==='string' && typeof m.sourceIdentity==='string' && typeof m.exportedAt==='string','manifest_identity');
|
|
37
37
|
insist(Number.isSafeInteger(m.partCount) && m.partCount>=0 && Number.isSafeInteger(m.epoch) && m.epoch>=0,'manifest_counts');
|
|
38
38
|
insist(Array.isArray(m.tables) && Array.isArray(m.objects) && Array.isArray(m.payloads) && Array.isArray(m.journal) && Array.isArray(m.requiredConfiguration),'manifest_inventory');
|
|
39
|
+
insist(m.schemaObjects===undefined || m.excludedTables===undefined || (Array.isArray(m.excludedTables) && m.excludedTables.every(name=>typeof name==='string') && new Set(m.excludedTables).size===m.excludedTables.length),'manifest_inventory');
|
|
39
40
|
// Applied product migrations, recorded by archives since additive restore.
|
|
40
41
|
insist(m.migrations===undefined || (Array.isArray(m.migrations) && m.migrations.every(x=>typeof x==='string')),'manifest_inventory');
|
|
41
42
|
// The ledger an export from a RER Platform installation records, by migration identity.
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { libraryMigrations } from './library-history.mjs';
|
|
1
2
|
import { all, statement, state, guard, job, save, partQueries, parts, readPart, dropCopyStatements, cleanCopyStatements, quote, stream, insist, run } from './storage.mjs';
|
|
2
3
|
import { stagePart } from './import-staging.mjs';
|
|
3
4
|
import { normalizeStaged } from './import-safety.mjs';
|
|
@@ -7,6 +8,7 @@ import { refusePolicyApprovals } from './policy-data.mjs';
|
|
|
7
8
|
import { refuseDevData } from './dev-data.mjs';
|
|
8
9
|
import { catalogue, validateRow, bindCell, writableColumns } from './catalogue.mjs';
|
|
9
10
|
import { platformTable, restorableManifest } from './bookkeeping.mjs';
|
|
11
|
+
import { withoutAbsentExcludedObjects } from './snapshot-objects.mjs';
|
|
10
12
|
import { canonicalJson, unbase64 } from './hash.mjs';
|
|
11
13
|
import { checkManifest, checkPart, ContentVerifier, ArchiveVerifier } from './format.mjs';
|
|
12
14
|
import { fence, mergeJournal, tombstone } from './journal.mjs';
|
|
@@ -15,13 +17,13 @@ import { createCopy,sidecar,readCopy,restoreCopy } from './copies.mjs';
|
|
|
15
17
|
import { excludedDependentGuards, migrationSeeded, seededCondition } from './restore-safety.mjs';
|
|
16
18
|
import { additiveBridge, sameSchemaMigrations, appliedMigrations, verifiedLegacySchema } from './bridge.mjs';
|
|
17
19
|
import { replayStatements } from './backfills.mjs';
|
|
18
|
-
import { checkArchivedLedgers, archivedHistory } from './ledgers.mjs';
|
|
20
|
+
import { checkArchivedLedgers, archivedHistory, archivedLedgerTables, ledgerReader, matchMigrationLedger } from './ledgers.mjs';
|
|
19
21
|
import { legacyEmptyGuards, legacyAuditStatements, requireLegacyEmpty } from './legacy-safety.mjs';
|
|
20
22
|
import { checkReferences } from './references.mjs';
|
|
21
23
|
import { rebuildDerivedStatements } from './derived.mjs';
|
|
22
24
|
// ledgers: the rows of each migration ledger table the archive carries, as the
|
|
23
25
|
// caller read them from it ({table:rows}); uploadPart binds them to its parts.
|
|
24
|
-
export async function createImport(context,manifest,actor='direct',legacyRecovery,ledgers,acknowledgeUnverifiedSchemaObjects=false) {
|
|
26
|
+
export async function createImport(context,manifest,actor='direct',legacyRecovery,ledgers,acknowledgeUnverifiedSchemaObjects=false,id=crypto.randomUUID()) {
|
|
25
27
|
const {db,config,buckets}=context;await checkManifest(manifest);checkArchivedLedgers(manifest,ledgers);
|
|
26
28
|
insist(manifest.product===config.product,'wrong_product');
|
|
27
29
|
// Explicit schema upgrades can reconcile rows, but cannot attest an interrupted deployment.
|
|
@@ -29,6 +31,7 @@ export async function createImport(context,manifest,actor='direct',legacyRecover
|
|
|
29
31
|
const original=manifest;manifest=await restorableManifest(original);
|
|
30
32
|
const skippedTables=original.tables.filter(table=>platformTable(table.name)).map(table=>table.name);
|
|
31
33
|
const cat=await catalogue(db,config),upgrade=config.acceptedSchemas[original.schema];
|
|
34
|
+
manifest=await withoutAbsentExcludedObjects(manifest,cat,config);
|
|
32
35
|
const legacy=original.version===2 && manifest.schemaObjects===undefined;
|
|
33
36
|
// An explicit acknowledgment keeps its existing recovery semantics. Without
|
|
34
37
|
// it, legacy v2 needs positive replay evidence for its recorded schema and
|
|
@@ -70,7 +73,7 @@ export async function createImport(context,manifest,actor='direct',legacyRecover
|
|
|
70
73
|
if(!(await importLeaseStatus(db,await job(db,previous,'import'))).stale)throw await importRunning(db,previous);
|
|
71
74
|
await abortImport(context,previous,{staleOnly:true});
|
|
72
75
|
}
|
|
73
|
-
const
|
|
76
|
+
const copies=Object.fromEntries(targets.map((t,i)=>[t.name,'rer_portability_stage_'+id.replaceAll('-','')+'_'+i]));
|
|
74
77
|
const row={id,kind:'import',actor,phase:'staging',completed:0,metadata:{manifest:original,...(unverified?{schemaUnverified:{reason:'acknowledged recovery of a version 2 archive without schemaObjects',sourceIdentity:original.sourceIdentity,exportedAt:original.exportedAt,acknowledgedAt:new Date().toISOString()}}:{}),...(skippedTables.length?{restoreManifest:manifest,skippedTables}:{}),copies,targets,triggers:cat.triggers,...(bridge.fromSchema?{bridge}:{}),...(replays?{replays}:{}),...(legacyRecovery?{legacyRecovery}: {}),...(ledgers?{ledgers}:{})}};
|
|
75
78
|
try {
|
|
76
79
|
await db.batch([guard(db,'(SELECT import_job FROM rer_portability_state WHERE id=1) IS NULL'),
|
|
@@ -273,3 +276,55 @@ export async function abortAbandonedImport(context) {
|
|
|
273
276
|
if(!id)return null;
|
|
274
277
|
await abortImport(context,id,{soleOperator:true});return id;
|
|
275
278
|
}
|
|
279
|
+
|
|
280
|
+
// A ledger carried in parts cannot authorize a durable job at create time.
|
|
281
|
+
// Collect this legacy upload in the service instance, authenticate it in full,
|
|
282
|
+
// then enter the ordinary durable import path. A lost collector is re-uploaded;
|
|
283
|
+
// it owns no destination data, lock, staging table or job to recover.
|
|
284
|
+
export function libraryImports(context,direct,migrationRoot) {
|
|
285
|
+
const pending=new Map();
|
|
286
|
+
const status=row=>({id:row.id,kind:'import',phase:row.phase,completed:row.parts.length,total:row.manifest.partCount,contentHash:null,requiredConfiguration:[]});
|
|
287
|
+
return {
|
|
288
|
+
...direct,
|
|
289
|
+
async create(manifest,actor='direct',options={}) {
|
|
290
|
+
if(manifest?.version!==2 || manifest.schemaObjects!==undefined || options.acknowledgeUnverifiedSchemaObjects || options.ledgers)return direct.create(manifest,actor,options);
|
|
291
|
+
await checkManifest(manifest);
|
|
292
|
+
if(!archivedLedgerTables(manifest).length)return direct.create(manifest,actor,options);
|
|
293
|
+
insist(manifest.product===context.config.product,'wrong_product');
|
|
294
|
+
await appliedMigrations(context.db);
|
|
295
|
+
insist(!(await statement(context.db,'SELECT import_job FROM rer_portability_state WHERE id=1').first())?.import_job && ![...pending.values()].some(row=>row.phase==='staging'),'import_running',409);
|
|
296
|
+
context.migrations??=await libraryMigrations(context.db,context.inputPolicy,migrationRoot);
|
|
297
|
+
const id=crypto.randomUUID();pending.set(id,{id,manifest:structuredClone(manifest),actor,parts:[],phase:'staging'});return id;
|
|
298
|
+
},
|
|
299
|
+
async uploadPart(id,index,part) {
|
|
300
|
+
const row=pending.get(id);if(!row)return direct.uploadPart(id,index,part);
|
|
301
|
+
insist(row.phase==='staging','import_phase',409);await checkPart(part,index,context.config.partBytes);
|
|
302
|
+
if(row.parts[index]){insist(row.parts[index].sha256===part.sha256,'part_conflict',409);return status(row);}
|
|
303
|
+
insist(index===row.parts.length && index<row.manifest.partCount,'part_order',409);
|
|
304
|
+
row.parts.push(structuredClone(part));return status(row);
|
|
305
|
+
},
|
|
306
|
+
async finalize(id,options={}) {
|
|
307
|
+
const row=pending.get(id);if(!row)return direct.finalize(id,options);
|
|
308
|
+
const {manifest,parts}=row;insist(row.phase==='staging','import_phase',409);insist(parts.length===manifest.partCount,'parts_missing');
|
|
309
|
+
const content=new ContentVerifier(manifest),archive=new ArchiveVerifier(manifest),reader=ledgerReader();reader.add(manifest);
|
|
310
|
+
for(const part of parts){archive.add(part);reader.add(part);for(const item of part.items)content.add(item);}
|
|
311
|
+
const digest=content.digest();
|
|
312
|
+
if(options.contentHash!==undefined)insist(digest===options.contentHash,'content_checksum');
|
|
313
|
+
insist(options.archiveHash===archive.digest(),'archive_checksum',409);
|
|
314
|
+
const ledgers=reader.ledgers(),history=archivedHistory(manifest,ledgers);
|
|
315
|
+
insist(history?.length && history.every(entry=>entry.sha256),'schema_mismatch:legacy_schema',409);
|
|
316
|
+
const replay=await context.migrations(),files=replay.names.map(name=>({name,sha256:replay.sha256?.(name),aliases:replay.aliases?.(name)}));
|
|
317
|
+
const recorded=matchMigrationLedger(history,files,replay);
|
|
318
|
+
insist(canonicalJson(recorded)===canonicalJson(replay.names.slice(0,recorded.length)),'schema_mismatch:migration_history',409);
|
|
319
|
+
const actual=await direct.create(manifest,row.actor,{ledgers,id});
|
|
320
|
+
pending.delete(id);
|
|
321
|
+
for(const part of parts)await direct.uploadPart(actual,part.index,part);
|
|
322
|
+
return direct.finalize(actual,options);
|
|
323
|
+
},
|
|
324
|
+
async status(id){return pending.has(id)?status(pending.get(id)):direct.status(id);},
|
|
325
|
+
async active(actor='direct'){const row=[...pending.values()].find(row=>row.actor===actor && row.phase==='staging');return row?status(row):direct.active(actor);},
|
|
326
|
+
async abort(id){const row=pending.get(id);if(!row)return direct.abort(id);row.phase='aborted';row.parts=[];return status(row);},
|
|
327
|
+
async lookup(id){return pending.has(id)?status(pending.get(id)):direct.lookup(id);},
|
|
328
|
+
async actor(id){return pending.get(id)?.actor??direct.actor(id);},
|
|
329
|
+
};
|
|
330
|
+
}
|
|
@@ -48,6 +48,8 @@ export interface MigrationReplay {names:string[];/** Seed-declared data-only Pla
|
|
|
48
48
|
/** Export job creation only; no record data, object keys or file paths. */
|
|
49
49
|
export interface ExportEvent {id:string;/** Opaque caller id; the direct/CLI sentinel is normalized to null. */actor:string|null;scope:'installation'}
|
|
50
50
|
export interface PortabilityOptions {
|
|
51
|
+
/** Node checkout root for automatic legacy evidence; defaults to cwd. */
|
|
52
|
+
migrationRoot?:string;
|
|
51
53
|
db:D1Database;buckets?:Record<string,R2Bucket>;policy:ArchivePolicy;migrations?:()=>Promise<MigrationReplay>;
|
|
52
54
|
/** Seed-owned audit SQL, appended to the snapshot/job batch. Throwing or SQL failure refuses creation. */
|
|
53
55
|
onExport?:(event:ExportEvent)=>[sql:string,args:unknown[]][];
|
|
@@ -3,7 +3,7 @@ import { policy, catalogue } from './catalogue.mjs';
|
|
|
3
3
|
import { wireDatabase } from '../d1-transport/index.mjs';
|
|
4
4
|
import { insist, job } from './storage.mjs';
|
|
5
5
|
import { createExport, advanceExport, exportStatus, downloadExport, abortExport } from './export.mjs';
|
|
6
|
-
import { createImport, uploadPart, finalizeImport, importStatus, activeImport, abortImport, abortAbandonedImport } from './import.mjs';
|
|
6
|
+
import { libraryImports, createImport, uploadPart, finalizeImport, importStatus, activeImport, abortImport, abortAbandonedImport } from './import.mjs';
|
|
7
7
|
import { erasureStatements, redactionStatements, invalidateTransfersStatements } from './journal.mjs';
|
|
8
8
|
export { archiveLines } from './lines.mjs';
|
|
9
9
|
export { excludedTables, excludedTable } from './catalogue.mjs';
|
|
@@ -11,18 +11,29 @@ export { readArchive, makePart, MAX_LINE_BYTES, MAX_PART_BYTES } from './format.
|
|
|
11
11
|
export { canonicalJson, sha256 } from './hash.mjs';
|
|
12
12
|
export { PortabilityError } from './storage.mjs';
|
|
13
13
|
export { createPortabilityHandler } from './http.mjs';
|
|
14
|
-
export async function createPortability({db,buckets={},policy:input,migrations,onExport}) {
|
|
14
|
+
export async function createPortability({db,buckets={},policy:input,migrations,onExport,migrationRoot}) {
|
|
15
15
|
insist(onExport===undefined || typeof onExport==='function','invalid_on_export');
|
|
16
16
|
db=wireDatabase(db);
|
|
17
17
|
const config=policy(input);await catalogue(db,config);
|
|
18
18
|
for(const name of Object.keys(config.buckets))insist(buckets[name] && typeof buckets[name].get==='function','missing_bucket:'+name);
|
|
19
19
|
await ensureImportLeases(db);
|
|
20
20
|
const context={db,buckets,config,migrations,onExport};
|
|
21
|
+
const importContext={...context,inputPolicy:input};
|
|
22
|
+
const {lookup,actor,...imports}=libraryImports(importContext,{
|
|
23
|
+
active:actor=>activeImport(importContext,actor),
|
|
24
|
+
create:(manifest,actor,options={})=>createImport(importContext,manifest,actor,undefined,options.ledgers,options.acknowledgeUnverifiedSchemaObjects,options.id),
|
|
25
|
+
uploadPart:(id,index,part)=>uploadPart(importContext,id,index,part),
|
|
26
|
+
finalize:(id,options)=>finalizeImport(importContext,id,options),
|
|
27
|
+
status:id=>importStatus(importContext,id),abort:id=>abortImport(importContext,id),
|
|
28
|
+
abortAbandoned:()=>abortAbandonedImport(importContext),
|
|
29
|
+
lookup:async id=>(await job(db,id)).kind==='export'?exportStatus(context,id):importStatus(importContext,id),
|
|
30
|
+
actor:id=>job(db,id).then(row=>row.actor),
|
|
31
|
+
},migrationRoot);
|
|
21
32
|
return {
|
|
22
33
|
exports:{create:actor=>createExport(context,actor),advance:(id,options)=>advanceExport(context,id,options),status:id=>exportStatus(context,id),download:id=>downloadExport(context,id),abort:id=>abortExport(context,id)},
|
|
23
|
-
imports
|
|
24
|
-
status:
|
|
25
|
-
actor
|
|
34
|
+
imports,
|
|
35
|
+
status:lookup,
|
|
36
|
+
actor,
|
|
26
37
|
erasureStatements:(table,key)=>erasureStatements(db,table,key),redactionStatements:()=>redactionStatements(db),invalidateTransfersStatements:()=>invalidateTransfersStatements(db),
|
|
27
38
|
};
|
|
28
39
|
}
|