@runeditrun/rer 0.2.91 → 0.2.94
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cli/README.md +1 -1
- package/cli/agent-door.mjs +20 -10
- package/cli/agent-history-database.mjs +82 -0
- package/cli/agent-history-isolation.mjs +68 -0
- package/cli/agent-history.mjs +58 -0
- package/cli/agent-journey-fixtures.mjs +3 -3
- package/cli/agent-network.mjs +12 -6
- package/cli/business/choices.mjs +3 -3
- package/cli/business/command.mjs +19 -0
- package/cli/business/setup-command.mjs +14 -1
- package/cli/deploy-command.mjs +19 -8
- package/cli/deploy.mjs +1 -1
- package/cli/export.mjs +1 -1
- package/cli/hosted-hosts.mjs +21 -0
- package/cli/hosted-output.mjs +91 -8
- package/cli/ledger-refusal.mjs +1 -1
- package/cli/lifecycle.mjs +5 -0
- package/cli/local.mjs +1 -1
- package/cli/port.mjs +1 -2
- package/cli/portability-evidence.d.mts +5 -0
- package/cli/portability-evidence.mjs +85 -0
- package/cli/portability-migrations.mjs +21 -9
- package/cli/portability-worker.mjs +4 -2
- package/edition-authoring/module-edges.mjs +14 -1
- package/foundation/runtime/contract-anchor.mjs +87 -0
- package/foundation/runtime/contract-docs.mjs +46 -16
- package/global/CHANGELOG.md +63 -0
- package/install.sh +4 -1
- package/modules/agent-door/conformance/crypto.mjs +14 -2
- package/modules/agent-door/conformance/injection.mjs +5 -3
- package/modules/agent-door/conformance/journey.mjs +27 -3
- package/modules/agent-door/conformance/negative.mjs +4 -1
- package/modules/agent-door/conformance/policy-phases.mjs +43 -1
- package/modules/agent-door/conformance/protocol.mjs +42 -7
- package/modules/agent-door/conformance/sponsors.mjs +4 -0
- package/modules/agent-door/conformance/surface.mjs +10 -0
- package/modules/deploy-kit/README.md +1 -1
- package/modules/deploy-kit/owner-cli.mjs +9 -1
- package/modules/deploy-kit/prepare.mjs +5 -2
- package/modules/erasure/README.md +4 -2
- package/modules/operator-auth/identity/oidc/README.md +2 -1
- package/modules/portability/README.md +84 -2
- package/modules/portability/backfills.mjs +96 -3
- package/modules/portability/bridge.mjs +3 -1
- package/modules/portability/catalogue.mjs +11 -1
- package/modules/portability/checkout-history.mjs +14 -0
- package/modules/portability/format.mjs +1 -0
- package/modules/portability/import.mjs +58 -3
- package/modules/portability/index.d.mts +3 -1
- package/modules/portability/index.mjs +16 -5
- package/modules/portability/library-history.mjs +147 -0
- package/modules/portability/migration-history.mjs +3 -2
- package/modules/portability/restore-safety.mjs +6 -3
- package/modules/portability/snapshot-objects.mjs +16 -0
- package/modules/portability/worker-history.d.mts +9 -0
- package/modules/portability/worker-history.mjs +29 -0
- package/onboarding/adapter.mjs +28 -14
- package/package.json +9 -1
|
@@ -4,6 +4,7 @@ import { execFileSync } from 'node:child_process';
|
|
|
4
4
|
import { resolve, join } from 'node:path';
|
|
5
5
|
import { pathToFileURL } from 'node:url';
|
|
6
6
|
import { parseArgs } from 'node:util';
|
|
7
|
+
import { AsyncLocalStorage } from 'node:async_hooks';
|
|
7
8
|
import { assert, hexId } from './validation.mjs';
|
|
8
9
|
import { regularBytes } from './artifact.mjs';
|
|
9
10
|
import { packageBuild } from './prepare.mjs';
|
|
@@ -21,6 +22,11 @@ import { publicOrigin } from './public-origin.mjs';
|
|
|
21
22
|
export const cronNotice = 'This Worker has a Cron Trigger. Cloudflare may take up to about two hours to start its first scheduled run on some accounts.';
|
|
22
23
|
const usage = 'usage: owner-cli.mjs deploy|dry-run|local-plan|health|remove|inspect|record-owner|signin|rekey --declaration <file> --environment <name> [--product <dir>] [--state <file>] [--outcome claimed|kept] | secret ... --set NAME [--set NAME]|--unset NAME [--unset NAME]';
|
|
23
24
|
|
|
25
|
+
// The public offline command also contains legacy hooks that call kit dry-run.
|
|
26
|
+
// Keep their plan local without changing direct kit callers or real deploys.
|
|
27
|
+
const localPlans = new AsyncLocalStorage();
|
|
28
|
+
export const withLocalDeployPlan = run => localPlans.run(true, run);
|
|
29
|
+
|
|
24
30
|
/** The owner's committed source and existing local build. Run the seed's build first. */
|
|
25
31
|
async function ownerArtifact(root, product) {
|
|
26
32
|
const git = (...args) => { try { return execFileSync('git',args,{cwd:root,encoding:'utf8',stdio:['ignore','pipe','pipe']}).trim(); } catch { throw new Error('owner-source-not-a-git-checkout'); } };
|
|
@@ -34,8 +40,10 @@ export async function main(argv = process.argv.slice(2), env = process.env, {fet
|
|
|
34
40
|
let parsed;
|
|
35
41
|
try { parsed = parseArgs({args:argv,allowPositionals:true,strict:true,options:{declaration:{type:'string'},environment:{type:'string'},product:{type:'string',default:''},state:{type:'string'},set:{type:'string',multiple:true},unset:{type:'string',multiple:true},outcome:{type:'string'}}}); }
|
|
36
42
|
catch { throw new Error('usage'); }
|
|
37
|
-
const {positionals:[
|
|
43
|
+
const {positionals:[requestedCommand,...rest],values} = parsed;
|
|
44
|
+
const command = localPlans.getStore() && requestedCommand === 'dry-run' ? 'local-plan' : requestedCommand;
|
|
38
45
|
if (!['deploy','dry-run','local-plan','health','secret','remove','inspect','record-owner','signin','rekey'].includes(command) || rest.length || !values.declaration || !values.environment) throw new Error('usage');
|
|
46
|
+
if (localPlans.getStore() && command !== 'local-plan') throw new Error('offline-deploy-refuses-provider-operation');
|
|
39
47
|
if ((command === 'secret') !== (values.set !== undefined || values.unset !== undefined) || values.set !== undefined && values.unset !== undefined) throw new Error('usage');
|
|
40
48
|
if (values.outcome !== undefined && command !== 'record-owner' || command === 'record-owner' && !['claimed','kept'].includes(values.outcome)) throw new Error('usage');
|
|
41
49
|
await refuseLocalModels(resolve(cwd));
|
|
@@ -7,6 +7,7 @@ import { assetsIgnore, collectFiles, regularBytes, validateArtifact, MAX_ARTIFAC
|
|
|
7
7
|
import { collectBuildModules } from './build-modules.mjs';
|
|
8
8
|
import { refuseLocalEnvFiles, refuseLocalModels } from './local-only.mjs';
|
|
9
9
|
import { readMigrationRewrites, withRewrites } from './migrations.mjs';
|
|
10
|
+
import { migrationDirectory } from '../../cli/portability-migrations.mjs';
|
|
10
11
|
|
|
11
12
|
export function assertCredentialFree(env) {
|
|
12
13
|
for (const [key,value] of Object.entries(env)) if (value && /(?:TOKEN|SECRET|PASSWORD|API_KEY|PRIVATE_KEY|AUTHORIZATION|COOKIE|CREDENTIAL)/i.test(key)) throw new Error('prepare-credential-environment-forbidden');
|
|
@@ -28,10 +29,12 @@ export async function packageBuild({root,source,runIdentity,productDirectory = '
|
|
|
28
29
|
const modules = await collectBuildModules(build,config,main);
|
|
29
30
|
const ignored = assetsIgnore(await regularBytes(join(product,'dist/client/.assetsignore'),65536).then(String,error => { if (error.code === 'ENOENT') return ''; throw error; }));
|
|
30
31
|
const assets = (await collectFiles(join(product,'dist/client'))).filter(file => !ignored(file.path));
|
|
32
|
+
const directory = await migrationDirectory(product,'DB',{fallback:'migrations/core'});
|
|
33
|
+
localPath(relative(product,directory));
|
|
31
34
|
let migrations = [];
|
|
32
|
-
for (const name of (await readdir(
|
|
35
|
+
for (const name of (await readdir(directory)).sort()) {
|
|
33
36
|
assert(/^[0-9]+_[a-zA-Z0-9_-]+\.sql$/.test(name),'unexpected-migration-file');
|
|
34
|
-
const sql = (await regularBytes(join(
|
|
37
|
+
const sql = (await regularBytes(join(directory,name),512000)).toString('utf8'); migrations.push({binding:'DB',name,sql,sha256:hash(sql)});
|
|
35
38
|
}
|
|
36
39
|
migrations = withRewrites(migrations, await readMigrationRewrites(product));
|
|
37
40
|
const artifact = {version:1,source,run:runIdentity,main,modules,assets,migrations}; validateArtifact(artifact);
|
|
@@ -25,12 +25,13 @@ retained exceptions and effects outside D1 and R2. Clauses are in
|
|
|
25
25
|
|
|
26
26
|
```js
|
|
27
27
|
import { defineErasure, createErasure } from './vendor/base/modules/erasure/index.mjs';
|
|
28
|
+
const fileColumn = 'storage_key';
|
|
28
29
|
export const person = defineErasure({
|
|
29
30
|
entity: 'person',
|
|
30
31
|
root: { table: 'people' },
|
|
31
32
|
edges: [
|
|
32
33
|
{ table: 'person_notes', from: 'people', on: { person_id: 'id' } },
|
|
33
|
-
{ table: 'person_files', from: 'people', on: { person_id: 'id' }, blob: { bucket: 'FILES', keyColumn:
|
|
34
|
+
{ table: 'person_files', from: 'people', on: { person_id: 'id' }, blob: { bucket: 'FILES', keyColumn: fileColumn } },
|
|
34
35
|
{ table: 'signals', from: 'people', where: 'instr(payload_json,?)>0', params: ['email'] },
|
|
35
36
|
],
|
|
36
37
|
redact: [{ table: 'messages', from: 'people', on: { author_id: 'id' }, set: { body: '' }, reason: 'transcript order is kept' }],
|
|
@@ -122,8 +123,9 @@ the declaration erases both together.
|
|
|
122
123
|
```js
|
|
123
124
|
import { rateEdges, rateWrite } from './vendor/base/modules/erasure/index.mjs';
|
|
124
125
|
const root = { table: 'people', column: 'id' };
|
|
126
|
+
const counterColumn = 'key';
|
|
125
127
|
// in the declaration
|
|
126
|
-
edges: [...rateEdges({ entity: 'person', root, counters: [{ table: 'rate_limits', keyColumn:
|
|
128
|
+
edges: [...rateEdges({ entity: 'person', root, counters: [{ table: 'rate_limits', keyColumn: counterColumn }] })],
|
|
127
129
|
// at every write whose key identifies a known person
|
|
128
130
|
const [counted] = await rateWrite(db, { entity: 'person', root, subject: person.id, rateTable: 'rate_limits', rateKey: key },
|
|
129
131
|
[db.prepare('INSERT INTO rate_limits(key,window,count) VALUES(?,?,1) ON CONFLICT(key) DO UPDATE SET count=count+1 RETURNING count').bind(key, window)]);
|
|
@@ -10,6 +10,7 @@ again as its next migration.
|
|
|
10
10
|
|
|
11
11
|
```js
|
|
12
12
|
import { createOperatorAuth, createAuthHandler, createSignInHandler } from './modules/operator-auth/index.mjs';
|
|
13
|
+
const clientAuthMethod = 'client_secret_basic';
|
|
13
14
|
const auth = createOperatorAuth({
|
|
14
15
|
db: env.DB, clientIp, mailSender,
|
|
15
16
|
// The runtime's fetch; on workerd use redirect 'manual' and refuse every 3xx.
|
|
@@ -22,7 +23,7 @@ const auth = createOperatorAuth({
|
|
|
22
23
|
providers: [
|
|
23
24
|
// The app's private key (a Worker secret) authenticates it: no shared secret exists.
|
|
24
25
|
{ id: 'auth', preset: 'auth', issuer: env.IDENTITY_PROVIDER, clientId: env.IDENTITY_CLIENT_ID, privateKey: JSON.parse(env.IDENTITY_PRIVATE_KEY) },
|
|
25
|
-
{ id: 'google', preset: 'google', clientId: env.GOOGLE_CLIENT_ID, tokenAuthMethod:
|
|
26
|
+
{ id: 'google', preset: 'google', clientId: env.GOOGLE_CLIENT_ID, tokenAuthMethod: clientAuthMethod, clientSecret: env.GOOGLE_CLIENT_SECRET },
|
|
26
27
|
{ id: 'corp', preset: 'okta', issuer: 'https://corp.okta.com', clientId: env.OKTA_CLIENT_ID, privateKey: JSON.parse(env.OKTA_PRIVATE_KEY),
|
|
27
28
|
subjectEmails: { '00u1abcd': 'operator@corp.example' } },
|
|
28
29
|
],
|
|
@@ -249,7 +249,15 @@ provider operation.
|
|
|
249
249
|
| Unknown hash/name, ambiguous hash/alias or conflicting name/hash | `migration_ledger:unknown`, `:ambiguous` or `:conflict`; use the matching checkout and declaration. |
|
|
250
250
|
|
|
251
251
|
Refusals throw `PortabilityError` with status 409 and a constant `code`;
|
|
252
|
-
ledger values never enter these diagnostics.
|
|
252
|
+
ledger values never enter these diagnostics. `migration` names a pending
|
|
253
|
+
checkout file, and `identity` may name a checkout
|
|
254
|
+
file, a declared alias or the known `rer_migrations` table. The worker appends
|
|
255
|
+
these safe identities to its public refusal token, retaining
|
|
256
|
+
`migration_ledger:unapplied:<file>`, `migration_ledger:multiple:rer_migrations`
|
|
257
|
+
and named `unknown`, `ambiguous` or `conflict` tokens only when the name exactly
|
|
258
|
+
equals a checkout filename or declared alias. Historical
|
|
259
|
+
`migration_ledger:unknown:initial-page.sql` stays category-only without that
|
|
260
|
+
provenance. Propagate failures and stop before
|
|
253
261
|
native apply. IO and provider failures also propagate. A successful check
|
|
254
262
|
creates no ledger, changes no row and does not attest unrecorded historical
|
|
255
263
|
bytes in a plain Wrangler ledger. It never silently skips pending migrations.
|
|
@@ -367,7 +375,7 @@ extra-payload recovery responsibilities and transfer erasure behavior still appl
|
|
|
367
375
|
The UTF-8 JSON-lines archive has a manifest, indexed checksummed parts and a
|
|
368
376
|
closing record. `readArchive(stream)` uses the shared bounded, fatal UTF-8 line
|
|
369
377
|
reader. The default maximum line size is 24 MiB; callers can set a smaller bound.
|
|
370
|
-
|
|
378
|
+
Current-archive paths do not materialize the whole archive; the legacy ledger collector described below buffers its initial upload. The manifest opens the archive and holds the final part count and payload hashes, so an export builds all its parts in the job before any byte is written; `exportArchive` and `importArchive` take an optional `progress` callback (`progress.mjs` turns its events into throttled lines with a time-left estimate). Readers enforce a 16 MiB
|
|
371
379
|
canonical UTF-8 part limit; `readArchive(stream,{partBytes})` can lower it.
|
|
372
380
|
Import uploads enforce the policy's part limit before any staging write.
|
|
373
381
|
A part has at most 1,000 rows;
|
|
@@ -400,6 +408,14 @@ Comments and harmless inter-token whitespace therefore compare equally, while
|
|
|
400
408
|
`x '31'` and `x'31'`, separated minus signs, numbers and combined operators stay
|
|
401
409
|
distinct. No regular expression rewrites SQL.
|
|
402
410
|
|
|
411
|
+
Restore may omit an archived schema object from its compatibility comparison
|
|
412
|
+
only when its table is in the archive's excluded inventory, the destination
|
|
413
|
+
policy also excludes it, neither side restores it, and the destination lacks
|
|
414
|
+
it. This permits transient export snapshots without installing source SQL.
|
|
415
|
+
Objects on restorable tables and excluded tables present at the destination
|
|
416
|
+
remain verified against the unchanged live and migration-replay catalogues;
|
|
417
|
+
the original manifest and parts still supply the archive digest.
|
|
418
|
+
|
|
403
419
|
Quoted identifiers (double quotes, backticks, brackets, or a single-quoted string
|
|
404
420
|
used as a name), strings, blobs, numbers, variables and ordinary identifiers keep
|
|
405
421
|
their exact spelling. Only unambiguous reserved keywords fold ASCII case.
|
|
@@ -548,6 +564,68 @@ archive's tables restore with their archived columns, later columns take their
|
|
|
548
564
|
defaults, and later tables stay empty unless their eligible seed or backfill is
|
|
549
565
|
replayed below. This applies to normal v2 import and to backup-only v1 recovery.
|
|
550
566
|
|
|
567
|
+
The Node in-process library door also verifies old v2 archives carrying a
|
|
568
|
+
hash-bearing migration ledger without caller-supplied replay or ledger rows.
|
|
569
|
+
`createPortability({db,policy})` reads the checkout's single D1 migration directory
|
|
570
|
+
from `wrangler.jsonc` (or use `migrationRoot` to name the checkout), matches its
|
|
571
|
+
files to the destination ledger, and collects the original parts in memory until
|
|
572
|
+
finalize authenticates their hashes and closing binding. The archived ledger must
|
|
573
|
+
be a hash-matched prefix; recorded replay must reproduce the archived tables and
|
|
574
|
+
full replay the complete destination identity before any durable import job is
|
|
575
|
+
created. Keep the same service instance through this initial upload; restarting
|
|
576
|
+
it requires uploading again. Acknowledged recovery and runner-supplied evidence
|
|
577
|
+
retain their existing paths.
|
|
578
|
+
|
|
579
|
+
Workers can opt into seed-owned build evidence. Add `legacyMigrationEvidence()`
|
|
580
|
+
from `@runeditrun/rer/portability-evidence` to the seed's existing Vite plugins.
|
|
581
|
+
It reads the declared D1 binding's `migrations_dir` through the same accessor
|
|
582
|
+
used by native migration checks and deployment packaging; pass `{binding:'DB'}`
|
|
583
|
+
to select a different binding. It never reads the live installation or an archive.
|
|
584
|
+
Import its generated module only in the archive's server adapter:
|
|
585
|
+
|
|
586
|
+
```js
|
|
587
|
+
import evidence from 'virtual:rer-legacy-migration-evidence';
|
|
588
|
+
import { workerMigrations } from '@runeditrun/rer/modules/portability/worker-history';
|
|
589
|
+
|
|
590
|
+
const service = await createPortability({
|
|
591
|
+
db, policy,
|
|
592
|
+
migrations: () => workerMigrations(db, policy, evidence),
|
|
593
|
+
});
|
|
594
|
+
```
|
|
595
|
+
|
|
596
|
+
For TypeScript, declare the virtual module in the seed's server declarations:
|
|
597
|
+
|
|
598
|
+
```ts
|
|
599
|
+
declare module 'virtual:rer-legacy-migration-evidence' {
|
|
600
|
+
const evidence: import('@runeditrun/rer/modules/portability/worker-history').MigrationEvidence;
|
|
601
|
+
export default evidence;
|
|
602
|
+
}
|
|
603
|
+
```
|
|
604
|
+
|
|
605
|
+
The generated JavaScript contains ordered checkout file hashes and statements,
|
|
606
|
+
deduplicated complete schema inventories for every prefix, and hashed identities
|
|
607
|
+
of bounded deterministic seeded rows. Volatile seeded rows cannot be vouched for;
|
|
608
|
+
their values are never bundled. Runtime policy projects each inventory through
|
|
609
|
+
the same schema identity code as live SQLite. The Worker adapter binds evidence
|
|
610
|
+
to the live hashed ledger and complete schema, then supplies the existing legacy
|
|
611
|
+
collector, archived-ledger validation, bridge and restore guards. Archive SQL is
|
|
612
|
+
never executed as evidence. Unsupported migration selections and missing evidence
|
|
613
|
+
fail closed with `schema_mismatch:legacy_evidence` (the public HTTP vocabulary
|
|
614
|
+
reports `schema_mismatch`). The ordinary automatic Node library path is unchanged.
|
|
615
|
+
|
|
616
|
+
The plugin runs for direct Vite builds and managed builds, rejects client imports,
|
|
617
|
+
and puts its module in the existing Worker bundle/package path. Seeds that do not
|
|
618
|
+
declare it generate no evidence module and need no migration or configuration
|
|
619
|
+
change. Keep the same service instance throughout the initial legacy upload.
|
|
620
|
+
|
|
621
|
+
Later data statements also qualify when a surviving trigger from that same
|
|
622
|
+
migration repeats the complete statement block, in order, on Base's restore
|
|
623
|
+
commit transition. Those statements run through that destination-owned trigger,
|
|
624
|
+
not a second finalize replay.
|
|
625
|
+
Absent tables count as empty only with an empty full replay and proof that every
|
|
626
|
+
later insert and possible trigger writer preserves emptiness; a bounded empty
|
|
627
|
+
copy/drop/rename rebuild and a SELECT from such an empty source carry no row.
|
|
628
|
+
|
|
551
629
|
The Base CLI runners (local Workers and container) pass `createPortability` a
|
|
552
630
|
`migrations` replayer (`migration-history.mjs`, Node only) built from the
|
|
553
631
|
destination's applied migration ledger and migration files, with each file's
|
|
@@ -1018,6 +1096,7 @@ modules/portability/schema-identity.mjs
|
|
|
1018
1096
|
modules/portability/schema-legacy.mjs
|
|
1019
1097
|
modules/portability/sqlite-tokens.mjs
|
|
1020
1098
|
modules/portability/bookkeeping.mjs
|
|
1099
|
+
modules/portability/snapshot-objects.mjs
|
|
1021
1100
|
modules/portability/storage.mjs
|
|
1022
1101
|
modules/portability/copies.mjs
|
|
1023
1102
|
modules/portability/derived.mjs
|
|
@@ -1027,6 +1106,9 @@ modules/portability/policy-data.mjs
|
|
|
1027
1106
|
modules/portability/generated.mjs
|
|
1028
1107
|
modules/portability/restore-safety.mjs
|
|
1029
1108
|
modules/portability/bridge.mjs
|
|
1109
|
+
modules/portability/library-history.mjs
|
|
1110
|
+
modules/portability/checkout-history.mjs
|
|
1111
|
+
modules/portability/migration-history.mjs
|
|
1030
1112
|
modules/portability/ledgers.mjs
|
|
1031
1113
|
modules/portability/migration-statements.mjs
|
|
1032
1114
|
modules/portability/sql-text.mjs
|
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
import { statement, quote, insist } from './storage.mjs';
|
|
2
|
-
import { tokens, folded } from './generated.mjs';
|
|
2
|
+
import { tokens, folded, identifier } from './generated.mjs';
|
|
3
3
|
import { changesData, dataStatement, reachingTables, triggerBodies, schemaChange, statementNames } from './migration-statements.mjs';
|
|
4
4
|
import { splitStatements } from './sql-text.mjs';
|
|
5
5
|
import { preservingRebuilds } from './rebuilds.mjs';
|
|
6
6
|
import { replayViews, viewReads } from './replay-views.mjs';
|
|
7
|
+
import { sqliteTokens } from './sqlite-tokens.mjs';
|
|
8
|
+
import { schemaSQL } from './schema-identity.mjs';
|
|
7
9
|
import { transientTables } from './transient-tables.mjs';
|
|
8
10
|
|
|
9
11
|
// The data statements of later migrations, judged by the tables they write
|
|
@@ -31,13 +33,24 @@ import { transientTables } from './transient-tables.mjs';
|
|
|
31
33
|
// statements after it is judged as the statements it holds.
|
|
32
34
|
// Returns the replays in migration order and the proven CHECK rebuilds, or refuses with the first migration
|
|
33
35
|
// (in `unapplied` order) holding a statement that cannot be admitted.
|
|
34
|
-
export function laterData({applied,unapplied,replay,catalogue,archived,empty,tables,secrets,manifest}) {
|
|
36
|
+
export function laterData({applied,unapplied,replay,catalogue,archived,empty,tables,secrets,manifest,replayEmpty=[]}) {
|
|
35
37
|
const split=name=>replay.statements(name).flatMap(splitStatements),fold=list=>list.map(folded);
|
|
36
38
|
const statements=applied.flatMap(split);
|
|
37
39
|
const restorable=fold(catalogue),carried=fold(archived);
|
|
38
40
|
const reached=reachingTables(statements,carried),writable=reachingTables(statements,restorable);
|
|
39
41
|
const later=unapplied.flatMap(migration=>split(migration).map((sql,index)=>({migration,index,sql,data:dataStatement(sql)})));
|
|
40
42
|
const {admitted,tables:rebuilt}=preservingRebuilds(later,manifest,tables,statements),views=replayViews(statements);
|
|
43
|
+
const restored=restoreCommitStatements(applied,split,later);
|
|
44
|
+
restored.forEach(s=>admitted.add(s));
|
|
45
|
+
// Renaming an uncarried table is metadata-only. Later row statements still
|
|
46
|
+
// face their own checks; carried table definitions still face the bridge.
|
|
47
|
+
for(const s of later) {
|
|
48
|
+
const list=sqliteTokens(s.sql),words=list.map(t=>folded(t.value));
|
|
49
|
+
if(words.length===6 && words[0]==='alter' && words[1]==='table' && words[3]==='rename' && words[4]==='to' && !carried.includes(words[2]) && !carried.includes(words[5]))admitted.add(s);
|
|
50
|
+
}
|
|
51
|
+
const absentEmpty=emptyAbsentTables(later,statements,carried,replayEmpty,admitted);
|
|
52
|
+
// An empty absent source contributes no row to a direct INSERT SELECT.
|
|
53
|
+
for(const s of later)if(s.data && emptySelect(s.sql,absentEmpty))admitted.add(s);
|
|
41
54
|
const {missing,plans:transients}=transientTables(statements,later,fold(tables),carried);
|
|
42
55
|
const current=fold(tables),known=[...current,...missing],outside=later.filter(s=>s.data && !writable.includes(s.data.target) && !transients.has(s.data.target)),tainted=new Set();
|
|
43
56
|
// Tables an admitted write fills from archived rows (directly, through its
|
|
@@ -61,7 +74,7 @@ export function laterData({applied,unapplied,replay,catalogue,archived,empty,tab
|
|
|
61
74
|
// Dependencies are checked below before any of the plan is selected.
|
|
62
75
|
const available=[...transients.keys(),...(!carried.includes(table)?[...writes.keys()].filter(name=>!carried.includes(name)):[])];
|
|
63
76
|
const allowUpdate=!transient && !carried.includes(table) && list.some(s=>['insert','replace'].includes(s.data.verb));
|
|
64
|
-
const options={table,allowUpdate,statements,restorable,writable,carried:[...carried,...available],tables:known,secrets,views,manifest};
|
|
77
|
+
const options={table,allowUpdate,statements,restorable,writable,carried:[...carried,...absentEmpty,...available],tables:known,secrets,views,manifest};
|
|
65
78
|
// Carried target rules must use the actual archive, not reconstructed reads.
|
|
66
79
|
const bad=carried.includes(table) && list.length>1?list:list.filter(s=>!replayable(s,{...options,targetCarried:carried.includes(table)}));
|
|
67
80
|
// Writes confined to tables created after the archive retain their
|
|
@@ -141,3 +154,83 @@ export function replayStatements(db,row) {
|
|
|
141
154
|
statement(db,`UPDATE rer_portability_jobs SET metadata=json_set(metadata,'$.bridge.replayed[${i}].rows',${r.schema?'0':'changes()'}) WHERE id=?`,row.id)])];
|
|
142
155
|
}
|
|
143
156
|
const autoincrement=sql=>tokens(sql).some(t=>!t.quoted && folded(t.text)==='autoincrement');
|
|
157
|
+
|
|
158
|
+
// Only a surviving destination-owned trigger on the actual restore transition
|
|
159
|
+
// can cover DML. The complete body must recur contiguously, in the same file
|
|
160
|
+
// and order, preserving literals and identifiers. Nothing is replayed twice.
|
|
161
|
+
function restoreCommitStatements(applied,split,later) {
|
|
162
|
+
const live=new Map();
|
|
163
|
+
for(const migration of applied)for(const sql of split(migration)) {
|
|
164
|
+
const list=tokens(sql),words=list.map(t=>folded(t.text));
|
|
165
|
+
if(!['create','drop'].includes(words[0]) || words[1]!=='trigger')continue;
|
|
166
|
+
const at=words[2]==='if'?(words[0]==='create'?5:4):2,name=folded(identifier(list[at]));
|
|
167
|
+
if(words[0]==='drop'){live.delete(name);continue;}
|
|
168
|
+
if(words[2]==='if' && live.has(name))continue;
|
|
169
|
+
live.set(name,{migration,sql,list,at});
|
|
170
|
+
}
|
|
171
|
+
const admitted=new Set();
|
|
172
|
+
const header='AFTER UPDATE OF import_job ON rer_portability_state WHEN OLD.import_job IS NOT NULL AND NEW.import_job IS NULL AND OLD.applying=1';
|
|
173
|
+
for(const {migration,sql,list,at} of live.values()) {
|
|
174
|
+
const begin=list.findIndex(t=>!t.quoted && folded(t.text)==='begin');
|
|
175
|
+
if(begin<0 || schemaChange(sql)!=='trigger' || schemaSQL(sql.slice(list[at].end,list[begin].start))!==schemaSQL(header))continue;
|
|
176
|
+
const end=list.at(-1);if(!/^end;?$/i.test(end.text))continue;
|
|
177
|
+
const body=splitStatements(sql.slice(list[begin].end,end.start));
|
|
178
|
+
if(!body.length || body.some(sql=>!dataStatement(sql)))continue;
|
|
179
|
+
const file=later.filter(s=>s.migration===migration),images=body.map(schemaSQL);
|
|
180
|
+
for(let i=0;i+body.length<=file.length;i++)if(images.every((image,k)=>file[i+k].data && schemaSQL(file[i+k].sql)===image))file.slice(i,i+body.length).forEach(s=>admitted.add(s));
|
|
181
|
+
}
|
|
182
|
+
return admitted;
|
|
183
|
+
}
|
|
184
|
+
// Prove an absent table stays empty rather than assuming every newly created
|
|
185
|
+
// table is empty. Full replay supplies the empty baseline; every later INSERT
|
|
186
|
+
// and every possible trigger writer must also preserve it. A local empty
|
|
187
|
+
// create/copy/drop/rename rebuild carries no row, including through its DROP.
|
|
188
|
+
function emptyAbsentTables(later,statements,carried,replayEmpty,admitted) {
|
|
189
|
+
const empty=new Set(replayEmpty.map(folded).filter(name=>!carried.includes(name)));
|
|
190
|
+
const rebuilds=[];
|
|
191
|
+
for(let i=0;i+3<later.length;i++) {
|
|
192
|
+
const block=later.slice(i,i+4),a=tokens(block[0].sql),b=tokens(block[1].sql),c=tokens(block[2].sql),d=tokens(block[3].sql);
|
|
193
|
+
const text=list=>list.map(t=>folded(identifier(t)));
|
|
194
|
+
const [aa,bb,cc,dd]=[a,b,c,d].map(text);
|
|
195
|
+
if(block.some(s=>s.migration!==block[0].migration) || aa[0]!=='create' || aa[1]!=='table' || a[3]?.text!=='(' || bb.length!==7 || bb[0]!=='insert' || bb[1]!=='into' || bb[2]!==aa[2] || bb[3]!=='select' || bb[4]!=='*' || bb[5]!=='from' || cc.join(' ')!==`drop table ${bb[6]}` || dd.join(' ')!==`alter table ${aa[2]} rename to ${bb[6]}` || !empty.has(bb[6]) || carried.includes(aa[2]))continue;
|
|
196
|
+
if(later.some(s=>!block.includes(s) && statementNames(s.sql).includes(aa[2])))continue;
|
|
197
|
+
rebuilds.push({block,source:bb[6]});
|
|
198
|
+
}
|
|
199
|
+
const copyStatements=new Set(rebuilds.flatMap(r=>r.block)),triggerTargets=new Set();
|
|
200
|
+
for(const sql of statements) {
|
|
201
|
+
const list=tokens(sql);if(folded(list[0]?.text??'')!=='create' || folded(list[1]?.text??'')!=='trigger')continue;
|
|
202
|
+
const on=list.findIndex(t=>!t.quoted && folded(t.text)==='on');
|
|
203
|
+
const begin=list.findIndex(t=>!t.quoted && folded(t.text)==='begin');
|
|
204
|
+
if(on<0 || begin<0 || schemaChange(sql)!=='trigger'){statementNames(sql).forEach(name=>triggerTargets.add(name));continue;}
|
|
205
|
+
for(const body of splitStatements(sql.slice(list[begin].end,list.at(-1).start))) {
|
|
206
|
+
const data=dataStatement(body);
|
|
207
|
+
if(data)triggerTargets.add(data.target);
|
|
208
|
+
else if(!/^\s*SELECT\b/i.test(body))statementNames(body).forEach(name=>triggerTargets.add(name));
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
for(let changed=true;changed;) {
|
|
212
|
+
changed=false;
|
|
213
|
+
for(const name of empty) {
|
|
214
|
+
const writes=later.filter(s=>s.data && s.data.target===name && !copyStatements.has(s) && ['insert','replace'].includes(s.data.verb));
|
|
215
|
+
const triggered=triggerTargets.has(name);
|
|
216
|
+
if(triggered || writes.some(s=>!emptySelect(s.sql,empty))) {empty.delete(name);changed=true;}
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
for(const {block,source} of rebuilds)if(empty.has(source))block.forEach(s=>admitted.add(s));
|
|
220
|
+
return [...empty];
|
|
221
|
+
}
|
|
222
|
+
function emptySelect(sql,empty) {
|
|
223
|
+
const list=sqliteTokens(sql),word=t=>['keyword','identifier'].includes(t?.type)?folded(t.value):undefined;
|
|
224
|
+
let depth=0,select=-1,from=-1;
|
|
225
|
+
for(let i=0;i<list.length;i++) {
|
|
226
|
+
const t=list[i],w=word(t);
|
|
227
|
+
if(!depth && ['union','except','intersect','right','full','returning'].includes(w))return false;
|
|
228
|
+
if(!depth && w==='select' && select<0)select=i;
|
|
229
|
+
if(!depth && w==='from' && select>=0 && from<0)from=i;
|
|
230
|
+
// An aggregate over an empty input can still produce one row. Permit
|
|
231
|
+
// only known scalar expressions and EXISTS before the outer FROM.
|
|
232
|
+
if(select>=0 && from<0 && list[i+1]?.text==='(' && w && !['coalesce','exists','in'].includes(w))return false;
|
|
233
|
+
if(t.text==='(')depth++;if(t.text===')')depth--;
|
|
234
|
+
}
|
|
235
|
+
return from>select && [...empty].includes(folded(list[from+1]?.value??''));
|
|
236
|
+
}
|
|
@@ -66,7 +66,9 @@ async function placement(db,manifest,cat,replay,ledger,history,legacyRecovery,un
|
|
|
66
66
|
const catalogue=cat.tables.map(t=>t.name),archivedTables=catalogue.filter(name=>manifest.tables.some(t=>t.name===name));
|
|
67
67
|
const empty=archivedTables.filter(name=>!manifest.tables.find(t=>t.name===name).count);
|
|
68
68
|
const tables=(await all(db,"SELECT name FROM sqlite_master WHERE type IN ('table','view')")).map(t=>t.name);
|
|
69
|
-
const
|
|
69
|
+
const baseline=await replay.rowImages?.(cat.tables.filter(t=>!archivedTables.includes(t.name)).map(t=>({name:t.name,columns:t.columns.map(c=>c.name)})),0);
|
|
70
|
+
const replayEmpty=baseline?[...baseline].filter(([,rows])=>!rows.length).map(([name])=>name):[];
|
|
71
|
+
const {replays,rebuilt}=laterData({applied,unapplied,replay,catalogue,archived:archivedTables,empty,tables,secrets:manifest.excludedColumns??{},manifest:manifest.tables,replayEmpty});
|
|
70
72
|
return {applied,earlier,replays,rebuilt};
|
|
71
73
|
}
|
|
72
74
|
// Old v2 writers recorded tables and applied migration names, but no behavioral
|
|
@@ -65,6 +65,10 @@ export function excludedTable(table,input={}) {
|
|
|
65
65
|
async function archived(db,config) {
|
|
66
66
|
const objects=await all(db,"SELECT name,sql,type,tbl_name FROM sqlite_master WHERE sql IS NOT NULL AND substr(lower(name),1,7) <> 'sqlite_' AND substr(lower(name),1,4) <> '_cf_' ORDER BY name");
|
|
67
67
|
const kinds=await all(db,'PRAGMA table_list');
|
|
68
|
+
return archivedInventory({objects,kinds},config);
|
|
69
|
+
}
|
|
70
|
+
function archivedInventory({objects:input,kinds},config) {
|
|
71
|
+
const objects=[...input];
|
|
68
72
|
const derived=config.derivedTables;
|
|
69
73
|
const excluded=objects.filter(x=>x.type==='table' && (excludedTable(x.name,config) || (!config.includeTables.includes(x.name) &&
|
|
70
74
|
derived.some(p=>x.name===p || (kinds.find(k=>k.name===x.name)?.type==='shadow' && x.name.startsWith(p+'_'))))));
|
|
@@ -76,7 +80,13 @@ async function archived(db,config) {
|
|
|
76
80
|
}
|
|
77
81
|
// The archive schema identity alone, for databases that need not satisfy the policy.
|
|
78
82
|
export async function schemaIdentity(db,config,{legacyExcludedObjects=false,tablesOnly=false}={}) {
|
|
79
|
-
|
|
83
|
+
return inventoryIdentity(await archived(db,config),{legacyExcludedObjects,tablesOnly});
|
|
84
|
+
}
|
|
85
|
+
// Build evidence and live SQLite use exactly the same policy projection.
|
|
86
|
+
export function inventorySchemaIdentity(inventory,config,options) {
|
|
87
|
+
return inventoryIdentity(archivedInventory(inventory,config),options);
|
|
88
|
+
}
|
|
89
|
+
function inventoryIdentity({entries,excluded,schemaObjects},{legacyExcludedObjects=false,tablesOnly=false}={}) {
|
|
80
90
|
// Acknowledged v1 recovery places an old backup using its historical
|
|
81
91
|
// projection; acknowledged recovery of a version 2 archive without an object
|
|
82
92
|
// inventory compares tables only. Current archives always retain the
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { catalogue, policy } from './catalogue.mjs';
|
|
2
|
+
import { destinationLedger, matchMigrationLedger } from './ledgers.mjs';
|
|
3
|
+
import { insist } from './storage.mjs';
|
|
4
|
+
|
|
5
|
+
// Both evidence adapters bind the seed's files to the live hashed history and
|
|
6
|
+
// complete schema before the shared legacy collector can accept an upload.
|
|
7
|
+
export async function checkoutHistory(db,input,files,replayFor) {
|
|
8
|
+
const ledger=await destinationLedger(db)??[];
|
|
9
|
+
insist(ledger.length && ledger.every(row=>row.sha256),'schema_mismatch:legacy_evidence',409);
|
|
10
|
+
const applied=matchMigrationLedger(ledger,files);
|
|
11
|
+
const replay=await replayFor(applied);
|
|
12
|
+
insist(await replay.schemaAfter(replay.names)===(await catalogue(db,policy(input))).identity,'schema_mismatch');
|
|
13
|
+
return replay;
|
|
14
|
+
}
|
|
@@ -36,6 +36,7 @@ async function checkManifestFields(m) {
|
|
|
36
36
|
insist(typeof m.product==='string' && typeof m.sourceIdentity==='string' && typeof m.exportedAt==='string','manifest_identity');
|
|
37
37
|
insist(Number.isSafeInteger(m.partCount) && m.partCount>=0 && Number.isSafeInteger(m.epoch) && m.epoch>=0,'manifest_counts');
|
|
38
38
|
insist(Array.isArray(m.tables) && Array.isArray(m.objects) && Array.isArray(m.payloads) && Array.isArray(m.journal) && Array.isArray(m.requiredConfiguration),'manifest_inventory');
|
|
39
|
+
insist(m.schemaObjects===undefined || m.excludedTables===undefined || (Array.isArray(m.excludedTables) && m.excludedTables.every(name=>typeof name==='string') && new Set(m.excludedTables).size===m.excludedTables.length),'manifest_inventory');
|
|
39
40
|
// Applied product migrations, recorded by archives since additive restore.
|
|
40
41
|
insist(m.migrations===undefined || (Array.isArray(m.migrations) && m.migrations.every(x=>typeof x==='string')),'manifest_inventory');
|
|
41
42
|
// The ledger an export from a RER Platform installation records, by migration identity.
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { libraryMigrations } from './library-history.mjs';
|
|
1
2
|
import { all, statement, state, guard, job, save, partQueries, parts, readPart, dropCopyStatements, cleanCopyStatements, quote, stream, insist, run } from './storage.mjs';
|
|
2
3
|
import { stagePart } from './import-staging.mjs';
|
|
3
4
|
import { normalizeStaged } from './import-safety.mjs';
|
|
@@ -7,6 +8,7 @@ import { refusePolicyApprovals } from './policy-data.mjs';
|
|
|
7
8
|
import { refuseDevData } from './dev-data.mjs';
|
|
8
9
|
import { catalogue, validateRow, bindCell, writableColumns } from './catalogue.mjs';
|
|
9
10
|
import { platformTable, restorableManifest } from './bookkeeping.mjs';
|
|
11
|
+
import { withoutAbsentExcludedObjects } from './snapshot-objects.mjs';
|
|
10
12
|
import { canonicalJson, unbase64 } from './hash.mjs';
|
|
11
13
|
import { checkManifest, checkPart, ContentVerifier, ArchiveVerifier } from './format.mjs';
|
|
12
14
|
import { fence, mergeJournal, tombstone } from './journal.mjs';
|
|
@@ -15,13 +17,13 @@ import { createCopy,sidecar,readCopy,restoreCopy } from './copies.mjs';
|
|
|
15
17
|
import { excludedDependentGuards, migrationSeeded, seededCondition } from './restore-safety.mjs';
|
|
16
18
|
import { additiveBridge, sameSchemaMigrations, appliedMigrations, verifiedLegacySchema } from './bridge.mjs';
|
|
17
19
|
import { replayStatements } from './backfills.mjs';
|
|
18
|
-
import { checkArchivedLedgers, archivedHistory } from './ledgers.mjs';
|
|
20
|
+
import { checkArchivedLedgers, archivedHistory, archivedLedgerTables, ledgerReader, matchMigrationLedger } from './ledgers.mjs';
|
|
19
21
|
import { legacyEmptyGuards, legacyAuditStatements, requireLegacyEmpty } from './legacy-safety.mjs';
|
|
20
22
|
import { checkReferences } from './references.mjs';
|
|
21
23
|
import { rebuildDerivedStatements } from './derived.mjs';
|
|
22
24
|
// ledgers: the rows of each migration ledger table the archive carries, as the
|
|
23
25
|
// caller read them from it ({table:rows}); uploadPart binds them to its parts.
|
|
24
|
-
export async function createImport(context,manifest,actor='direct',legacyRecovery,ledgers,acknowledgeUnverifiedSchemaObjects=false) {
|
|
26
|
+
export async function createImport(context,manifest,actor='direct',legacyRecovery,ledgers,acknowledgeUnverifiedSchemaObjects=false,id=crypto.randomUUID()) {
|
|
25
27
|
const {db,config,buckets}=context;await checkManifest(manifest);checkArchivedLedgers(manifest,ledgers);
|
|
26
28
|
insist(manifest.product===config.product,'wrong_product');
|
|
27
29
|
// Explicit schema upgrades can reconcile rows, but cannot attest an interrupted deployment.
|
|
@@ -29,6 +31,7 @@ export async function createImport(context,manifest,actor='direct',legacyRecover
|
|
|
29
31
|
const original=manifest;manifest=await restorableManifest(original);
|
|
30
32
|
const skippedTables=original.tables.filter(table=>platformTable(table.name)).map(table=>table.name);
|
|
31
33
|
const cat=await catalogue(db,config),upgrade=config.acceptedSchemas[original.schema];
|
|
34
|
+
manifest=await withoutAbsentExcludedObjects(manifest,cat,config);
|
|
32
35
|
const legacy=original.version===2 && manifest.schemaObjects===undefined;
|
|
33
36
|
// An explicit acknowledgment keeps its existing recovery semantics. Without
|
|
34
37
|
// it, legacy v2 needs positive replay evidence for its recorded schema and
|
|
@@ -70,7 +73,7 @@ export async function createImport(context,manifest,actor='direct',legacyRecover
|
|
|
70
73
|
if(!(await importLeaseStatus(db,await job(db,previous,'import'))).stale)throw await importRunning(db,previous);
|
|
71
74
|
await abortImport(context,previous,{staleOnly:true});
|
|
72
75
|
}
|
|
73
|
-
const
|
|
76
|
+
const copies=Object.fromEntries(targets.map((t,i)=>[t.name,'rer_portability_stage_'+id.replaceAll('-','')+'_'+i]));
|
|
74
77
|
const row={id,kind:'import',actor,phase:'staging',completed:0,metadata:{manifest:original,...(unverified?{schemaUnverified:{reason:'acknowledged recovery of a version 2 archive without schemaObjects',sourceIdentity:original.sourceIdentity,exportedAt:original.exportedAt,acknowledgedAt:new Date().toISOString()}}:{}),...(skippedTables.length?{restoreManifest:manifest,skippedTables}:{}),copies,targets,triggers:cat.triggers,...(bridge.fromSchema?{bridge}:{}),...(replays?{replays}:{}),...(legacyRecovery?{legacyRecovery}: {}),...(ledgers?{ledgers}:{})}};
|
|
75
78
|
try {
|
|
76
79
|
await db.batch([guard(db,'(SELECT import_job FROM rer_portability_state WHERE id=1) IS NULL'),
|
|
@@ -273,3 +276,55 @@ export async function abortAbandonedImport(context) {
|
|
|
273
276
|
if(!id)return null;
|
|
274
277
|
await abortImport(context,id,{soleOperator:true});return id;
|
|
275
278
|
}
|
|
279
|
+
|
|
280
|
+
// A ledger carried in parts cannot authorize a durable job at create time.
|
|
281
|
+
// Collect this legacy upload in the service instance, authenticate it in full,
|
|
282
|
+
// then enter the ordinary durable import path. A lost collector is re-uploaded;
|
|
283
|
+
// it owns no destination data, lock, staging table or job to recover.
|
|
284
|
+
export function libraryImports(context,direct,migrationRoot) {
|
|
285
|
+
const pending=new Map();
|
|
286
|
+
const status=row=>({id:row.id,kind:'import',phase:row.phase,completed:row.parts.length,total:row.manifest.partCount,contentHash:null,requiredConfiguration:[]});
|
|
287
|
+
return {
|
|
288
|
+
...direct,
|
|
289
|
+
async create(manifest,actor='direct',options={}) {
|
|
290
|
+
if(manifest?.version!==2 || manifest.schemaObjects!==undefined || options.acknowledgeUnverifiedSchemaObjects || options.ledgers)return direct.create(manifest,actor,options);
|
|
291
|
+
await checkManifest(manifest);
|
|
292
|
+
if(!archivedLedgerTables(manifest).length)return direct.create(manifest,actor,options);
|
|
293
|
+
insist(manifest.product===context.config.product,'wrong_product');
|
|
294
|
+
await appliedMigrations(context.db);
|
|
295
|
+
insist(!(await statement(context.db,'SELECT import_job FROM rer_portability_state WHERE id=1').first())?.import_job && ![...pending.values()].some(row=>row.phase==='staging'),'import_running',409);
|
|
296
|
+
context.migrations??=await libraryMigrations(context.db,context.inputPolicy,migrationRoot);
|
|
297
|
+
const id=crypto.randomUUID();pending.set(id,{id,manifest:structuredClone(manifest),actor,parts:[],phase:'staging'});return id;
|
|
298
|
+
},
|
|
299
|
+
async uploadPart(id,index,part) {
|
|
300
|
+
const row=pending.get(id);if(!row)return direct.uploadPart(id,index,part);
|
|
301
|
+
insist(row.phase==='staging','import_phase',409);await checkPart(part,index,context.config.partBytes);
|
|
302
|
+
if(row.parts[index]){insist(row.parts[index].sha256===part.sha256,'part_conflict',409);return status(row);}
|
|
303
|
+
insist(index===row.parts.length && index<row.manifest.partCount,'part_order',409);
|
|
304
|
+
row.parts.push(structuredClone(part));return status(row);
|
|
305
|
+
},
|
|
306
|
+
async finalize(id,options={}) {
|
|
307
|
+
const row=pending.get(id);if(!row)return direct.finalize(id,options);
|
|
308
|
+
const {manifest,parts}=row;insist(row.phase==='staging','import_phase',409);insist(parts.length===manifest.partCount,'parts_missing');
|
|
309
|
+
const content=new ContentVerifier(manifest),archive=new ArchiveVerifier(manifest),reader=ledgerReader();reader.add(manifest);
|
|
310
|
+
for(const part of parts){archive.add(part);reader.add(part);for(const item of part.items)content.add(item);}
|
|
311
|
+
const digest=content.digest();
|
|
312
|
+
if(options.contentHash!==undefined)insist(digest===options.contentHash,'content_checksum');
|
|
313
|
+
insist(options.archiveHash===archive.digest(),'archive_checksum',409);
|
|
314
|
+
const ledgers=reader.ledgers(),history=archivedHistory(manifest,ledgers);
|
|
315
|
+
insist(history?.length && history.every(entry=>entry.sha256),'schema_mismatch:legacy_schema',409);
|
|
316
|
+
const replay=await context.migrations(),files=replay.names.map(name=>({name,sha256:replay.sha256?.(name),aliases:replay.aliases?.(name)}));
|
|
317
|
+
const recorded=matchMigrationLedger(history,files,replay);
|
|
318
|
+
insist(canonicalJson(recorded)===canonicalJson(replay.names.slice(0,recorded.length)),'schema_mismatch:migration_history',409);
|
|
319
|
+
const actual=await direct.create(manifest,row.actor,{ledgers,id});
|
|
320
|
+
pending.delete(id);
|
|
321
|
+
for(const part of parts)await direct.uploadPart(actual,part.index,part);
|
|
322
|
+
return direct.finalize(actual,options);
|
|
323
|
+
},
|
|
324
|
+
async status(id){return pending.has(id)?status(pending.get(id)):direct.status(id);},
|
|
325
|
+
async active(actor='direct'){const row=[...pending.values()].find(row=>row.actor===actor && row.phase==='staging');return row?status(row):direct.active(actor);},
|
|
326
|
+
async abort(id){const row=pending.get(id);if(!row)return direct.abort(id);row.phase='aborted';row.parts=[];return status(row);},
|
|
327
|
+
async lookup(id){return pending.has(id)?status(pending.get(id)):direct.lookup(id);},
|
|
328
|
+
async actor(id){return pending.get(id)?.actor??direct.actor(id);},
|
|
329
|
+
};
|
|
330
|
+
}
|
|
@@ -44,10 +44,12 @@ export interface Status {schemaUnverified?:SchemaUnverifiedReceipt;id:string;kin
|
|
|
44
44
|
/** Later additive migrations an older archive was restored across. */
|
|
45
45
|
export interface Bridge {fromSchema:string;toSchema:string;migrations:string[];tables:string[];columns:Record<string,string[]>;/** Later backfills of tables the archive lacks, replayed at finalize in this order; rows is null until the apply commits. */replayed?:{migration:string;statement:number;table:string;rows:number|null}[]}
|
|
46
46
|
/** A runner's replay of the destination's applied migrations: the archive schema identity after a chosen set, and each migration's statements. */
|
|
47
|
-
export interface MigrationReplay {names:string[];/** Seed-declared data-only Platform entries without migration files. */dataOnly?:string[];/** Names generated by the seed's deployment specification for this file. */aliases?(name:string):string[]|undefined;statements(name:string):string[];/** The migration file's SHA-256 hex, where the runner read the file. */sha256?(name:string):string|undefined;schemaAfter(names:string[],options?:{tablesOnly?:boolean}):Promise<string>}
|
|
47
|
+
export interface MigrationReplay {names:string[];/** Seed-declared data-only Platform entries without migration files. */dataOnly?:string[];/** Names generated by the seed's deployment specification for this file. */aliases?(name:string):string[]|undefined;statements(name:string):string[];/** The migration file's SHA-256 hex, where the runner read the file. */sha256?(name:string):string|undefined;schemaAfter(names:string[],options?:{tablesOnly?:boolean;legacyExcludedObjects?:boolean}):Promise<string>}
|
|
48
48
|
/** Export job creation only; no record data, object keys or file paths. */
|
|
49
49
|
export interface ExportEvent {id:string;/** Opaque caller id; the direct/CLI sentinel is normalized to null. */actor:string|null;scope:'installation'}
|
|
50
50
|
export interface PortabilityOptions {
|
|
51
|
+
/** Node checkout root for automatic legacy evidence; defaults to cwd. */
|
|
52
|
+
migrationRoot?:string;
|
|
51
53
|
db:D1Database;buckets?:Record<string,R2Bucket>;policy:ArchivePolicy;migrations?:()=>Promise<MigrationReplay>;
|
|
52
54
|
/** Seed-owned audit SQL, appended to the snapshot/job batch. Throwing or SQL failure refuses creation. */
|
|
53
55
|
onExport?:(event:ExportEvent)=>[sql:string,args:unknown[]][];
|
|
@@ -3,7 +3,7 @@ import { policy, catalogue } from './catalogue.mjs';
|
|
|
3
3
|
import { wireDatabase } from '../d1-transport/index.mjs';
|
|
4
4
|
import { insist, job } from './storage.mjs';
|
|
5
5
|
import { createExport, advanceExport, exportStatus, downloadExport, abortExport } from './export.mjs';
|
|
6
|
-
import { createImport, uploadPart, finalizeImport, importStatus, activeImport, abortImport, abortAbandonedImport } from './import.mjs';
|
|
6
|
+
import { libraryImports, createImport, uploadPart, finalizeImport, importStatus, activeImport, abortImport, abortAbandonedImport } from './import.mjs';
|
|
7
7
|
import { erasureStatements, redactionStatements, invalidateTransfersStatements } from './journal.mjs';
|
|
8
8
|
export { archiveLines } from './lines.mjs';
|
|
9
9
|
export { excludedTables, excludedTable } from './catalogue.mjs';
|
|
@@ -11,18 +11,29 @@ export { readArchive, makePart, MAX_LINE_BYTES, MAX_PART_BYTES } from './format.
|
|
|
11
11
|
export { canonicalJson, sha256 } from './hash.mjs';
|
|
12
12
|
export { PortabilityError } from './storage.mjs';
|
|
13
13
|
export { createPortabilityHandler } from './http.mjs';
|
|
14
|
-
export async function createPortability({db,buckets={},policy:input,migrations,onExport}) {
|
|
14
|
+
export async function createPortability({db,buckets={},policy:input,migrations,onExport,migrationRoot}) {
|
|
15
15
|
insist(onExport===undefined || typeof onExport==='function','invalid_on_export');
|
|
16
16
|
db=wireDatabase(db);
|
|
17
17
|
const config=policy(input);await catalogue(db,config);
|
|
18
18
|
for(const name of Object.keys(config.buckets))insist(buckets[name] && typeof buckets[name].get==='function','missing_bucket:'+name);
|
|
19
19
|
await ensureImportLeases(db);
|
|
20
20
|
const context={db,buckets,config,migrations,onExport};
|
|
21
|
+
const importContext={...context,inputPolicy:input};
|
|
22
|
+
const {lookup,actor,...imports}=libraryImports(importContext,{
|
|
23
|
+
active:actor=>activeImport(importContext,actor),
|
|
24
|
+
create:(manifest,actor,options={})=>createImport(importContext,manifest,actor,undefined,options.ledgers,options.acknowledgeUnverifiedSchemaObjects,options.id),
|
|
25
|
+
uploadPart:(id,index,part)=>uploadPart(importContext,id,index,part),
|
|
26
|
+
finalize:(id,options)=>finalizeImport(importContext,id,options),
|
|
27
|
+
status:id=>importStatus(importContext,id),abort:id=>abortImport(importContext,id),
|
|
28
|
+
abortAbandoned:()=>abortAbandonedImport(importContext),
|
|
29
|
+
lookup:async id=>(await job(db,id)).kind==='export'?exportStatus(context,id):importStatus(importContext,id),
|
|
30
|
+
actor:id=>job(db,id).then(row=>row.actor),
|
|
31
|
+
},migrationRoot);
|
|
21
32
|
return {
|
|
22
33
|
exports:{create:actor=>createExport(context,actor),advance:(id,options)=>advanceExport(context,id,options),status:id=>exportStatus(context,id),download:id=>downloadExport(context,id),abort:id=>abortExport(context,id)},
|
|
23
|
-
imports
|
|
24
|
-
status:
|
|
25
|
-
actor
|
|
34
|
+
imports,
|
|
35
|
+
status:lookup,
|
|
36
|
+
actor,
|
|
26
37
|
erasureStatements:(table,key)=>erasureStatements(db,table,key),redactionStatements:()=>redactionStatements(db),invalidateTransfersStatements:()=>invalidateTransfersStatements(db),
|
|
27
38
|
};
|
|
28
39
|
}
|