@runeditrun/rer 0.2.67 → 0.2.69
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cli/business/vendors/README.md +16 -4
- package/cli/business/vendors/common.mjs +49 -28
- package/cli/business/vendors/credential-names.mjs +124 -0
- package/cli/import-summary.mjs +7 -0
- package/cli/inspect.mjs +29 -1
- package/cli/output.mjs +2 -0
- package/cli/portability-worker.mjs +10 -4
- package/cli/portability.mjs +1 -0
- package/global/CHANGELOG.md +15 -0
- package/modules/operator-auth/dev/index.mjs +2 -1
- package/modules/operator-auth/signin/index.mjs +2 -2
- package/modules/operator-auth/signin/one-click.mjs +8 -3
- package/modules/portability/README.md +102 -11
- package/modules/portability/bridge.mjs +44 -29
- package/modules/portability/catalogue.mjs +33 -3
- package/modules/portability/files.mjs +9 -3
- package/modules/portability/http.mjs +2 -1
- package/modules/portability/import-lease.mjs +99 -0
- package/modules/portability/import-safety.mjs +80 -0
- package/modules/portability/import.mjs +47 -16
- package/modules/portability/index.d.mts +10 -4
- package/modules/portability/index.mjs +4 -2
- package/modules/portability/ledgers.mjs +83 -0
- package/modules/portability/legacy-files.mjs +7 -4
- package/modules/portability/migration-history.mjs +3 -2
- package/modules/portability/migration-statements.mjs +83 -9
- package/modules/release/README.md +14 -6
- package/modules/release/publication-scan.mjs +13 -7
- package/package.json +1 -1
|
@@ -41,6 +41,8 @@ schema at construction. Apply the bookkeeping migration first. Policy fields:
|
|
|
41
41
|
| `derivedTables` | Virtual/FTS tables to regenerate; their shadow tables are excluded. |
|
|
42
42
|
| `secretColumns` | Per-table nullable base columns omitted at export and written as NULL. Primary keys and generated columns cannot be secret. |
|
|
43
43
|
| `exportRow(table,row)` | Pure, synchronous, idempotent transformation of encoded base cells. Restore writes its result verbatim; SQLite recomputes generated cells. |
|
|
44
|
+
| `importSafety` | Per-table, per-column nonempty finite sets of safe encoded cells, paired with `importRow`. Only writable non-key columns may be declared. |
|
|
45
|
+
| `importRow(table,row)` | Pure, synchronous, deterministic, idempotent normalization of staged rows in declared tables, before `validateStaged` and apply. Changes are bounded by `importSafety`. |
|
|
44
46
|
| `replaceableTables` | Bootstrap rows replaced in the atomic restore batch. Rows the destination's own migrations insert need no entry where the runner passes `migrations` (see "Migration-seeded rows"). Populated excluded dependents reachable through modifying FK actions refuse replacement. |
|
|
45
47
|
| `requiredConfiguration` | Setting names returned after restore for the owner to re-enter. |
|
|
46
48
|
| `validateStaged({db,tables,read,manifest})` | Read-only product validation before any object write; false or a thrown error refuses restore. `tables` names the staging tables for bounded SQL queries; `read` is a convenience for small tables. |
|
|
@@ -55,7 +57,7 @@ belong to the current runtime or environment and never travel:
|
|
|
55
57
|
| Tables (exact names or prefixes ending in `%`) | Reason |
|
|
56
58
|
| --- | --- |
|
|
57
59
|
| `sqlite_%`, `_cf_%` | SQLite and Cloudflare internals. |
|
|
58
|
-
| `d1_migrations`, `_rer_deploy_migrations`, `_rer_portability_migrations`, `rer_migrations` | Destination migration history.
|
|
60
|
+
| `d1_migrations`, `_rer_deploy_migrations`, `_rer_portability_migrations`, `rer_migrations` | Destination migration history. Base uses the D1 and `_rer_` ledgers; RER Platform's runner writes `rer_migrations`. An old archive that still carries its source's ledger is placed by it (see below); its rows are never restored. |
|
|
59
61
|
| `rer_portability_%` | Transfer jobs, snapshots, staging, journals and recovery bookkeeping. |
|
|
60
62
|
| `rer_mail_development_outbox`, `rer_ai_development_outbox` | Local development effect captures. |
|
|
61
63
|
| `rer_dependency_keys`, `rer_dependency_pending`, `rer_dependency_codes` | Provider secrets and transient linking credentials. |
|
|
@@ -151,6 +153,49 @@ the bytes. Large objects span parts and restore through multipart upload with at
|
|
|
151
153
|
most one 5 MiB upload buffer. Extra payload consumers own their own transaction,
|
|
152
154
|
idempotence and recovery; D1 cannot roll back Durable Object or external state.
|
|
153
155
|
|
|
156
|
+
### Bounded import normalization
|
|
157
|
+
|
|
158
|
+
Use `importSafety` with `importRow` when restored records must start in a safe
|
|
159
|
+
state, even if an older archive contains active connections or credentials:
|
|
160
|
+
|
|
161
|
+
```js
|
|
162
|
+
export default {
|
|
163
|
+
product: 'my-product',
|
|
164
|
+
importSafety: {connections: {status: ['inactive'], token: [null]}},
|
|
165
|
+
importRow(table, row) {
|
|
166
|
+
return {...row, status: 'inactive', token: null};
|
|
167
|
+
},
|
|
168
|
+
};
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
The hook is called only for tables declared in `importSafety`, after staging
|
|
172
|
+
and before `validateStaged` or apply. Mandatory development/provider-data refusals
|
|
173
|
+
run first; normalization cannot bypass them. It must return exactly the writable columns
|
|
174
|
+
of its input row, preserve primary keys and every undeclared column, and change
|
|
175
|
+
declared columns only to their listed safe values. Unchanged values need not be
|
|
176
|
+
in the safe set. The hook must be pure, synchronous, deterministic and idempotent;
|
|
177
|
+
keep external effects out of it. Generated columns recompute from the normalized
|
|
178
|
+
base cells. Without this pair, import behavior is unchanged.
|
|
179
|
+
|
|
180
|
+
Construction refuses an unpaired hook or declaration, unknown table or column
|
|
181
|
+
names, primary keys, generated columns, empty safe sets and malformed encoded
|
|
182
|
+
cells. Safe values use the ordinary cell encoding: null, strings, finite numbers
|
|
183
|
+
or `{blob:base64}`. The destination policy supplies the hook; an archive cannot
|
|
184
|
+
supply executable normalization.
|
|
185
|
+
|
|
186
|
+
The import job persistently journals the summary in `metadata.safetyChanges`.
|
|
187
|
+
`imports.status(id)` and the `importArchive` receipt expose it as `safetyChanges`,
|
|
188
|
+
an array of `{table,column,value,rows}`. Each entry counts actual changed rows for
|
|
189
|
+
that column and safe replacement value; a row changed in two columns contributes
|
|
190
|
+
to two entries. The summary contains no original values or row records. Durable
|
|
191
|
+
checkpoints keep retries from counting changes twice. Human `rer import` output
|
|
192
|
+
prints these counts too, for example `3 automations imported with status="inactive"`.
|
|
193
|
+
|
|
194
|
+
Archive integrity checks still verify the original archived contents. This
|
|
195
|
+
summary is separate from the erasure journal: normalization does not bypass its
|
|
196
|
+
fence or change journal adoption. Existing atomic D1 apply, owned-object rollback,
|
|
197
|
+
extra-payload recovery responsibilities and transfer erasure behavior still apply.
|
|
198
|
+
|
|
154
199
|
## Archive and durable jobs
|
|
155
200
|
|
|
156
201
|
The UTF-8 JSON-lines archive has a manifest, indexed checksummed parts and a
|
|
@@ -210,16 +255,30 @@ table). This applies to normal v2 import and to backup-only v1 recovery.
|
|
|
210
255
|
|
|
211
256
|
The Base CLI runners (local Workers and container) pass `createPortability` a
|
|
212
257
|
`migrations` replayer (`migration-history.mjs`, Node only) built from the
|
|
213
|
-
destination's applied migration ledger and migration files
|
|
214
|
-
|
|
258
|
+
destination's applied migration ledger and migration files, with each file's
|
|
259
|
+
SHA-256. The ledger is `_rer_deploy_migrations`, `d1_migrations` (Wrangler or
|
|
260
|
+
the deploy kit) or, on an installation RER Platform installed, its exact
|
|
261
|
+
`rer_migrations(name, sha256, applied_at)` ledger, read in application order
|
|
262
|
+
(rowid) and refused while it holds a partially applied file. Import replays
|
|
263
|
+
them into a scratch database to find the archive's place in that history:
|
|
215
264
|
|
|
216
265
|
- Archives record the applied migration names in `manifest.migrations` (since
|
|
217
266
|
additive restore). Every recorded migration must be applied here, and
|
|
218
267
|
replaying exactly those must give the archive's schema identity.
|
|
219
|
-
- Older
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
268
|
+
- Older archives that still carry their source's migration ledger as a table
|
|
269
|
+
(`rer_migrations` and the other ledgers above, from before they became
|
|
270
|
+
bookkeeping) are placed by it. Every name it records must be applied here,
|
|
271
|
+
with the same file hash (the destination ledger's, else the replayed file's),
|
|
272
|
+
or the import is refused with `schema_mismatch:migration_history:<migration>`
|
|
273
|
+
naming the first that differs; replaying exactly those must give the archive's
|
|
274
|
+
schema identity. Every other applied migration is later, wherever the
|
|
275
|
+
destination's history orders it. `importArchive` reads the ledger rows from
|
|
276
|
+
the verified file and passes them to `imports.create(manifest, actor,
|
|
277
|
+
{ledgers})`; staging refuses any part whose ledger rows differ from them.
|
|
278
|
+
- Other older v2 and all v1 archives do not record them; the latest prefix of
|
|
279
|
+
the applied migrations that gives the archive's schema identity is used.
|
|
280
|
+
Migrations that change no archived table (indexes, excluded tables) cannot be
|
|
281
|
+
told apart there and are reported as later.
|
|
223
282
|
|
|
224
283
|
Restored rows never pass through the later migrations, so each of them may
|
|
225
284
|
contain only `CREATE TABLE` (not `AS SELECT`), `CREATE INDEX` and
|
|
@@ -227,7 +286,11 @@ contain only `CREATE TABLE` (not `AS SELECT`), `CREATE INDEX` and
|
|
|
227
286
|
restorable tables. A later migration with any other statement that names a
|
|
228
287
|
restorable table, such as an `INSERT`, `UPDATE` or `DELETE` that purges or
|
|
229
288
|
backfills rows or a trigger, is refused with `schema_mismatch`; statements that
|
|
230
|
-
touch only excluded tables are allowed.
|
|
289
|
+
touch only excluded tables are allowed. An archived table the archive holds no
|
|
290
|
+
rows of restores empty, so an `UPDATE` or `DELETE` of it changes nothing and is
|
|
291
|
+
allowed unless it also names a table the archive holds rows of (which it could
|
|
292
|
+
read into its writes); an `INSERT` into it, or any other statement naming it,
|
|
293
|
+
is still refused. For archives without a migration record,
|
|
231
294
|
every migration after the earliest prefix that gives the archive's schema is
|
|
232
295
|
checked, since the archive may predate any of them.
|
|
233
296
|
|
|
@@ -371,8 +434,32 @@ bridge). The batch repeats the check. Any other row refuses with
|
|
|
371
434
|
export, reset and import commands to run. A Worker import has no replay, so a
|
|
372
435
|
hosted restore still needs such tables in `replaceableTables`.
|
|
373
436
|
|
|
374
|
-
An empty destination must be reserved for restore: pause destination writers
|
|
375
|
-
|
|
437
|
+
An empty destination must be reserved for restore: pause destination writers.
|
|
438
|
+
Product writes to source tables can continue. Import upload, finalize and abort
|
|
439
|
+
requests are serialized by a durable request claim.
|
|
440
|
+
|
|
441
|
+
Imports hold a **five-minute inactivity lease**, created with the job and renewed
|
|
442
|
+
by every upload/finalize request, then every 30 seconds while that request is
|
|
443
|
+
working (including verification, R2 transfers and product callbacks). This is a
|
|
444
|
+
recovery window, not a total restore limit: the measured 20,000-row round trip
|
|
445
|
+
below already takes 171 seconds, and callbacks have no wall-time bound. A fresh
|
|
446
|
+
import returns `409 import_running` with `jobId` and ISO `abortableAt`. The owner
|
|
447
|
+
can find its active job with `GET /api/portability/imports` (`{active:status}`
|
|
448
|
+
or `{active:null}`), and inspect `GET .../imports/:id`: `abortableAt` and `stale` describe
|
|
449
|
+
its lease. The existing authenticated `POST .../imports/:id/abort` (or CLI
|
|
450
|
+
`abort --resume ID`) discards stale staging and owned objects. A live request
|
|
451
|
+
cannot be aborted through that route. An expired upload/finalize returns
|
|
452
|
+
`409 import_lease_expired`; start the restore again.
|
|
453
|
+
|
|
454
|
+
The next import automatically aborts a stale job through that same cleanup path
|
|
455
|
+
before claiming the destination. Applying jobs use the existing owned-object
|
|
456
|
+
recovery: product rows publish in one fenced D1 transaction, so an interrupted
|
|
457
|
+
pre-commit apply has no product rows to undo; a committed job stays complete,
|
|
458
|
+
including after a lost acknowledgement. Cleanup never deletes foreign objects.
|
|
459
|
+
Lease bookkeeping is created privately on existing destinations without a product
|
|
460
|
+
migration; an older unfinished job receives one five-minute grace period. The
|
|
461
|
+
local CLI's `abortAbandoned` remains restricted to a sole operator holding its
|
|
462
|
+
checkout lock; hosted callers must use the lease-aware create/status/abort paths.
|
|
376
463
|
R2 has no compare-and-swap transaction covering bucket emptiness and writes.
|
|
377
464
|
An interrupted applying job can be finalized again; ownership survives restart.
|
|
378
465
|
Keep job status until the client acknowledges completion. Abort completed exports
|
|
@@ -398,7 +485,8 @@ are `{error:code}`. Request bodies are bounded; `maxBodyBytes` can lower the bou
|
|
|
398
485
|
| `POST imports` | Create with `{confirmed:true,manifest}`. |
|
|
399
486
|
| `PUT imports/:id/parts/:index` | Upload part. |
|
|
400
487
|
| `POST imports/:id/finalize` | Apply with `{archiveHash,contentHash?,laterJournal?}`. |
|
|
401
|
-
| `GET imports
|
|
488
|
+
| `GET imports` | Active import belonging to this actor, or null. |
|
|
489
|
+
| `GET imports/:id` | Status including lease expiry and stale flag. |
|
|
402
490
|
| `POST imports/:id/abort` | Clean up unfinished import. |
|
|
403
491
|
|
|
404
492
|
```sh
|
|
@@ -496,11 +584,14 @@ modules/portability/provider-data.mjs
|
|
|
496
584
|
modules/portability/generated.mjs
|
|
497
585
|
modules/portability/restore-safety.mjs
|
|
498
586
|
modules/portability/bridge.mjs
|
|
587
|
+
modules/portability/ledgers.mjs
|
|
499
588
|
modules/portability/migration-statements.mjs
|
|
500
589
|
modules/portability/legacy-safety.mjs
|
|
501
590
|
modules/portability/export.mjs
|
|
502
591
|
modules/portability/import.mjs
|
|
592
|
+
modules/portability/import-lease.mjs
|
|
503
593
|
modules/portability/import-staging.mjs
|
|
594
|
+
modules/portability/import-safety.mjs
|
|
504
595
|
modules/portability/payloads.mjs
|
|
505
596
|
modules/portability/journal.mjs
|
|
506
597
|
modules/portability/http.mjs
|
|
@@ -1,35 +1,31 @@
|
|
|
1
1
|
import { all, quote, insist } from './storage.mjs';
|
|
2
2
|
import { canonicalJson } from './hash.mjs';
|
|
3
3
|
import { tokens, identifier, folded } from './generated.mjs';
|
|
4
|
-
import { changesData } from './migration-statements.mjs';
|
|
4
|
+
import { changesData, reachingTables } from './migration-statements.mjs';
|
|
5
|
+
import { destinationLedger } from './ledgers.mjs';
|
|
5
6
|
|
|
6
|
-
// Applied product migrations in the ledger's own order. A
|
|
7
|
-
// ledger may hold an unacknowledged migration whose effects are
|
|
8
|
-
// an archive nor a restore can attest its history until that
|
|
7
|
+
// Applied product migrations in the ledger's own order (ledgers.mjs). A
|
|
8
|
+
// state-bearing ledger may hold an unacknowledged migration whose effects are
|
|
9
|
+
// unknown; neither an archive nor a restore can attest its history until that
|
|
10
|
+
// is resolved.
|
|
9
11
|
export async function appliedMigrations(db) {
|
|
10
|
-
|
|
11
|
-
if(ledgers.includes('_rer_deploy_migrations'))return (await all(db,"SELECT name FROM _rer_deploy_migrations WHERE state='applied' ORDER BY name")).map(x=>x.name);
|
|
12
|
-
if(!ledgers.includes('d1_migrations'))return;
|
|
13
|
-
const columns=(await all(db,'PRAGMA table_info(d1_migrations)')).map(x=>x.name);
|
|
14
|
-
const tracked=columns.includes('state') || columns.includes('sha256');
|
|
15
|
-
if(tracked)insist(columns.includes('state') && columns.includes('sha256'),'schema_mismatch:migration_history',409);
|
|
16
|
-
const rows=await all(db,`SELECT name${tracked?',state':''} FROM d1_migrations ORDER BY id`);
|
|
17
|
-
if(tracked)insist(rows.every(row=>row.state==='applied'),'schema_mismatch:uncertain_migrations',409);
|
|
18
|
-
return rows.map(row=>row.name);
|
|
12
|
+
return (await destinationLedger(db))?.map(m=>m.name);
|
|
19
13
|
}
|
|
20
14
|
// Where an archive sits in the destination's applied migrations: those its
|
|
21
15
|
// rows passed through (earlier) and those applied after it (unapplied).
|
|
22
16
|
// migrations() is the runner's replay of the destination's applied migrations
|
|
23
17
|
// (migration-history.mjs). The archive's schema must be the replay of the
|
|
24
|
-
// migrations it records or, for archives that predate that record, of
|
|
25
|
-
//
|
|
26
|
-
//
|
|
27
|
-
// the schema.
|
|
28
|
-
async function placement(manifest,cat,replay,
|
|
18
|
+
// migrations it records or, for archives that predate that record, of those
|
|
19
|
+
// its own migration ledger records or, without one, of an earlier prefix.
|
|
20
|
+
// Restored rows never pass through a later migration, so one that writes or
|
|
21
|
+
// rewrites restorable data is refused, whether or not it changes the schema.
|
|
22
|
+
async function placement(manifest,cat,replay,ledger,history) {
|
|
23
|
+
const applied=ledger.map(m=>m.name);
|
|
29
24
|
insist(canonicalJson(replay.names)===canonicalJson(applied) && await replay.schemaAfter(applied)===cat.identity,'schema_mismatch');
|
|
25
|
+
const recorded=manifest.migrations ?? archivedApplied(history?.(),ledger,replay);
|
|
30
26
|
let earlier,unapplied;
|
|
31
|
-
if(
|
|
32
|
-
earlier=applied.filter(name=>
|
|
27
|
+
if(recorded) {
|
|
28
|
+
earlier=applied.filter(name=>recorded.includes(name));
|
|
33
29
|
insist(await replay.schemaAfter(earlier)===manifest.schema,'schema_mismatch');
|
|
34
30
|
unapplied=applied.filter(name=>!earlier.includes(name));
|
|
35
31
|
}else{
|
|
@@ -44,12 +40,31 @@ async function placement(manifest,cat,replay,applied) {
|
|
|
44
40
|
unapplied=applied.slice(matching[0]);
|
|
45
41
|
}
|
|
46
42
|
// Only rows the archive carries can skip a later migration's change. A table
|
|
47
|
-
// the archive does not hold is restored empty, so
|
|
48
|
-
//
|
|
43
|
+
// the archive does not hold is restored empty, so backfills that name only
|
|
44
|
+
// such tables (a newly added module's own) change no restored row, unless a
|
|
45
|
+
// trigger from any applied migration carries their writes into one that is
|
|
46
|
+
// restorable. Trigger DDL itself changes no row, nor does an UPDATE or DELETE
|
|
47
|
+
// of a table the archive holds no rows of (migration-statements.mjs).
|
|
49
48
|
const restorable=cat.tables.map(t=>t.name).filter(name=>manifest.tables.some(t=>t.name===name));
|
|
50
|
-
|
|
49
|
+
const reached=reachingTables(applied.flatMap(name=>replay.statements(name)),restorable);
|
|
50
|
+
const empty=restorable.filter(name=>!manifest.tables.find(t=>t.name===name).count);
|
|
51
|
+
for(const name of unapplied)insist(!replay.statements(name).some(sql=>changesData(sql,reached,empty)),'schema_mismatch:migration_data:'+name);
|
|
51
52
|
return {earlier};
|
|
52
53
|
}
|
|
54
|
+
// An archive without a migration record that carries its source's migration
|
|
55
|
+
// ledger (an old archive from before that ledger became bookkeeping) records
|
|
56
|
+
// exactly which migrations its rows passed through. Each must be one the
|
|
57
|
+
// destination applied, with the same file hash where the ledger records one:
|
|
58
|
+
// the destination ledger's, else the replayed file's. The ledger itself is
|
|
59
|
+
// never restored (bookkeeping.mjs). history: [{name,sha256?}] from
|
|
60
|
+
// ledgers.mjs archivedHistory, undefined when the archive carries no ledger.
|
|
61
|
+
function archivedApplied(history,ledger,replay) {
|
|
62
|
+
for(const m of history??[]) {
|
|
63
|
+
const applied=ledger.find(x=>x.name===m.name);
|
|
64
|
+
insist(applied && (m.sha256===undefined || m.sha256===(applied.sha256??replay.sha256?.(m.name))),'schema_mismatch:migration_history:'+m.name);
|
|
65
|
+
}
|
|
66
|
+
return history?.map(m=>m.name);
|
|
67
|
+
}
|
|
53
68
|
// An archive with the destination's schema identity can still predate later
|
|
54
69
|
// migrations that changed only data (a purge or a backfill) or added an index.
|
|
55
70
|
// Those that change restorable data are refused like any other. Without a
|
|
@@ -57,21 +72,21 @@ async function placement(manifest,cat,replay,applied) {
|
|
|
57
72
|
// migrations the destination's ledger has outgrown is refused. An archive that
|
|
58
73
|
// records a migration the destination never applied is newer than it, even
|
|
59
74
|
// when that migration left the schema as it was (an index, a data change).
|
|
60
|
-
export async function sameSchemaMigrations(db,manifest,cat,migrations) {
|
|
61
|
-
const
|
|
75
|
+
export async function sameSchemaMigrations(db,manifest,cat,migrations,history) {
|
|
76
|
+
const ledger=await destinationLedger(db),applied=ledger?.map(m=>m.name);
|
|
62
77
|
if(!applied)return;
|
|
63
78
|
insist(!manifest.migrations || manifest.migrations.every(name=>applied.includes(name)),'schema_mismatch:migrations');
|
|
64
79
|
if(!migrations){insist(!manifest.migrations || applied.every(name=>manifest.migrations.includes(name)),'schema_mismatch:migrations');return;}
|
|
65
|
-
await placement(manifest,cat,await migrations(),
|
|
80
|
+
await placement(manifest,cat,await migrations(),ledger,history);
|
|
66
81
|
}
|
|
67
82
|
// An archive made before later migrations restores only when those migrations
|
|
68
83
|
// are additive: new tables, new columns that take a default, or indexes (which
|
|
69
84
|
// archives never carry). Anything else stays a schema_mismatch refusal.
|
|
70
|
-
export async function additiveBridge(db,manifest,cat,migrations) {
|
|
85
|
+
export async function additiveBridge(db,manifest,cat,migrations,history) {
|
|
71
86
|
insist(migrations,'schema_mismatch');
|
|
72
|
-
const replay=await migrations(),
|
|
87
|
+
const replay=await migrations(),ledger=await destinationLedger(db)??[],applied=ledger.map(m=>m.name);
|
|
73
88
|
insist(!manifest.migrations || manifest.migrations.every(name=>applied.includes(name)),'schema_mismatch:migrations');
|
|
74
|
-
const {earlier}=await placement(manifest,cat,replay,
|
|
89
|
+
const {earlier}=await placement(manifest,cat,replay,ledger,history);
|
|
75
90
|
const columns={};
|
|
76
91
|
for(const table of manifest.tables) {
|
|
77
92
|
const target=cat.tables.find(t=>t.name===table.name),label=':'+table.name;
|
|
@@ -3,18 +3,40 @@ import { canonicalJson, sha256, base64, unbase64, utf8 } from './hash.mjs';
|
|
|
3
3
|
import { platformTables, platformTable, identifier, matches } from './bookkeeping.mjs';
|
|
4
4
|
/** Platform bookkeeping is mandatory; operator identity remains a seed choice. */
|
|
5
5
|
export const excludedTables=Object.freeze([...platformTables,'oa_%','oc_%']);
|
|
6
|
-
const fields=['product','excludeTables','includeTables','derivedTables','secretColumns','exportRow','replaceableTables','requiredConfiguration','validateStaged','acceptedSchemas','buckets','extraPayloads','partBytes'];
|
|
6
|
+
const fields=['product','excludeTables','includeTables','derivedTables','secretColumns','exportRow','importSafety','importRow','replaceableTables','requiredConfiguration','validateStaged','acceptedSchemas','buckets','extraPayloads','partBytes'];
|
|
7
7
|
const name=value=>typeof value==='string' && value.length>0 && value.length<=512 && !/[\x00-\x1f]/.test(value);
|
|
8
8
|
const names=value=>Array.isArray(value) && value.every(name) && new Set(value).size===value.length;
|
|
9
|
+
const record=value=>value && typeof value==='object' && !Array.isArray(value) && [Object.prototype,null].includes(Object.getPrototypeOf(value));
|
|
10
|
+
const safeCell=value=>{
|
|
11
|
+
if(value===null || typeof value==='string' || (typeof value==='number' && Number.isFinite(value)))return true;
|
|
12
|
+
if(!record(value) || Object.keys(value).length!==1 || typeof value.blob!=='string')return false;
|
|
13
|
+
try{unbase64(value.blob);return true;}catch{return false;}
|
|
14
|
+
};
|
|
9
15
|
export function policy(input) {
|
|
10
16
|
insist(input && typeof input==='object' && Object.keys(input).every(k=>fields.includes(k)) && name(input.product),'invalid_policy');
|
|
11
|
-
const result={excludeTables:[],includeTables:[],derivedTables:[],secretColumns:{},replaceableTables:[],requiredConfiguration:[],acceptedSchemas:{},buckets:{},extraPayloads:{},partBytes:16*1024*1024,...input};
|
|
17
|
+
const result={excludeTables:[],includeTables:[],derivedTables:[],secretColumns:{},importSafety:{},replaceableTables:[],requiredConfiguration:[],acceptedSchemas:{},buckets:{},extraPayloads:{},partBytes:16*1024*1024,...input};
|
|
12
18
|
for(const key of ['excludeTables','includeTables','derivedTables','replaceableTables','requiredConfiguration']) insist(names(result[key]),'invalid_policy_'+key);
|
|
13
19
|
for(const table of result.includeTables) if(platformTable(table)) {
|
|
14
20
|
const error=new PortabilityError('platform_table:'+table);
|
|
15
21
|
error.message='includeTables cannot include platform-owned table "'+table+'".';throw error;
|
|
16
22
|
}
|
|
17
|
-
for(const key of ['exportRow','validateStaged']) insist(result[key]===undefined || typeof result[key]==='function','invalid_policy_'+key);
|
|
23
|
+
for(const key of ['exportRow','importRow','validateStaged']) insist(result[key]===undefined || typeof result[key]==='function','invalid_policy_'+key);
|
|
24
|
+
insist(record(result.importSafety),'invalid_policy_importSafety');
|
|
25
|
+
const safety=Object.create(null);
|
|
26
|
+
for(const [table,columns] of Object.entries(result.importSafety)) {
|
|
27
|
+
insist(name(table) && record(columns) && Object.keys(columns).length>0,'invalid_import_safety:'+table);
|
|
28
|
+
const declared=Object.create(null);
|
|
29
|
+
for(const [column,values] of Object.entries(columns)) {
|
|
30
|
+
insist(name(column) && Array.isArray(values) && values.length>0,'invalid_import_safety:'+table+':'+column);
|
|
31
|
+
const cells=Array.from(values,value=>record(value)?{...value}:value);
|
|
32
|
+
insist(cells.every(safeCell) && new Set(cells.map(canonicalJson)).size===cells.length,'invalid_import_safety:'+table+':'+column);
|
|
33
|
+
// The hook's closure cannot expand the declaration while it runs.
|
|
34
|
+
declared[column]=Object.freeze(cells.map(value=>value && typeof value==='object'?Object.freeze({...value}):value));
|
|
35
|
+
}
|
|
36
|
+
safety[table]=Object.freeze(declared);
|
|
37
|
+
}
|
|
38
|
+
insist(Boolean(result.importRow)===(Object.keys(safety).length>0),'invalid_policy_importSafety_pair');
|
|
39
|
+
result.importSafety=Object.freeze(safety);
|
|
18
40
|
insist(Number.isSafeInteger(result.partBytes) && result.partBytes>=1024 && result.partBytes<=16*1024*1024,'invalid_part_limit');
|
|
19
41
|
for(const [table,columns] of Object.entries(result.secretColumns)) insist(name(table) && names(columns),'invalid_secret_columns');
|
|
20
42
|
for(const [identity,entry] of Object.entries(result.acceptedSchemas)) insist(/^[a-f0-9]{64}$/.test(identity) && typeof entry.upgradeRow==='function','invalid_accepted_schema');
|
|
@@ -63,6 +85,14 @@ export async function catalogue(db,config) {
|
|
|
63
85
|
const entry=tables.find(t=>t.name===table); insist(entry,'secret_table_missing:'+table);
|
|
64
86
|
for(const key of list) {const column=entry.columns.find(c=>c.name===key);insist(column && !column.notnull && !column.pk && !column.generated,'invalid_secret_column:'+table+':'+key);}
|
|
65
87
|
}
|
|
88
|
+
for(const [table,columns] of Object.entries(config.importSafety)) {
|
|
89
|
+
const entry=tables.find(t=>t.name===table);insist(entry,'import_safety_table_missing:'+table);
|
|
90
|
+
for(const [key,values] of Object.entries(columns)) {
|
|
91
|
+
const column=entry.columns.find(c=>c.name===key);
|
|
92
|
+
insist(column && !column.generated && !column.pk,'invalid_import_safety_column:'+table+':'+key);
|
|
93
|
+
for(const value of values)validateRow({...entry,columns:[column]},{[key]:value});
|
|
94
|
+
}
|
|
95
|
+
}
|
|
66
96
|
for(const table of [...config.replaceableTables,...config.includeTables]) insist(tables.some(x=>x.name===table),'policy_table_missing:'+table);
|
|
67
97
|
for(const spec of Object.values(config.buckets)) if(spec.objectReferences) {
|
|
68
98
|
const {table,keyColumn}=spec.objectReferences;
|
|
@@ -5,6 +5,7 @@ import { Readable } from 'node:stream';
|
|
|
5
5
|
import { randomUUID } from 'node:crypto';
|
|
6
6
|
import { readArchive } from './index.mjs';
|
|
7
7
|
import { canonicalJson } from './hash.mjs';
|
|
8
|
+
import { ledgerReader } from './ledgers.mjs';
|
|
8
9
|
export async function* fileRecords(path,reader=readArchive) {
|
|
9
10
|
const file = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW);
|
|
10
11
|
try { yield* reader(Readable.toWeb(file.createReadStream({ autoClose: false }))); }
|
|
@@ -62,15 +63,20 @@ export async function exportArchive(service, path, transient, progress) {
|
|
|
62
63
|
export async function importArchive(service, path, transient, journal, progress) {
|
|
63
64
|
let id, manifest, closing, complete = false, staged = 0;
|
|
64
65
|
try {
|
|
65
|
-
// Validate both complete archives before creating any restore state.
|
|
66
|
+
// Validate both complete archives before creating any restore state. An old
|
|
67
|
+
// archive's own migration ledger places it in the destination's history
|
|
68
|
+
// (bridge.mjs); import binds the rows read here to the parts it stages.
|
|
66
69
|
let later;
|
|
70
|
+
const ledger = ledgerReader();
|
|
67
71
|
progress?.({ phase: 'checking', direction: 'import' });
|
|
68
72
|
for (const file of [path, journal].filter(Boolean)) for await (const record of fileRecords(file)) {
|
|
69
73
|
if (file === journal && record.kind === 'manifest') later = { sourceIdentity: record.sourceIdentity, epoch: record.epoch, journal: record.journal };
|
|
74
|
+
if (file === path) ledger.add(record);
|
|
70
75
|
}
|
|
76
|
+
const ledgers = ledger.ledgers();
|
|
71
77
|
for await (const record of fileRecords(path)) {
|
|
72
78
|
if (record.kind === 'manifest') {
|
|
73
|
-
manifest = record; id = await service.imports.create(manifest);
|
|
79
|
+
manifest = record; id = await service.imports.create(manifest, undefined, ledgers ? { ledgers } : {});
|
|
74
80
|
const { skippedTables } = await service.imports.status(id);
|
|
75
81
|
if (skippedTables?.length) progress?.({ phase: 'skipped', tables: skippedTables });
|
|
76
82
|
}
|
|
@@ -83,6 +89,6 @@ export async function importArchive(service, path, transient, journal, progress)
|
|
|
83
89
|
}
|
|
84
90
|
progress?.({ phase: 'finishing', total: manifest.partCount });
|
|
85
91
|
const status = await service.imports.finalize(id, { contentHash: closing.contentHash, archiveHash: closing.archiveHash, ...(later ? { laterJournal: later } : {}) }); complete = true;
|
|
86
|
-
return { ...receipt(manifest, closing, transient), ...(status.bridged ? { bridged: status.bridged } : {}), ...(status.skippedTables ? { skippedTables: status.skippedTables } : {}) };
|
|
92
|
+
return { ...receipt(manifest, closing, transient), ...(status.bridged ? { bridged: status.bridged } : {}), ...(status.skippedTables ? { skippedTables: status.skippedTables } : {}), ...(status.safetyChanges ? { safetyChanges: status.safetyChanges } : {}) };
|
|
87
93
|
} finally { if (id && !complete) await service.imports.abort(id); }
|
|
88
94
|
}
|
|
@@ -25,6 +25,7 @@ export async function createPortabilityHandler({db,buckets,policy,appOrigin,base
|
|
|
25
25
|
if(id)insist(await service.actor(id)===actor,'job_not_found',404);
|
|
26
26
|
insist(['exports','imports'].includes(collection) && route.length<=4,'route_not_found',404);
|
|
27
27
|
const api=service[collection];
|
|
28
|
+
if(collection==='imports' && request.method==='GET' && !id)return json({active:await api.active(actor)});
|
|
28
29
|
if(request.method==='POST' && !id) {
|
|
29
30
|
const body=await boundedJson(request,maxBodyBytes);
|
|
30
31
|
if(collection==='imports'){insist(body.confirmed===true,'confirmation_required');return json({id:await api.create(body.manifest,actor)},201);}
|
|
@@ -37,6 +38,6 @@ export async function createPortabilityHandler({db,buckets,policy,appOrigin,base
|
|
|
37
38
|
if(collection==='imports' && request.method==='PUT' && id && action==='parts' && /^(0|[1-9]\d*)$/.test(index??''))return json(await api.uploadPart(id,Number(index),await boundedJson(request,maxBodyBytes)));
|
|
38
39
|
if(collection==='imports' && request.method==='POST' && id && action==='finalize' && !index)return json(await api.finalize(id,await boundedJson(request,maxBodyBytes)));
|
|
39
40
|
throw new PortabilityError('route_not_found',404);
|
|
40
|
-
}catch(e){return json({error:e instanceof PortabilityError?e.code:'operation_failed'},e instanceof PortabilityError?e.status:500);}
|
|
41
|
+
}catch(e){return json({error:e instanceof PortabilityError?e.code:'operation_failed',...(e instanceof PortabilityError && e.code==='import_running'?{jobId:e.jobId,abortableAt:e.abortableAt}:{})},e instanceof PortabilityError?e.status:500);}
|
|
41
42
|
};
|
|
42
43
|
}
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
import { statement, guard, job, PortabilityError } from './storage.mjs';
|
|
2
|
+
|
|
3
|
+
// Inactivity, not total restore time: the measured 20k-row round trip takes
|
|
4
|
+
// 171s (README). Finalize also awaits unbounded product hooks and multipart R2
|
|
5
|
+
// work, so the owning request renews every 30s throughout those awaits.
|
|
6
|
+
export const IMPORT_LEASE_MS=5*60*1000;
|
|
7
|
+
const RENEW_MS=30*1000;
|
|
8
|
+
const active=row=>['staging','applying'].includes(row.phase);
|
|
9
|
+
|
|
10
|
+
// Private transfer bookkeeping can be added to already-installed destinations
|
|
11
|
+
// without changing product migrations or the archive's schema identity. Keep
|
|
12
|
+
// it separate from metadata: staging/normalization compare that JSON by CAS.
|
|
13
|
+
export async function ensureImportLeases(db) {
|
|
14
|
+
await db.prepare(`CREATE TABLE IF NOT EXISTS rer_portability_import_leases (
|
|
15
|
+
job TEXT PRIMARY KEY REFERENCES rer_portability_jobs(id) ON DELETE CASCADE,
|
|
16
|
+
expires_at INTEGER NOT NULL, holder TEXT
|
|
17
|
+
)`).run();
|
|
18
|
+
// An import left by an older runtime gets one conservative grace period.
|
|
19
|
+
await statement(db,`INSERT OR IGNORE INTO rer_portability_import_leases(job,expires_at)
|
|
20
|
+
SELECT j.id,? FROM rer_portability_jobs j JOIN rer_portability_state s ON s.import_job=j.id
|
|
21
|
+
WHERE j.kind='import' AND j.phase IN ('staging','applying')`,Date.now()+IMPORT_LEASE_MS).run();
|
|
22
|
+
}
|
|
23
|
+
export const createImportLease=(db,id)=>statement(db,'INSERT INTO rer_portability_import_leases(job,expires_at) VALUES(?,?)',id,Date.now()+IMPORT_LEASE_MS);
|
|
24
|
+
const leaseRow=(db,id)=>statement(db,'SELECT * FROM rer_portability_import_leases WHERE job=?',id).first();
|
|
25
|
+
export async function importLeaseStatus(db,row) {
|
|
26
|
+
const lease=await leaseRow(db,row.id);
|
|
27
|
+
return {abortableAt:active(row)&&lease?new Date(lease.expires_at).toISOString():null,stale:active(row)&&!!lease&&lease.expires_at<=Date.now()};
|
|
28
|
+
}
|
|
29
|
+
export async function importRunning(db,id) {
|
|
30
|
+
const error=new PortabilityError('import_running',409);
|
|
31
|
+
error.jobId=id;error.abortableAt=(await importLeaseStatus(db,await job(db,id,'import'))).abortableAt;
|
|
32
|
+
return error;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
// Upload, finalize and abort each claim one request. Renewal and takeover are
|
|
36
|
+
// conditional writes on that claim; an expired request cannot revive itself.
|
|
37
|
+
// Every mutating D1 batch is fenced by the same claim, including publication.
|
|
38
|
+
export async function withImportLease(context,id,operation,{abort=false,staleOnly=false,soleOperator=false}={}) {
|
|
39
|
+
const db=context.db,holder=crypto.randomUUID(),now=Date.now(),lease=await leaseRow(db,id);
|
|
40
|
+
if(!lease)throw new PortabilityError('import_lease_expired',409);
|
|
41
|
+
if(staleOnly && lease.expires_at>now)throw await importRunning(db,id);
|
|
42
|
+
if(!abort && lease.expires_at<=now)throw new PortabilityError('import_lease_expired',409);
|
|
43
|
+
if(!soleOperator && lease.holder && lease.expires_at>now)throw await importRunning(db,id);
|
|
44
|
+
// Like staging/publication, claim by a transactional SQL guard. Transport
|
|
45
|
+
// adapters need not expose provider-specific UPDATE result metadata.
|
|
46
|
+
try {
|
|
47
|
+
await db.batch([guard(db,`EXISTS(SELECT 1 FROM rer_portability_import_leases WHERE job=? AND expires_at=? AND holder IS ?)
|
|
48
|
+
AND EXISTS(SELECT 1 FROM rer_portability_state WHERE id=1 AND import_job=?)`,id,lease.expires_at,lease.holder,id),
|
|
49
|
+
statement(db,'UPDATE rer_portability_import_leases SET holder=?,expires_at=? WHERE job=?',holder,now+IMPORT_LEASE_MS,id),
|
|
50
|
+
db.prepare('DELETE FROM rer_portability_guard')]);
|
|
51
|
+
}catch(error){
|
|
52
|
+
const current=await leaseRow(db,id);
|
|
53
|
+
if(current?.holder!==lease.holder || current?.expires_at!==lease.expires_at)throw await importRunning(db,id);
|
|
54
|
+
throw error;
|
|
55
|
+
}
|
|
56
|
+
let stopped=false,timer,pending=Promise.resolve(),failure;
|
|
57
|
+
const owned=async()=>{
|
|
58
|
+
const current=await leaseRow(db,id);
|
|
59
|
+
return current?.holder===holder && current.expires_at>Date.now();
|
|
60
|
+
};
|
|
61
|
+
const renew=async()=>{
|
|
62
|
+
const at=Date.now();
|
|
63
|
+
try {
|
|
64
|
+
await db.batch([guard(db,'EXISTS(SELECT 1 FROM rer_portability_import_leases WHERE job=? AND holder=? AND expires_at>?)',id,holder,at),
|
|
65
|
+
statement(db,'UPDATE rer_portability_import_leases SET expires_at=? WHERE job=?',at+IMPORT_LEASE_MS,id),
|
|
66
|
+
db.prepare('DELETE FROM rer_portability_guard')]);
|
|
67
|
+
}catch(error){if(!await owned())throw new PortabilityError('import_lease_expired',409);throw error;}
|
|
68
|
+
};
|
|
69
|
+
const schedule=()=>{
|
|
70
|
+
timer=setTimeout(()=>{
|
|
71
|
+
pending=renew().catch(error=>{failure=error;}).finally(()=>{if(!stopped && !failure)schedule();});
|
|
72
|
+
return pending;
|
|
73
|
+
},RENEW_MS);
|
|
74
|
+
};
|
|
75
|
+
const fence=()=>guard(db,`EXISTS(SELECT 1 FROM rer_portability_import_leases l
|
|
76
|
+
JOIN rer_portability_state s ON s.import_job=l.job WHERE s.id=1 AND l.job=? AND l.holder=? AND l.expires_at>?)`,id,holder,Date.now());
|
|
77
|
+
const native=new WeakMap();
|
|
78
|
+
const batch=async statements=>{
|
|
79
|
+
if(failure)throw failure;
|
|
80
|
+
try {
|
|
81
|
+
const result=await db.batch([fence(),...statements.map(s=>native.get(s)??s),db.prepare('DELETE FROM rer_portability_guard')]);
|
|
82
|
+
return result.slice(1,-1);
|
|
83
|
+
}catch(error){if(!await owned())throw new PortabilityError('import_lease_expired',409);throw error;}
|
|
84
|
+
};
|
|
85
|
+
const prepare=sql=>{
|
|
86
|
+
const wrap=raw=>{const wrapped=new Proxy(raw,{get(target,key){
|
|
87
|
+
if(key==='bind')return (...values)=>wrap(target.bind(...values));
|
|
88
|
+
if(key==='run')return async()=> (await batch([target]))[0];
|
|
89
|
+
const value=target[key];return typeof value==='function'?value.bind(target):value;
|
|
90
|
+
}});native.set(wrapped,raw);return wrapped;};
|
|
91
|
+
return wrap(db.prepare(sql));
|
|
92
|
+
};
|
|
93
|
+
schedule();
|
|
94
|
+
try{return await operation({...context,db:{prepare,batch},importLease:{owned}});}
|
|
95
|
+
finally {
|
|
96
|
+
stopped=true;clearTimeout(timer);await pending;
|
|
97
|
+
await statement(db,'UPDATE rer_portability_import_leases SET holder=NULL WHERE job=? AND holder=?',id,holder).run();
|
|
98
|
+
}
|
|
99
|
+
}
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
import { encodedRow, validateRow, bindCell } from './catalogue.mjs';
|
|
2
|
+
import { readCopy, copyRowid } from './copies.mjs';
|
|
3
|
+
import { canonicalJson } from './hash.mjs';
|
|
4
|
+
import { statement, insist, quote } from './storage.mjs';
|
|
5
|
+
import { RPC_LIMIT, rpcBytes } from '../d1-transport/index.mjs';
|
|
6
|
+
|
|
7
|
+
const same=(a,b)=>canonicalJson(a)===canonicalJson(b);
|
|
8
|
+
const location=(table,column)=>table.name+(column===undefined?'':':'+column);
|
|
9
|
+
function checkResult(config,table,before,after) {
|
|
10
|
+
insist(after && typeof after==='object' && !Array.isArray(after) && typeof after.then!=='function','import_row_must_be_pure:'+table.name,409);
|
|
11
|
+
const keys=Reflect.ownKeys(after),original=Object.keys(before);
|
|
12
|
+
const structural=[...keys.filter(k=>!original.includes(k)),...original.filter(k=>!Object.prototype.propertyIsEnumerable.call(after,k))];
|
|
13
|
+
insist(!structural.length,'import_row_columns:'+location(table,String(structural[0])),409);
|
|
14
|
+
validateRow(table,after);
|
|
15
|
+
for(const column of original) {
|
|
16
|
+
insist(after[column]!==undefined,'import_row_columns:'+location(table,column),409);
|
|
17
|
+
if(same(before[column],after[column]))continue;
|
|
18
|
+
insist(!table.columns.some(c=>c.name===column && c.pk),'import_row_key:'+location(table,column),409);
|
|
19
|
+
const allowed=config.importSafety[table.name][column];
|
|
20
|
+
insist(allowed,'import_row_undeclared:'+location(table,column),409);
|
|
21
|
+
insist(allowed.some(value=>same(value,after[column])),'import_row_unsafe_value:'+location(table,column),409);
|
|
22
|
+
}
|
|
23
|
+
return structuredClone(after);
|
|
24
|
+
}
|
|
25
|
+
function mapImportRow(config,table,before) {
|
|
26
|
+
const apply=row=>checkResult(config,table,row,config.importRow(table.name,structuredClone(row)));
|
|
27
|
+
const after=apply(before),again=apply(before),twice=apply(after);
|
|
28
|
+
for(const [result,reason] of [[again,'deterministic'],[twice,'idempotent']]) {
|
|
29
|
+
const column=Object.keys(before).find(key=>!same(after[key],result[key]));
|
|
30
|
+
insist(column===undefined,'import_row_not_'+reason+':'+location(table,column),409);
|
|
31
|
+
}
|
|
32
|
+
return after;
|
|
33
|
+
}
|
|
34
|
+
function countChanges(summary,table,before,after) {
|
|
35
|
+
for(const column of Object.keys(before))if(!same(before[column],after[column])) {
|
|
36
|
+
let entry=summary.find(x=>x.table===table.name && x.column===column && same(x.value,after[column]));
|
|
37
|
+
if(!entry){entry={table:table.name,column,value:after[column],rows:0};summary.push(entry);}
|
|
38
|
+
entry.rows++;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
const guardSql=`INSERT INTO rer_portability_guard(value) SELECT CASE WHEN (
|
|
42
|
+
EXISTS(SELECT 1 FROM rer_portability_jobs WHERE id=? AND metadata=?)
|
|
43
|
+
AND (SELECT import_job FROM rer_portability_state WHERE id=1)=?
|
|
44
|
+
) THEN 1 ELSE 0 END`;
|
|
45
|
+
const checkpoint=(row,before,metadata,queries)=>[
|
|
46
|
+
{sql:guardSql,params:[row.id,before,row.id]},...queries,
|
|
47
|
+
{sql:'UPDATE rer_portability_jobs SET phase=?,completed=?,metadata=? WHERE id=?',params:[row.phase,row.completed,canonicalJson(metadata),row.id]},
|
|
48
|
+
{sql:'DELETE FROM rer_portability_guard'},
|
|
49
|
+
];
|
|
50
|
+
// Each page's staged updates, cursor and value-free change counts commit together.
|
|
51
|
+
// A resumed finalize never transforms an already-normalized row or recounts it.
|
|
52
|
+
export async function normalizeStaged(context,row) {
|
|
53
|
+
const {db,config}=context;if(!config.importRow)return;
|
|
54
|
+
for(const table of row.metadata.targets.filter(t=>Object.hasOwn(config.importSafety,t.name))) {
|
|
55
|
+
const copy=row.metadata.copies[table.name],count=row.metadata.stagedCounts?.[table.name]??0;
|
|
56
|
+
let offset=row.metadata.safetyProgress?.[table.name]??0;
|
|
57
|
+
while(offset<count) {
|
|
58
|
+
let before=canonicalJson(row.metadata),queries=[];
|
|
59
|
+
const commit=async()=>{
|
|
60
|
+
await db.batch(checkpoint(row,before,row.metadata,queries).map(q=>statement(db,q.sql,...(q.params??[]))));
|
|
61
|
+
before=canonicalJson(row.metadata);queries=[];
|
|
62
|
+
};
|
|
63
|
+
for(const {row:raw,meta} of await readCopy(db,table,copy,offset,100,true)) {
|
|
64
|
+
const original=encodedRow(table,raw),after=mapImportRow(config,table,original);
|
|
65
|
+
const columns=Object.keys(original).filter(key=>!same(original[key],after[key]));
|
|
66
|
+
insist(Number.isSafeInteger(meta.rer_portability_copy_rowid),'copy_rowid');
|
|
67
|
+
const query=columns.length?{sql:`UPDATE ${quote(copy)} SET ${columns.map(c=>quote(c)+'=?').join(',')} WHERE ${copyRowid(table)}=${meta.rer_portability_copy_rowid}`,params:columns.map(c=>bindCell(after[c]))}:undefined;
|
|
68
|
+
// Reserve room for a metadata checkpoint and its concurrency guard.
|
|
69
|
+
const summary=structuredClone(row.metadata.safetyChanges??[]);countChanges(summary,table,original,after);
|
|
70
|
+
const metadata={...row.metadata,safetyChanges:summary,safetyProgress:{...row.metadata.safetyProgress,[table.name]:offset+1}};
|
|
71
|
+
const pending=[...queries,...(query?[query]:[])];
|
|
72
|
+
if(rpcBytes(checkpoint(row,before,metadata,pending))>RPC_LIMIT && queries.length)await commit();
|
|
73
|
+
insist(rpcBytes(checkpoint(row,before,metadata,[...queries,...(query?[query]:[])]))<=RPC_LIMIT,'D1_RPC_TRANSPORT_LIMIT',413);
|
|
74
|
+
if(query)queries.push(query);
|
|
75
|
+
row.metadata=metadata;offset++;
|
|
76
|
+
}
|
|
77
|
+
await commit();
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
}
|