@runeditrun/rer 0.2.66 → 0.2.68

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. package/cli/business/vendors/README.md +38 -2
  2. package/cli/business/vendors/calendly.mjs +2 -2
  3. package/cli/business/vendors/common.mjs +94 -0
  4. package/cli/business/vendors/credential-names.mjs +124 -0
  5. package/cli/business/vendors/pagerduty.mjs +6 -15
  6. package/cli/business/vendors/sentry.mjs +4 -3
  7. package/cli/business/vendors/uptimerobot.mjs +7 -6
  8. package/cli/import-summary.mjs +7 -0
  9. package/cli/output.mjs +2 -0
  10. package/cli/portability-worker.mjs +10 -4
  11. package/cli/portability.mjs +1 -0
  12. package/cli/production-mail.mjs +4 -1
  13. package/global/CHANGELOG.md +14 -0
  14. package/modules/deploy-kit/d1-import.mjs +3 -1
  15. package/modules/deploy-kit/migrations.mjs +1 -0
  16. package/modules/deploy-kit/owner-cli.mjs +1 -1
  17. package/modules/deploy-kit/provider.mjs +16 -4
  18. package/modules/portability/README.md +73 -8
  19. package/modules/portability/bridge.mjs +44 -29
  20. package/modules/portability/catalogue.mjs +33 -3
  21. package/modules/portability/files.mjs +9 -3
  22. package/modules/portability/import-safety.mjs +80 -0
  23. package/modules/portability/import.mjs +13 -7
  24. package/modules/portability/index.d.mts +9 -3
  25. package/modules/portability/index.mjs +1 -1
  26. package/modules/portability/ledgers.mjs +83 -0
  27. package/modules/portability/legacy-files.mjs +7 -4
  28. package/modules/portability/migration-history.mjs +3 -2
  29. package/modules/portability/migration-statements.mjs +83 -9
  30. package/modules/release/README.md +25 -7
  31. package/modules/release/cleanup.mjs +16 -0
  32. package/modules/release/produce.mjs +11 -4
  33. package/modules/release/publication-scan.mjs +18 -8
  34. package/package.json +1 -1
  35. package/scripts/publish-check/README.md +5 -2
  36. package/scripts/publish-check/destination.py +8 -1
  37. package/scripts/publish-check/git_history.py +7 -6
  38. package/scripts/publish-check/history_hydration.py +32 -9
@@ -41,6 +41,8 @@ schema at construction. Apply the bookkeeping migration first. Policy fields:
41
41
  | `derivedTables` | Virtual/FTS tables to regenerate; their shadow tables are excluded. |
42
42
  | `secretColumns` | Per-table nullable base columns omitted at export and written as NULL. Primary keys and generated columns cannot be secret. |
43
43
  | `exportRow(table,row)` | Pure, synchronous, idempotent transformation of encoded base cells. Restore writes its result verbatim; SQLite recomputes generated cells. |
44
+ | `importSafety` | Per-table, per-column nonempty finite sets of safe encoded cells, paired with `importRow`. Only writable non-key columns may be declared. |
45
+ | `importRow(table,row)` | Pure, synchronous, deterministic, idempotent normalization of staged rows in declared tables, before `validateStaged` and apply. Changes are bounded by `importSafety`. |
44
46
  | `replaceableTables` | Bootstrap rows replaced in the atomic restore batch. Rows the destination's own migrations insert need no entry where the runner passes `migrations` (see "Migration-seeded rows"). Populated excluded dependents reachable through modifying FK actions refuse replacement. |
45
47
  | `requiredConfiguration` | Setting names returned after restore for the owner to re-enter. |
46
48
  | `validateStaged({db,tables,read,manifest})` | Read-only product validation before any object write; false or a thrown error refuses restore. `tables` names the staging tables for bounded SQL queries; `read` is a convenience for small tables. |
@@ -55,7 +57,7 @@ belong to the current runtime or environment and never travel:
55
57
  | Tables (exact names or prefixes ending in `%`) | Reason |
56
58
  | --- | --- |
57
59
  | `sqlite_%`, `_cf_%` | SQLite and Cloudflare internals. |
58
- | `d1_migrations`, `_rer_deploy_migrations`, `_rer_portability_migrations`, `rer_migrations` | Destination migration history. Current Base uses the D1 and `_rer_` ledgers; `rer_migrations` is excluded for historical compatibility. |
60
+ | `d1_migrations`, `_rer_deploy_migrations`, `_rer_portability_migrations`, `rer_migrations` | Destination migration history. Base uses the D1 and `_rer_` ledgers; RER Platform's runner writes `rer_migrations`. An old archive that still carries its source's ledger is placed by it (see below); its rows are never restored. |
59
61
  | `rer_portability_%` | Transfer jobs, snapshots, staging, journals and recovery bookkeeping. |
60
62
  | `rer_mail_development_outbox`, `rer_ai_development_outbox` | Local development effect captures. |
61
63
  | `rer_dependency_keys`, `rer_dependency_pending`, `rer_dependency_codes` | Provider secrets and transient linking credentials. |
@@ -151,6 +153,49 @@ the bytes. Large objects span parts and restore through multipart upload with at
151
153
  most one 5 MiB upload buffer. Extra payload consumers own their own transaction,
152
154
  idempotence and recovery; D1 cannot roll back Durable Object or external state.
153
155
 
156
+ ### Bounded import normalization
157
+
158
+ Use `importSafety` with `importRow` when restored records must start in a safe
159
+ state, even if an older archive contains active connections or credentials:
160
+
161
+ ```js
162
+ export default {
163
+ product: 'my-product',
164
+ importSafety: {connections: {status: ['inactive'], token: [null]}},
165
+ importRow(table, row) {
166
+ return {...row, status: 'inactive', token: null};
167
+ },
168
+ };
169
+ ```
170
+
171
+ The hook is called only for tables declared in `importSafety`, after staging
172
+ and before `validateStaged` or apply. Mandatory development/provider-data refusals
173
+ run first; normalization cannot bypass them. It must return exactly the writable columns
174
+ of its input row, preserve primary keys and every undeclared column, and change
175
+ declared columns only to their listed safe values. Unchanged values need not be
176
+ in the safe set. The hook must be pure, synchronous, deterministic and idempotent;
177
+ keep external effects out of it. Generated columns recompute from the normalized
178
+ base cells. Without this pair, import behavior is unchanged.
179
+
180
+ Construction refuses an unpaired hook or declaration, unknown table or column
181
+ names, primary keys, generated columns, empty safe sets and malformed encoded
182
+ cells. Safe values use the ordinary cell encoding: null, strings, finite numbers
183
+ or `{blob:base64}`. The destination policy supplies the hook; an archive cannot
184
+ supply executable normalization.
185
+
186
+ The import job persistently journals the summary in `metadata.safetyChanges`.
187
+ `imports.status(id)` and the `importArchive` receipt expose it as `safetyChanges`,
188
+ an array of `{table,column,value,rows}`. Each entry counts actual changed rows for
189
+ that column and safe replacement value; a row changed in two columns contributes
190
+ to two entries. The summary contains no original values or row records. Durable
191
+ checkpoints keep retries from counting changes twice. Human `rer import` output
192
+ prints these counts too, for example `3 automations imported with status="inactive"`.
193
+
194
+ Archive integrity checks still verify the original archived contents. This
195
+ summary is separate from the erasure journal: normalization does not bypass its
196
+ fence or change journal adoption. Existing atomic D1 apply, owned-object rollback,
197
+ extra-payload recovery responsibilities and transfer erasure behavior still apply.
198
+
154
199
  ## Archive and durable jobs
155
200
 
156
201
  The UTF-8 JSON-lines archive has a manifest, indexed checksummed parts and a
@@ -210,16 +255,30 @@ table). This applies to normal v2 import and to backup-only v1 recovery.
210
255
 
211
256
  The Base CLI runners (local Workers and container) pass `createPortability` a
212
257
  `migrations` replayer (`migration-history.mjs`, Node only) built from the
213
- destination's applied migration ledger and migration files. Import replays them
214
- into a scratch database to find the archive's place in that history:
258
+ destination's applied migration ledger and migration files, with each file's
259
+ SHA-256. The ledger is `_rer_deploy_migrations`, `d1_migrations` (Wrangler or
260
+ the deploy kit) or, on an installation RER Platform installed, its exact
261
+ `rer_migrations(name, sha256, applied_at)` ledger, read in application order
262
+ (rowid) and refused while it holds a partially applied file. Import replays
263
+ them into a scratch database to find the archive's place in that history:
215
264
 
216
265
  - Archives record the applied migration names in `manifest.migrations` (since
217
266
  additive restore). Every recorded migration must be applied here, and
218
267
  replaying exactly those must give the archive's schema identity.
219
- - Older v2 and all v1 archives do not record them; the latest prefix of the
220
- applied migrations that gives the archive's schema identity is used. Migrations
221
- that change no archived table (indexes, excluded tables) cannot be told apart
222
- there and are reported as later.
268
+ - Older archives that still carry their source's migration ledger as a table
269
+ (`rer_migrations` and the other ledgers above, from before they became
270
+ bookkeeping) are placed by it. Every name it records must be applied here,
271
+ with the same file hash (the destination ledger's, else the replayed file's),
272
+ or the import is refused with `schema_mismatch:migration_history:<migration>`
273
+ naming the first that differs; replaying exactly those must give the archive's
274
+ schema identity. Every other applied migration is later, wherever the
275
+ destination's history orders it. `importArchive` reads the ledger rows from
276
+ the verified file and passes them to `imports.create(manifest, actor,
277
+ {ledgers})`; staging refuses any part whose ledger rows differ from them.
278
+ - Other older v2 and all v1 archives do not record them; the latest prefix of
279
+ the applied migrations that gives the archive's schema identity is used.
280
+ Migrations that change no archived table (indexes, excluded tables) cannot be
281
+ told apart there and are reported as later.
223
282
 
224
283
  Restored rows never pass through the later migrations, so each of them may
225
284
  contain only `CREATE TABLE` (not `AS SELECT`), `CREATE INDEX` and
@@ -227,7 +286,11 @@ contain only `CREATE TABLE` (not `AS SELECT`), `CREATE INDEX` and
227
286
  restorable tables. A later migration with any other statement that names a
228
287
  restorable table, such as an `INSERT`, `UPDATE` or `DELETE` that purges or
229
288
  backfills rows or a trigger, is refused with `schema_mismatch`; statements that
230
- touch only excluded tables are allowed. For archives without a migration record,
289
+ touch only excluded tables are allowed. An archived table the archive holds no
290
+ rows of restores empty, so an `UPDATE` or `DELETE` of it changes nothing and is
291
+ allowed unless it also names a table the archive holds rows of (which it could
292
+ read into its writes); an `INSERT` into it, or any other statement naming it,
293
+ is still refused. For archives without a migration record,
231
294
  every migration after the earliest prefix that gives the archive's schema is
232
295
  checked, since the archive may predate any of them.
233
296
 
@@ -496,11 +559,13 @@ modules/portability/provider-data.mjs
496
559
  modules/portability/generated.mjs
497
560
  modules/portability/restore-safety.mjs
498
561
  modules/portability/bridge.mjs
562
+ modules/portability/ledgers.mjs
499
563
  modules/portability/migration-statements.mjs
500
564
  modules/portability/legacy-safety.mjs
501
565
  modules/portability/export.mjs
502
566
  modules/portability/import.mjs
503
567
  modules/portability/import-staging.mjs
568
+ modules/portability/import-safety.mjs
504
569
  modules/portability/payloads.mjs
505
570
  modules/portability/journal.mjs
506
571
  modules/portability/http.mjs
@@ -1,35 +1,31 @@
1
1
  import { all, quote, insist } from './storage.mjs';
2
2
  import { canonicalJson } from './hash.mjs';
3
3
  import { tokens, identifier, folded } from './generated.mjs';
4
- import { changesData } from './migration-statements.mjs';
4
+ import { changesData, reachingTables } from './migration-statements.mjs';
5
+ import { destinationLedger } from './ledgers.mjs';
5
6
 
6
- // Applied product migrations in the ledger's own order. A state-bearing D1
7
- // ledger may hold an unacknowledged migration whose effects are unknown; neither
8
- // an archive nor a restore can attest its history until that is resolved.
7
+ // Applied product migrations in the ledger's own order (ledgers.mjs). A
8
+ // state-bearing ledger may hold an unacknowledged migration whose effects are
9
+ // unknown; neither an archive nor a restore can attest its history until that
10
+ // is resolved.
9
11
  export async function appliedMigrations(db) {
10
- const ledgers=(await all(db,"SELECT name FROM sqlite_master WHERE type='table' AND name IN ('_rer_deploy_migrations','d1_migrations')")).map(x=>x.name);
11
- if(ledgers.includes('_rer_deploy_migrations'))return (await all(db,"SELECT name FROM _rer_deploy_migrations WHERE state='applied' ORDER BY name")).map(x=>x.name);
12
- if(!ledgers.includes('d1_migrations'))return;
13
- const columns=(await all(db,'PRAGMA table_info(d1_migrations)')).map(x=>x.name);
14
- const tracked=columns.includes('state') || columns.includes('sha256');
15
- if(tracked)insist(columns.includes('state') && columns.includes('sha256'),'schema_mismatch:migration_history',409);
16
- const rows=await all(db,`SELECT name${tracked?',state':''} FROM d1_migrations ORDER BY id`);
17
- if(tracked)insist(rows.every(row=>row.state==='applied'),'schema_mismatch:uncertain_migrations',409);
18
- return rows.map(row=>row.name);
12
+ return (await destinationLedger(db))?.map(m=>m.name);
19
13
  }
20
14
  // Where an archive sits in the destination's applied migrations: those its
21
15
  // rows passed through (earlier) and those applied after it (unapplied).
22
16
  // migrations() is the runner's replay of the destination's applied migrations
23
17
  // (migration-history.mjs). The archive's schema must be the replay of the
24
- // migrations it records or, for archives that predate that record, of an
25
- // earlier prefix. Restored rows never pass through a later migration, so one
26
- // that writes or rewrites restorable data is refused, whether or not it changes
27
- // the schema.
28
- async function placement(manifest,cat,replay,applied) {
18
+ // migrations it records or, for archives that predate that record, of those
19
+ // its own migration ledger records or, without one, of an earlier prefix.
20
+ // Restored rows never pass through a later migration, so one that writes or
21
+ // rewrites restorable data is refused, whether or not it changes the schema.
22
+ async function placement(manifest,cat,replay,ledger,history) {
23
+ const applied=ledger.map(m=>m.name);
29
24
  insist(canonicalJson(replay.names)===canonicalJson(applied) && await replay.schemaAfter(applied)===cat.identity,'schema_mismatch');
25
+ const recorded=manifest.migrations ?? archivedApplied(history?.(),ledger,replay);
30
26
  let earlier,unapplied;
31
- if(manifest.migrations) {
32
- earlier=applied.filter(name=>manifest.migrations.includes(name));
27
+ if(recorded) {
28
+ earlier=applied.filter(name=>recorded.includes(name));
33
29
  insist(await replay.schemaAfter(earlier)===manifest.schema,'schema_mismatch');
34
30
  unapplied=applied.filter(name=>!earlier.includes(name));
35
31
  }else{
@@ -44,12 +40,31 @@ async function placement(manifest,cat,replay,applied) {
44
40
  unapplied=applied.slice(matching[0]);
45
41
  }
46
42
  // Only rows the archive carries can skip a later migration's change. A table
47
- // the archive does not hold is restored empty, so triggers and backfills that
48
- // name only such tables (a newly added module's own) change no restored row.
43
+ // the archive does not hold is restored empty, so backfills that name only
44
+ // such tables (a newly added module's own) change no restored row, unless a
45
+ // trigger from any applied migration carries their writes into one that is
46
+ // restorable. Trigger DDL itself changes no row, nor does an UPDATE or DELETE
47
+ // of a table the archive holds no rows of (migration-statements.mjs).
49
48
  const restorable=cat.tables.map(t=>t.name).filter(name=>manifest.tables.some(t=>t.name===name));
50
- for(const name of unapplied)insist(!replay.statements(name).some(sql=>changesData(sql,restorable)),'schema_mismatch:migration_data:'+name);
49
+ const reached=reachingTables(applied.flatMap(name=>replay.statements(name)),restorable);
50
+ const empty=restorable.filter(name=>!manifest.tables.find(t=>t.name===name).count);
51
+ for(const name of unapplied)insist(!replay.statements(name).some(sql=>changesData(sql,reached,empty)),'schema_mismatch:migration_data:'+name);
51
52
  return {earlier};
52
53
  }
54
+ // An archive without a migration record that carries its source's migration
55
+ // ledger (an old archive from before that ledger became bookkeeping) records
56
+ // exactly which migrations its rows passed through. Each must be one the
57
+ // destination applied, with the same file hash where the ledger records one:
58
+ // the destination ledger's, else the replayed file's. The ledger itself is
59
+ // never restored (bookkeeping.mjs). history: [{name,sha256?}] from
60
+ // ledgers.mjs archivedHistory, undefined when the archive carries no ledger.
61
+ function archivedApplied(history,ledger,replay) {
62
+ for(const m of history??[]) {
63
+ const applied=ledger.find(x=>x.name===m.name);
64
+ insist(applied && (m.sha256===undefined || m.sha256===(applied.sha256??replay.sha256?.(m.name))),'schema_mismatch:migration_history:'+m.name);
65
+ }
66
+ return history?.map(m=>m.name);
67
+ }
53
68
  // An archive with the destination's schema identity can still predate later
54
69
  // migrations that changed only data (a purge or a backfill) or added an index.
55
70
  // Those that change restorable data are refused like any other. Without a
@@ -57,21 +72,21 @@ async function placement(manifest,cat,replay,applied) {
57
72
  // migrations the destination's ledger has outgrown is refused. An archive that
58
73
  // records a migration the destination never applied is newer than it, even
59
74
  // when that migration left the schema as it was (an index, a data change).
60
- export async function sameSchemaMigrations(db,manifest,cat,migrations) {
61
- const applied=await appliedMigrations(db);
75
+ export async function sameSchemaMigrations(db,manifest,cat,migrations,history) {
76
+ const ledger=await destinationLedger(db),applied=ledger?.map(m=>m.name);
62
77
  if(!applied)return;
63
78
  insist(!manifest.migrations || manifest.migrations.every(name=>applied.includes(name)),'schema_mismatch:migrations');
64
79
  if(!migrations){insist(!manifest.migrations || applied.every(name=>manifest.migrations.includes(name)),'schema_mismatch:migrations');return;}
65
- await placement(manifest,cat,await migrations(),applied);
80
+ await placement(manifest,cat,await migrations(),ledger,history);
66
81
  }
67
82
  // An archive made before later migrations restores only when those migrations
68
83
  // are additive: new tables, new columns that take a default, or indexes (which
69
84
  // archives never carry). Anything else stays a schema_mismatch refusal.
70
- export async function additiveBridge(db,manifest,cat,migrations) {
85
+ export async function additiveBridge(db,manifest,cat,migrations,history) {
71
86
  insist(migrations,'schema_mismatch');
72
- const replay=await migrations(),applied=await appliedMigrations(db)??[];
87
+ const replay=await migrations(),ledger=await destinationLedger(db)??[],applied=ledger.map(m=>m.name);
73
88
  insist(!manifest.migrations || manifest.migrations.every(name=>applied.includes(name)),'schema_mismatch:migrations');
74
- const {earlier}=await placement(manifest,cat,replay,applied);
89
+ const {earlier}=await placement(manifest,cat,replay,ledger,history);
75
90
  const columns={};
76
91
  for(const table of manifest.tables) {
77
92
  const target=cat.tables.find(t=>t.name===table.name),label=':'+table.name;
@@ -3,18 +3,40 @@ import { canonicalJson, sha256, base64, unbase64, utf8 } from './hash.mjs';
3
3
  import { platformTables, platformTable, identifier, matches } from './bookkeeping.mjs';
4
4
  /** Platform bookkeeping is mandatory; operator identity remains a seed choice. */
5
5
  export const excludedTables=Object.freeze([...platformTables,'oa_%','oc_%']);
6
- const fields=['product','excludeTables','includeTables','derivedTables','secretColumns','exportRow','replaceableTables','requiredConfiguration','validateStaged','acceptedSchemas','buckets','extraPayloads','partBytes'];
6
+ const fields=['product','excludeTables','includeTables','derivedTables','secretColumns','exportRow','importSafety','importRow','replaceableTables','requiredConfiguration','validateStaged','acceptedSchemas','buckets','extraPayloads','partBytes'];
7
7
  const name=value=>typeof value==='string' && value.length>0 && value.length<=512 && !/[\x00-\x1f]/.test(value);
8
8
  const names=value=>Array.isArray(value) && value.every(name) && new Set(value).size===value.length;
9
+ const record=value=>value && typeof value==='object' && !Array.isArray(value) && [Object.prototype,null].includes(Object.getPrototypeOf(value));
10
+ const safeCell=value=>{
11
+ if(value===null || typeof value==='string' || (typeof value==='number' && Number.isFinite(value)))return true;
12
+ if(!record(value) || Object.keys(value).length!==1 || typeof value.blob!=='string')return false;
13
+ try{unbase64(value.blob);return true;}catch{return false;}
14
+ };
9
15
  export function policy(input) {
10
16
  insist(input && typeof input==='object' && Object.keys(input).every(k=>fields.includes(k)) && name(input.product),'invalid_policy');
11
- const result={excludeTables:[],includeTables:[],derivedTables:[],secretColumns:{},replaceableTables:[],requiredConfiguration:[],acceptedSchemas:{},buckets:{},extraPayloads:{},partBytes:16*1024*1024,...input};
17
+ const result={excludeTables:[],includeTables:[],derivedTables:[],secretColumns:{},importSafety:{},replaceableTables:[],requiredConfiguration:[],acceptedSchemas:{},buckets:{},extraPayloads:{},partBytes:16*1024*1024,...input};
12
18
  for(const key of ['excludeTables','includeTables','derivedTables','replaceableTables','requiredConfiguration']) insist(names(result[key]),'invalid_policy_'+key);
13
19
  for(const table of result.includeTables) if(platformTable(table)) {
14
20
  const error=new PortabilityError('platform_table:'+table);
15
21
  error.message='includeTables cannot include platform-owned table "'+table+'".';throw error;
16
22
  }
17
- for(const key of ['exportRow','validateStaged']) insist(result[key]===undefined || typeof result[key]==='function','invalid_policy_'+key);
23
+ for(const key of ['exportRow','importRow','validateStaged']) insist(result[key]===undefined || typeof result[key]==='function','invalid_policy_'+key);
24
+ insist(record(result.importSafety),'invalid_policy_importSafety');
25
+ const safety=Object.create(null);
26
+ for(const [table,columns] of Object.entries(result.importSafety)) {
27
+ insist(name(table) && record(columns) && Object.keys(columns).length>0,'invalid_import_safety:'+table);
28
+ const declared=Object.create(null);
29
+ for(const [column,values] of Object.entries(columns)) {
30
+ insist(name(column) && Array.isArray(values) && values.length>0,'invalid_import_safety:'+table+':'+column);
31
+ const cells=Array.from(values,value=>record(value)?{...value}:value);
32
+ insist(cells.every(safeCell) && new Set(cells.map(canonicalJson)).size===cells.length,'invalid_import_safety:'+table+':'+column);
33
+ // The hook's closure cannot expand the declaration while it runs.
34
+ declared[column]=Object.freeze(cells.map(value=>value && typeof value==='object'?Object.freeze({...value}):value));
35
+ }
36
+ safety[table]=Object.freeze(declared);
37
+ }
38
+ insist(Boolean(result.importRow)===(Object.keys(safety).length>0),'invalid_policy_importSafety_pair');
39
+ result.importSafety=Object.freeze(safety);
18
40
  insist(Number.isSafeInteger(result.partBytes) && result.partBytes>=1024 && result.partBytes<=16*1024*1024,'invalid_part_limit');
19
41
  for(const [table,columns] of Object.entries(result.secretColumns)) insist(name(table) && names(columns),'invalid_secret_columns');
20
42
  for(const [identity,entry] of Object.entries(result.acceptedSchemas)) insist(/^[a-f0-9]{64}$/.test(identity) && typeof entry.upgradeRow==='function','invalid_accepted_schema');
@@ -63,6 +85,14 @@ export async function catalogue(db,config) {
63
85
  const entry=tables.find(t=>t.name===table); insist(entry,'secret_table_missing:'+table);
64
86
  for(const key of list) {const column=entry.columns.find(c=>c.name===key);insist(column && !column.notnull && !column.pk && !column.generated,'invalid_secret_column:'+table+':'+key);}
65
87
  }
88
+ for(const [table,columns] of Object.entries(config.importSafety)) {
89
+ const entry=tables.find(t=>t.name===table);insist(entry,'import_safety_table_missing:'+table);
90
+ for(const [key,values] of Object.entries(columns)) {
91
+ const column=entry.columns.find(c=>c.name===key);
92
+ insist(column && !column.generated && !column.pk,'invalid_import_safety_column:'+table+':'+key);
93
+ for(const value of values)validateRow({...entry,columns:[column]},{[key]:value});
94
+ }
95
+ }
66
96
  for(const table of [...config.replaceableTables,...config.includeTables]) insist(tables.some(x=>x.name===table),'policy_table_missing:'+table);
67
97
  for(const spec of Object.values(config.buckets)) if(spec.objectReferences) {
68
98
  const {table,keyColumn}=spec.objectReferences;
@@ -5,6 +5,7 @@ import { Readable } from 'node:stream';
5
5
  import { randomUUID } from 'node:crypto';
6
6
  import { readArchive } from './index.mjs';
7
7
  import { canonicalJson } from './hash.mjs';
8
+ import { ledgerReader } from './ledgers.mjs';
8
9
  export async function* fileRecords(path,reader=readArchive) {
9
10
  const file = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW);
10
11
  try { yield* reader(Readable.toWeb(file.createReadStream({ autoClose: false }))); }
@@ -62,15 +63,20 @@ export async function exportArchive(service, path, transient, progress) {
62
63
  export async function importArchive(service, path, transient, journal, progress) {
63
64
  let id, manifest, closing, complete = false, staged = 0;
64
65
  try {
65
- // Validate both complete archives before creating any restore state.
66
+ // Validate both complete archives before creating any restore state. An old
67
+ // archive's own migration ledger places it in the destination's history
68
+ // (bridge.mjs); import binds the rows read here to the parts it stages.
66
69
  let later;
70
+ const ledger = ledgerReader();
67
71
  progress?.({ phase: 'checking', direction: 'import' });
68
72
  for (const file of [path, journal].filter(Boolean)) for await (const record of fileRecords(file)) {
69
73
  if (file === journal && record.kind === 'manifest') later = { sourceIdentity: record.sourceIdentity, epoch: record.epoch, journal: record.journal };
74
+ if (file === path) ledger.add(record);
70
75
  }
76
+ const ledgers = ledger.ledgers();
71
77
  for await (const record of fileRecords(path)) {
72
78
  if (record.kind === 'manifest') {
73
- manifest = record; id = await service.imports.create(manifest);
79
+ manifest = record; id = await service.imports.create(manifest, undefined, ledgers ? { ledgers } : {});
74
80
  const { skippedTables } = await service.imports.status(id);
75
81
  if (skippedTables?.length) progress?.({ phase: 'skipped', tables: skippedTables });
76
82
  }
@@ -83,6 +89,6 @@ export async function importArchive(service, path, transient, journal, progress)
83
89
  }
84
90
  progress?.({ phase: 'finishing', total: manifest.partCount });
85
91
  const status = await service.imports.finalize(id, { contentHash: closing.contentHash, archiveHash: closing.archiveHash, ...(later ? { laterJournal: later } : {}) }); complete = true;
86
- return { ...receipt(manifest, closing, transient), ...(status.bridged ? { bridged: status.bridged } : {}), ...(status.skippedTables ? { skippedTables: status.skippedTables } : {}) };
92
+ return { ...receipt(manifest, closing, transient), ...(status.bridged ? { bridged: status.bridged } : {}), ...(status.skippedTables ? { skippedTables: status.skippedTables } : {}), ...(status.safetyChanges ? { safetyChanges: status.safetyChanges } : {}) };
87
93
  } finally { if (id && !complete) await service.imports.abort(id); }
88
94
  }
@@ -0,0 +1,80 @@
1
+ import { encodedRow, validateRow, bindCell } from './catalogue.mjs';
2
+ import { readCopy, copyRowid } from './copies.mjs';
3
+ import { canonicalJson } from './hash.mjs';
4
+ import { statement, insist, quote } from './storage.mjs';
5
+ import { RPC_LIMIT, rpcBytes } from '../d1-transport/index.mjs';
6
+
7
+ const same=(a,b)=>canonicalJson(a)===canonicalJson(b);
8
+ const location=(table,column)=>table.name+(column===undefined?'':':'+column);
9
+ function checkResult(config,table,before,after) {
10
+ insist(after && typeof after==='object' && !Array.isArray(after) && typeof after.then!=='function','import_row_must_be_pure:'+table.name,409);
11
+ const keys=Reflect.ownKeys(after),original=Object.keys(before);
12
+ const structural=[...keys.filter(k=>!original.includes(k)),...original.filter(k=>!Object.prototype.propertyIsEnumerable.call(after,k))];
13
+ insist(!structural.length,'import_row_columns:'+location(table,String(structural[0])),409);
14
+ validateRow(table,after);
15
+ for(const column of original) {
16
+ insist(after[column]!==undefined,'import_row_columns:'+location(table,column),409);
17
+ if(same(before[column],after[column]))continue;
18
+ insist(!table.columns.some(c=>c.name===column && c.pk),'import_row_key:'+location(table,column),409);
19
+ const allowed=config.importSafety[table.name][column];
20
+ insist(allowed,'import_row_undeclared:'+location(table,column),409);
21
+ insist(allowed.some(value=>same(value,after[column])),'import_row_unsafe_value:'+location(table,column),409);
22
+ }
23
+ return structuredClone(after);
24
+ }
25
+ function mapImportRow(config,table,before) {
26
+ const apply=row=>checkResult(config,table,row,config.importRow(table.name,structuredClone(row)));
27
+ const after=apply(before),again=apply(before),twice=apply(after);
28
+ for(const [result,reason] of [[again,'deterministic'],[twice,'idempotent']]) {
29
+ const column=Object.keys(before).find(key=>!same(after[key],result[key]));
30
+ insist(column===undefined,'import_row_not_'+reason+':'+location(table,column),409);
31
+ }
32
+ return after;
33
+ }
34
+ function countChanges(summary,table,before,after) {
35
+ for(const column of Object.keys(before))if(!same(before[column],after[column])) {
36
+ let entry=summary.find(x=>x.table===table.name && x.column===column && same(x.value,after[column]));
37
+ if(!entry){entry={table:table.name,column,value:after[column],rows:0};summary.push(entry);}
38
+ entry.rows++;
39
+ }
40
+ }
41
+ const guardSql=`INSERT INTO rer_portability_guard(value) SELECT CASE WHEN (
42
+ EXISTS(SELECT 1 FROM rer_portability_jobs WHERE id=? AND metadata=?)
43
+ AND (SELECT import_job FROM rer_portability_state WHERE id=1)=?
44
+ ) THEN 1 ELSE 0 END`;
45
+ const checkpoint=(row,before,metadata,queries)=>[
46
+ {sql:guardSql,params:[row.id,before,row.id]},...queries,
47
+ {sql:'UPDATE rer_portability_jobs SET phase=?,completed=?,metadata=? WHERE id=?',params:[row.phase,row.completed,canonicalJson(metadata),row.id]},
48
+ {sql:'DELETE FROM rer_portability_guard'},
49
+ ];
50
+ // Each page's staged updates, cursor and value-free change counts commit together.
51
+ // A resumed finalize never transforms an already-normalized row or recounts it.
52
+ export async function normalizeStaged(context,row) {
53
+ const {db,config}=context;if(!config.importRow)return;
54
+ for(const table of row.metadata.targets.filter(t=>Object.hasOwn(config.importSafety,t.name))) {
55
+ const copy=row.metadata.copies[table.name],count=row.metadata.stagedCounts?.[table.name]??0;
56
+ let offset=row.metadata.safetyProgress?.[table.name]??0;
57
+ while(offset<count) {
58
+ let before=canonicalJson(row.metadata),queries=[];
59
+ const commit=async()=>{
60
+ await db.batch(checkpoint(row,before,row.metadata,queries).map(q=>statement(db,q.sql,...(q.params??[]))));
61
+ before=canonicalJson(row.metadata);queries=[];
62
+ };
63
+ for(const {row:raw,meta} of await readCopy(db,table,copy,offset,100,true)) {
64
+ const original=encodedRow(table,raw),after=mapImportRow(config,table,original);
65
+ const columns=Object.keys(original).filter(key=>!same(original[key],after[key]));
66
+ insist(Number.isSafeInteger(meta.rer_portability_copy_rowid),'copy_rowid');
67
+ const query=columns.length?{sql:`UPDATE ${quote(copy)} SET ${columns.map(c=>quote(c)+'=?').join(',')} WHERE ${copyRowid(table)}=${meta.rer_portability_copy_rowid}`,params:columns.map(c=>bindCell(after[c]))}:undefined;
68
+ // Reserve room for a metadata checkpoint and its concurrency guard.
69
+ const summary=structuredClone(row.metadata.safetyChanges??[]);countChanges(summary,table,original,after);
70
+ const metadata={...row.metadata,safetyChanges:summary,safetyProgress:{...row.metadata.safetyProgress,[table.name]:offset+1}};
71
+ const pending=[...queries,...(query?[query]:[])];
72
+ if(rpcBytes(checkpoint(row,before,metadata,pending))>RPC_LIMIT && queries.length)await commit();
73
+ insist(rpcBytes(checkpoint(row,before,metadata,[...queries,...(query?[query]:[])]))<=RPC_LIMIT,'D1_RPC_TRANSPORT_LIMIT',413);
74
+ if(query)queries.push(query);
75
+ row.metadata=metadata;offset++;
76
+ }
77
+ await commit();
78
+ }
79
+ }
80
+ }
@@ -1,5 +1,6 @@
1
1
  import { all, statement, state, guard, job, save, partQueries, parts, readPart, cleanCopies, quote, stream, insist, run } from './storage.mjs';
2
2
  import { stagePart } from './import-staging.mjs';
3
+ import { normalizeStaged } from './import-safety.mjs';
3
4
  import { refuseProviderData } from './provider-data.mjs';
4
5
  import { refuseDevData } from './dev-data.mjs';
5
6
  import { catalogue, validateRow, bindCell, writableColumns } from './catalogue.mjs';
@@ -11,18 +12,21 @@ import { writeObject } from './payloads.mjs';
11
12
  import { createCopy,sidecar,readCopy,restoreCopy } from './copies.mjs';
12
13
  import { excludedDependentGuards, migrationSeeded, seededCondition } from './restore-safety.mjs';
13
14
  import { additiveBridge, sameSchemaMigrations, appliedMigrations } from './bridge.mjs';
15
+ import { checkArchivedLedgers, archivedHistory } from './ledgers.mjs';
14
16
  import { legacyEmptyGuards, legacyAuditStatements, requireLegacyEmpty } from './legacy-safety.mjs';
15
- export async function createImport(context,manifest,actor='direct',legacyRecovery) {
16
- const {db,config,buckets}=context;await checkManifest(manifest);
17
+ // ledgers: the rows of each migration ledger table the archive carries, as the
18
+ // caller read them from it ({table:rows}); uploadPart binds them to its parts.
19
+ export async function createImport(context,manifest,actor='direct',legacyRecovery,ledgers) {
20
+ const {db,config,buckets}=context;await checkManifest(manifest);checkArchivedLedgers(manifest,ledgers);
17
21
  insist(manifest.product===config.product,'wrong_product');
18
22
  // Explicit schema upgrades can reconcile rows, but cannot attest an interrupted deployment.
19
23
  await appliedMigrations(db);await state(db);
20
24
  const original=manifest;manifest=await restorableManifest(original);
21
25
  const skippedTables=original.tables.filter(table=>platformTable(table.name)).map(table=>table.name);
22
26
  const cat=await catalogue(db,config),upgrade=config.acceptedSchemas[original.schema];
23
- const same=manifest.schema===cat.identity;
24
- const bridge=same || upgrade?undefined:await additiveBridge(db,manifest,cat,context.migrations);
25
- if(same && !upgrade)await sameSchemaMigrations(db,manifest,cat,context.migrations);
27
+ const same=manifest.schema===cat.identity,history=()=>archivedHistory(original,ledgers);
28
+ const bridge=same || upgrade?undefined:await additiveBridge(db,manifest,cat,context.migrations,history);
29
+ if(same && !upgrade)await sameSchemaMigrations(db,manifest,cat,context.migrations,history);
26
30
  if(!bridge)insist(canonicalJson(manifest.tables.map(x=>x.name))===canonicalJson(cat.tables.map(x=>x.name)),'table_inventory');
27
31
  if(!upgrade && !bridge)for(const table of manifest.tables) {
28
32
  const target=cat.tables.find(t=>t.name===table.name);
@@ -36,7 +40,7 @@ export async function createImport(context,manifest,actor='direct',legacyRecover
36
40
  for(const o of manifest.objects) insist(buckets[o.bucket] && config.buckets[o.bucket] && o.key.startsWith(config.buckets[o.bucket].prefix??''),'bucket_inventory');
37
41
  insist(canonicalJson(manifest.payloads.map(x=>x.name))===canonicalJson(Object.keys(config.extraPayloads).sort()),'payload_inventory');
38
42
  const id=crypto.randomUUID(),copies=Object.fromEntries(targets.map((t,i)=>[t.name,'rer_portability_stage_'+id.replaceAll('-','')+'_'+i]));
39
- const row={id,kind:'import',actor,phase:'staging',completed:0,metadata:{manifest:original,...(skippedTables.length?{restoreManifest:manifest,skippedTables}:{}),copies,targets,triggers:cat.triggers,...(bridge?{bridge}:{}),...(legacyRecovery?{legacyRecovery}: {})}};
43
+ const row={id,kind:'import',actor,phase:'staging',completed:0,metadata:{manifest:original,...(skippedTables.length?{restoreManifest:manifest,skippedTables}:{}),copies,targets,triggers:cat.triggers,...(bridge?{bridge}:{}),...(legacyRecovery?{legacyRecovery}: {}),...(ledgers?{ledgers}:{})}};
40
44
  try {
41
45
  await db.batch([guard(db,'(SELECT import_job FROM rer_portability_state WHERE id=1) IS NULL'),
42
46
  statement(db,'UPDATE rer_portability_state SET import_job=? WHERE id=1',id),
@@ -55,6 +59,7 @@ export async function uploadPart(context,id,index,part) {
55
59
  if(item.kind==='row') {
56
60
  const table=manifest.tables.find(x=>x.name===item.table);insist(table && Number.isSafeInteger(item.index) && item.index>=0 && item.index<table.count,'row_inventory');
57
61
  validateRow(table,item.row,manifest.excludedColumns[table.name]??[]);
62
+ if(row.metadata.ledgers?.[table.name])insist(canonicalJson(item.row)===canonicalJson(row.metadata.ledgers[table.name][item.index]),'ledger_inventory');
58
63
  if(platformTable(table.name))continue;
59
64
  const target=row.metadata.targets.find(x=>x.name===table.name);
60
65
  const data=upgrade?upgrade.upgradeRow(table.name,structuredClone(item.row)):item.row;
@@ -158,6 +163,7 @@ export async function finalizeImport(context,id,{contentHash,archiveHash,laterJo
158
163
  await bucketsEmpty(context,row);
159
164
  await refuseDevData(db,row.metadata.copies,row.metadata.targets);
160
165
  await refuseProviderData(db,row.metadata.copies,row.metadata.targets);
166
+ await normalizeStaged(context,row);
161
167
  if(config.validateStaged) {
162
168
  const read=async table=>{insist(row.metadata.copies[table],'staged_table');const target=row.metadata.targets.find(t=>t.name===table);return (await readCopy(db,target,row.metadata.copies[table],0,row.metadata.manifest.tables.find(t=>t.name===table)?.count??0)).map(x=>x.row);};
163
169
  insist(await config.validateStaged({db,tables:{...row.metadata.copies},read,manifest})!==false,'staged_validation');
@@ -193,7 +199,7 @@ export async function finalizeImport(context,id,{contentHash,archiveHash,laterJo
193
199
  }
194
200
  export async function importStatus(context,id) {
195
201
  const row=await job(context.db,id,'import');return {id,kind:'import',phase:row.phase,completed:row.completed,total:row.metadata.manifest.partCount,
196
- contentHash:row.metadata.contentHash??null,requiredConfiguration:row.phase==='complete'?row.metadata.manifest.requiredConfiguration:[],...(row.metadata.bridge?{bridged:row.metadata.bridge}:{}),...(row.metadata.skippedTables?{skippedTables:row.metadata.skippedTables}:{})};
202
+ contentHash:row.metadata.contentHash??null,requiredConfiguration:row.phase==='complete'?row.metadata.manifest.requiredConfiguration:[],...(row.metadata.bridge?{bridged:row.metadata.bridge}:{}),...(row.metadata.skippedTables?{skippedTables:row.metadata.skippedTables}:{}),...(row.metadata.safetyChanges?{safetyChanges:row.metadata.safetyChanges}:{})};
197
203
  }
198
204
  export async function abortImport(context,id) {
199
205
  const row=await job(context.db,id,'import');insist(row.phase!=='complete','import_complete',409);
@@ -22,21 +22,27 @@ export interface Staged {
22
22
  export interface ArchivePolicy {
23
23
  product: string; excludeTables?: string[]; includeTables?: string[]; derivedTables?: string[];
24
24
  secretColumns?: Record<string,string[]>; exportRow?: (table:string,row:Row)=>Row;
25
+ /** Nonempty finite safe-value sets for writable non-key columns; requires importRow. */
26
+ importSafety?: Record<string,Record<string,Cell[]>>;
27
+ /** Pure, synchronous, deterministic and idempotent; called only for importSafety tables before validateStaged. */
28
+ importRow?: (table:string,row:Row)=>Row;
25
29
  replaceableTables?: string[]; requiredConfiguration?: string[]; validateStaged?: (staged:Staged)=>boolean|void|Promise<boolean|void>;
26
30
  acceptedSchemas?: Record<string,{upgradeRow:(table:string,row:Row)=>Row}>;
27
31
  buckets?: Record<string,{prefix?:string;objectReferences?:{table:string;keyColumn:string}}>;
28
32
  extraPayloads?: Record<string,{produce:()=>Uint8Array|ReadableStream<Uint8Array>|Promise<Uint8Array|ReadableStream<Uint8Array>>;consume:(source:ReadableStream<Uint8Array>,descriptor:PayloadDescriptor)=>void|Promise<void>}>;
29
33
  partBytes?: number;
30
34
  }
31
- export interface Status {id:string;kind:'export'|'import';phase:'building'|'staging'|'applying'|'complete'|'aborted';completed:number;total:number|null;/** Export only: bytes of the parts built so far. */bytes?:number;contentHash:string|null;requiredConfiguration?:string[];bridged?:Bridge;/** Platform bookkeeping ignored in an older archive. */skippedTables?:string[]}
35
+ /** Actual changed rows grouped by table, column and safe replacement value; never original values. */
36
+ export interface SafetyChange {table:string;column:string;value:Cell;rows:number}
37
+ export interface Status {id:string;kind:'export'|'import';phase:'building'|'staging'|'applying'|'complete'|'aborted';completed:number;total:number|null;/** Export only: bytes of the parts built so far. */bytes?:number;contentHash:string|null;requiredConfiguration?:string[];bridged?:Bridge;/** Platform bookkeeping ignored in an older archive. */skippedTables?:string[];/** Durable summary of bounded import normalization. */safetyChanges?:SafetyChange[]}
32
38
  /** Later additive migrations an older archive was restored across. */
33
39
  export interface Bridge {fromSchema:string;toSchema:string;migrations:string[];tables:string[];columns:Record<string,string[]>}
34
40
  /** A runner's replay of the destination's applied migrations: the archive schema identity after a chosen set, and each migration's statements. */
35
- export interface MigrationReplay {names:string[];statements(name:string):string[];schemaAfter(names:string[]):Promise<string>}
41
+ export interface MigrationReplay {names:string[];statements(name:string):string[];/** The migration file's SHA-256 hex, where the runner read the file. */sha256?(name:string):string|undefined;schemaAfter(names:string[]):Promise<string>}
36
42
  export interface PortabilityOptions {db:D1Database;buckets?:Record<string,R2Bucket>;policy:ArchivePolicy;migrations?:()=>Promise<MigrationReplay>}
37
43
  export interface Portability {
38
44
  exports: {create(actor?:string):Promise<string>;advance(id:string,options?:{expectedPart?:number;maxParts?:number}):Promise<Status>;status(id:string):Promise<Status>;download(id:string):Promise<ReadableStream<Uint8Array>>;abort(id:string):Promise<Status>};
39
- imports: {create(manifest:Manifest,actor?:string):Promise<string>;uploadPart(id:string,index:number,part:Part):Promise<Status>;finalize(id:string,options:{archiveHash:string;contentHash?:string;laterJournal?:LaterJournal}):Promise<Status>;status(id:string):Promise<Status>;abort(id:string):Promise<Status>;/** Aborts the import left staged by a crashed restore and returns its id; call only as the destination's sole operator. */abortAbandoned():Promise<string|null>};
45
+ imports: {/** ledgers: the rows of each migration ledger table an old archive carries, as read from it; they place it in the destination's history and must equal its parts. */create(manifest:Manifest,actor?:string,options?:{ledgers?:Record<string,Row[]>}):Promise<string>;uploadPart(id:string,index:number,part:Part):Promise<Status>;finalize(id:string,options:{archiveHash:string;contentHash?:string;laterJournal?:LaterJournal}):Promise<Status>;status(id:string):Promise<Status>;abort(id:string):Promise<Status>;/** Aborts the import left staged by a crashed restore and returns its id; call only as the destination's sole operator. */abortAbandoned():Promise<string|null>};
40
46
  status(id:string):Promise<Status>;actor(id:string):Promise<string>;
41
47
  erasureStatements(table:string,primaryKey:Record<string,Cell>):Promise<D1PreparedStatement[]>;
42
48
  redactionStatements():Promise<D1PreparedStatement[]>;
@@ -17,7 +17,7 @@ export async function createPortability({db,buckets={},policy:input,migrations})
17
17
  const context={db,buckets,config,migrations};
18
18
  return {
19
19
  exports:{create:actor=>createExport(context,actor),advance:(id,options)=>advanceExport(context,id,options),status:id=>exportStatus(context,id),download:id=>downloadExport(context,id),abort:id=>abortExport(context,id)},
20
- imports:{create:(manifest,actor)=>createImport(context,manifest,actor),uploadPart:(id,index,part)=>uploadPart(context,id,index,part),finalize:(id,options)=>finalizeImport(context,id,options),status:id=>importStatus(context,id),abort:id=>abortImport(context,id),abortAbandoned:()=>abortAbandonedImport(context)},
20
+ imports:{create:(manifest,actor,options={})=>createImport(context,manifest,actor,undefined,options.ledgers),uploadPart:(id,index,part)=>uploadPart(context,id,index,part),finalize:(id,options)=>finalizeImport(context,id,options),status:id=>importStatus(context,id),abort:id=>abortImport(context,id),abortAbandoned:()=>abortAbandonedImport(context)},
21
21
  status:async id=>(await job(db,id)).kind==='export'?exportStatus(context,id):importStatus(context,id),
22
22
  actor:async id=>(await job(db,id)).actor,
23
23
  erasureStatements:(table,key)=>erasureStatements(db,table,key),redactionStatements:()=>redactionStatements(db),invalidateTransfersStatements:()=>invalidateTransfersStatements(db),