@runeditrun/rer 0.2.78 → 0.2.80

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (116) hide show
  1. package/README.md +17 -3
  2. package/cli/README.md +25 -13
  3. package/cli/authoring.mjs +14 -0
  4. package/cli/contract.mjs +25 -5
  5. package/cli/demo-host.mjs +4 -2
  6. package/cli/dev-identity.mjs +2 -2
  7. package/cli/export.mjs +1 -1
  8. package/cli/inspect.mjs +38 -2
  9. package/cli/main.mjs +8 -2
  10. package/cli/managed-server.mjs +1 -1
  11. package/cli/parse.mjs +7 -6
  12. package/cli/portability-worker.mjs +2 -1
  13. package/cli/portability.mjs +6 -2
  14. package/cli/production-auth.mjs +2 -1
  15. package/cli/production-mail.mjs +36 -9
  16. package/cli/project.mjs +5 -14
  17. package/cli/release.mjs +2 -1
  18. package/cli/server-proxy.mjs +30 -8
  19. package/cli/site-preview.mjs +4 -1
  20. package/cli/site.mjs +3 -2
  21. package/cli/upstream-agent.mjs +14 -0
  22. package/edition-authoring/adopt.mjs +39 -0
  23. package/edition-authoring/base-acknowledgements.mjs +24 -0
  24. package/edition-authoring/base-copies.mjs +418 -0
  25. package/edition-authoring/cli.mjs +85 -0
  26. package/edition-authoring/copy-checkout.mjs +69 -0
  27. package/edition-authoring/files.mjs +15 -2
  28. package/edition-authoring/lock.mjs +48 -0
  29. package/edition-authoring/offline-git.mjs +28 -0
  30. package/edition-authoring/parent-comparison.mjs +77 -0
  31. package/edition-authoring/projection-lock.mjs +38 -0
  32. package/edition-authoring/release.mjs +48 -0
  33. package/edition-authoring/sources.mjs +114 -0
  34. package/edition-authoring/updates.mjs +105 -0
  35. package/global/CHANGELOG.md +17 -0
  36. package/global/README.md +17 -3
  37. package/global/cli/install.mjs +3 -1
  38. package/global/main.mjs +9 -12
  39. package/modules/container/build.mjs +1 -1
  40. package/modules/d1-transport/index.mjs +68 -21
  41. package/modules/demo/README.md +8 -2
  42. package/modules/demo/declaration.mjs +21 -0
  43. package/modules/demo/input-path.mjs +18 -0
  44. package/modules/demo/label.d.mts +1 -1
  45. package/modules/demo/label.mjs +10 -5
  46. package/modules/demo/navigation.mjs +32 -0
  47. package/modules/demo/run-url.mjs +9 -0
  48. package/modules/demo/vite.d.mts +7 -2
  49. package/modules/demo/vite.mjs +17 -2
  50. package/modules/demo-host/README.md +5 -3
  51. package/modules/demo-host/native/visitor.mjs +1 -1
  52. package/modules/demo-host/page-style.mjs +2 -2
  53. package/modules/demo-host/pages.mjs +3 -3
  54. package/modules/demo-host/plan.mjs +6 -4
  55. package/modules/demo-host/theme.mjs +5 -2
  56. package/modules/deploy-kit/README.md +19 -15
  57. package/modules/deploy-kit/owner-cli.mjs +1 -1
  58. package/modules/deploy-kit/readiness-window.mjs +3 -3
  59. package/modules/erasure/README.md +10 -5
  60. package/modules/import-kit/src/d1/README.md +15 -9
  61. package/modules/import-kit/src/d1/apply.ts +10 -6
  62. package/modules/import-kit/src/d1/atomic-prefix.ts +64 -0
  63. package/modules/import-kit/src/d1/budget.ts +48 -26
  64. package/modules/import-kit/src/d1/compressed-plan.ts +9 -2
  65. package/modules/import-kit/src/d1/database.ts +2 -1
  66. package/modules/import-kit/src/d1/jobs.ts +2 -1
  67. package/modules/import-kit/src/d1/plans.ts +4 -2
  68. package/modules/import-kit/src/d1/verified-plan.ts +63 -0
  69. package/modules/integrations/core/README.md +2 -2
  70. package/modules/mail/README.md +4 -3
  71. package/modules/mail/pnpm-lock.yaml +16 -0
  72. package/modules/operator-auth/accounts/native.mjs +3 -1
  73. package/modules/operator-auth/accounts/provider-control.mjs +7 -4
  74. package/modules/operator-auth/accounts/provider-setup.mjs +34 -0
  75. package/modules/operator-auth/accounts/settings-page.mjs +13 -4
  76. package/modules/operator-auth/conformance/README.md +1 -1
  77. package/modules/operator-auth/dev/index.mjs +22 -4
  78. package/modules/operator-auth/dev/refusal.mjs +19 -0
  79. package/modules/operator-auth/docs/README.md +6 -5
  80. package/modules/operator-auth/docs/tanstack-start.md +1 -1
  81. package/modules/operator-auth/docs/vite-spa.md +5 -6
  82. package/modules/operator-auth/index.d.mts +1 -1
  83. package/modules/operator-auth/page.mjs +14 -0
  84. package/modules/operator-auth/representation.mjs +83 -0
  85. package/modules/operator-auth/routes.mjs +25 -2
  86. package/modules/portability/README.md +91 -14
  87. package/modules/portability/bookkeeping.mjs +3 -3
  88. package/modules/portability/bridge.mjs +5 -4
  89. package/modules/portability/catalogue.mjs +12 -8
  90. package/modules/portability/cli.mjs +5 -1
  91. package/modules/portability/export.mjs +5 -11
  92. package/modules/portability/format.mjs +7 -1
  93. package/modules/portability/http-errors.mjs +51 -0
  94. package/modules/portability/http.mjs +4 -1
  95. package/modules/portability/import-lease.mjs +13 -2
  96. package/modules/portability/import.mjs +10 -0
  97. package/modules/portability/index.d.mts +3 -0
  98. package/modules/portability/references.mjs +20 -0
  99. package/modules/portability/schema-identity.mjs +13 -0
  100. package/modules/portability/schema-legacy.mjs +21 -0
  101. package/modules/portability/sqlite-tokens.mjs +121 -0
  102. package/modules/release/README.md +20 -9
  103. package/modules/release/github-verify.mjs +30 -9
  104. package/modules/site/docs-search/search.css +1 -1
  105. package/onboarding/README.md +16 -4
  106. package/onboarding/adapter.mjs +5 -1
  107. package/onboarding/schema/SETUP.schema.json +26 -1
  108. package/onboarding/schema/business-field-inputs.mjs +37 -0
  109. package/onboarding/schema/json-schema.mjs +6 -2
  110. package/onboarding/schema/setup-validation.mjs +20 -3
  111. package/onboarding/server.mjs +1 -1
  112. package/onboarding/setup-manifest.mjs +2 -0
  113. package/onboarding/start-origin-diagnostic.mjs +24 -0
  114. package/onboarding/ui/style.css +9 -9
  115. package/package.json +1 -1
  116. package/scripts/publish-check/README.md +1 -1
@@ -31,7 +31,8 @@ payload consumers, and prove the integrated round trip.
31
31
  ## Policy
32
32
 
33
33
  `await createPortability({db, buckets, policy})` validates policy and the current
34
- schema at construction. Apply the bookkeeping migration first. Policy fields:
34
+ schema at construction. Apply the bookkeeping migration first. Export refuses
35
+ with HTTP 409 `import_applying` while an import is applying. Policy fields:
35
36
 
36
37
  | Field | Meaning |
37
38
  | --- | --- |
@@ -47,7 +48,7 @@ schema at construction. Apply the bookkeeping migration first. Policy fields:
47
48
  | `requiredConfiguration` | Setting names returned after restore for the owner to re-enter. |
48
49
  | `validateStaged({db,tables,read,manifest})` | Read-only product validation before any object write; false or a thrown error refuses restore. `tables` names the staging tables for bounded SQL queries; `read` is a convenience for small tables. |
49
50
  | `acceptedSchemas` | Map from an earlier full schema hash to `{upgradeRow(table,row)}`. Upgrade functions are pure and synchronous. No older schema is accepted by default. |
50
- | `buckets` | Binding names mapped to `{prefix?,objectReferences?:{table,keyColumn}}`. Prefixes isolate an object namespace. References require exactly the distinct referenced keys. |
51
+ | `buckets` | Binding names mapped to `{prefix?,objectReferences?:{table,keyColumn}}`. Prefixes isolate an object namespace. References require exactly the distinct referenced keys: export checks its snapshot, and restore its staged rows (after `importRow`, before any object write), that each key is nonempty, under the prefix, not absolute and without `.` or `..` segments, and that the keys are exactly the bucket's archived objects, none missing and none extra; otherwise `object_references:<binding>`. |
51
52
  | `extraPayloads` | Named `{produce,consume}` pairs. Producers return bytes or a byte stream. Consumers receive a verified byte stream and descriptor before D1 commit. |
52
53
  | `partBytes` | Maximum part size, default 16 MiB, configurable from 1 KiB to 16 MiB. |
53
54
 
@@ -107,7 +108,8 @@ and declaring both `policy` and `policyModule` fail closed. Serializable object
107
108
  schema-upgrade and payload callbacks) are refused instead of being dropped by
108
109
  JSON transport. Container transfer currently refuses `policyModule` explicitly;
109
110
  it continues to accept serializable object policies. Malformed primary or later
110
- journal JSON is reported once as `archive_json` with exit 2, before local data
111
+ journal JSON is reported once as `invalid_archive` with exit 2 and a re-export
112
+ recovery command, before local data
111
113
  migrations, without echoing archive content or parser diagnostics.
112
114
 
113
115
  An installation migrated by Platform has `rer_migrations(name, sha256,
@@ -207,7 +209,7 @@ RER's native `localPortability` invokes this same entry point in its
207
209
  history again for transfer.
208
210
 
209
211
  The implementation ships in the RER CLI, not the Worker portability module.
210
- Package-mode seeds need a future `@runeditrun/rer` release containing this
212
+ Package-mode seeds need a pinned `@runeditrun/rer` release containing this
211
213
  export and an exact pin/lock update; version 0.2.77 does not expose it.
212
214
  Vendored seeds need a reviewed Base CLI-copy refresh with
213
215
  `node scripts/export-cli.mjs SEED_ROOT` from the selected Base source and their
@@ -256,7 +258,7 @@ secret base cells become NULL before generated snapshots are normalized. A
256
258
  generated column cannot itself be redacted: exclude the table or change the
257
259
  product schema/policy if its computed value must be secret. Ordinary-column
258
260
  metadata and row encoding are unchanged. Ordinary-table archives using the
259
- authenticated format continue to import into the same schema; see the version
261
+ v2 integrity-checked format continue to import into the same schema; see the version
260
262
  1 refusal below.
261
263
 
262
264
  An archived INTEGER-affinity base cell outside JavaScript's safe integer range fails before
@@ -330,6 +332,59 @@ parts also preserve the original rowid, including gaps, for restoration and
330
332
  erasure matching. Export time,
331
333
  source identity, job ids and boundaries do not affect equivalence.
332
334
 
335
+ Schema identity covers table definitions (including constraints and generated
336
+ expressions), explicit indexes (including uniqueness and partial predicates),
337
+ views and triggers. Indexes and triggers on excluded tables are excluded with
338
+ their tables, including platform bookkeeping tables. The complete additional
339
+ definitions travel in `schemaObjects`. A single SQLite lexer reads the original
340
+ SQL before removing trivia. Identity records each token's class and canonical
341
+ value, plus a deterministic spelling: a single space separates neighbours
342
+ exactly when concatenating them would change their tokenization. The complete
343
+ joined stream is tokenized again and must reproduce those same tokens.
344
+ Comments and harmless inter-token whitespace therefore compare equally, while
345
+ `x '31'` and `x'31'`, separated minus signs, numbers and combined operators stay
346
+ distinct. No regular expression rewrites SQL.
347
+
348
+ Quoted identifiers (double quotes, backticks, brackets, or a single-quoted string
349
+ used as a name), strings, blobs, numbers, variables and ordinary identifiers keep
350
+ their exact spelling. Only unambiguous reserved keywords fold ASCII case.
351
+ Fallback/JOIN/window keywords retain their spelling: for example, SQLite can
352
+ interpret `DEFAULT ABORT` and `DEFAULT abort` as different string defaults.
353
+ Unicode case and quote styles are never normalized. The lexer follows SQLite's
354
+ [`sqlite3GetToken`](https://github.com/sqlite/sqlite/blob/master/src/tokenize.c)
355
+ and the keyword fallback rules in
356
+ [`parse.y`](https://github.com/sqlite/sqlite/blob/master/src/parse.y).
357
+ The original DDL is authenticated by `archiveHash`.
358
+
359
+ Unsupported or ambiguous SQL refuses with `schema_mismatch:unsupported_sql` and
360
+ the existing original-installation re-export/recovery instructions; it never
361
+ receives a shared or empty identity. This includes NUL and ill-formed Unicode,
362
+ BOM at token start, vertical-tab whitespace, unterminated quotes/brackets/block
363
+ comments (including bare `/*`), malformed/odd-length blobs, invalid numeric
364
+ separators or numeric identifier suffixes, malformed named/Tcl variables, and
365
+ unknown ASCII punctuation. Valid decimal/hex/exponent/underscore numbers,
366
+ SQLite's longest operators, question-mark and named/Tcl variables, LF/EOF line
367
+ comments, and terminated block comments are recognized. SQLite may accept some
368
+ of the refused whitespace or EOF-comment forms; re-export with supported DDL
369
+ rather than silently discarding their bytes. Virtual tables remain unsupported.
370
+
371
+ Older version 2 archives first verify their original raw-DDL, table-token, or
372
+ complete-object-token schema checksum and complete archive proof. A read-only
373
+ historical checksum codec projects the real lexer's tokens into those old wire
374
+ encodings. Its lossy result never decides compatibility: complete definitions
375
+ are always compared through the new typed token identity. Compatibility then uses
376
+ any complete `schemaObjects` inventory the archive carries, without rewriting
377
+ its authenticated manifest or parts. Without that inventory, the original
378
+ presence or absence of indexes, views and triggers cannot be established from
379
+ table SQL or migration names alone, even when the destination has no such
380
+ objects. Normal restore refuses `schema_mismatch` and the CLI gives the existing
381
+ re-export recovery instructions. This also catches removal of an old object's
382
+ definition rather than assuming missing evidence means the objects match.
383
+ Re-export from the original installation to carry the complete definitions.
384
+ A new archive explicitly records even an empty object inventory, so later
385
+ additive migrations can still be placed normally. The separate acknowledged
386
+ v1 recovery path remains an explicitly unverified recovery, as documented below.
387
+
333
388
  New exports use archive version 2. The closing `archiveHash` is SHA-256 of
334
389
  canonical JSON `{manifest,parts}`, where `manifest` is the complete manifest
335
390
  and `parts` is the ordered array of verified part SHA-256 strings. This binds
@@ -468,7 +523,9 @@ a view resolved from applied migration history), selects `*` or
468
523
  a secret column, when a trigger on it or a foreign key action from it reaches a
469
524
  catalogue table, when any other later statement writes it (directly or through
470
525
  a trigger) or changes its columns other than by `ADD COLUMN` (allowed only when
471
- every replay names its columns), or when the statement cannot be bounded.
526
+ every replay names its columns), when the statement cannot be bounded, or when
527
+ it writes a table that a bucket's `objectReferences` names: references are
528
+ checked on the staged rows, before any object write.
472
529
  A failed or interrupted apply leaves no replayed row; finalizing again replays
473
530
  from the start.
474
531
  Read-only trigger bodies, including RAISE-only guards, add no write dependency.
@@ -612,8 +669,8 @@ part; different content returns 409. Reassembled bytes pass the part checksum
612
669
  before completion advances. Staged data stays private until finalization.
613
670
  Finalize requires the closing `archiveHash` and verifies all
614
671
  parts, semantic contents, destination
615
- emptiness, erasure and product invariants. R2 ownership is recorded after the
616
- empty-namespace check. Verified object writes precede one atomic D1 batch, with
672
+ emptiness, erasure, object references and product invariants. R2 ownership
673
+ is recorded after the empty-namespace check. Verified object writes precede one atomic D1 batch, with
617
674
  deferred FKs, trigger suppression/recreation, replaceable rows, staged inserts,
618
675
  journal adoption and completion. FK cycles and self-references restore together.
619
676
  An unsuccessful commit removes only owned objects. A lost commit acknowledgement
@@ -640,7 +697,10 @@ by every upload/finalize request, then every 30 seconds while that request is
640
697
  working (including verification, R2 transfers and product callbacks). This is a
641
698
  recovery window, not a total restore limit: the measured 20,000-row round trip
642
699
  below already takes 171 seconds, and callbacks have no wall-time bound. A fresh
643
- import returns `409 import_running` with `jobId` and ISO `abortableAt`. The owner
700
+ import returns `409 import_running`. Only a validated internal lease refusal,
701
+ after authorization and for the actor who owns that import, includes `jobId`
702
+ and ISO `abortableAt`. Authorization failures and other actors receive no job or
703
+ lease metadata, even if an error carries the same code or fields. The owner
644
704
  can find its active job with `GET /api/portability/imports` (`{active:status}`
645
705
  or `{active:null}`), and inspect `GET .../imports/:id`: `abortableAt` and `stale` describe
646
706
  its lease. The existing authenticated `POST .../imports/:id/abort` (or CLI
@@ -671,6 +731,17 @@ returns an async dispatcher, or null for foreign paths. Default base path is
671
731
  an actor id or null. Jobs belong to their creating actor. Unsafe methods require
672
732
  the exact configured Origin. Responses have `Cache-Control: no-store` and errors
673
733
  are `{error:code}`. Request bodies are bounded; `maxBodyBytes` can lower the bound.
734
+ Typed authorization errors retain their HTTP status, directly or through a
735
+ bounded, cycle-safe `cause` chain. Codes come from the existing fixed public
736
+ vocabulary (for example `403 step_up_required`); unknown codes become
737
+ `operation_failed`. Portability diagnostics expose the known code prefix only,
738
+ never caller-supplied suffixes. Aggregate error collections are not authority
739
+ wrappers. Native diagnostics remain private:
740
+ transient D1 availability failures use `503 operation_failed`, transport timeouts
741
+ use `504 operation_failed`, and unexpected faults remain `500 operation_failed`.
742
+ The hosted CLI aborts a completed export after saving and verifying the local
743
+ archive, releasing its snapshots and parts. Interrupted downloads retain their
744
+ job for resume.
674
745
 
675
746
  | Method / relative path | Action |
676
747
  | --- | --- |
@@ -749,13 +820,14 @@ product erasure or remove product objects; the product handles attachments and
749
820
  any extra state. Do not retain a previously downloaded archive as erasure proof;
750
821
  supply the owner's later journal when restoring old offline data.
751
822
 
752
- ## Measured capacity
823
+ ## Historical measured capacity
753
824
 
754
- One local Miniflare D1/R2 run exported, restored and exported 20,000 rows across
755
- five tables in 171.169 seconds, with 625 parts at a configured 4 KiB bound and a
825
+ An unverified historical local Miniflare D1/R2 report recorded export, restore
826
+ and export of 20,000 rows across five tables in 171.169 seconds, with 625 parts at a configured 4 KiB bound and a
756
827
  1,979,471-byte archive. Restore used one atomic D1 commit. The verification run
757
- reports the measurements again. These measurements are test evidence, not a production
758
- capacity claim. Real D1 CPU, query, statement and storage limits, object mutation,
828
+ can produce fresh measurements; no corresponding historical benchmark receipt
829
+ is retained in this repository. These figures do not establish current production
830
+ capacity. Real D1 CPU, query, statement and storage limits, object mutation,
759
831
  destination writers and extra-payload recovery need product-level verification.
760
832
 
761
833
  ## Adopting
@@ -772,7 +844,11 @@ modules/portability/index.d.mts
772
844
  modules/portability/lines.mjs
773
845
  modules/portability/hash.mjs
774
846
  modules/portability/format.mjs
847
+ modules/portability/references.mjs
775
848
  modules/portability/catalogue.mjs
849
+ modules/portability/schema-identity.mjs
850
+ modules/portability/schema-legacy.mjs
851
+ modules/portability/sqlite-tokens.mjs
776
852
  modules/portability/bookkeeping.mjs
777
853
  modules/portability/storage.mjs
778
854
  modules/portability/copies.mjs
@@ -798,6 +874,7 @@ modules/portability/import-safety.mjs
798
874
  modules/portability/payloads.mjs
799
875
  modules/portability/journal.mjs
800
876
  modules/portability/http.mjs
877
+ modules/portability/http-errors.mjs
801
878
  modules/portability/http-client.mjs
802
879
  modules/portability/cli.mjs
803
880
  modules/portability/cli-options.mjs
@@ -1,4 +1,4 @@
1
- import { canonicalJson, sha256 } from './hash.mjs';
1
+ import { schemaIdentityOf } from './schema-identity.mjs';
2
2
  import { insist } from './storage.mjs';
3
3
 
4
4
  // Installation authority and transient working state, never product data.
@@ -23,8 +23,8 @@ export async function restorableManifest(manifest) {
23
23
  for(const table of manifest.tables)if(identifier(table.name)==='ik_d1_assertions')
24
24
  insist(table.count===0,'invalid_bookkeeping_rows:'+table.name,409);
25
25
  const tables=manifest.tables.filter(table=>!platformTable(table.name));
26
- if(tables.length===manifest.tables.length)return manifest;
27
- return {...manifest,tables,schema:await sha256(canonicalJson(tables.map(({name,sql})=>({name,sql})))),
26
+ const schemaObjects=manifest.schemaObjects?.filter(object=>!manifest.tables.some(table=>platformTable(table.name) && table.name===object.table));
27
+ return {...manifest,tables,...(schemaObjects?{schemaObjects}:{}),schema:await schemaIdentityOf(tables,schemaObjects),
28
28
  excludedColumns:Object.fromEntries(Object.entries(manifest.excludedColumns).filter(([name])=>!platformTable(name)))};
29
29
  }
30
30
  export const skippedTablesLine=tables=>'Skipped platform bookkeeping tables: '+tables.join(', ')+'.';
@@ -4,6 +4,7 @@ import { tokens, identifier, folded } from './generated.mjs';
4
4
  import { laterData } from './backfills.mjs';
5
5
  import { destinationLedger, matchMigrationLedger } from './ledgers.mjs';
6
6
  import { checkRebuild } from './rebuilds.mjs';
7
+ import { schemaSQL } from './schema-identity.mjs';
7
8
 
8
9
  // Applied product migrations in the ledger's own order (ledgers.mjs). A
9
10
  // state-bearing ledger may hold an unacknowledged migration whose effects are
@@ -38,7 +39,7 @@ async function placement(db,manifest,cat,replay,ledger,history) {
38
39
  unapplied=applied.filter(name=>!earlier.includes(name));
39
40
  }else{
40
41
  // Unrecorded: the latest matching prefix. Migrations that change no archived
41
- // table (indexes, excluded tables, data-only changes) cannot be told apart
42
+ // definition (excluded tables or data-only changes) cannot be told apart
42
43
  // and count as later. The source may have stopped at any matching prefix,
43
44
  // so every migration after the earliest one may be missing from its rows.
44
45
  const matching=[];
@@ -62,7 +63,7 @@ async function placement(db,manifest,cat,replay,ledger,history) {
62
63
  return {applied,earlier,replays,rebuilt};
63
64
  }
64
65
  // An archive with the destination's schema identity can still predate later
65
- // migrations that changed only data (a purge or a backfill) or added an index.
66
+ // migrations that changed only data (a purge or a backfill).
66
67
  // Those that change restorable data are refused like any other. Without a
67
68
  // replayer the statements cannot be read, so an archive whose recorded
68
69
  // migrations the destination's ledger has outgrown is refused. An archive that
@@ -82,7 +83,7 @@ export async function sameSchemaMigrations(db,manifest,cat,migrations,history) {
82
83
  }
83
84
  // An archive made before later migrations restores only when those migrations
84
85
  // are additive, or a proven row-preserving CHECK rebuild: new tables, new
85
- // columns that take a default, or indexes (which archives never carry).
86
+ // columns that take a default, or later schema objects established by replay.
86
87
  // Anything else stays a schema_mismatch refusal.
87
88
  export async function additiveBridge(db,manifest,cat,migrations,history) {
88
89
  insist(migrations,'schema_mismatch');
@@ -95,7 +96,7 @@ export async function additiveBridge(db,manifest,cat,migrations,history) {
95
96
  const kept=target.columns.slice(0,table.columns.length),added=target.columns.slice(table.columns.length);
96
97
  insist(canonicalJson(kept)===canonicalJson(table.columns) && table.rowidAlias===target.rowidAlias && table.dependencies.every(d=>target.dependencies.includes(d)),'schema_mismatch:columns'+label);
97
98
  const rebuiltChecks=rebuilt.has(folded(table.name)) && checkRebuild(table.sql,target.sql);
98
- if(!added.length){insist(table.sql===target.sql || rebuiltChecks,'schema_mismatch:definition'+label);continue;}
99
+ if(!added.length){insist(schemaSQL(table.sql)===schemaSQL(target.sql) || rebuiltChecks,'schema_mismatch:definition'+label);continue;}
99
100
  const declared=await all(db,`PRAGMA table_xinfo(${quote(table.name)})`);
100
101
  insist(added.every(c=>!c.pk && !c.generated && (!c.notnull || declared.find(d=>d.name===c.name)?.dflt_value!=null)),'schema_mismatch:column_default'+label);
101
102
  insist(appended(table.sql,target.sql,added.map(c=>c.name)),'schema_mismatch:definition'+label);
@@ -1,5 +1,6 @@
1
1
  import { all, insist, quote, PortabilityError } from './storage.mjs';
2
- import { canonicalJson, sha256, base64, unbase64, utf8 } from './hash.mjs';
2
+ import { canonicalJson, base64, unbase64, utf8 } from './hash.mjs';
3
+ import { schemaIdentityOf } from './schema-identity.mjs';
3
4
  import { platformTables, platformTable, identifier, matches } from './bookkeeping.mjs';
4
5
  /** Platform bookkeeping is mandatory; operator identity remains a seed choice. */
5
6
  export const excludedTables=Object.freeze([...platformTables,'oa_%','oc_%']);
@@ -57,14 +58,17 @@ async function archived(db,config) {
57
58
  const derived=config.derivedTables;
58
59
  const excluded=objects.filter(x=>x.type==='table' && (excludedTable(x.name,config) || (!config.includeTables.includes(x.name) &&
59
60
  derived.some(p=>x.name===p || (kinds.find(k=>k.name===x.name)?.type==='shadow' && x.name.startsWith(p+'_'))))));
60
- objects.sort((a,b)=>a.name<b.name?-1:a.name>b.name?1:0);
61
- return {objects,kinds,excluded,entries:objects.filter(x=>x.type==='table' && !excluded.some(y=>y.name===x.name))};
61
+ objects.sort((a,b)=>a.name<b.name?-1:a.name>b.name?1:a.type<b.type?-1:a.type>b.type?1:0);
62
+ const schemaObjects=objects.filter(x=>['index','view','trigger'].includes(x.type)
63
+ && !excluded.some(t=>t.name===x.tbl_name)).map(({name,type,tbl_name,sql})=>({name,type,table:tbl_name,sql}));
64
+ return {objects,kinds,excluded,schemaObjects,entries:objects.filter(x=>x.type==='table' && !excluded.some(y=>y.name===x.name))};
62
65
  }
63
- const identityOf=tables=>sha256(canonicalJson(tables.map(({name,sql})=>({name,sql}))));
64
66
  // The archive schema identity alone, for databases that need not satisfy the policy.
65
- export async function schemaIdentity(db,config) { return identityOf((await archived(db,config)).entries); }
67
+ export async function schemaIdentity(db,config) {
68
+ const {entries,schemaObjects}=await archived(db,config);return schemaIdentityOf(entries,schemaObjects);
69
+ }
66
70
  export async function catalogue(db,config) {
67
- const {objects,kinds,excluded,entries}=await archived(db,config);
71
+ const {objects,kinds,excluded,entries,schemaObjects}=await archived(db,config);
68
72
  const tables=[];
69
73
  for(const entry of entries) {
70
74
  insist(!/^CREATE\s+VIRTUAL\s+TABLE/i.test(entry.sql) && kinds.find(x=>x.name===entry.name)?.type!=='shadow','virtual_table:'+entry.name);
@@ -99,8 +103,8 @@ export async function catalogue(db,config) {
99
103
  insist(tables.find(x=>x.name===table)?.columns.some(c=>c.name===keyColumn),'reference_column_missing');
100
104
  insist(!config.secretColumns[table]?.includes(keyColumn),'reference_column_excluded');
101
105
  }
102
- const identity=await identityOf(tables);
103
- return {identity,tables,excludedTables:excluded.map(x=>x.name).sort(),triggers:objects.filter(x=>x.type==='trigger' && tables.some(t=>t.name===x.tbl_name))};
106
+ const identity=await schemaIdentityOf(tables,schemaObjects);
107
+ return {identity,tables,schemaObjects,excludedTables:excluded.map(x=>x.name).sort(),triggers:objects.filter(x=>x.type==='trigger' && tables.some(t=>t.name===x.tbl_name))};
104
108
  }
105
109
  export function affinity(type) {
106
110
  const text=type.toUpperCase();
@@ -24,7 +24,11 @@ export async function main(args) {
24
24
  const records=await client.download(id),file=await privateFile(opts.out+'.partial',Boolean(opts.resume));
25
25
  try{for await(const record of records)await file.writeFile(canonicalJson(record)+'\n');await file.sync();}
26
26
  finally{await file.close();}
27
- await promote(opts.out+'.partial',opts.out);process.stdout.write('Export complete.\n');return;
27
+ await promote(opts.out+'.partial',opts.out);
28
+ // The verified local archive is durable: acknowledge it and release the
29
+ // hosted job's snapshots/parts. Failed downloads keep their resumable job.
30
+ await get('exports/'+id+'/abort','POST',{});
31
+ process.stdout.write('Export complete.\n');return;
28
32
  }
29
33
  if(opts.command==='import') {
30
34
  const source=Readable.toWeb(createReadStream(opts.from));let id=opts.resume,closing;
@@ -1,5 +1,6 @@
1
- import { all, statement, state, guard, job, save, partStatements, parts, readPart, cleanCopies, quote, stream, insist, run } from './storage.mjs';
1
+ import { statement, state, guard, job, save, partStatements, parts, readPart, cleanCopies, quote, stream, insist, run } from './storage.mjs';
2
2
  import { catalogue, encodedRow } from './catalogue.mjs';
3
+ import { checkReferences } from './references.mjs';
3
4
  import { createCopy,readCopy,normalizeCopy } from './copies.mjs';
4
5
  import { canonicalJson, utf8, base64, StreamHash } from './hash.mjs';
5
6
  import { FORMAT, makePart, checkPart, ContentVerifier, ArchiveVerifier } from './format.mjs';
@@ -23,7 +24,7 @@ export async function createExport(context,actor='direct') {
23
24
  objects.sort((a,b)=>a.bucket<b.bucket?-1:a.bucket>b.bucket?1:a.key<b.key?-1:a.key>b.key?1:0);
24
25
  const copies=Object.fromEntries(cat.tables.map((t,i)=>[t.name,'rer_portability_snap_'+id.replaceAll('-','')+'_'+i]));
25
26
  const migrations=await appliedMigrations(db);
26
- const manifest={kind:'manifest',format:FORMAT,version:2,product:config.product,exportedAt:new Date().toISOString(),sourceIdentity:s.source_identity,schema:cat.identity,tables:cat.tables,objects,payloads:[],excludedTables:cat.excludedTables,excludedColumns:config.secretColumns,requiredConfiguration:config.requiredConfiguration,partCount:0,epoch:s.epoch,journal:await journal(db,s.source_identity),...(migrations?{migrations}:{})};
27
+ const manifest={kind:'manifest',format:FORMAT,version:2,product:config.product,exportedAt:new Date().toISOString(),sourceIdentity:s.source_identity,schema:cat.identity,tables:cat.tables,schemaObjects:cat.schemaObjects,objects,payloads:[],excludedTables:cat.excludedTables,excludedColumns:config.secretColumns,requiredConfiguration:config.requiredConfiguration,partCount:0,epoch:s.epoch,journal:await journal(db,s.source_identity),...(migrations?{migrations}:{})};
27
28
  const row={id,kind:'export',actor,phase:'building',completed:0,metadata:{manifest,copies,cursor:{table:0,row:0,binary:0,offset:0},partBytes:config.partBytes}};
28
29
  await db.batch([guard(db,'EXISTS(SELECT 1 FROM rer_portability_state WHERE id=1 AND applying=0 AND source_identity=? AND epoch=?)',s.source_identity,s.epoch),
29
30
  ...cat.tables.flatMap(t=>createCopy(db,t,copies[t.name],true)),statement(db,'INSERT INTO rer_portability_jobs(id,kind,actor,phase,metadata) VALUES(?,?,?,?,?)',id,'export',actor,row.phase,canonicalJson(row.metadata)),db.prepare('DELETE FROM rer_portability_guard')]);
@@ -36,17 +37,10 @@ export async function createExport(context,actor='direct') {
36
37
  const descriptor=await capturePayload(db,id,manifest.payloads.length,await producer.produce());manifest.payloads.push({name,...descriptor});
37
38
  }
38
39
  await run(db,'UPDATE rer_portability_jobs SET metadata=? WHERE id=?',canonicalJson(row.metadata),id);
39
- await checkReferences(context,row);
40
+ await checkReferences(context,copies,objects);
40
41
  return id;
41
42
  }catch(e){await abortExport(context,id);throw e;}
42
43
  }
43
- async function checkReferences({db,config},row) {
44
- for(const [binding,spec] of Object.entries(config.buckets)) if(spec.objectReferences) {
45
- const {table,keyColumn}=spec.objectReferences,keys=new Set((await all(db,`SELECT DISTINCT ${quote(keyColumn)} AS key FROM ${quote(row.metadata.copies[table])} WHERE ${quote(keyColumn)} IS NOT NULL`)).map(x=>x.key));
46
- const objects=row.metadata.manifest.objects.filter(x=>x.bucket===binding);
47
- insist(objects.length===keys.size && objects.every(x=>keys.has(x.key)),'object_references:'+binding);
48
- }
49
- }
50
44
  async function nextItems(context,row) {
51
45
  const {db,config,buckets}=context,{manifest,cursor,copies,partBytes}=row.metadata;
52
46
  while(cursor.table<manifest.tables.length && cursor.row===manifest.tables[cursor.table].count){cursor.table++;cursor.row=0;}
@@ -106,7 +100,7 @@ export async function advanceExport(context,id,{expectedPart,maxParts=1}={}) {
106
100
  if(expectedPart!==undefined)expectedPart++;
107
101
  }else{
108
102
  // References are checked again against the archived, normalized snapshot.
109
- await checkReferences(context,row);
103
+ await checkReferences(context,row.metadata.copies,row.metadata.manifest.objects);
110
104
  row.metadata.manifest.partCount=row.completed;
111
105
  const verifier=new ContentVerifier(row.metadata.manifest),archive=new ArchiveVerifier(row.metadata.manifest);
112
106
  for await(const part of parts(context.db,id,row.completed)){archive.add(part);for(const item of part.items)verifier.add(item);}
@@ -2,6 +2,8 @@ import { archiveLines } from './lines.mjs';
2
2
  import { canonicalJson, sha256, StreamHash, unbase64, utf8 } from './hash.mjs';
3
3
  import { insist, PortabilityError } from './storage.mjs';
4
4
  import { validateRow } from './catalogue.mjs';
5
+ import { schemaIdentityOf, rawSchemaIdentityOf, tableSchemaIdentityOf } from './schema-identity.mjs';
6
+ import { legacySchemaIdentities } from './schema-legacy.mjs';
5
7
  export const FORMAT='runeditrun-portability';
6
8
  export const MAX_LINE_BYTES=24*1024*1024;
7
9
  export const MAX_PART_BYTES=16*1024*1024;
@@ -36,8 +38,12 @@ async function checkManifestFields(m) {
36
38
  insist(Array.isArray(m.tables) && Array.isArray(m.objects) && Array.isArray(m.payloads) && Array.isArray(m.journal) && Array.isArray(m.requiredConfiguration),'manifest_inventory');
37
39
  // Applied product migrations, recorded by archives since additive restore.
38
40
  insist(m.migrations===undefined || (Array.isArray(m.migrations) && m.migrations.every(x=>typeof x==='string')),'manifest_inventory');
39
- insist(m.schema===await sha256(canonicalJson(m.tables.map(({name,sql})=>({name,sql})))),'schema_checksum');
40
41
  insist(m.tables.every((t,i)=>typeof t.name==='string' && typeof t.sql==='string' && Array.isArray(t.columns) && Array.isArray(t.dependencies) && Number.isSafeInteger(t.count) && t.count>=0 && (!i || m.tables[i-1].name<t.name)),'table_inventory');
42
+ insist(m.schemaObjects===undefined || (Array.isArray(m.schemaObjects) && m.schemaObjects.every((o,i)=>
43
+ o && typeof o.name==='string' && ['index','view','trigger'].includes(o.type) && typeof o.table==='string' && typeof o.sql==='string'
44
+ && (!i || m.schemaObjects[i-1].name<o.name || (m.schemaObjects[i-1].name===o.name && m.schemaObjects[i-1].type<o.type)))),'manifest_inventory');
45
+ insist(m.schema===await schemaIdentityOf(m.tables,m.schemaObjects) || m.schema===await tableSchemaIdentityOf(m.tables)
46
+ || m.schema===await rawSchemaIdentityOf(m.tables) || (await legacySchemaIdentities(m.tables,m.schemaObjects)).includes(m.schema),'schema_checksum');
41
47
  for(const collection of [m.objects,m.payloads]) {
42
48
  const keys=new Set();
43
49
  for(const entry of collection) {
@@ -0,0 +1,51 @@
1
+ import { importRunningMetadata } from './import-lease.mjs';
2
+
3
+ // A code's spelling is not evidence that it is public. Keep the established
4
+ // vocabulary closed, and discard diagnostic suffixes containing schema/input.
5
+ const publicCodes = new Set(`
6
+ D1_RPC_TRANSPORT_LIMIT advance_limit archive_checksum archive_format archive_incomplete archive_json
7
+ binary_checksum binary_inventory binary_order binary_size body_too_large bucket_inventory bucket_not_empty
8
+ closing_missing column_inventory configuration_inventory confirmation_required content_checksum copy_inventory copy_rowid
9
+ data_after_closing dev_data erased_row erasure_epoch excluded_destination_dependents excluded_foreign_key
10
+ export_incomplete export_phase export_row_must_be_pure export_row_not_idempotent extra_part generated_sql
11
+ import_applying import_complete import_lease_expired import_phase import_row_columns import_row_key import_row_must_be_pure
12
+ import_row_not_deterministic import_row_not_idempotent import_row_undeclared import_row_unsafe_value import_running import_safety_table_missing
13
+ invalid_accepted_schema invalid_bookkeeping_rows invalid_bucket_policy invalid_bucket_prefix invalid_extra_payload
14
+ invalid_handler_configuration invalid_import_safety invalid_import_safety_column invalid_journal invalid_json invalid_object_references
15
+ invalid_part_limit invalid_policy invalid_policy_importSafety invalid_policy_importSafety_pair invalid_secret_column invalid_secret_columns
16
+ invalid_policy_excludeTables invalid_policy_includeTables invalid_policy_derivedTables invalid_policy_replaceableTables
17
+ invalid_policy_requiredConfiguration invalid_policy_exportRow invalid_policy_importRow invalid_policy_validateStaged
18
+ job_not_found later_journal ledger_inventory legacy_acknowledgement_required legacy_archive_unverifiable legacy_bucket_not_empty
19
+ legacy_output_exists legacy_output_required legacy_restored_export_failed legacy_target_not_empty legacy_v1_required live_policy_approval
20
+ manifest_counts manifest_identity manifest_inventory migration_ledger missing_bucket object_changed object_references
21
+ object_write_checksum object_write_failed origin_refused part_byte_limit part_checksum part_conflict part_count part_missing
22
+ part_order part_row_limit parts_missing payload_changed payload_inventory platform_table policy_table_missing
23
+ provider_relink_required provider_session_data provider_sign_in_must_be_off provider_transient_data reference_column_excluded reference_column_missing
24
+ route_not_found row_columns row_inventory row_not_null row_order row_part_too_large row_type rowid_inventory rows_missing
25
+ schema_checksum schema_mismatch secret_inventory secret_table_missing staged_table staged_validation table_inventory target_not_empty
26
+ transfer_sql_limit unauthorized unsafe_integer unsupported_column unsupported_copy_rowid unsupported_restore_width unsupported_rowid
27
+ upgrade_row_must_be_pure virtual_table wrong_product
28
+ unauthenticated forbidden owner_required authority_changed step_up_required rate_limited operation_failed
29
+ `.trim().split(/\s+/));
30
+ const publicCode = code => typeof code === 'string' && publicCodes.has(code.split(':')[0]) ? code.split(':')[0] : 'operation_failed';
31
+
32
+ // Public errors use the existing {error:code} vocabulary. Native messages can
33
+ // contain SQL, row values or credentials; inspect causes only to choose status.
34
+ export function httpFailure(error,{actor=null}={}) {
35
+ const seen = new Set();let typed,nativeStatus;
36
+ for (let cause = error; cause && !seen.has(cause) && seen.size < 5; cause = cause.cause) {
37
+ seen.add(cause);
38
+ if (Number.isInteger(cause.status) && cause.status >= 400 && cause.status <= 599) {
39
+ const code = publicCode(cause.code);
40
+ // A generic server wrapper must not turn a known authority refusal into 500.
41
+ if (!typed || typed.status >= 500 && cause.status < 500) typed = { status: cause.status, body: { error: code,
42
+ ...(cause.status === 409 && cause.code === 'import_running' ? importRunningMetadata(cause,actor) : {}) } };
43
+ }
44
+ if (cause.name === 'TimeoutError' || ['ETIMEDOUT', 'UND_ERR_CONNECT_TIMEOUT', 'UND_ERR_HEADERS_TIMEOUT'].includes(cause.code))
45
+ nativeStatus = 504;
46
+ if (['ECONNRESET', 'ECONNREFUSED', 'SQLITE_BUSY', 'SQLITE_LOCKED'].includes(cause.code)
47
+ || /\bD1_ERROR\b/i.test(cause.message ?? '') && /overloaded|temporarily unavailable|reset|timed? out|timeout|SQLITE_BUSY|SQLITE_LOCKED|database is locked/i.test(cause.message))
48
+ nativeStatus ??= 503;
49
+ }
50
+ return typed ?? { status: nativeStatus ?? 500, body: { error: 'operation_failed' } };
51
+ }
@@ -1,6 +1,7 @@
1
1
  import { createPortability } from './index.mjs';
2
2
  import { insist, PortabilityError } from './storage.mjs';
3
3
  import { MAX_LINE_BYTES } from './format.mjs';
4
+ import { httpFailure } from './http-errors.mjs';
4
5
  export async function boundedJson(request,limit) {
5
6
  const reader=request.body?.getReader();if(!reader)return {};
6
7
  const chunks=[];let size=0;
@@ -19,10 +20,12 @@ export async function createPortabilityHandler({db,buckets,policy,appOrigin,base
19
20
  if(url.pathname!==basePath && !url.pathname.startsWith(basePath+'/'))return null;
20
21
  const route=url.pathname.slice(basePath.length+1).split('/'),[collection,id,action,index]=route;
21
22
  const operation=collection==='exports'?'export':collection==='imports'?'import':'unknown';
23
+ let authorizedActor=null;
22
24
  try {
23
25
  const actor=await authorize(request,{operation,jobId:id??null});insist(typeof actor==='string' && actor.length>0,'unauthorized',401);
24
26
  if(!['GET','HEAD'].includes(request.method))insist(request.headers.get('Origin')===appOrigin,'origin_refused',403);
25
27
  if(id)insist(await service.actor(id)===actor,'job_not_found',404);
28
+ authorizedActor=actor;
26
29
  insist(['exports','imports'].includes(collection) && route.length<=4,'route_not_found',404);
27
30
  const api=service[collection];
28
31
  if(collection==='imports' && request.method==='GET' && !id)return json({active:await api.active(actor)});
@@ -38,6 +41,6 @@ export async function createPortabilityHandler({db,buckets,policy,appOrigin,base
38
41
  if(collection==='imports' && request.method==='PUT' && id && action==='parts' && /^(0|[1-9]\d*)$/.test(index??''))return json(await api.uploadPart(id,Number(index),await boundedJson(request,maxBodyBytes)));
39
42
  if(collection==='imports' && request.method==='POST' && id && action==='finalize' && !index)return json(await api.finalize(id,await boundedJson(request,maxBodyBytes)));
40
43
  throw new PortabilityError('route_not_found',404);
41
- }catch(e){return json({error:e instanceof PortabilityError?e.code:'operation_failed',...(e instanceof PortabilityError && e.code==='import_running'?{jobId:e.jobId,abortableAt:e.abortableAt}:{})},e instanceof PortabilityError?e.status:500);}
44
+ }catch(e){const failure=httpFailure(e,{actor:authorizedActor});return json(failure.body,failure.status);}
42
45
  };
43
46
  }
@@ -6,6 +6,13 @@ import { statement, guard, job, PortabilityError } from './storage.mjs';
6
6
  export const IMPORT_LEASE_MS=5*60*1000;
7
7
  const RENEW_MS=30*1000;
8
8
  const active=row=>['staging','applying'].includes(row.phase);
9
+ const runningMetadata=new WeakMap();
10
+ // Only the lease constructor can approve these details. Mutable Error fields,
11
+ // prototypes and diagnostic code suffixes cannot establish that provenance.
12
+ export const importRunningMetadata=(error,actor)=>{
13
+ const running=runningMetadata.get(error);
14
+ return typeof actor==='string' && actor===running?.actor?running.details:undefined;
15
+ };
9
16
 
10
17
  // Private transfer bookkeeping can be added to already-installed destinations
11
18
  // without changing product migrations or the archive's schema identity. Keep
@@ -27,8 +34,12 @@ export async function importLeaseStatus(db,row) {
27
34
  return {abortableAt:active(row)&&lease?new Date(lease.expires_at).toISOString():null,stale:active(row)&&!!lease&&lease.expires_at<=Date.now()};
28
35
  }
29
36
  export async function importRunning(db,id) {
30
- const error=new PortabilityError('import_running',409);
31
- error.jobId=id;error.abortableAt=(await importLeaseStatus(db,await job(db,id,'import'))).abortableAt;
37
+ const error=new PortabilityError('import_running',409),row=await job(db,id,'import');
38
+ error.jobId=id;error.abortableAt=(await importLeaseStatus(db,row)).abortableAt;
39
+ if(typeof id==='string' && id.length>0 && id.length<=100 && !/[^a-zA-Z0-9-]/.test(id)
40
+ && (error.abortableAt===null || typeof error.abortableAt==='string' && Number.isFinite(Date.parse(error.abortableAt))
41
+ && new Date(error.abortableAt).toISOString()===error.abortableAt))
42
+ runningMetadata.set(error,Object.freeze({actor:row.actor,details:Object.freeze({jobId:id,abortableAt:error.abortableAt})}));
32
43
  return error;
33
44
  }
34
45
 
@@ -17,6 +17,7 @@ import { additiveBridge, sameSchemaMigrations, appliedMigrations } from './bridg
17
17
  import { replayStatements } from './backfills.mjs';
18
18
  import { checkArchivedLedgers, archivedHistory } from './ledgers.mjs';
19
19
  import { legacyEmptyGuards, legacyAuditStatements, requireLegacyEmpty } from './legacy-safety.mjs';
20
+ import { checkReferences } from './references.mjs';
20
21
  // ledgers: the rows of each migration ledger table the archive carries, as the
21
22
  // caller read them from it ({table:rows}); uploadPart binds them to its parts.
22
23
  export async function createImport(context,manifest,actor='direct',legacyRecovery,ledgers) {
@@ -27,8 +28,15 @@ export async function createImport(context,manifest,actor='direct',legacyRecover
27
28
  const original=manifest;manifest=await restorableManifest(original);
28
29
  const skippedTables=original.tables.filter(table=>platformTable(table.name)).map(table=>table.name);
29
30
  const cat=await catalogue(db,config),upgrade=config.acceptedSchemas[original.schema];
31
+ // Table SQL and migration names cannot establish missing trigger/view/index
32
+ // definitions. Never infer that an old archive's absent inventory is empty.
33
+ insist(manifest.schemaObjects!==undefined || legacyRecovery,'schema_mismatch:legacy_schema',409);
30
34
  const same=manifest.schema===cat.identity,history=()=>archivedHistory(original,ledgers);
31
35
  const {replays,...bridge}=upgrade?{}:same?(await sameSchemaMigrations(db,manifest,cat,context.migrations,history)??{}):await additiveBridge(db,manifest,cat,context.migrations,history);
36
+ // Finalize checks object references on the staged rows; a later migration
37
+ // replayed into a reference table would publish rows that check never saw.
38
+ const referenced=replays?.find(r=>Object.values(config.buckets).some(spec=>spec.objectReferences?.table===r.table));
39
+ insist(!referenced,'schema_mismatch:migration_data:'+referenced?.migration);
32
40
  if(!bridge.fromSchema)insist(canonicalJson(manifest.tables.map(x=>x.name))===canonicalJson(cat.tables.map(x=>x.name)),'table_inventory');
33
41
  if(!upgrade && !bridge.fromSchema)for(const table of manifest.tables) {
34
42
  const target=cat.tables.find(t=>t.name===table.name);
@@ -181,6 +189,8 @@ async function finalizeLeasedImport(context,id,{contentHash,archiveHash,laterJou
181
189
  await refusePolicyApprovals(db,row.metadata.copies);
182
190
  await normalizeStaged(context,row);
183
191
  await refusePolicyApprovals(db,row.metadata.copies);
192
+ // The rows publication restores may name only this archive's objects, in their namespace.
193
+ await checkReferences(context,row.metadata.copies,manifest.objects);
184
194
  if(config.validateStaged) {
185
195
  const read=async table=>{insist(row.metadata.copies[table],'staged_table:'+table);const target=row.metadata.targets.find(t=>t.name===table);return (await readCopy(db,target,row.metadata.copies[table],0,row.metadata.manifest.tables.find(t=>t.name===table)?.count??0)).map(x=>x.row);};
186
196
  insist(await config.validateStaged({db,tables:{...row.metadata.copies},read,manifest})!==false,'staged_validation');
@@ -2,11 +2,14 @@ export type Cell = null | string | number | { blob: string };
2
2
  export type Row = Record<string, Cell>;
3
3
  export interface Column { name: string; type: string; notnull: number; pk: number; generated?: 'virtual'|'stored' }
4
4
  export interface Table { name: string; sql: string; columns: Column[]; dependencies: string[]; count: number; rowidAlias: string|null }
5
+ export interface SchemaObject { name: string; type: 'index'|'view'|'trigger'; table: string; sql: string }
5
6
  export interface ObjectDescriptor { bucket: string; key: string; size: number; sha256: string; httpMetadata: Omit<R2HTTPMetadata,'cacheExpiry'> & {cacheExpiry?:string}; customMetadata: Record<string,string> }
6
7
  export interface PayloadDescriptor { name: string; size: number; sha256: string }
7
8
  export interface Manifest {
8
9
  kind: 'manifest'; format: 'runeditrun-portability'; version: 2; product: string; exportedAt: string; sourceIdentity: string;
9
10
  schema: string; tables: Table[]; objects: ObjectDescriptor[]; payloads: PayloadDescriptor[];
11
+ /** Complete behavioral object definitions; absent from older table-only archives. */
12
+ schemaObjects?: SchemaObject[];
10
13
  excludedTables: string[]; excludedColumns: Record<string,string[]>; requiredConfiguration: string[]; partCount: number; epoch: number; journal: string[];
11
14
  /** Applied product migrations; absent from archives made before additive restore. */
12
15
  migrations?: string[];
@@ -0,0 +1,20 @@
1
+ import { all, quote, insist } from './storage.mjs';
2
+ // A referenced key stays in its bucket's namespace however a reader joins or
3
+ // resolves it: nonempty, under the declared prefix, not absolute and with no
4
+ // '.' or '..' segment.
5
+ const namespaced=(prefix,key)=>typeof key==='string' && key!=='' && key.startsWith(prefix) && !/^[\\/]/.test(key) && !key.split(/[\\/]/).some(part=>part==='.' || part==='..');
6
+ // Export checks its snapshot and restore its staged rows, before any object
7
+ // write: for every bucket that declares references, the distinct referenced keys
8
+ // lie in its namespace and are exactly its archived objects, none missing and
9
+ // none extra. A table restored across a later ADD COLUMN stages only archived
10
+ // columns; publication gives its rows the destination's default there.
11
+ export async function checkReferences({db,config},copies,objects) {
12
+ for(const [binding,spec] of Object.entries(config.buckets)) if(spec.objectReferences) {
13
+ const {table,keyColumn}=spec.objectReferences,copy=copies[table];
14
+ const column=async name=>(await all(db,`PRAGMA table_xinfo(${quote(name)})`)).find(c=>c.name===keyColumn);
15
+ const value=await column(copy)?quote(keyColumn):'('+((await column(table))?.dflt_value??'NULL')+')';
16
+ const keys=new Set((await all(db,`SELECT DISTINCT ${value} COLLATE BINARY AS key FROM ${quote(copy)} WHERE ${value} IS NOT NULL`)).map(x=>x.key));
17
+ const listed=objects.filter(x=>x.bucket===binding);
18
+ insist([...keys].every(key=>namespaced(spec.prefix??'',key)) && listed.length===keys.size && listed.every(x=>keys.has(x.key)),'object_references:'+binding);
19
+ }
20
+ }
@@ -0,0 +1,13 @@
1
+ import { canonicalTokens } from './sqlite-tokens.mjs';
2
+ import { canonicalJson, sha256 } from './hash.mjs';
3
+ export { rawSchemaIdentityOf } from './schema-legacy.mjs';
4
+
5
+ // Typed, lossless lexical identity. Only trivia and proven-safe keyword case
6
+ // are normalized; the same lexer proves every removed separator is harmless.
7
+ export const schemaSQL = sql => canonicalJson(canonicalTokens(sql));
8
+ const tableDefinitions = tables => tables.map(({ name, sql }) => ({ name, sql: schemaSQL(sql) }));
9
+ export const schemaIdentityOf = (tables, objects = []) => sha256(canonicalJson({
10
+ tables: tableDefinitions(tables),
11
+ objects: objects.map(({ name, type, table, sql }) => ({ name, type, table, sql: schemaSQL(sql) })),
12
+ }));
13
+ export const tableSchemaIdentityOf = tables => sha256(canonicalJson(tableDefinitions(tables)));
@@ -0,0 +1,21 @@
1
+ import { sqliteTokens } from './sqlite-tokens.mjs';
2
+ import { canonicalJson, sha256 } from './hash.mjs';
3
+
4
+ // Read-only wire codecs for 4280971d/537b088b/9352a46d checksums. These
5
+ // lossy representations authenticate historical manifests ONLY; they never
6
+ // decide whether two schemas match. All compatibility uses schemaIdentityOf.
7
+ function legacySQL(sql) {
8
+ return canonicalJson(sqliteTokens(sql).flatMap(token => {
9
+ if (token.type === 'blob') return [token.text[0], token.text.slice(1)];
10
+ if (token.type === 'quoted-identifier' || token.type === 'string') return [token.text];
11
+ return token.text.match(/[\p{L}\p{N}_$]+|->>|->|==|!=|<>|<=|>=|<<|>>|\|\||./gu);
12
+ }));
13
+ }
14
+ export const rawSchemaIdentityOf = tables => sha256(canonicalJson(tables.map(({name,sql}) => ({name,sql}))));
15
+ export async function legacySchemaIdentities(tables, objects = []) {
16
+ const definitions = tables.map(({name,sql}) => ({name,sql:legacySQL(sql)}));
17
+ return Promise.all([
18
+ sha256(canonicalJson(definitions)),
19
+ sha256(canonicalJson({tables:definitions,objects:objects.map(({name,type,table,sql}) => ({name,type,table,sql:legacySQL(sql)}))})),
20
+ ]);
21
+ }