@runeditrun/rer 0.2.79 → 0.2.81

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (147) hide show
  1. package/README.md +17 -3
  2. package/cli/README.md +25 -13
  3. package/cli/authoring.mjs +14 -0
  4. package/cli/contract.mjs +25 -5
  5. package/cli/demo-host.mjs +4 -2
  6. package/cli/deploy-needs.mjs +4 -2
  7. package/cli/dev-identity.mjs +2 -2
  8. package/cli/export.mjs +1 -1
  9. package/cli/inspect.mjs +38 -2
  10. package/cli/ledger-refusal.mjs +18 -0
  11. package/cli/main.mjs +8 -2
  12. package/cli/managed-server.mjs +1 -1
  13. package/cli/parse.mjs +7 -6
  14. package/cli/portability-migrations.mjs +30 -11
  15. package/cli/portability-worker.mjs +4 -1
  16. package/cli/portability.mjs +8 -2
  17. package/cli/production-auth.mjs +2 -1
  18. package/cli/production-mail.mjs +36 -9
  19. package/cli/project.mjs +5 -14
  20. package/cli/release.mjs +6 -5
  21. package/cli/site-hosting.mjs +90 -0
  22. package/cli/site-preview.mjs +4 -1
  23. package/cli/site-render.mjs +3 -2
  24. package/cli/site.mjs +8 -4
  25. package/edition-authoring/adopt.mjs +39 -0
  26. package/edition-authoring/base-acknowledgements.mjs +24 -0
  27. package/edition-authoring/base-copies.mjs +427 -0
  28. package/edition-authoring/cli.mjs +85 -0
  29. package/edition-authoring/copy-checkout.mjs +69 -0
  30. package/edition-authoring/files.mjs +15 -2
  31. package/edition-authoring/lock.mjs +48 -0
  32. package/edition-authoring/offline-git.mjs +28 -0
  33. package/edition-authoring/parent-comparison.mjs +77 -0
  34. package/edition-authoring/projection-lock.mjs +38 -0
  35. package/edition-authoring/release.mjs +48 -0
  36. package/edition-authoring/runtime-dependencies.mjs +108 -0
  37. package/edition-authoring/sources.mjs +114 -0
  38. package/edition-authoring/updates.mjs +105 -0
  39. package/global/CHANGELOG.md +24 -0
  40. package/global/README.md +17 -3
  41. package/global/cli/install.mjs +3 -1
  42. package/global/main.mjs +9 -12
  43. package/modules/container/build.mjs +1 -1
  44. package/modules/d1-transport/index.mjs +68 -21
  45. package/modules/demo/README.md +16 -2
  46. package/modules/demo/declaration.mjs +21 -0
  47. package/modules/demo/input-path.mjs +18 -0
  48. package/modules/demo/label.d.mts +1 -1
  49. package/modules/demo/label.mjs +10 -5
  50. package/modules/demo/navigation.mjs +148 -0
  51. package/modules/demo/run-url.mjs +9 -0
  52. package/modules/demo/vite.d.mts +7 -2
  53. package/modules/demo/vite.mjs +17 -2
  54. package/modules/demo-host/README.md +5 -3
  55. package/modules/demo-host/page-style.mjs +2 -2
  56. package/modules/demo-host/pages.mjs +3 -3
  57. package/modules/demo-host/plan.mjs +6 -4
  58. package/modules/demo-host/theme.mjs +5 -2
  59. package/modules/dependencies/auth.mjs +2 -2
  60. package/modules/deploy-kit/README.md +19 -15
  61. package/modules/deploy-kit/owner-cli.mjs +1 -1
  62. package/modules/deploy-kit/owner.mjs +8 -1
  63. package/modules/deploy-kit/readiness-window.mjs +3 -3
  64. package/modules/erasure/README.md +10 -5
  65. package/modules/import-kit/src/d1/README.md +15 -9
  66. package/modules/import-kit/src/d1/apply-effects.ts +33 -0
  67. package/modules/import-kit/src/d1/apply.ts +46 -14
  68. package/modules/import-kit/src/d1/atomic-prefix.ts +64 -0
  69. package/modules/import-kit/src/d1/budget.ts +48 -26
  70. package/modules/import-kit/src/d1/compressed-plan.ts +9 -2
  71. package/modules/import-kit/src/d1/database.ts +2 -1
  72. package/modules/import-kit/src/d1/jobs.ts +2 -1
  73. package/modules/import-kit/src/d1/plans.ts +4 -2
  74. package/modules/import-kit/src/d1/verified-plan.ts +63 -0
  75. package/modules/import-kit/src/types.ts +3 -1
  76. package/modules/integrations/core/README.md +2 -2
  77. package/modules/mail/README.md +4 -3
  78. package/modules/mail/pnpm-lock.yaml +16 -0
  79. package/modules/operator-auth/README.md +92 -3
  80. package/modules/operator-auth/accounts/admin.mjs +52 -20
  81. package/modules/operator-auth/accounts/audit.mjs +24 -0
  82. package/modules/operator-auth/accounts/business-team.mjs +91 -0
  83. package/modules/operator-auth/accounts/domain-audit.mjs +24 -0
  84. package/modules/operator-auth/accounts/native.mjs +6 -2
  85. package/modules/operator-auth/accounts/provider-control.mjs +7 -4
  86. package/modules/operator-auth/accounts/provider-setup.mjs +34 -0
  87. package/modules/operator-auth/accounts/settings-page.mjs +26 -4
  88. package/modules/operator-auth/accounts/status.mjs +7 -2
  89. package/modules/operator-auth/accounts/store.mjs +32 -6
  90. package/modules/operator-auth/business-settings.mjs +40 -0
  91. package/modules/operator-auth/conformance/README.md +1 -1
  92. package/modules/operator-auth/dev/index.mjs +22 -4
  93. package/modules/operator-auth/dev/refusal.mjs +19 -0
  94. package/modules/operator-auth/docs/README.md +6 -5
  95. package/modules/operator-auth/docs/tanstack-start.md +1 -1
  96. package/modules/operator-auth/docs/vite-spa.md +5 -6
  97. package/modules/operator-auth/identity/email-link/index.mjs +1 -1
  98. package/modules/operator-auth/identity/oidc/admin-audit.mjs +31 -0
  99. package/modules/operator-auth/identity/oidc/index.mjs +12 -5
  100. package/modules/operator-auth/identity/oidc/store.mjs +2 -0
  101. package/modules/operator-auth/index.d.mts +41 -2
  102. package/modules/operator-auth/page.mjs +14 -0
  103. package/modules/operator-auth/representation.mjs +83 -0
  104. package/modules/operator-auth/routes.mjs +31 -3
  105. package/modules/operator-auth/service.mjs +8 -3
  106. package/modules/operator-auth/signin/index.mjs +1 -1
  107. package/modules/portability/README.md +173 -15
  108. package/modules/portability/bookkeeping.mjs +3 -3
  109. package/modules/portability/bridge.mjs +5 -4
  110. package/modules/portability/catalogue.mjs +12 -8
  111. package/modules/portability/cli.mjs +5 -1
  112. package/modules/portability/export.mjs +10 -14
  113. package/modules/portability/format.mjs +12 -1
  114. package/modules/portability/http-errors.mjs +51 -0
  115. package/modules/portability/http.mjs +4 -1
  116. package/modules/portability/import-lease.mjs +13 -2
  117. package/modules/portability/import.mjs +10 -0
  118. package/modules/portability/index.d.mts +13 -2
  119. package/modules/portability/index.mjs +3 -2
  120. package/modules/portability/ledgers.mjs +27 -4
  121. package/modules/portability/references.mjs +20 -0
  122. package/modules/portability/schema-identity.mjs +13 -0
  123. package/modules/portability/schema-legacy.mjs +21 -0
  124. package/modules/portability/sqlite-tokens.mjs +121 -0
  125. package/modules/release/README.md +94 -11
  126. package/modules/release/github-verify.mjs +30 -9
  127. package/modules/release/sign.mjs +4 -2
  128. package/modules/release/signature.mjs +14 -7
  129. package/modules/release/signing-cli.mjs +2 -1
  130. package/modules/release/statement.mjs +2 -2
  131. package/modules/release/vectors/generate.mjs +1 -0
  132. package/modules/release/vectors/publisher-projects.mjs +133 -0
  133. package/modules/release/vectors/release-signature-vectors.json +5195 -0
  134. package/modules/release/verify.mjs +5 -1
  135. package/modules/site/docs-search/search.css +1 -1
  136. package/onboarding/README.md +16 -4
  137. package/onboarding/adapter.mjs +5 -1
  138. package/onboarding/schema/SETUP.schema.json +68 -1
  139. package/onboarding/schema/business-field-inputs.mjs +36 -0
  140. package/onboarding/schema/json-schema.mjs +6 -2
  141. package/onboarding/schema/setup-validation.mjs +23 -3
  142. package/onboarding/server.mjs +1 -1
  143. package/onboarding/setup-manifest.mjs +2 -0
  144. package/onboarding/start-origin-diagnostic.mjs +24 -0
  145. package/onboarding/ui/style.css +9 -9
  146. package/package.json +1 -1
  147. package/scripts/publish-check/README.md +1 -1
@@ -31,7 +31,8 @@ payload consumers, and prove the integrated round trip.
31
31
  ## Policy
32
32
 
33
33
  `await createPortability({db, buckets, policy})` validates policy and the current
34
- schema at construction. Apply the bookkeeping migration first. Policy fields:
34
+ schema at construction. Apply the bookkeeping migration first. Export refuses
35
+ with HTTP 409 `import_applying` while an import is applying. Policy fields:
35
36
 
36
37
  | Field | Meaning |
37
38
  | --- | --- |
@@ -47,7 +48,7 @@ schema at construction. Apply the bookkeeping migration first. Policy fields:
47
48
  | `requiredConfiguration` | Setting names returned after restore for the owner to re-enter. |
48
49
  | `validateStaged({db,tables,read,manifest})` | Read-only product validation before any object write; false or a thrown error refuses restore. `tables` names the staging tables for bounded SQL queries; `read` is a convenience for small tables. |
49
50
  | `acceptedSchemas` | Map from an earlier full schema hash to `{upgradeRow(table,row)}`. Upgrade functions are pure and synchronous. No older schema is accepted by default. |
50
- | `buckets` | Binding names mapped to `{prefix?,objectReferences?:{table,keyColumn}}`. Prefixes isolate an object namespace. References require exactly the distinct referenced keys. |
51
+ | `buckets` | Binding names mapped to `{prefix?,objectReferences?:{table,keyColumn}}`. Prefixes isolate an object namespace. References require exactly the distinct referenced keys: export checks its snapshot, and restore its staged rows (after `importRow`, before any object write), that each key is nonempty, under the prefix, not absolute and without `.` or `..` segments, and that the keys are exactly the bucket's archived objects, none missing and none extra; otherwise `object_references:<binding>`. |
51
52
  | `extraPayloads` | Named `{produce,consume}` pairs. Producers return bytes or a byte stream. Consumers receive a verified byte stream and descriptor before D1 commit. |
52
53
  | `partBytes` | Maximum part size, default 16 MiB, configurable from 1 KiB to 16 MiB. |
53
54
 
@@ -107,7 +108,8 @@ and declaring both `policy` and `policyModule` fail closed. Serializable object
107
108
  schema-upgrade and payload callbacks) are refused instead of being dropped by
108
109
  JSON transport. Container transfer currently refuses `policyModule` explicitly;
109
110
  it continues to accept serializable object policies. Malformed primary or later
110
- journal JSON is reported once as `archive_json` with exit 2, before local data
111
+ journal JSON is reported once as `invalid_archive` with exit 2 and a re-export
112
+ recovery command, before local data
111
113
  migrations, without echoing archive content or parser diagnostics.
112
114
 
113
115
  An installation migrated by Platform has `rer_migrations(name, sha256,
@@ -131,6 +133,39 @@ Custom replay runners can supply each plan entry's `aliases` and pass
131
133
  `{dataOnly}` as `migrationReplay`'s third argument. Verified archive ledger
132
134
  hashes take precedence over name-only placement where available.
133
135
 
136
+ ### Leaving RER Managed: the Platform ledger rule
137
+
138
+ A Managed installation's D1 keeps its history in `rer_migrations`, which the
139
+ Platform runner writes; a native installation uses `d1_migrations`. The ledger
140
+ is deployment history, never product schema or data. Base maps it by migration
141
+ identity, name and SHA-256 of the file:
142
+
143
+ - Export from a Platform-ledger source records `manifest.migrationLedger` (see
144
+ "Migration history" below) and never archives `rer_migrations` as a table.
145
+ - Import into a native installation matches every entry to exactly one
146
+ checkout file by hash; a name may confirm that file but cannot point at
147
+ another. A file the checkout does not have or whose content differs refuses
148
+ as `migration_ledger:unknown:<name>`, a name on one file with the hash of
149
+ another as `migration_ledger:conflict:<name>`, and the CLI prints the fix.
150
+ A checkout file the Platform history lacks refuses at the preflight as
151
+ `migration_ledger:unapplied`, naming the file; apply it with Platform first.
152
+ - Adapters with their own archive format (not Base's) call `mapMigrationLedger`
153
+ instead of comparing ledger controllers: it returns the entries named by the
154
+ checkout's files, or refuses with the same constant codes.
155
+
156
+ ```js
157
+ import { mapMigrationLedger } from '@runeditrun/rer/portability-migrations';
158
+
159
+ // Export: the source's ledger rows (rer_migrations or not), as native identities.
160
+ const ledger = await mapMigrationLedger({ root, binding: 'DB', platformMigrations, entries: rows });
161
+ // Import: the archive's entries must map without refusal; write no ledger rows.
162
+ await mapMigrationLedger({ root, binding: 'DB', platformMigrations, entries: archive.ledger });
163
+ ```
164
+
165
+ Call `nativeMigrationPreflight` before each native apply as below: a sole,
166
+ fully applied `rer_migrations` returns `{nativeApply:false}` for export and
167
+ import alike, so a seed's own guard must not require `d1_migrations`.
168
+
134
169
  ### Before an adapter's native migrations
135
170
 
136
171
  Seed adapters sometimes prepare schema before calling `localPortability`.
@@ -190,7 +225,7 @@ provider operation.
190
225
  | Compatible `d1_migrations`, every recorded name exactly equal to its checkout filename | `{nativeApply:true}`; Wrangler may apply its pending files. Plain Wrangler names remain supported; tracked history must also match hashes and applied states. |
191
226
  | Native alias-only or hash-only relocation, or an unmapped native data-only row | `migration_ledger:incompatible`; restore the matching checkout or reconcile through the owning runner before native apply. |
192
227
  | Sole, fully applied matching `rer_migrations` | `{nativeApply:false}`; preserve Platform's ledger and skip Wrangler. |
193
- | Any pending checkout file with Platform history | `migration_ledger:unapplied`; apply it through the owning runner first. |
228
+ | Any pending checkout file with Platform history | `migration_ledger:unapplied`, naming the file (`error.migration`); apply it through the owning runner first. |
194
229
  | Multiple ledgers, even empty | `migration_ledger:multiple`; reconcile history through its owning runner. |
195
230
  | Legacy `_rer_deploy_migrations` alone | `migration_ledger:runner`; use its owning runner, which can adopt the history. |
196
231
  | Invalid ledger shape, row or binding | `migration_ledger:incompatible` or `migration_ledger:binding`. |
@@ -207,7 +242,7 @@ RER's native `localPortability` invokes this same entry point in its
207
242
  history again for transfer.
208
243
 
209
244
  The implementation ships in the RER CLI, not the Worker portability module.
210
- Package-mode seeds need a future `@runeditrun/rer` release containing this
245
+ Package-mode seeds need a pinned `@runeditrun/rer` release containing this
211
246
  export and an exact pin/lock update; version 0.2.77 does not expose it.
212
247
  Vendored seeds need a reviewed Base CLI-copy refresh with
213
248
  `node scripts/export-cli.mjs SEED_ROOT` from the selected Base source and their
@@ -256,7 +291,7 @@ secret base cells become NULL before generated snapshots are normalized. A
256
291
  generated column cannot itself be redacted: exclude the table or change the
257
292
  product schema/policy if its computed value must be secret. Ordinary-column
258
293
  metadata and row encoding are unchanged. Ordinary-table archives using the
259
- authenticated format continue to import into the same schema; see the version
294
+ v2 integrity-checked format continue to import into the same schema; see the version
260
295
  1 refusal below.
261
296
 
262
297
  An archived INTEGER-affinity base cell outside JavaScript's safe integer range fails before
@@ -330,6 +365,59 @@ parts also preserve the original rowid, including gaps, for restoration and
330
365
  erasure matching. Export time,
331
366
  source identity, job ids and boundaries do not affect equivalence.
332
367
 
368
+ Schema identity covers table definitions (including constraints and generated
369
+ expressions), explicit indexes (including uniqueness and partial predicates),
370
+ views and triggers. Indexes and triggers on excluded tables are excluded with
371
+ their tables, including platform bookkeeping tables. The complete additional
372
+ definitions travel in `schemaObjects`. A single SQLite lexer reads the original
373
+ SQL before removing trivia. Identity records each token's class and canonical
374
+ value, plus a deterministic spelling: a single space separates neighbours
375
+ exactly when concatenating them would change their tokenization. The complete
376
+ joined stream is tokenized again and must reproduce those same tokens.
377
+ Comments and harmless inter-token whitespace therefore compare equally, while
378
+ `x '31'` and `x'31'`, separated minus signs, numbers and combined operators stay
379
+ distinct. No regular expression rewrites SQL.
380
+
381
+ Quoted identifiers (double quotes, backticks, brackets, or a single-quoted string
382
+ used as a name), strings, blobs, numbers, variables and ordinary identifiers keep
383
+ their exact spelling. Only unambiguous reserved keywords fold ASCII case.
384
+ Fallback/JOIN/window keywords retain their spelling: for example, SQLite can
385
+ interpret `DEFAULT ABORT` and `DEFAULT abort` as different string defaults.
386
+ Unicode case and quote styles are never normalized. The lexer follows SQLite's
387
+ [`sqlite3GetToken`](https://github.com/sqlite/sqlite/blob/master/src/tokenize.c)
388
+ and the keyword fallback rules in
389
+ [`parse.y`](https://github.com/sqlite/sqlite/blob/master/src/parse.y).
390
+ The original DDL is authenticated by `archiveHash`.
391
+
392
+ Unsupported or ambiguous SQL refuses with `schema_mismatch:unsupported_sql` and
393
+ the existing original-installation re-export/recovery instructions; it never
394
+ receives a shared or empty identity. This includes NUL and ill-formed Unicode,
395
+ BOM at token start, vertical-tab whitespace, unterminated quotes/brackets/block
396
+ comments (including bare `/*`), malformed/odd-length blobs, invalid numeric
397
+ separators or numeric identifier suffixes, malformed named/Tcl variables, and
398
+ unknown ASCII punctuation. Valid decimal/hex/exponent/underscore numbers,
399
+ SQLite's longest operators, question-mark and named/Tcl variables, LF/EOF line
400
+ comments, and terminated block comments are recognized. SQLite may accept some
401
+ of the refused whitespace or EOF-comment forms; re-export with supported DDL
402
+ rather than silently discarding their bytes. Virtual tables remain unsupported.
403
+
404
+ Older version 2 archives first verify their original raw-DDL, table-token, or
405
+ complete-object-token schema checksum and complete archive proof. A read-only
406
+ historical checksum codec projects the real lexer's tokens into those old wire
407
+ encodings. Its lossy result never decides compatibility: complete definitions
408
+ are always compared through the new typed token identity. Compatibility then uses
409
+ any complete `schemaObjects` inventory the archive carries, without rewriting
410
+ its authenticated manifest or parts. Without that inventory, the original
411
+ presence or absence of indexes, views and triggers cannot be established from
412
+ table SQL or migration names alone, even when the destination has no such
413
+ objects. Normal restore refuses `schema_mismatch` and the CLI gives the existing
414
+ re-export recovery instructions. This also catches removal of an old object's
415
+ definition rather than assuming missing evidence means the objects match.
416
+ Re-export from the original installation to carry the complete definitions.
417
+ A new archive explicitly records even an empty object inventory, so later
418
+ additive migrations can still be placed normally. The separate acknowledged
419
+ v1 recovery path remains an explicitly unverified recovery, as documented below.
420
+
333
421
  New exports use archive version 2. The closing `archiveHash` is SHA-256 of
334
422
  canonical JSON `{manifest,parts}`, where `manifest` is the complete manifest
335
423
  and `parts` is the ordered array of verified part SHA-256 strings. This binds
@@ -385,6 +473,18 @@ them into a scratch database to find the archive's place in that history:
385
473
  - Archives record the applied migration names in `manifest.migrations` (since
386
474
  additive restore). Every recorded migration must be applied here, and
387
475
  replaying exactly those must give the archive's schema identity.
476
+ - Archives exported from an installation RER Platform migrated (a Managed
477
+ installation leaving for native) also carry `manifest.migrationLedger`:
478
+ `{origin:'rer_migrations',entries:[{name,sha256}]}` in application order,
479
+ each entry named by the exporting checkout's file with that SHA-256, so
480
+ `manifest.migrations` lists native file names whatever prefix Platform
481
+ recorded. Where the exporter has no checkout files (a hosted export), the
482
+ ledger's own names are kept beside their hashes. Import resolves each entry
483
+ by hash exactly as it does an archived ledger table (next item), so a
484
+ Platform prefix needs no declaration and a changed file refuses. Platform's
485
+ ledger is never imported as product tables. Native sources record no
486
+ `migrationLedger`; their archives are unchanged, and v1 and earlier v2
487
+ archives remain importable.
388
488
  - Older archives that still carry their source's migration ledger as a table
389
489
  (`rer_migrations` and the other ledgers above, from before they became
390
490
  bookkeeping) are placed by it. Every name it records must be applied here,
@@ -468,7 +568,9 @@ a view resolved from applied migration history), selects `*` or
468
568
  a secret column, when a trigger on it or a foreign key action from it reaches a
469
569
  catalogue table, when any other later statement writes it (directly or through
470
570
  a trigger) or changes its columns other than by `ADD COLUMN` (allowed only when
471
- every replay names its columns), or when the statement cannot be bounded.
571
+ every replay names its columns), when the statement cannot be bounded, or when
572
+ it writes a table that a bucket's `objectReferences` names: references are
573
+ checked on the staged rows, before any object write.
472
574
  A failed or interrupted apply leaves no replayed row; finalizing again replays
473
575
  from the start.
474
576
  Read-only trigger bodies, including RAISE-only guards, add no write dependency.
@@ -586,6 +688,42 @@ its v2 output succeeds; do not reset the recovered installation to retry.
586
688
  `exports.create(actor?)` copies all tables in one D1 batch. Product writes may
587
689
  continue after that snapshot. R2 listing records size and etag; changes or missing
588
690
  objects fail subsequent reads. D1 and R2 do not share a cross-binding transaction.
691
+
692
+ The direct service factory `createPortability` accepts an optional synchronous
693
+ `onExport({id, actor, scope})` callback returning `[sql, args][]`. Its SQL is
694
+ appended to that same snapshot/job-creating D1 batch. A thrown callback or failed
695
+ statement refuses creation; failed SQL rolls back the job, snapshots and audit
696
+ rows together. With no hook, export behavior is unchanged. This records job
697
+ creation, not successful download: a later build, file-write or download failure
698
+ does not erase a committed creation audit.
699
+
700
+ This hook applies to exports created through that service, including the file
701
+ adapter used by CLI exports. `createPortabilityHandler` does not accept or
702
+ forward `onExport`; the HTTP export handler has no audit hook.
703
+
704
+ `id` is the export job id, `scope` is the constant `'installation'`, and `actor`
705
+ is the opaque caller id, with the direct/CLI sentinel `'direct'` normalized to
706
+ `null`. Job ownership still stores `'direct'` for existing direct callers.
707
+ The callback receives no names, emails, IdP identifiers, tokens, record data,
708
+ object keys or file paths. Pass only opaque operator ids as authenticated actors.
709
+ The seed owns audit storage, migrations, retention, erasure and archive policy;
710
+ Base adds no audit table. For a CLI installation export, wire the seed's existing
711
+ `operation_audit` schema to action `installation.export`, actor_id `NULL` and
712
+ feature `rer export`, for example:
713
+
714
+ ```js
715
+ const service = await createPortability({db, buckets, policy,
716
+ onExport: ({id, actor}) => [[
717
+ 'INSERT INTO operation_audit(id,action,actor_id,feature) VALUES(?,?,?,?)',
718
+ [id, 'installation.export', actor, 'rer export'],
719
+ ]],
720
+ });
721
+ ```
722
+
723
+ Adapt the statement to the seed's schema and use the service in its existing
724
+ export adapter. Hooks return SQL only; external side effects cannot share the
725
+ D1 transaction. Do not write these events to the erasure tombstone journal.
726
+
589
727
  Transfer bookkeeping lives in sidecar tables, so product tables up to D1's
590
728
  100-column limit retain the full column and binding capacity. A keyless table
591
729
  with 100 columns or generated columns restores its rowids with two updates in
@@ -612,8 +750,8 @@ part; different content returns 409. Reassembled bytes pass the part checksum
612
750
  before completion advances. Staged data stays private until finalization.
613
751
  Finalize requires the closing `archiveHash` and verifies all
614
752
  parts, semantic contents, destination
615
- emptiness, erasure and product invariants. R2 ownership is recorded after the
616
- empty-namespace check. Verified object writes precede one atomic D1 batch, with
753
+ emptiness, erasure, object references and product invariants. R2 ownership
754
+ is recorded after the empty-namespace check. Verified object writes precede one atomic D1 batch, with
617
755
  deferred FKs, trigger suppression/recreation, replaceable rows, staged inserts,
618
756
  journal adoption and completion. FK cycles and self-references restore together.
619
757
  An unsuccessful commit removes only owned objects. A lost commit acknowledgement
@@ -640,7 +778,10 @@ by every upload/finalize request, then every 30 seconds while that request is
640
778
  working (including verification, R2 transfers and product callbacks). This is a
641
779
  recovery window, not a total restore limit: the measured 20,000-row round trip
642
780
  below already takes 171 seconds, and callbacks have no wall-time bound. A fresh
643
- import returns `409 import_running` with `jobId` and ISO `abortableAt`. The owner
781
+ import returns `409 import_running`. Only a validated internal lease refusal,
782
+ after authorization and for the actor who owns that import, includes `jobId`
783
+ and ISO `abortableAt`. Authorization failures and other actors receive no job or
784
+ lease metadata, even if an error carries the same code or fields. The owner
644
785
  can find its active job with `GET /api/portability/imports` (`{active:status}`
645
786
  or `{active:null}`), and inspect `GET .../imports/:id`: `abortableAt` and `stale` describe
646
787
  its lease. The existing authenticated `POST .../imports/:id/abort` (or CLI
@@ -671,6 +812,17 @@ returns an async dispatcher, or null for foreign paths. Default base path is
671
812
  an actor id or null. Jobs belong to their creating actor. Unsafe methods require
672
813
  the exact configured Origin. Responses have `Cache-Control: no-store` and errors
673
814
  are `{error:code}`. Request bodies are bounded; `maxBodyBytes` can lower the bound.
815
+ Typed authorization errors retain their HTTP status, directly or through a
816
+ bounded, cycle-safe `cause` chain. Codes come from the existing fixed public
817
+ vocabulary (for example `403 step_up_required`); unknown codes become
818
+ `operation_failed`. Portability diagnostics expose the known code prefix only,
819
+ never caller-supplied suffixes. Aggregate error collections are not authority
820
+ wrappers. Native diagnostics remain private:
821
+ transient D1 availability failures use `503 operation_failed`, transport timeouts
822
+ use `504 operation_failed`, and unexpected faults remain `500 operation_failed`.
823
+ The hosted CLI aborts a completed export after saving and verifying the local
824
+ archive, releasing its snapshots and parts. Interrupted downloads retain their
825
+ job for resume.
674
826
 
675
827
  | Method / relative path | Action |
676
828
  | --- | --- |
@@ -749,13 +901,14 @@ product erasure or remove product objects; the product handles attachments and
749
901
  any extra state. Do not retain a previously downloaded archive as erasure proof;
750
902
  supply the owner's later journal when restoring old offline data.
751
903
 
752
- ## Measured capacity
904
+ ## Historical measured capacity
753
905
 
754
- One local Miniflare D1/R2 run exported, restored and exported 20,000 rows across
755
- five tables in 171.169 seconds, with 625 parts at a configured 4 KiB bound and a
906
+ An unverified historical local Miniflare D1/R2 report recorded export, restore
907
+ and export of 20,000 rows across five tables in 171.169 seconds, with 625 parts at a configured 4 KiB bound and a
756
908
  1,979,471-byte archive. Restore used one atomic D1 commit. The verification run
757
- reports the measurements again. These measurements are test evidence, not a production
758
- capacity claim. Real D1 CPU, query, statement and storage limits, object mutation,
909
+ can produce fresh measurements; no corresponding historical benchmark receipt
910
+ is retained in this repository. These figures do not establish current production
911
+ capacity. Real D1 CPU, query, statement and storage limits, object mutation,
759
912
  destination writers and extra-payload recovery need product-level verification.
760
913
 
761
914
  ## Adopting
@@ -772,7 +925,11 @@ modules/portability/index.d.mts
772
925
  modules/portability/lines.mjs
773
926
  modules/portability/hash.mjs
774
927
  modules/portability/format.mjs
928
+ modules/portability/references.mjs
775
929
  modules/portability/catalogue.mjs
930
+ modules/portability/schema-identity.mjs
931
+ modules/portability/schema-legacy.mjs
932
+ modules/portability/sqlite-tokens.mjs
776
933
  modules/portability/bookkeeping.mjs
777
934
  modules/portability/storage.mjs
778
935
  modules/portability/copies.mjs
@@ -798,6 +955,7 @@ modules/portability/import-safety.mjs
798
955
  modules/portability/payloads.mjs
799
956
  modules/portability/journal.mjs
800
957
  modules/portability/http.mjs
958
+ modules/portability/http-errors.mjs
801
959
  modules/portability/http-client.mjs
802
960
  modules/portability/cli.mjs
803
961
  modules/portability/cli-options.mjs
@@ -1,4 +1,4 @@
1
- import { canonicalJson, sha256 } from './hash.mjs';
1
+ import { schemaIdentityOf } from './schema-identity.mjs';
2
2
  import { insist } from './storage.mjs';
3
3
 
4
4
  // Installation authority and transient working state, never product data.
@@ -23,8 +23,8 @@ export async function restorableManifest(manifest) {
23
23
  for(const table of manifest.tables)if(identifier(table.name)==='ik_d1_assertions')
24
24
  insist(table.count===0,'invalid_bookkeeping_rows:'+table.name,409);
25
25
  const tables=manifest.tables.filter(table=>!platformTable(table.name));
26
- if(tables.length===manifest.tables.length)return manifest;
27
- return {...manifest,tables,schema:await sha256(canonicalJson(tables.map(({name,sql})=>({name,sql})))),
26
+ const schemaObjects=manifest.schemaObjects?.filter(object=>!manifest.tables.some(table=>platformTable(table.name) && table.name===object.table));
27
+ return {...manifest,tables,...(schemaObjects?{schemaObjects}:{}),schema:await schemaIdentityOf(tables,schemaObjects),
28
28
  excludedColumns:Object.fromEntries(Object.entries(manifest.excludedColumns).filter(([name])=>!platformTable(name)))};
29
29
  }
30
30
  export const skippedTablesLine=tables=>'Skipped platform bookkeeping tables: '+tables.join(', ')+'.';
@@ -4,6 +4,7 @@ import { tokens, identifier, folded } from './generated.mjs';
4
4
  import { laterData } from './backfills.mjs';
5
5
  import { destinationLedger, matchMigrationLedger } from './ledgers.mjs';
6
6
  import { checkRebuild } from './rebuilds.mjs';
7
+ import { schemaSQL } from './schema-identity.mjs';
7
8
 
8
9
  // Applied product migrations in the ledger's own order (ledgers.mjs). A
9
10
  // state-bearing ledger may hold an unacknowledged migration whose effects are
@@ -38,7 +39,7 @@ async function placement(db,manifest,cat,replay,ledger,history) {
38
39
  unapplied=applied.filter(name=>!earlier.includes(name));
39
40
  }else{
40
41
  // Unrecorded: the latest matching prefix. Migrations that change no archived
41
- // table (indexes, excluded tables, data-only changes) cannot be told apart
42
+ // definition (excluded tables or data-only changes) cannot be told apart
42
43
  // and count as later. The source may have stopped at any matching prefix,
43
44
  // so every migration after the earliest one may be missing from its rows.
44
45
  const matching=[];
@@ -62,7 +63,7 @@ async function placement(db,manifest,cat,replay,ledger,history) {
62
63
  return {applied,earlier,replays,rebuilt};
63
64
  }
64
65
  // An archive with the destination's schema identity can still predate later
65
- // migrations that changed only data (a purge or a backfill) or added an index.
66
+ // migrations that changed only data (a purge or a backfill).
66
67
  // Those that change restorable data are refused like any other. Without a
67
68
  // replayer the statements cannot be read, so an archive whose recorded
68
69
  // migrations the destination's ledger has outgrown is refused. An archive that
@@ -82,7 +83,7 @@ export async function sameSchemaMigrations(db,manifest,cat,migrations,history) {
82
83
  }
83
84
  // An archive made before later migrations restores only when those migrations
84
85
  // are additive, or a proven row-preserving CHECK rebuild: new tables, new
85
- // columns that take a default, or indexes (which archives never carry).
86
+ // columns that take a default, or later schema objects established by replay.
86
87
  // Anything else stays a schema_mismatch refusal.
87
88
  export async function additiveBridge(db,manifest,cat,migrations,history) {
88
89
  insist(migrations,'schema_mismatch');
@@ -95,7 +96,7 @@ export async function additiveBridge(db,manifest,cat,migrations,history) {
95
96
  const kept=target.columns.slice(0,table.columns.length),added=target.columns.slice(table.columns.length);
96
97
  insist(canonicalJson(kept)===canonicalJson(table.columns) && table.rowidAlias===target.rowidAlias && table.dependencies.every(d=>target.dependencies.includes(d)),'schema_mismatch:columns'+label);
97
98
  const rebuiltChecks=rebuilt.has(folded(table.name)) && checkRebuild(table.sql,target.sql);
98
- if(!added.length){insist(table.sql===target.sql || rebuiltChecks,'schema_mismatch:definition'+label);continue;}
99
+ if(!added.length){insist(schemaSQL(table.sql)===schemaSQL(target.sql) || rebuiltChecks,'schema_mismatch:definition'+label);continue;}
99
100
  const declared=await all(db,`PRAGMA table_xinfo(${quote(table.name)})`);
100
101
  insist(added.every(c=>!c.pk && !c.generated && (!c.notnull || declared.find(d=>d.name===c.name)?.dflt_value!=null)),'schema_mismatch:column_default'+label);
101
102
  insist(appended(table.sql,target.sql,added.map(c=>c.name)),'schema_mismatch:definition'+label);
@@ -1,5 +1,6 @@
1
1
  import { all, insist, quote, PortabilityError } from './storage.mjs';
2
- import { canonicalJson, sha256, base64, unbase64, utf8 } from './hash.mjs';
2
+ import { canonicalJson, base64, unbase64, utf8 } from './hash.mjs';
3
+ import { schemaIdentityOf } from './schema-identity.mjs';
3
4
  import { platformTables, platformTable, identifier, matches } from './bookkeeping.mjs';
4
5
  /** Platform bookkeeping is mandatory; operator identity remains a seed choice. */
5
6
  export const excludedTables=Object.freeze([...platformTables,'oa_%','oc_%']);
@@ -57,14 +58,17 @@ async function archived(db,config) {
57
58
  const derived=config.derivedTables;
58
59
  const excluded=objects.filter(x=>x.type==='table' && (excludedTable(x.name,config) || (!config.includeTables.includes(x.name) &&
59
60
  derived.some(p=>x.name===p || (kinds.find(k=>k.name===x.name)?.type==='shadow' && x.name.startsWith(p+'_'))))));
60
- objects.sort((a,b)=>a.name<b.name?-1:a.name>b.name?1:0);
61
- return {objects,kinds,excluded,entries:objects.filter(x=>x.type==='table' && !excluded.some(y=>y.name===x.name))};
61
+ objects.sort((a,b)=>a.name<b.name?-1:a.name>b.name?1:a.type<b.type?-1:a.type>b.type?1:0);
62
+ const schemaObjects=objects.filter(x=>['index','view','trigger'].includes(x.type)
63
+ && !excluded.some(t=>t.name===x.tbl_name)).map(({name,type,tbl_name,sql})=>({name,type,table:tbl_name,sql}));
64
+ return {objects,kinds,excluded,schemaObjects,entries:objects.filter(x=>x.type==='table' && !excluded.some(y=>y.name===x.name))};
62
65
  }
63
- const identityOf=tables=>sha256(canonicalJson(tables.map(({name,sql})=>({name,sql}))));
64
66
  // The archive schema identity alone, for databases that need not satisfy the policy.
65
- export async function schemaIdentity(db,config) { return identityOf((await archived(db,config)).entries); }
67
+ export async function schemaIdentity(db,config) {
68
+ const {entries,schemaObjects}=await archived(db,config);return schemaIdentityOf(entries,schemaObjects);
69
+ }
66
70
  export async function catalogue(db,config) {
67
- const {objects,kinds,excluded,entries}=await archived(db,config);
71
+ const {objects,kinds,excluded,entries,schemaObjects}=await archived(db,config);
68
72
  const tables=[];
69
73
  for(const entry of entries) {
70
74
  insist(!/^CREATE\s+VIRTUAL\s+TABLE/i.test(entry.sql) && kinds.find(x=>x.name===entry.name)?.type!=='shadow','virtual_table:'+entry.name);
@@ -99,8 +103,8 @@ export async function catalogue(db,config) {
99
103
  insist(tables.find(x=>x.name===table)?.columns.some(c=>c.name===keyColumn),'reference_column_missing');
100
104
  insist(!config.secretColumns[table]?.includes(keyColumn),'reference_column_excluded');
101
105
  }
102
- const identity=await identityOf(tables);
103
- return {identity,tables,excludedTables:excluded.map(x=>x.name).sort(),triggers:objects.filter(x=>x.type==='trigger' && tables.some(t=>t.name===x.tbl_name))};
106
+ const identity=await schemaIdentityOf(tables,schemaObjects);
107
+ return {identity,tables,schemaObjects,excludedTables:excluded.map(x=>x.name).sort(),triggers:objects.filter(x=>x.type==='trigger' && tables.some(t=>t.name===x.tbl_name))};
104
108
  }
105
109
  export function affinity(type) {
106
110
  const text=type.toUpperCase();
@@ -24,7 +24,11 @@ export async function main(args) {
24
24
  const records=await client.download(id),file=await privateFile(opts.out+'.partial',Boolean(opts.resume));
25
25
  try{for await(const record of records)await file.writeFile(canonicalJson(record)+'\n');await file.sync();}
26
26
  finally{await file.close();}
27
- await promote(opts.out+'.partial',opts.out);process.stdout.write('Export complete.\n');return;
27
+ await promote(opts.out+'.partial',opts.out);
28
+ // The verified local archive is durable: acknowledge it and release the
29
+ // hosted job's snapshots/parts. Failed downloads keep their resumable job.
30
+ await get('exports/'+id+'/abort','POST',{});
31
+ process.stdout.write('Export complete.\n');return;
28
32
  }
29
33
  if(opts.command==='import') {
30
34
  const source=Readable.toWeb(createReadStream(opts.from));let id=opts.resume,closing;
@@ -1,10 +1,11 @@
1
- import { all, statement, state, guard, job, save, partStatements, parts, readPart, cleanCopies, quote, stream, insist, run } from './storage.mjs';
1
+ import { statement, state, guard, job, save, partStatements, parts, readPart, cleanCopies, quote, stream, insist, run } from './storage.mjs';
2
2
  import { catalogue, encodedRow } from './catalogue.mjs';
3
+ import { checkReferences } from './references.mjs';
3
4
  import { createCopy,readCopy,normalizeCopy } from './copies.mjs';
4
5
  import { canonicalJson, utf8, base64, StreamHash } from './hash.mjs';
5
6
  import { FORMAT, makePart, checkPart, ContentVerifier, ArchiveVerifier } from './format.mjs';
6
7
  import { journal } from './journal.mjs';
7
- import { appliedMigrations } from './bridge.mjs';
8
+ import { readLedger, platformOrigin } from './ledgers.mjs';
8
9
  import { capturePayload, payloadRange } from './payloads.mjs';
9
10
  import { dropProviderData } from './provider-data.mjs';
10
11
  import { dropDevData } from './dev-data.mjs';
@@ -22,11 +23,13 @@ export async function createExport(context,actor='direct') {
22
23
  }
23
24
  objects.sort((a,b)=>a.bucket<b.bucket?-1:a.bucket>b.bucket?1:a.key<b.key?-1:a.key>b.key?1:0);
24
25
  const copies=Object.fromEntries(cat.tables.map((t,i)=>[t.name,'rer_portability_snap_'+id.replaceAll('-','')+'_'+i]));
25
- const migrations=await appliedMigrations(db);
26
- const manifest={kind:'manifest',format:FORMAT,version:2,product:config.product,exportedAt:new Date().toISOString(),sourceIdentity:s.source_identity,schema:cat.identity,tables:cat.tables,objects,payloads:[],excludedTables:cat.excludedTables,excludedColumns:config.secretColumns,requiredConfiguration:config.requiredConfiguration,partCount:0,epoch:s.epoch,journal:await journal(db,s.source_identity),...(migrations?{migrations}:{})};
26
+ const ledger=await readLedger(db),origin=await platformOrigin(ledger,context.migrations),migrations=origin?origin.entries.map(entry=>entry.name):ledger?.entries.map(entry=>entry.name);
27
+ const manifest={kind:'manifest',format:FORMAT,version:2,product:config.product,exportedAt:new Date().toISOString(),sourceIdentity:s.source_identity,schema:cat.identity,tables:cat.tables,schemaObjects:cat.schemaObjects,objects,payloads:[],excludedTables:cat.excludedTables,excludedColumns:config.secretColumns,requiredConfiguration:config.requiredConfiguration,partCount:0,epoch:s.epoch,journal:await journal(db,s.source_identity),...(migrations?{migrations}:{}),...(origin?{migrationLedger:origin}:{})};
27
28
  const row={id,kind:'export',actor,phase:'building',completed:0,metadata:{manifest,copies,cursor:{table:0,row:0,binary:0,offset:0},partBytes:config.partBytes}};
29
+ const audit=context.onExport?context.onExport({id,actor:actor==='direct'?null:actor,scope:'installation'}):[];
28
30
  await db.batch([guard(db,'EXISTS(SELECT 1 FROM rer_portability_state WHERE id=1 AND applying=0 AND source_identity=? AND epoch=?)',s.source_identity,s.epoch),
29
- ...cat.tables.flatMap(t=>createCopy(db,t,copies[t.name],true)),statement(db,'INSERT INTO rer_portability_jobs(id,kind,actor,phase,metadata) VALUES(?,?,?,?,?)',id,'export',actor,row.phase,canonicalJson(row.metadata)),db.prepare('DELETE FROM rer_portability_guard')]);
31
+ ...cat.tables.flatMap(t=>createCopy(db,t,copies[t.name],true)),statement(db,'INSERT INTO rer_portability_jobs(id,kind,actor,phase,metadata) VALUES(?,?,?,?,?)',id,'export',actor,row.phase,canonicalJson(row.metadata)),db.prepare('DELETE FROM rer_portability_guard'),
32
+ ...audit.map(([sql,args])=>statement(db,sql,...args))]);
30
33
  try {
31
34
  await dropDevData(db,cat.tables,copies);
32
35
  await dropProviderData(db,cat.tables,copies);
@@ -36,17 +39,10 @@ export async function createExport(context,actor='direct') {
36
39
  const descriptor=await capturePayload(db,id,manifest.payloads.length,await producer.produce());manifest.payloads.push({name,...descriptor});
37
40
  }
38
41
  await run(db,'UPDATE rer_portability_jobs SET metadata=? WHERE id=?',canonicalJson(row.metadata),id);
39
- await checkReferences(context,row);
42
+ await checkReferences(context,copies,objects);
40
43
  return id;
41
44
  }catch(e){await abortExport(context,id);throw e;}
42
45
  }
43
- async function checkReferences({db,config},row) {
44
- for(const [binding,spec] of Object.entries(config.buckets)) if(spec.objectReferences) {
45
- const {table,keyColumn}=spec.objectReferences,keys=new Set((await all(db,`SELECT DISTINCT ${quote(keyColumn)} AS key FROM ${quote(row.metadata.copies[table])} WHERE ${quote(keyColumn)} IS NOT NULL`)).map(x=>x.key));
46
- const objects=row.metadata.manifest.objects.filter(x=>x.bucket===binding);
47
- insist(objects.length===keys.size && objects.every(x=>keys.has(x.key)),'object_references:'+binding);
48
- }
49
- }
50
46
  async function nextItems(context,row) {
51
47
  const {db,config,buckets}=context,{manifest,cursor,copies,partBytes}=row.metadata;
52
48
  while(cursor.table<manifest.tables.length && cursor.row===manifest.tables[cursor.table].count){cursor.table++;cursor.row=0;}
@@ -106,7 +102,7 @@ export async function advanceExport(context,id,{expectedPart,maxParts=1}={}) {
106
102
  if(expectedPart!==undefined)expectedPart++;
107
103
  }else{
108
104
  // References are checked again against the archived, normalized snapshot.
109
- await checkReferences(context,row);
105
+ await checkReferences(context,row.metadata.copies,row.metadata.manifest.objects);
110
106
  row.metadata.manifest.partCount=row.completed;
111
107
  const verifier=new ContentVerifier(row.metadata.manifest),archive=new ArchiveVerifier(row.metadata.manifest);
112
108
  for await(const part of parts(context.db,id,row.completed)){archive.add(part);for(const item of part.items)verifier.add(item);}
@@ -2,6 +2,8 @@ import { archiveLines } from './lines.mjs';
2
2
  import { canonicalJson, sha256, StreamHash, unbase64, utf8 } from './hash.mjs';
3
3
  import { insist, PortabilityError } from './storage.mjs';
4
4
  import { validateRow } from './catalogue.mjs';
5
+ import { schemaIdentityOf, rawSchemaIdentityOf, tableSchemaIdentityOf } from './schema-identity.mjs';
6
+ import { legacySchemaIdentities } from './schema-legacy.mjs';
5
7
  export const FORMAT='runeditrun-portability';
6
8
  export const MAX_LINE_BYTES=24*1024*1024;
7
9
  export const MAX_PART_BYTES=16*1024*1024;
@@ -36,8 +38,17 @@ async function checkManifestFields(m) {
36
38
  insist(Array.isArray(m.tables) && Array.isArray(m.objects) && Array.isArray(m.payloads) && Array.isArray(m.journal) && Array.isArray(m.requiredConfiguration),'manifest_inventory');
37
39
  // Applied product migrations, recorded by archives since additive restore.
38
40
  insist(m.migrations===undefined || (Array.isArray(m.migrations) && m.migrations.every(x=>typeof x==='string')),'manifest_inventory');
39
- insist(m.schema===await sha256(canonicalJson(m.tables.map(({name,sql})=>({name,sql})))),'schema_checksum');
41
+ // The ledger an export from a RER Platform installation records, by migration identity.
42
+ const ledger=m.migrationLedger;
43
+ insist(ledger===undefined || (ledger && typeof ledger==='object' && Object.keys(ledger).sort().join()==='entries,origin' && ledger.origin==='rer_migrations' && Array.isArray(ledger.entries)
44
+ && ledger.entries.every(e=>e && typeof e==='object' && Object.keys(e).sort().join()==='name,sha256' && typeof e.name==='string' && /^[a-f0-9]{64}$/.test(e.sha256))
45
+ && canonicalJson(m.migrations)===canonicalJson(ledger.entries.map(e=>e.name)) && new Set(m.migrations).size===m.migrations.length),'manifest_inventory');
40
46
  insist(m.tables.every((t,i)=>typeof t.name==='string' && typeof t.sql==='string' && Array.isArray(t.columns) && Array.isArray(t.dependencies) && Number.isSafeInteger(t.count) && t.count>=0 && (!i || m.tables[i-1].name<t.name)),'table_inventory');
47
+ insist(m.schemaObjects===undefined || (Array.isArray(m.schemaObjects) && m.schemaObjects.every((o,i)=>
48
+ o && typeof o.name==='string' && ['index','view','trigger'].includes(o.type) && typeof o.table==='string' && typeof o.sql==='string'
49
+ && (!i || m.schemaObjects[i-1].name<o.name || (m.schemaObjects[i-1].name===o.name && m.schemaObjects[i-1].type<o.type)))),'manifest_inventory');
50
+ insist(m.schema===await schemaIdentityOf(m.tables,m.schemaObjects) || m.schema===await tableSchemaIdentityOf(m.tables)
51
+ || m.schema===await rawSchemaIdentityOf(m.tables) || (await legacySchemaIdentities(m.tables,m.schemaObjects)).includes(m.schema),'schema_checksum');
41
52
  for(const collection of [m.objects,m.payloads]) {
42
53
  const keys=new Set();
43
54
  for(const entry of collection) {
@@ -0,0 +1,51 @@
1
+ import { importRunningMetadata } from './import-lease.mjs';
2
+
3
+ // A code's spelling is not evidence that it is public. Keep the established
4
+ // vocabulary closed, and discard diagnostic suffixes containing schema/input.
5
+ const publicCodes = new Set(`
6
+ D1_RPC_TRANSPORT_LIMIT advance_limit archive_checksum archive_format archive_incomplete archive_json
7
+ binary_checksum binary_inventory binary_order binary_size body_too_large bucket_inventory bucket_not_empty
8
+ closing_missing column_inventory configuration_inventory confirmation_required content_checksum copy_inventory copy_rowid
9
+ data_after_closing dev_data erased_row erasure_epoch excluded_destination_dependents excluded_foreign_key
10
+ export_incomplete export_phase export_row_must_be_pure export_row_not_idempotent extra_part generated_sql
11
+ import_applying import_complete import_lease_expired import_phase import_row_columns import_row_key import_row_must_be_pure
12
+ import_row_not_deterministic import_row_not_idempotent import_row_undeclared import_row_unsafe_value import_running import_safety_table_missing
13
+ invalid_accepted_schema invalid_bookkeeping_rows invalid_bucket_policy invalid_bucket_prefix invalid_extra_payload
14
+ invalid_handler_configuration invalid_import_safety invalid_import_safety_column invalid_journal invalid_json invalid_object_references
15
+ invalid_part_limit invalid_policy invalid_policy_importSafety invalid_policy_importSafety_pair invalid_secret_column invalid_secret_columns
16
+ invalid_policy_excludeTables invalid_policy_includeTables invalid_policy_derivedTables invalid_policy_replaceableTables
17
+ invalid_policy_requiredConfiguration invalid_policy_exportRow invalid_policy_importRow invalid_policy_validateStaged
18
+ job_not_found later_journal ledger_inventory legacy_acknowledgement_required legacy_archive_unverifiable legacy_bucket_not_empty
19
+ legacy_output_exists legacy_output_required legacy_restored_export_failed legacy_target_not_empty legacy_v1_required live_policy_approval
20
+ manifest_counts manifest_identity manifest_inventory migration_ledger missing_bucket object_changed object_references
21
+ object_write_checksum object_write_failed origin_refused part_byte_limit part_checksum part_conflict part_count part_missing
22
+ part_order part_row_limit parts_missing payload_changed payload_inventory platform_table policy_table_missing
23
+ provider_relink_required provider_session_data provider_sign_in_must_be_off provider_transient_data reference_column_excluded reference_column_missing
24
+ route_not_found row_columns row_inventory row_not_null row_order row_part_too_large row_type rowid_inventory rows_missing
25
+ schema_checksum schema_mismatch secret_inventory secret_table_missing staged_table staged_validation table_inventory target_not_empty
26
+ transfer_sql_limit unauthorized unsafe_integer unsupported_column unsupported_copy_rowid unsupported_restore_width unsupported_rowid
27
+ upgrade_row_must_be_pure virtual_table wrong_product
28
+ unauthenticated forbidden owner_required authority_changed step_up_required rate_limited operation_failed
29
+ `.trim().split(/\s+/));
30
+ const publicCode = code => typeof code === 'string' && publicCodes.has(code.split(':')[0]) ? code.split(':')[0] : 'operation_failed';
31
+
32
+ // Public errors use the existing {error:code} vocabulary. Native messages can
33
+ // contain SQL, row values or credentials; inspect causes only to choose status.
34
+ export function httpFailure(error,{actor=null}={}) {
35
+ const seen = new Set();let typed,nativeStatus;
36
+ for (let cause = error; cause && !seen.has(cause) && seen.size < 5; cause = cause.cause) {
37
+ seen.add(cause);
38
+ if (Number.isInteger(cause.status) && cause.status >= 400 && cause.status <= 599) {
39
+ const code = publicCode(cause.code);
40
+ // A generic server wrapper must not turn a known authority refusal into 500.
41
+ if (!typed || typed.status >= 500 && cause.status < 500) typed = { status: cause.status, body: { error: code,
42
+ ...(cause.status === 409 && cause.code === 'import_running' ? importRunningMetadata(cause,actor) : {}) } };
43
+ }
44
+ if (cause.name === 'TimeoutError' || ['ETIMEDOUT', 'UND_ERR_CONNECT_TIMEOUT', 'UND_ERR_HEADERS_TIMEOUT'].includes(cause.code))
45
+ nativeStatus = 504;
46
+ if (['ECONNRESET', 'ECONNREFUSED', 'SQLITE_BUSY', 'SQLITE_LOCKED'].includes(cause.code)
47
+ || /\bD1_ERROR\b/i.test(cause.message ?? '') && /overloaded|temporarily unavailable|reset|timed? out|timeout|SQLITE_BUSY|SQLITE_LOCKED|database is locked/i.test(cause.message))
48
+ nativeStatus ??= 503;
49
+ }
50
+ return typed ?? { status: nativeStatus ?? 500, body: { error: 'operation_failed' } };
51
+ }
@@ -1,6 +1,7 @@
1
1
  import { createPortability } from './index.mjs';
2
2
  import { insist, PortabilityError } from './storage.mjs';
3
3
  import { MAX_LINE_BYTES } from './format.mjs';
4
+ import { httpFailure } from './http-errors.mjs';
4
5
  export async function boundedJson(request,limit) {
5
6
  const reader=request.body?.getReader();if(!reader)return {};
6
7
  const chunks=[];let size=0;
@@ -19,10 +20,12 @@ export async function createPortabilityHandler({db,buckets,policy,appOrigin,base
19
20
  if(url.pathname!==basePath && !url.pathname.startsWith(basePath+'/'))return null;
20
21
  const route=url.pathname.slice(basePath.length+1).split('/'),[collection,id,action,index]=route;
21
22
  const operation=collection==='exports'?'export':collection==='imports'?'import':'unknown';
23
+ let authorizedActor=null;
22
24
  try {
23
25
  const actor=await authorize(request,{operation,jobId:id??null});insist(typeof actor==='string' && actor.length>0,'unauthorized',401);
24
26
  if(!['GET','HEAD'].includes(request.method))insist(request.headers.get('Origin')===appOrigin,'origin_refused',403);
25
27
  if(id)insist(await service.actor(id)===actor,'job_not_found',404);
28
+ authorizedActor=actor;
26
29
  insist(['exports','imports'].includes(collection) && route.length<=4,'route_not_found',404);
27
30
  const api=service[collection];
28
31
  if(collection==='imports' && request.method==='GET' && !id)return json({active:await api.active(actor)});
@@ -38,6 +41,6 @@ export async function createPortabilityHandler({db,buckets,policy,appOrigin,base
38
41
  if(collection==='imports' && request.method==='PUT' && id && action==='parts' && /^(0|[1-9]\d*)$/.test(index??''))return json(await api.uploadPart(id,Number(index),await boundedJson(request,maxBodyBytes)));
39
42
  if(collection==='imports' && request.method==='POST' && id && action==='finalize' && !index)return json(await api.finalize(id,await boundedJson(request,maxBodyBytes)));
40
43
  throw new PortabilityError('route_not_found',404);
41
- }catch(e){return json({error:e instanceof PortabilityError?e.code:'operation_failed',...(e instanceof PortabilityError && e.code==='import_running'?{jobId:e.jobId,abortableAt:e.abortableAt}:{})},e instanceof PortabilityError?e.status:500);}
44
+ }catch(e){const failure=httpFailure(e,{actor:authorizedActor});return json(failure.body,failure.status);}
42
45
  };
43
46
  }