@runeditrun/rer 0.2.79 → 0.2.81
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -3
- package/cli/README.md +25 -13
- package/cli/authoring.mjs +14 -0
- package/cli/contract.mjs +25 -5
- package/cli/demo-host.mjs +4 -2
- package/cli/deploy-needs.mjs +4 -2
- package/cli/dev-identity.mjs +2 -2
- package/cli/export.mjs +1 -1
- package/cli/inspect.mjs +38 -2
- package/cli/ledger-refusal.mjs +18 -0
- package/cli/main.mjs +8 -2
- package/cli/managed-server.mjs +1 -1
- package/cli/parse.mjs +7 -6
- package/cli/portability-migrations.mjs +30 -11
- package/cli/portability-worker.mjs +4 -1
- package/cli/portability.mjs +8 -2
- package/cli/production-auth.mjs +2 -1
- package/cli/production-mail.mjs +36 -9
- package/cli/project.mjs +5 -14
- package/cli/release.mjs +6 -5
- package/cli/site-hosting.mjs +90 -0
- package/cli/site-preview.mjs +4 -1
- package/cli/site-render.mjs +3 -2
- package/cli/site.mjs +8 -4
- package/edition-authoring/adopt.mjs +39 -0
- package/edition-authoring/base-acknowledgements.mjs +24 -0
- package/edition-authoring/base-copies.mjs +427 -0
- package/edition-authoring/cli.mjs +85 -0
- package/edition-authoring/copy-checkout.mjs +69 -0
- package/edition-authoring/files.mjs +15 -2
- package/edition-authoring/lock.mjs +48 -0
- package/edition-authoring/offline-git.mjs +28 -0
- package/edition-authoring/parent-comparison.mjs +77 -0
- package/edition-authoring/projection-lock.mjs +38 -0
- package/edition-authoring/release.mjs +48 -0
- package/edition-authoring/runtime-dependencies.mjs +108 -0
- package/edition-authoring/sources.mjs +114 -0
- package/edition-authoring/updates.mjs +105 -0
- package/global/CHANGELOG.md +24 -0
- package/global/README.md +17 -3
- package/global/cli/install.mjs +3 -1
- package/global/main.mjs +9 -12
- package/modules/container/build.mjs +1 -1
- package/modules/d1-transport/index.mjs +68 -21
- package/modules/demo/README.md +16 -2
- package/modules/demo/declaration.mjs +21 -0
- package/modules/demo/input-path.mjs +18 -0
- package/modules/demo/label.d.mts +1 -1
- package/modules/demo/label.mjs +10 -5
- package/modules/demo/navigation.mjs +148 -0
- package/modules/demo/run-url.mjs +9 -0
- package/modules/demo/vite.d.mts +7 -2
- package/modules/demo/vite.mjs +17 -2
- package/modules/demo-host/README.md +5 -3
- package/modules/demo-host/page-style.mjs +2 -2
- package/modules/demo-host/pages.mjs +3 -3
- package/modules/demo-host/plan.mjs +6 -4
- package/modules/demo-host/theme.mjs +5 -2
- package/modules/dependencies/auth.mjs +2 -2
- package/modules/deploy-kit/README.md +19 -15
- package/modules/deploy-kit/owner-cli.mjs +1 -1
- package/modules/deploy-kit/owner.mjs +8 -1
- package/modules/deploy-kit/readiness-window.mjs +3 -3
- package/modules/erasure/README.md +10 -5
- package/modules/import-kit/src/d1/README.md +15 -9
- package/modules/import-kit/src/d1/apply-effects.ts +33 -0
- package/modules/import-kit/src/d1/apply.ts +46 -14
- package/modules/import-kit/src/d1/atomic-prefix.ts +64 -0
- package/modules/import-kit/src/d1/budget.ts +48 -26
- package/modules/import-kit/src/d1/compressed-plan.ts +9 -2
- package/modules/import-kit/src/d1/database.ts +2 -1
- package/modules/import-kit/src/d1/jobs.ts +2 -1
- package/modules/import-kit/src/d1/plans.ts +4 -2
- package/modules/import-kit/src/d1/verified-plan.ts +63 -0
- package/modules/import-kit/src/types.ts +3 -1
- package/modules/integrations/core/README.md +2 -2
- package/modules/mail/README.md +4 -3
- package/modules/mail/pnpm-lock.yaml +16 -0
- package/modules/operator-auth/README.md +92 -3
- package/modules/operator-auth/accounts/admin.mjs +52 -20
- package/modules/operator-auth/accounts/audit.mjs +24 -0
- package/modules/operator-auth/accounts/business-team.mjs +91 -0
- package/modules/operator-auth/accounts/domain-audit.mjs +24 -0
- package/modules/operator-auth/accounts/native.mjs +6 -2
- package/modules/operator-auth/accounts/provider-control.mjs +7 -4
- package/modules/operator-auth/accounts/provider-setup.mjs +34 -0
- package/modules/operator-auth/accounts/settings-page.mjs +26 -4
- package/modules/operator-auth/accounts/status.mjs +7 -2
- package/modules/operator-auth/accounts/store.mjs +32 -6
- package/modules/operator-auth/business-settings.mjs +40 -0
- package/modules/operator-auth/conformance/README.md +1 -1
- package/modules/operator-auth/dev/index.mjs +22 -4
- package/modules/operator-auth/dev/refusal.mjs +19 -0
- package/modules/operator-auth/docs/README.md +6 -5
- package/modules/operator-auth/docs/tanstack-start.md +1 -1
- package/modules/operator-auth/docs/vite-spa.md +5 -6
- package/modules/operator-auth/identity/email-link/index.mjs +1 -1
- package/modules/operator-auth/identity/oidc/admin-audit.mjs +31 -0
- package/modules/operator-auth/identity/oidc/index.mjs +12 -5
- package/modules/operator-auth/identity/oidc/store.mjs +2 -0
- package/modules/operator-auth/index.d.mts +41 -2
- package/modules/operator-auth/page.mjs +14 -0
- package/modules/operator-auth/representation.mjs +83 -0
- package/modules/operator-auth/routes.mjs +31 -3
- package/modules/operator-auth/service.mjs +8 -3
- package/modules/operator-auth/signin/index.mjs +1 -1
- package/modules/portability/README.md +173 -15
- package/modules/portability/bookkeeping.mjs +3 -3
- package/modules/portability/bridge.mjs +5 -4
- package/modules/portability/catalogue.mjs +12 -8
- package/modules/portability/cli.mjs +5 -1
- package/modules/portability/export.mjs +10 -14
- package/modules/portability/format.mjs +12 -1
- package/modules/portability/http-errors.mjs +51 -0
- package/modules/portability/http.mjs +4 -1
- package/modules/portability/import-lease.mjs +13 -2
- package/modules/portability/import.mjs +10 -0
- package/modules/portability/index.d.mts +13 -2
- package/modules/portability/index.mjs +3 -2
- package/modules/portability/ledgers.mjs +27 -4
- package/modules/portability/references.mjs +20 -0
- package/modules/portability/schema-identity.mjs +13 -0
- package/modules/portability/schema-legacy.mjs +21 -0
- package/modules/portability/sqlite-tokens.mjs +121 -0
- package/modules/release/README.md +94 -11
- package/modules/release/github-verify.mjs +30 -9
- package/modules/release/sign.mjs +4 -2
- package/modules/release/signature.mjs +14 -7
- package/modules/release/signing-cli.mjs +2 -1
- package/modules/release/statement.mjs +2 -2
- package/modules/release/vectors/generate.mjs +1 -0
- package/modules/release/vectors/publisher-projects.mjs +133 -0
- package/modules/release/vectors/release-signature-vectors.json +5195 -0
- package/modules/release/verify.mjs +5 -1
- package/modules/site/docs-search/search.css +1 -1
- package/onboarding/README.md +16 -4
- package/onboarding/adapter.mjs +5 -1
- package/onboarding/schema/SETUP.schema.json +68 -1
- package/onboarding/schema/business-field-inputs.mjs +36 -0
- package/onboarding/schema/json-schema.mjs +6 -2
- package/onboarding/schema/setup-validation.mjs +23 -3
- package/onboarding/server.mjs +1 -1
- package/onboarding/setup-manifest.mjs +2 -0
- package/onboarding/start-origin-diagnostic.mjs +24 -0
- package/onboarding/ui/style.css +9 -9
- package/package.json +1 -1
- package/scripts/publish-check/README.md +1 -1
|
@@ -31,7 +31,8 @@ payload consumers, and prove the integrated round trip.
|
|
|
31
31
|
## Policy
|
|
32
32
|
|
|
33
33
|
`await createPortability({db, buckets, policy})` validates policy and the current
|
|
34
|
-
schema at construction. Apply the bookkeeping migration first.
|
|
34
|
+
schema at construction. Apply the bookkeeping migration first. Export refuses
|
|
35
|
+
with HTTP 409 `import_applying` while an import is applying. Policy fields:
|
|
35
36
|
|
|
36
37
|
| Field | Meaning |
|
|
37
38
|
| --- | --- |
|
|
@@ -47,7 +48,7 @@ schema at construction. Apply the bookkeeping migration first. Policy fields:
|
|
|
47
48
|
| `requiredConfiguration` | Setting names returned after restore for the owner to re-enter. |
|
|
48
49
|
| `validateStaged({db,tables,read,manifest})` | Read-only product validation before any object write; false or a thrown error refuses restore. `tables` names the staging tables for bounded SQL queries; `read` is a convenience for small tables. |
|
|
49
50
|
| `acceptedSchemas` | Map from an earlier full schema hash to `{upgradeRow(table,row)}`. Upgrade functions are pure and synchronous. No older schema is accepted by default. |
|
|
50
|
-
| `buckets` | Binding names mapped to `{prefix?,objectReferences?:{table,keyColumn}}`. Prefixes isolate an object namespace. References require exactly the distinct referenced keys
|
|
51
|
+
| `buckets` | Binding names mapped to `{prefix?,objectReferences?:{table,keyColumn}}`. Prefixes isolate an object namespace. References require exactly the distinct referenced keys: export checks its snapshot, and restore its staged rows (after `importRow`, before any object write), that each key is nonempty, under the prefix, not absolute and without `.` or `..` segments, and that the keys are exactly the bucket's archived objects, none missing and none extra; otherwise `object_references:<binding>`. |
|
|
51
52
|
| `extraPayloads` | Named `{produce,consume}` pairs. Producers return bytes or a byte stream. Consumers receive a verified byte stream and descriptor before D1 commit. |
|
|
52
53
|
| `partBytes` | Maximum part size, default 16 MiB, configurable from 1 KiB to 16 MiB. |
|
|
53
54
|
|
|
@@ -107,7 +108,8 @@ and declaring both `policy` and `policyModule` fail closed. Serializable object
|
|
|
107
108
|
schema-upgrade and payload callbacks) are refused instead of being dropped by
|
|
108
109
|
JSON transport. Container transfer currently refuses `policyModule` explicitly;
|
|
109
110
|
it continues to accept serializable object policies. Malformed primary or later
|
|
110
|
-
journal JSON is reported once as `
|
|
111
|
+
journal JSON is reported once as `invalid_archive` with exit 2 and a re-export
|
|
112
|
+
recovery command, before local data
|
|
111
113
|
migrations, without echoing archive content or parser diagnostics.
|
|
112
114
|
|
|
113
115
|
An installation migrated by Platform has `rer_migrations(name, sha256,
|
|
@@ -131,6 +133,39 @@ Custom replay runners can supply each plan entry's `aliases` and pass
|
|
|
131
133
|
`{dataOnly}` as `migrationReplay`'s third argument. Verified archive ledger
|
|
132
134
|
hashes take precedence over name-only placement where available.
|
|
133
135
|
|
|
136
|
+
### Leaving RER Managed: the Platform ledger rule
|
|
137
|
+
|
|
138
|
+
A Managed installation's D1 keeps its history in `rer_migrations`, which the
|
|
139
|
+
Platform runner writes; a native installation uses `d1_migrations`. The ledger
|
|
140
|
+
is deployment history, never product schema or data. Base maps it by migration
|
|
141
|
+
identity, name and SHA-256 of the file:
|
|
142
|
+
|
|
143
|
+
- Export from a Platform-ledger source records `manifest.migrationLedger` (see
|
|
144
|
+
"Migration history" below) and never archives `rer_migrations` as a table.
|
|
145
|
+
- Import into a native installation matches every entry to exactly one
|
|
146
|
+
checkout file by hash; a name may confirm that file but cannot point at
|
|
147
|
+
another. A file the checkout does not have or whose content differs refuses
|
|
148
|
+
as `migration_ledger:unknown:<name>`, a name on one file with the hash of
|
|
149
|
+
another as `migration_ledger:conflict:<name>`, and the CLI prints the fix.
|
|
150
|
+
A checkout file the Platform history lacks refuses at the preflight as
|
|
151
|
+
`migration_ledger:unapplied`, naming the file; apply it with Platform first.
|
|
152
|
+
- Adapters with their own archive format (not Base's) call `mapMigrationLedger`
|
|
153
|
+
instead of comparing ledger controllers: it returns the entries named by the
|
|
154
|
+
checkout's files, or refuses with the same constant codes.
|
|
155
|
+
|
|
156
|
+
```js
|
|
157
|
+
import { mapMigrationLedger } from '@runeditrun/rer/portability-migrations';
|
|
158
|
+
|
|
159
|
+
// Export: the source's ledger rows (rer_migrations or not), as native identities.
|
|
160
|
+
const ledger = await mapMigrationLedger({ root, binding: 'DB', platformMigrations, entries: rows });
|
|
161
|
+
// Import: the archive's entries must map without refusal; write no ledger rows.
|
|
162
|
+
await mapMigrationLedger({ root, binding: 'DB', platformMigrations, entries: archive.ledger });
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
Call `nativeMigrationPreflight` before each native apply as below: a sole,
|
|
166
|
+
fully applied `rer_migrations` returns `{nativeApply:false}` for export and
|
|
167
|
+
import alike, so a seed's own guard must not require `d1_migrations`.
|
|
168
|
+
|
|
134
169
|
### Before an adapter's native migrations
|
|
135
170
|
|
|
136
171
|
Seed adapters sometimes prepare schema before calling `localPortability`.
|
|
@@ -190,7 +225,7 @@ provider operation.
|
|
|
190
225
|
| Compatible `d1_migrations`, every recorded name exactly equal to its checkout filename | `{nativeApply:true}`; Wrangler may apply its pending files. Plain Wrangler names remain supported; tracked history must also match hashes and applied states. |
|
|
191
226
|
| Native alias-only or hash-only relocation, or an unmapped native data-only row | `migration_ledger:incompatible`; restore the matching checkout or reconcile through the owning runner before native apply. |
|
|
192
227
|
| Sole, fully applied matching `rer_migrations` | `{nativeApply:false}`; preserve Platform's ledger and skip Wrangler. |
|
|
193
|
-
| Any pending checkout file with Platform history | `migration_ledger:unapplied
|
|
228
|
+
| Any pending checkout file with Platform history | `migration_ledger:unapplied`, naming the file (`error.migration`); apply it through the owning runner first. |
|
|
194
229
|
| Multiple ledgers, even empty | `migration_ledger:multiple`; reconcile history through its owning runner. |
|
|
195
230
|
| Legacy `_rer_deploy_migrations` alone | `migration_ledger:runner`; use its owning runner, which can adopt the history. |
|
|
196
231
|
| Invalid ledger shape, row or binding | `migration_ledger:incompatible` or `migration_ledger:binding`. |
|
|
@@ -207,7 +242,7 @@ RER's native `localPortability` invokes this same entry point in its
|
|
|
207
242
|
history again for transfer.
|
|
208
243
|
|
|
209
244
|
The implementation ships in the RER CLI, not the Worker portability module.
|
|
210
|
-
Package-mode seeds need a
|
|
245
|
+
Package-mode seeds need a pinned `@runeditrun/rer` release containing this
|
|
211
246
|
export and an exact pin/lock update; version 0.2.77 does not expose it.
|
|
212
247
|
Vendored seeds need a reviewed Base CLI-copy refresh with
|
|
213
248
|
`node scripts/export-cli.mjs SEED_ROOT` from the selected Base source and their
|
|
@@ -256,7 +291,7 @@ secret base cells become NULL before generated snapshots are normalized. A
|
|
|
256
291
|
generated column cannot itself be redacted: exclude the table or change the
|
|
257
292
|
product schema/policy if its computed value must be secret. Ordinary-column
|
|
258
293
|
metadata and row encoding are unchanged. Ordinary-table archives using the
|
|
259
|
-
|
|
294
|
+
v2 integrity-checked format continue to import into the same schema; see the version
|
|
260
295
|
1 refusal below.
|
|
261
296
|
|
|
262
297
|
An archived INTEGER-affinity base cell outside JavaScript's safe integer range fails before
|
|
@@ -330,6 +365,59 @@ parts also preserve the original rowid, including gaps, for restoration and
|
|
|
330
365
|
erasure matching. Export time,
|
|
331
366
|
source identity, job ids and boundaries do not affect equivalence.
|
|
332
367
|
|
|
368
|
+
Schema identity covers table definitions (including constraints and generated
|
|
369
|
+
expressions), explicit indexes (including uniqueness and partial predicates),
|
|
370
|
+
views and triggers. Indexes and triggers on excluded tables are excluded with
|
|
371
|
+
their tables, including platform bookkeeping tables. The complete additional
|
|
372
|
+
definitions travel in `schemaObjects`. A single SQLite lexer reads the original
|
|
373
|
+
SQL before removing trivia. Identity records each token's class and canonical
|
|
374
|
+
value, plus a deterministic spelling: a single space separates neighbours
|
|
375
|
+
exactly when concatenating them would change their tokenization. The complete
|
|
376
|
+
joined stream is tokenized again and must reproduce those same tokens.
|
|
377
|
+
Comments and harmless inter-token whitespace therefore compare equally, while
|
|
378
|
+
`x '31'` and `x'31'`, separated minus signs, numbers and combined operators stay
|
|
379
|
+
distinct. No regular expression rewrites SQL.
|
|
380
|
+
|
|
381
|
+
Quoted identifiers (double quotes, backticks, brackets, or a single-quoted string
|
|
382
|
+
used as a name), strings, blobs, numbers, variables and ordinary identifiers keep
|
|
383
|
+
their exact spelling. Only unambiguous reserved keywords fold ASCII case.
|
|
384
|
+
Fallback/JOIN/window keywords retain their spelling: for example, SQLite can
|
|
385
|
+
interpret `DEFAULT ABORT` and `DEFAULT abort` as different string defaults.
|
|
386
|
+
Unicode case and quote styles are never normalized. The lexer follows SQLite's
|
|
387
|
+
[`sqlite3GetToken`](https://github.com/sqlite/sqlite/blob/master/src/tokenize.c)
|
|
388
|
+
and the keyword fallback rules in
|
|
389
|
+
[`parse.y`](https://github.com/sqlite/sqlite/blob/master/src/parse.y).
|
|
390
|
+
The original DDL is authenticated by `archiveHash`.
|
|
391
|
+
|
|
392
|
+
Unsupported or ambiguous SQL refuses with `schema_mismatch:unsupported_sql` and
|
|
393
|
+
the existing original-installation re-export/recovery instructions; it never
|
|
394
|
+
receives a shared or empty identity. This includes NUL and ill-formed Unicode,
|
|
395
|
+
BOM at token start, vertical-tab whitespace, unterminated quotes/brackets/block
|
|
396
|
+
comments (including bare `/*`), malformed/odd-length blobs, invalid numeric
|
|
397
|
+
separators or numeric identifier suffixes, malformed named/Tcl variables, and
|
|
398
|
+
unknown ASCII punctuation. Valid decimal/hex/exponent/underscore numbers,
|
|
399
|
+
SQLite's longest operators, question-mark and named/Tcl variables, LF/EOF line
|
|
400
|
+
comments, and terminated block comments are recognized. SQLite may accept some
|
|
401
|
+
of the refused whitespace or EOF-comment forms; re-export with supported DDL
|
|
402
|
+
rather than silently discarding their bytes. Virtual tables remain unsupported.
|
|
403
|
+
|
|
404
|
+
Older version 2 archives first verify their original raw-DDL, table-token, or
|
|
405
|
+
complete-object-token schema checksum and complete archive proof. A read-only
|
|
406
|
+
historical checksum codec projects the real lexer's tokens into those old wire
|
|
407
|
+
encodings. Its lossy result never decides compatibility: complete definitions
|
|
408
|
+
are always compared through the new typed token identity. Compatibility then uses
|
|
409
|
+
any complete `schemaObjects` inventory the archive carries, without rewriting
|
|
410
|
+
its authenticated manifest or parts. Without that inventory, the original
|
|
411
|
+
presence or absence of indexes, views and triggers cannot be established from
|
|
412
|
+
table SQL or migration names alone, even when the destination has no such
|
|
413
|
+
objects. Normal restore refuses `schema_mismatch` and the CLI gives the existing
|
|
414
|
+
re-export recovery instructions. This also catches removal of an old object's
|
|
415
|
+
definition rather than assuming missing evidence means the objects match.
|
|
416
|
+
Re-export from the original installation to carry the complete definitions.
|
|
417
|
+
A new archive explicitly records even an empty object inventory, so later
|
|
418
|
+
additive migrations can still be placed normally. The separate acknowledged
|
|
419
|
+
v1 recovery path remains an explicitly unverified recovery, as documented below.
|
|
420
|
+
|
|
333
421
|
New exports use archive version 2. The closing `archiveHash` is SHA-256 of
|
|
334
422
|
canonical JSON `{manifest,parts}`, where `manifest` is the complete manifest
|
|
335
423
|
and `parts` is the ordered array of verified part SHA-256 strings. This binds
|
|
@@ -385,6 +473,18 @@ them into a scratch database to find the archive's place in that history:
|
|
|
385
473
|
- Archives record the applied migration names in `manifest.migrations` (since
|
|
386
474
|
additive restore). Every recorded migration must be applied here, and
|
|
387
475
|
replaying exactly those must give the archive's schema identity.
|
|
476
|
+
- Archives exported from an installation RER Platform migrated (a Managed
|
|
477
|
+
installation leaving for native) also carry `manifest.migrationLedger`:
|
|
478
|
+
`{origin:'rer_migrations',entries:[{name,sha256}]}` in application order,
|
|
479
|
+
each entry named by the exporting checkout's file with that SHA-256, so
|
|
480
|
+
`manifest.migrations` lists native file names whatever prefix Platform
|
|
481
|
+
recorded. Where the exporter has no checkout files (a hosted export), the
|
|
482
|
+
ledger's own names are kept beside their hashes. Import resolves each entry
|
|
483
|
+
by hash exactly as it does an archived ledger table (next item), so a
|
|
484
|
+
Platform prefix needs no declaration and a changed file refuses. Platform's
|
|
485
|
+
ledger is never imported as product tables. Native sources record no
|
|
486
|
+
`migrationLedger`; their archives are unchanged, and v1 and earlier v2
|
|
487
|
+
archives remain importable.
|
|
388
488
|
- Older archives that still carry their source's migration ledger as a table
|
|
389
489
|
(`rer_migrations` and the other ledgers above, from before they became
|
|
390
490
|
bookkeeping) are placed by it. Every name it records must be applied here,
|
|
@@ -468,7 +568,9 @@ a view resolved from applied migration history), selects `*` or
|
|
|
468
568
|
a secret column, when a trigger on it or a foreign key action from it reaches a
|
|
469
569
|
catalogue table, when any other later statement writes it (directly or through
|
|
470
570
|
a trigger) or changes its columns other than by `ADD COLUMN` (allowed only when
|
|
471
|
-
every replay names its columns),
|
|
571
|
+
every replay names its columns), when the statement cannot be bounded, or when
|
|
572
|
+
it writes a table that a bucket's `objectReferences` names: references are
|
|
573
|
+
checked on the staged rows, before any object write.
|
|
472
574
|
A failed or interrupted apply leaves no replayed row; finalizing again replays
|
|
473
575
|
from the start.
|
|
474
576
|
Read-only trigger bodies, including RAISE-only guards, add no write dependency.
|
|
@@ -586,6 +688,42 @@ its v2 output succeeds; do not reset the recovered installation to retry.
|
|
|
586
688
|
`exports.create(actor?)` copies all tables in one D1 batch. Product writes may
|
|
587
689
|
continue after that snapshot. R2 listing records size and etag; changes or missing
|
|
588
690
|
objects fail subsequent reads. D1 and R2 do not share a cross-binding transaction.
|
|
691
|
+
|
|
692
|
+
The direct service factory `createPortability` accepts an optional synchronous
|
|
693
|
+
`onExport({id, actor, scope})` callback returning `[sql, args][]`. Its SQL is
|
|
694
|
+
appended to that same snapshot/job-creating D1 batch. A thrown callback or failed
|
|
695
|
+
statement refuses creation; failed SQL rolls back the job, snapshots and audit
|
|
696
|
+
rows together. With no hook, export behavior is unchanged. This records job
|
|
697
|
+
creation, not successful download: a later build, file-write or download failure
|
|
698
|
+
does not erase a committed creation audit.
|
|
699
|
+
|
|
700
|
+
This hook applies to exports created through that service, including the file
|
|
701
|
+
adapter used by CLI exports. `createPortabilityHandler` does not accept or
|
|
702
|
+
forward `onExport`; the HTTP export handler has no audit hook.
|
|
703
|
+
|
|
704
|
+
`id` is the export job id, `scope` is the constant `'installation'`, and `actor`
|
|
705
|
+
is the opaque caller id, with the direct/CLI sentinel `'direct'` normalized to
|
|
706
|
+
`null`. Job ownership still stores `'direct'` for existing direct callers.
|
|
707
|
+
The callback receives no names, emails, IdP identifiers, tokens, record data,
|
|
708
|
+
object keys or file paths. Pass only opaque operator ids as authenticated actors.
|
|
709
|
+
The seed owns audit storage, migrations, retention, erasure and archive policy;
|
|
710
|
+
Base adds no audit table. For a CLI installation export, wire the seed's existing
|
|
711
|
+
`operation_audit` schema to action `installation.export`, actor_id `NULL` and
|
|
712
|
+
feature `rer export`, for example:
|
|
713
|
+
|
|
714
|
+
```js
|
|
715
|
+
const service = await createPortability({db, buckets, policy,
|
|
716
|
+
onExport: ({id, actor}) => [[
|
|
717
|
+
'INSERT INTO operation_audit(id,action,actor_id,feature) VALUES(?,?,?,?)',
|
|
718
|
+
[id, 'installation.export', actor, 'rer export'],
|
|
719
|
+
]],
|
|
720
|
+
});
|
|
721
|
+
```
|
|
722
|
+
|
|
723
|
+
Adapt the statement to the seed's schema and use the service in its existing
|
|
724
|
+
export adapter. Hooks return SQL only; external side effects cannot share the
|
|
725
|
+
D1 transaction. Do not write these events to the erasure tombstone journal.
|
|
726
|
+
|
|
589
727
|
Transfer bookkeeping lives in sidecar tables, so product tables up to D1's
|
|
590
728
|
100-column limit retain the full column and binding capacity. A keyless table
|
|
591
729
|
with 100 columns or generated columns restores its rowids with two updates in
|
|
@@ -612,8 +750,8 @@ part; different content returns 409. Reassembled bytes pass the part checksum
|
|
|
612
750
|
before completion advances. Staged data stays private until finalization.
|
|
613
751
|
Finalize requires the closing `archiveHash` and verifies all
|
|
614
752
|
parts, semantic contents, destination
|
|
615
|
-
emptiness, erasure and product invariants. R2 ownership
|
|
616
|
-
empty-namespace check. Verified object writes precede one atomic D1 batch, with
|
|
753
|
+
emptiness, erasure, object references and product invariants. R2 ownership
|
|
754
|
+
is recorded after the empty-namespace check. Verified object writes precede one atomic D1 batch, with
|
|
617
755
|
deferred FKs, trigger suppression/recreation, replaceable rows, staged inserts,
|
|
618
756
|
journal adoption and completion. FK cycles and self-references restore together.
|
|
619
757
|
An unsuccessful commit removes only owned objects. A lost commit acknowledgement
|
|
@@ -640,7 +778,10 @@ by every upload/finalize request, then every 30 seconds while that request is
|
|
|
640
778
|
working (including verification, R2 transfers and product callbacks). This is a
|
|
641
779
|
recovery window, not a total restore limit: the measured 20,000-row round trip
|
|
642
780
|
below already takes 171 seconds, and callbacks have no wall-time bound. A fresh
|
|
643
|
-
import returns `409 import_running
|
|
781
|
+
import returns `409 import_running`. Only a validated internal lease refusal,
|
|
782
|
+
after authorization and for the actor who owns that import, includes `jobId`
|
|
783
|
+
and ISO `abortableAt`. Authorization failures and other actors receive no job or
|
|
784
|
+
lease metadata, even if an error carries the same code or fields. The owner
|
|
644
785
|
can find its active job with `GET /api/portability/imports` (`{active:status}`
|
|
645
786
|
or `{active:null}`), and inspect `GET .../imports/:id`: `abortableAt` and `stale` describe
|
|
646
787
|
its lease. The existing authenticated `POST .../imports/:id/abort` (or CLI
|
|
@@ -671,6 +812,17 @@ returns an async dispatcher, or null for foreign paths. Default base path is
|
|
|
671
812
|
an actor id or null. Jobs belong to their creating actor. Unsafe methods require
|
|
672
813
|
the exact configured Origin. Responses have `Cache-Control: no-store` and errors
|
|
673
814
|
are `{error:code}`. Request bodies are bounded; `maxBodyBytes` can lower the bound.
|
|
815
|
+
Typed authorization errors retain their HTTP status, directly or through a
|
|
816
|
+
bounded, cycle-safe `cause` chain. Codes come from the existing fixed public
|
|
817
|
+
vocabulary (for example `403 step_up_required`); unknown codes become
|
|
818
|
+
`operation_failed`. Portability diagnostics expose the known code prefix only,
|
|
819
|
+
never caller-supplied suffixes. Aggregate error collections are not authority
|
|
820
|
+
wrappers. Native diagnostics remain private:
|
|
821
|
+
transient D1 availability failures use `503 operation_failed`, transport timeouts
|
|
822
|
+
use `504 operation_failed`, and unexpected faults remain `500 operation_failed`.
|
|
823
|
+
The hosted CLI aborts a completed export after saving and verifying the local
|
|
824
|
+
archive, releasing its snapshots and parts. Interrupted downloads retain their
|
|
825
|
+
job for resume.
|
|
674
826
|
|
|
675
827
|
| Method / relative path | Action |
|
|
676
828
|
| --- | --- |
|
|
@@ -749,13 +901,14 @@ product erasure or remove product objects; the product handles attachments and
|
|
|
749
901
|
any extra state. Do not retain a previously downloaded archive as erasure proof;
|
|
750
902
|
supply the owner's later journal when restoring old offline data.
|
|
751
903
|
|
|
752
|
-
##
|
|
904
|
+
## Historical measured capacity
|
|
753
905
|
|
|
754
|
-
|
|
755
|
-
five tables in 171.169 seconds, with 625 parts at a configured 4 KiB bound and a
|
|
906
|
+
An unverified historical local Miniflare D1/R2 report recorded export, restore
|
|
907
|
+
and export of 20,000 rows across five tables in 171.169 seconds, with 625 parts at a configured 4 KiB bound and a
|
|
756
908
|
1,979,471-byte archive. Restore used one atomic D1 commit. The verification run
|
|
757
|
-
|
|
758
|
-
|
|
909
|
+
can produce fresh measurements; no corresponding historical benchmark receipt
|
|
910
|
+
is retained in this repository. These figures do not establish current production
|
|
911
|
+
capacity. Real D1 CPU, query, statement and storage limits, object mutation,
|
|
759
912
|
destination writers and extra-payload recovery need product-level verification.
|
|
760
913
|
|
|
761
914
|
## Adopting
|
|
@@ -772,7 +925,11 @@ modules/portability/index.d.mts
|
|
|
772
925
|
modules/portability/lines.mjs
|
|
773
926
|
modules/portability/hash.mjs
|
|
774
927
|
modules/portability/format.mjs
|
|
928
|
+
modules/portability/references.mjs
|
|
775
929
|
modules/portability/catalogue.mjs
|
|
930
|
+
modules/portability/schema-identity.mjs
|
|
931
|
+
modules/portability/schema-legacy.mjs
|
|
932
|
+
modules/portability/sqlite-tokens.mjs
|
|
776
933
|
modules/portability/bookkeeping.mjs
|
|
777
934
|
modules/portability/storage.mjs
|
|
778
935
|
modules/portability/copies.mjs
|
|
@@ -798,6 +955,7 @@ modules/portability/import-safety.mjs
|
|
|
798
955
|
modules/portability/payloads.mjs
|
|
799
956
|
modules/portability/journal.mjs
|
|
800
957
|
modules/portability/http.mjs
|
|
958
|
+
modules/portability/http-errors.mjs
|
|
801
959
|
modules/portability/http-client.mjs
|
|
802
960
|
modules/portability/cli.mjs
|
|
803
961
|
modules/portability/cli-options.mjs
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { schemaIdentityOf } from './schema-identity.mjs';
|
|
2
2
|
import { insist } from './storage.mjs';
|
|
3
3
|
|
|
4
4
|
// Installation authority and transient working state, never product data.
|
|
@@ -23,8 +23,8 @@ export async function restorableManifest(manifest) {
|
|
|
23
23
|
for(const table of manifest.tables)if(identifier(table.name)==='ik_d1_assertions')
|
|
24
24
|
insist(table.count===0,'invalid_bookkeeping_rows:'+table.name,409);
|
|
25
25
|
const tables=manifest.tables.filter(table=>!platformTable(table.name));
|
|
26
|
-
|
|
27
|
-
return {...manifest,tables,schema:await
|
|
26
|
+
const schemaObjects=manifest.schemaObjects?.filter(object=>!manifest.tables.some(table=>platformTable(table.name) && table.name===object.table));
|
|
27
|
+
return {...manifest,tables,...(schemaObjects?{schemaObjects}:{}),schema:await schemaIdentityOf(tables,schemaObjects),
|
|
28
28
|
excludedColumns:Object.fromEntries(Object.entries(manifest.excludedColumns).filter(([name])=>!platformTable(name)))};
|
|
29
29
|
}
|
|
30
30
|
export const skippedTablesLine=tables=>'Skipped platform bookkeeping tables: '+tables.join(', ')+'.';
|
|
@@ -4,6 +4,7 @@ import { tokens, identifier, folded } from './generated.mjs';
|
|
|
4
4
|
import { laterData } from './backfills.mjs';
|
|
5
5
|
import { destinationLedger, matchMigrationLedger } from './ledgers.mjs';
|
|
6
6
|
import { checkRebuild } from './rebuilds.mjs';
|
|
7
|
+
import { schemaSQL } from './schema-identity.mjs';
|
|
7
8
|
|
|
8
9
|
// Applied product migrations in the ledger's own order (ledgers.mjs). A
|
|
9
10
|
// state-bearing ledger may hold an unacknowledged migration whose effects are
|
|
@@ -38,7 +39,7 @@ async function placement(db,manifest,cat,replay,ledger,history) {
|
|
|
38
39
|
unapplied=applied.filter(name=>!earlier.includes(name));
|
|
39
40
|
}else{
|
|
40
41
|
// Unrecorded: the latest matching prefix. Migrations that change no archived
|
|
41
|
-
//
|
|
42
|
+
// definition (excluded tables or data-only changes) cannot be told apart
|
|
42
43
|
// and count as later. The source may have stopped at any matching prefix,
|
|
43
44
|
// so every migration after the earliest one may be missing from its rows.
|
|
44
45
|
const matching=[];
|
|
@@ -62,7 +63,7 @@ async function placement(db,manifest,cat,replay,ledger,history) {
|
|
|
62
63
|
return {applied,earlier,replays,rebuilt};
|
|
63
64
|
}
|
|
64
65
|
// An archive with the destination's schema identity can still predate later
|
|
65
|
-
// migrations that changed only data (a purge or a backfill)
|
|
66
|
+
// migrations that changed only data (a purge or a backfill).
|
|
66
67
|
// Those that change restorable data are refused like any other. Without a
|
|
67
68
|
// replayer the statements cannot be read, so an archive whose recorded
|
|
68
69
|
// migrations the destination's ledger has outgrown is refused. An archive that
|
|
@@ -82,7 +83,7 @@ export async function sameSchemaMigrations(db,manifest,cat,migrations,history) {
|
|
|
82
83
|
}
|
|
83
84
|
// An archive made before later migrations restores only when those migrations
|
|
84
85
|
// are additive, or a proven row-preserving CHECK rebuild: new tables, new
|
|
85
|
-
// columns that take a default, or
|
|
86
|
+
// columns that take a default, or later schema objects established by replay.
|
|
86
87
|
// Anything else stays a schema_mismatch refusal.
|
|
87
88
|
export async function additiveBridge(db,manifest,cat,migrations,history) {
|
|
88
89
|
insist(migrations,'schema_mismatch');
|
|
@@ -95,7 +96,7 @@ export async function additiveBridge(db,manifest,cat,migrations,history) {
|
|
|
95
96
|
const kept=target.columns.slice(0,table.columns.length),added=target.columns.slice(table.columns.length);
|
|
96
97
|
insist(canonicalJson(kept)===canonicalJson(table.columns) && table.rowidAlias===target.rowidAlias && table.dependencies.every(d=>target.dependencies.includes(d)),'schema_mismatch:columns'+label);
|
|
97
98
|
const rebuiltChecks=rebuilt.has(folded(table.name)) && checkRebuild(table.sql,target.sql);
|
|
98
|
-
if(!added.length){insist(table.sql===target.sql || rebuiltChecks,'schema_mismatch:definition'+label);continue;}
|
|
99
|
+
if(!added.length){insist(schemaSQL(table.sql)===schemaSQL(target.sql) || rebuiltChecks,'schema_mismatch:definition'+label);continue;}
|
|
99
100
|
const declared=await all(db,`PRAGMA table_xinfo(${quote(table.name)})`);
|
|
100
101
|
insist(added.every(c=>!c.pk && !c.generated && (!c.notnull || declared.find(d=>d.name===c.name)?.dflt_value!=null)),'schema_mismatch:column_default'+label);
|
|
101
102
|
insist(appended(table.sql,target.sql,added.map(c=>c.name)),'schema_mismatch:definition'+label);
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { all, insist, quote, PortabilityError } from './storage.mjs';
|
|
2
|
-
import { canonicalJson,
|
|
2
|
+
import { canonicalJson, base64, unbase64, utf8 } from './hash.mjs';
|
|
3
|
+
import { schemaIdentityOf } from './schema-identity.mjs';
|
|
3
4
|
import { platformTables, platformTable, identifier, matches } from './bookkeeping.mjs';
|
|
4
5
|
/** Platform bookkeeping is mandatory; operator identity remains a seed choice. */
|
|
5
6
|
export const excludedTables=Object.freeze([...platformTables,'oa_%','oc_%']);
|
|
@@ -57,14 +58,17 @@ async function archived(db,config) {
|
|
|
57
58
|
const derived=config.derivedTables;
|
|
58
59
|
const excluded=objects.filter(x=>x.type==='table' && (excludedTable(x.name,config) || (!config.includeTables.includes(x.name) &&
|
|
59
60
|
derived.some(p=>x.name===p || (kinds.find(k=>k.name===x.name)?.type==='shadow' && x.name.startsWith(p+'_'))))));
|
|
60
|
-
objects.sort((a,b)=>a.name<b.name?-1:a.name>b.name?1:0);
|
|
61
|
-
|
|
61
|
+
objects.sort((a,b)=>a.name<b.name?-1:a.name>b.name?1:a.type<b.type?-1:a.type>b.type?1:0);
|
|
62
|
+
const schemaObjects=objects.filter(x=>['index','view','trigger'].includes(x.type)
|
|
63
|
+
&& !excluded.some(t=>t.name===x.tbl_name)).map(({name,type,tbl_name,sql})=>({name,type,table:tbl_name,sql}));
|
|
64
|
+
return {objects,kinds,excluded,schemaObjects,entries:objects.filter(x=>x.type==='table' && !excluded.some(y=>y.name===x.name))};
|
|
62
65
|
}
|
|
63
|
-
const identityOf=tables=>sha256(canonicalJson(tables.map(({name,sql})=>({name,sql}))));
|
|
64
66
|
// The archive schema identity alone, for databases that need not satisfy the policy.
|
|
65
|
-
export async function schemaIdentity(db,config) {
|
|
67
|
+
export async function schemaIdentity(db,config) {
|
|
68
|
+
const {entries,schemaObjects}=await archived(db,config);return schemaIdentityOf(entries,schemaObjects);
|
|
69
|
+
}
|
|
66
70
|
export async function catalogue(db,config) {
|
|
67
|
-
const {objects,kinds,excluded,entries}=await archived(db,config);
|
|
71
|
+
const {objects,kinds,excluded,entries,schemaObjects}=await archived(db,config);
|
|
68
72
|
const tables=[];
|
|
69
73
|
for(const entry of entries) {
|
|
70
74
|
insist(!/^CREATE\s+VIRTUAL\s+TABLE/i.test(entry.sql) && kinds.find(x=>x.name===entry.name)?.type!=='shadow','virtual_table:'+entry.name);
|
|
@@ -99,8 +103,8 @@ export async function catalogue(db,config) {
|
|
|
99
103
|
insist(tables.find(x=>x.name===table)?.columns.some(c=>c.name===keyColumn),'reference_column_missing');
|
|
100
104
|
insist(!config.secretColumns[table]?.includes(keyColumn),'reference_column_excluded');
|
|
101
105
|
}
|
|
102
|
-
const identity=await
|
|
103
|
-
return {identity,tables,excludedTables:excluded.map(x=>x.name).sort(),triggers:objects.filter(x=>x.type==='trigger' && tables.some(t=>t.name===x.tbl_name))};
|
|
106
|
+
const identity=await schemaIdentityOf(tables,schemaObjects);
|
|
107
|
+
return {identity,tables,schemaObjects,excludedTables:excluded.map(x=>x.name).sort(),triggers:objects.filter(x=>x.type==='trigger' && tables.some(t=>t.name===x.tbl_name))};
|
|
104
108
|
}
|
|
105
109
|
export function affinity(type) {
|
|
106
110
|
const text=type.toUpperCase();
|
|
@@ -24,7 +24,11 @@ export async function main(args) {
|
|
|
24
24
|
const records=await client.download(id),file=await privateFile(opts.out+'.partial',Boolean(opts.resume));
|
|
25
25
|
try{for await(const record of records)await file.writeFile(canonicalJson(record)+'\n');await file.sync();}
|
|
26
26
|
finally{await file.close();}
|
|
27
|
-
await promote(opts.out+'.partial',opts.out);
|
|
27
|
+
await promote(opts.out+'.partial',opts.out);
|
|
28
|
+
// The verified local archive is durable: acknowledge it and release the
|
|
29
|
+
// hosted job's snapshots/parts. Failed downloads keep their resumable job.
|
|
30
|
+
await get('exports/'+id+'/abort','POST',{});
|
|
31
|
+
process.stdout.write('Export complete.\n');return;
|
|
28
32
|
}
|
|
29
33
|
if(opts.command==='import') {
|
|
30
34
|
const source=Readable.toWeb(createReadStream(opts.from));let id=opts.resume,closing;
|
|
@@ -1,10 +1,11 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { statement, state, guard, job, save, partStatements, parts, readPart, cleanCopies, quote, stream, insist, run } from './storage.mjs';
|
|
2
2
|
import { catalogue, encodedRow } from './catalogue.mjs';
|
|
3
|
+
import { checkReferences } from './references.mjs';
|
|
3
4
|
import { createCopy,readCopy,normalizeCopy } from './copies.mjs';
|
|
4
5
|
import { canonicalJson, utf8, base64, StreamHash } from './hash.mjs';
|
|
5
6
|
import { FORMAT, makePart, checkPart, ContentVerifier, ArchiveVerifier } from './format.mjs';
|
|
6
7
|
import { journal } from './journal.mjs';
|
|
7
|
-
import {
|
|
8
|
+
import { readLedger, platformOrigin } from './ledgers.mjs';
|
|
8
9
|
import { capturePayload, payloadRange } from './payloads.mjs';
|
|
9
10
|
import { dropProviderData } from './provider-data.mjs';
|
|
10
11
|
import { dropDevData } from './dev-data.mjs';
|
|
@@ -22,11 +23,13 @@ export async function createExport(context,actor='direct') {
|
|
|
22
23
|
}
|
|
23
24
|
objects.sort((a,b)=>a.bucket<b.bucket?-1:a.bucket>b.bucket?1:a.key<b.key?-1:a.key>b.key?1:0);
|
|
24
25
|
const copies=Object.fromEntries(cat.tables.map((t,i)=>[t.name,'rer_portability_snap_'+id.replaceAll('-','')+'_'+i]));
|
|
25
|
-
const
|
|
26
|
-
const manifest={kind:'manifest',format:FORMAT,version:2,product:config.product,exportedAt:new Date().toISOString(),sourceIdentity:s.source_identity,schema:cat.identity,tables:cat.tables,objects,payloads:[],excludedTables:cat.excludedTables,excludedColumns:config.secretColumns,requiredConfiguration:config.requiredConfiguration,partCount:0,epoch:s.epoch,journal:await journal(db,s.source_identity),...(migrations?{migrations}:{})};
|
|
26
|
+
const ledger=await readLedger(db),origin=await platformOrigin(ledger,context.migrations),migrations=origin?origin.entries.map(entry=>entry.name):ledger?.entries.map(entry=>entry.name);
|
|
27
|
+
const manifest={kind:'manifest',format:FORMAT,version:2,product:config.product,exportedAt:new Date().toISOString(),sourceIdentity:s.source_identity,schema:cat.identity,tables:cat.tables,schemaObjects:cat.schemaObjects,objects,payloads:[],excludedTables:cat.excludedTables,excludedColumns:config.secretColumns,requiredConfiguration:config.requiredConfiguration,partCount:0,epoch:s.epoch,journal:await journal(db,s.source_identity),...(migrations?{migrations}:{}),...(origin?{migrationLedger:origin}:{})};
|
|
27
28
|
const row={id,kind:'export',actor,phase:'building',completed:0,metadata:{manifest,copies,cursor:{table:0,row:0,binary:0,offset:0},partBytes:config.partBytes}};
|
|
29
|
+
const audit=context.onExport?context.onExport({id,actor:actor==='direct'?null:actor,scope:'installation'}):[];
|
|
28
30
|
await db.batch([guard(db,'EXISTS(SELECT 1 FROM rer_portability_state WHERE id=1 AND applying=0 AND source_identity=? AND epoch=?)',s.source_identity,s.epoch),
|
|
29
|
-
...cat.tables.flatMap(t=>createCopy(db,t,copies[t.name],true)),statement(db,'INSERT INTO rer_portability_jobs(id,kind,actor,phase,metadata) VALUES(?,?,?,?,?)',id,'export',actor,row.phase,canonicalJson(row.metadata)),db.prepare('DELETE FROM rer_portability_guard')
|
|
31
|
+
...cat.tables.flatMap(t=>createCopy(db,t,copies[t.name],true)),statement(db,'INSERT INTO rer_portability_jobs(id,kind,actor,phase,metadata) VALUES(?,?,?,?,?)',id,'export',actor,row.phase,canonicalJson(row.metadata)),db.prepare('DELETE FROM rer_portability_guard'),
|
|
32
|
+
...audit.map(([sql,args])=>statement(db,sql,...args))]);
|
|
30
33
|
try {
|
|
31
34
|
await dropDevData(db,cat.tables,copies);
|
|
32
35
|
await dropProviderData(db,cat.tables,copies);
|
|
@@ -36,17 +39,10 @@ export async function createExport(context,actor='direct') {
|
|
|
36
39
|
const descriptor=await capturePayload(db,id,manifest.payloads.length,await producer.produce());manifest.payloads.push({name,...descriptor});
|
|
37
40
|
}
|
|
38
41
|
await run(db,'UPDATE rer_portability_jobs SET metadata=? WHERE id=?',canonicalJson(row.metadata),id);
|
|
39
|
-
await checkReferences(context,
|
|
42
|
+
await checkReferences(context,copies,objects);
|
|
40
43
|
return id;
|
|
41
44
|
}catch(e){await abortExport(context,id);throw e;}
|
|
42
45
|
}
|
|
43
|
-
async function checkReferences({db,config},row) {
|
|
44
|
-
for(const [binding,spec] of Object.entries(config.buckets)) if(spec.objectReferences) {
|
|
45
|
-
const {table,keyColumn}=spec.objectReferences,keys=new Set((await all(db,`SELECT DISTINCT ${quote(keyColumn)} AS key FROM ${quote(row.metadata.copies[table])} WHERE ${quote(keyColumn)} IS NOT NULL`)).map(x=>x.key));
|
|
46
|
-
const objects=row.metadata.manifest.objects.filter(x=>x.bucket===binding);
|
|
47
|
-
insist(objects.length===keys.size && objects.every(x=>keys.has(x.key)),'object_references:'+binding);
|
|
48
|
-
}
|
|
49
|
-
}
|
|
50
46
|
async function nextItems(context,row) {
|
|
51
47
|
const {db,config,buckets}=context,{manifest,cursor,copies,partBytes}=row.metadata;
|
|
52
48
|
while(cursor.table<manifest.tables.length && cursor.row===manifest.tables[cursor.table].count){cursor.table++;cursor.row=0;}
|
|
@@ -106,7 +102,7 @@ export async function advanceExport(context,id,{expectedPart,maxParts=1}={}) {
|
|
|
106
102
|
if(expectedPart!==undefined)expectedPart++;
|
|
107
103
|
}else{
|
|
108
104
|
// References are checked again against the archived, normalized snapshot.
|
|
109
|
-
await checkReferences(context,row);
|
|
105
|
+
await checkReferences(context,row.metadata.copies,row.metadata.manifest.objects);
|
|
110
106
|
row.metadata.manifest.partCount=row.completed;
|
|
111
107
|
const verifier=new ContentVerifier(row.metadata.manifest),archive=new ArchiveVerifier(row.metadata.manifest);
|
|
112
108
|
for await(const part of parts(context.db,id,row.completed)){archive.add(part);for(const item of part.items)verifier.add(item);}
|
|
@@ -2,6 +2,8 @@ import { archiveLines } from './lines.mjs';
|
|
|
2
2
|
import { canonicalJson, sha256, StreamHash, unbase64, utf8 } from './hash.mjs';
|
|
3
3
|
import { insist, PortabilityError } from './storage.mjs';
|
|
4
4
|
import { validateRow } from './catalogue.mjs';
|
|
5
|
+
import { schemaIdentityOf, rawSchemaIdentityOf, tableSchemaIdentityOf } from './schema-identity.mjs';
|
|
6
|
+
import { legacySchemaIdentities } from './schema-legacy.mjs';
|
|
5
7
|
export const FORMAT='runeditrun-portability';
|
|
6
8
|
export const MAX_LINE_BYTES=24*1024*1024;
|
|
7
9
|
export const MAX_PART_BYTES=16*1024*1024;
|
|
@@ -36,8 +38,17 @@ async function checkManifestFields(m) {
|
|
|
36
38
|
insist(Array.isArray(m.tables) && Array.isArray(m.objects) && Array.isArray(m.payloads) && Array.isArray(m.journal) && Array.isArray(m.requiredConfiguration),'manifest_inventory');
|
|
37
39
|
// Applied product migrations, recorded by archives since additive restore.
|
|
38
40
|
insist(m.migrations===undefined || (Array.isArray(m.migrations) && m.migrations.every(x=>typeof x==='string')),'manifest_inventory');
|
|
39
|
-
|
|
41
|
+
// The ledger an export from a RER Platform installation records, by migration identity.
|
|
42
|
+
const ledger=m.migrationLedger;
|
|
43
|
+
insist(ledger===undefined || (ledger && typeof ledger==='object' && Object.keys(ledger).sort().join()==='entries,origin' && ledger.origin==='rer_migrations' && Array.isArray(ledger.entries)
|
|
44
|
+
&& ledger.entries.every(e=>e && typeof e==='object' && Object.keys(e).sort().join()==='name,sha256' && typeof e.name==='string' && /^[a-f0-9]{64}$/.test(e.sha256))
|
|
45
|
+
&& canonicalJson(m.migrations)===canonicalJson(ledger.entries.map(e=>e.name)) && new Set(m.migrations).size===m.migrations.length),'manifest_inventory');
|
|
40
46
|
insist(m.tables.every((t,i)=>typeof t.name==='string' && typeof t.sql==='string' && Array.isArray(t.columns) && Array.isArray(t.dependencies) && Number.isSafeInteger(t.count) && t.count>=0 && (!i || m.tables[i-1].name<t.name)),'table_inventory');
|
|
47
|
+
insist(m.schemaObjects===undefined || (Array.isArray(m.schemaObjects) && m.schemaObjects.every((o,i)=>
|
|
48
|
+
o && typeof o.name==='string' && ['index','view','trigger'].includes(o.type) && typeof o.table==='string' && typeof o.sql==='string'
|
|
49
|
+
&& (!i || m.schemaObjects[i-1].name<o.name || (m.schemaObjects[i-1].name===o.name && m.schemaObjects[i-1].type<o.type)))),'manifest_inventory');
|
|
50
|
+
insist(m.schema===await schemaIdentityOf(m.tables,m.schemaObjects) || m.schema===await tableSchemaIdentityOf(m.tables)
|
|
51
|
+
|| m.schema===await rawSchemaIdentityOf(m.tables) || (await legacySchemaIdentities(m.tables,m.schemaObjects)).includes(m.schema),'schema_checksum');
|
|
41
52
|
for(const collection of [m.objects,m.payloads]) {
|
|
42
53
|
const keys=new Set();
|
|
43
54
|
for(const entry of collection) {
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import { importRunningMetadata } from './import-lease.mjs';
|
|
2
|
+
|
|
3
|
+
// A code's spelling is not evidence that it is public. Keep the established
|
|
4
|
+
// vocabulary closed, and discard diagnostic suffixes containing schema/input.
|
|
5
|
+
const publicCodes = new Set(`
|
|
6
|
+
D1_RPC_TRANSPORT_LIMIT advance_limit archive_checksum archive_format archive_incomplete archive_json
|
|
7
|
+
binary_checksum binary_inventory binary_order binary_size body_too_large bucket_inventory bucket_not_empty
|
|
8
|
+
closing_missing column_inventory configuration_inventory confirmation_required content_checksum copy_inventory copy_rowid
|
|
9
|
+
data_after_closing dev_data erased_row erasure_epoch excluded_destination_dependents excluded_foreign_key
|
|
10
|
+
export_incomplete export_phase export_row_must_be_pure export_row_not_idempotent extra_part generated_sql
|
|
11
|
+
import_applying import_complete import_lease_expired import_phase import_row_columns import_row_key import_row_must_be_pure
|
|
12
|
+
import_row_not_deterministic import_row_not_idempotent import_row_undeclared import_row_unsafe_value import_running import_safety_table_missing
|
|
13
|
+
invalid_accepted_schema invalid_bookkeeping_rows invalid_bucket_policy invalid_bucket_prefix invalid_extra_payload
|
|
14
|
+
invalid_handler_configuration invalid_import_safety invalid_import_safety_column invalid_journal invalid_json invalid_object_references
|
|
15
|
+
invalid_part_limit invalid_policy invalid_policy_importSafety invalid_policy_importSafety_pair invalid_secret_column invalid_secret_columns
|
|
16
|
+
invalid_policy_excludeTables invalid_policy_includeTables invalid_policy_derivedTables invalid_policy_replaceableTables
|
|
17
|
+
invalid_policy_requiredConfiguration invalid_policy_exportRow invalid_policy_importRow invalid_policy_validateStaged
|
|
18
|
+
job_not_found later_journal ledger_inventory legacy_acknowledgement_required legacy_archive_unverifiable legacy_bucket_not_empty
|
|
19
|
+
legacy_output_exists legacy_output_required legacy_restored_export_failed legacy_target_not_empty legacy_v1_required live_policy_approval
|
|
20
|
+
manifest_counts manifest_identity manifest_inventory migration_ledger missing_bucket object_changed object_references
|
|
21
|
+
object_write_checksum object_write_failed origin_refused part_byte_limit part_checksum part_conflict part_count part_missing
|
|
22
|
+
part_order part_row_limit parts_missing payload_changed payload_inventory platform_table policy_table_missing
|
|
23
|
+
provider_relink_required provider_session_data provider_sign_in_must_be_off provider_transient_data reference_column_excluded reference_column_missing
|
|
24
|
+
route_not_found row_columns row_inventory row_not_null row_order row_part_too_large row_type rowid_inventory rows_missing
|
|
25
|
+
schema_checksum schema_mismatch secret_inventory secret_table_missing staged_table staged_validation table_inventory target_not_empty
|
|
26
|
+
transfer_sql_limit unauthorized unsafe_integer unsupported_column unsupported_copy_rowid unsupported_restore_width unsupported_rowid
|
|
27
|
+
upgrade_row_must_be_pure virtual_table wrong_product
|
|
28
|
+
unauthenticated forbidden owner_required authority_changed step_up_required rate_limited operation_failed
|
|
29
|
+
`.trim().split(/\s+/));
|
|
30
|
+
const publicCode = code => typeof code === 'string' && publicCodes.has(code.split(':')[0]) ? code.split(':')[0] : 'operation_failed';
|
|
31
|
+
|
|
32
|
+
// Public errors use the existing {error:code} vocabulary. Native messages can
|
|
33
|
+
// contain SQL, row values or credentials; inspect causes only to choose status.
|
|
34
|
+
export function httpFailure(error,{actor=null}={}) {
|
|
35
|
+
const seen = new Set();let typed,nativeStatus;
|
|
36
|
+
for (let cause = error; cause && !seen.has(cause) && seen.size < 5; cause = cause.cause) {
|
|
37
|
+
seen.add(cause);
|
|
38
|
+
if (Number.isInteger(cause.status) && cause.status >= 400 && cause.status <= 599) {
|
|
39
|
+
const code = publicCode(cause.code);
|
|
40
|
+
// A generic server wrapper must not turn a known authority refusal into 500.
|
|
41
|
+
if (!typed || typed.status >= 500 && cause.status < 500) typed = { status: cause.status, body: { error: code,
|
|
42
|
+
...(cause.status === 409 && cause.code === 'import_running' ? importRunningMetadata(cause,actor) : {}) } };
|
|
43
|
+
}
|
|
44
|
+
if (cause.name === 'TimeoutError' || ['ETIMEDOUT', 'UND_ERR_CONNECT_TIMEOUT', 'UND_ERR_HEADERS_TIMEOUT'].includes(cause.code))
|
|
45
|
+
nativeStatus = 504;
|
|
46
|
+
if (['ECONNRESET', 'ECONNREFUSED', 'SQLITE_BUSY', 'SQLITE_LOCKED'].includes(cause.code)
|
|
47
|
+
|| /\bD1_ERROR\b/i.test(cause.message ?? '') && /overloaded|temporarily unavailable|reset|timed? out|timeout|SQLITE_BUSY|SQLITE_LOCKED|database is locked/i.test(cause.message))
|
|
48
|
+
nativeStatus ??= 503;
|
|
49
|
+
}
|
|
50
|
+
return typed ?? { status: nativeStatus ?? 500, body: { error: 'operation_failed' } };
|
|
51
|
+
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { createPortability } from './index.mjs';
|
|
2
2
|
import { insist, PortabilityError } from './storage.mjs';
|
|
3
3
|
import { MAX_LINE_BYTES } from './format.mjs';
|
|
4
|
+
import { httpFailure } from './http-errors.mjs';
|
|
4
5
|
export async function boundedJson(request,limit) {
|
|
5
6
|
const reader=request.body?.getReader();if(!reader)return {};
|
|
6
7
|
const chunks=[];let size=0;
|
|
@@ -19,10 +20,12 @@ export async function createPortabilityHandler({db,buckets,policy,appOrigin,base
|
|
|
19
20
|
if(url.pathname!==basePath && !url.pathname.startsWith(basePath+'/'))return null;
|
|
20
21
|
const route=url.pathname.slice(basePath.length+1).split('/'),[collection,id,action,index]=route;
|
|
21
22
|
const operation=collection==='exports'?'export':collection==='imports'?'import':'unknown';
|
|
23
|
+
let authorizedActor=null;
|
|
22
24
|
try {
|
|
23
25
|
const actor=await authorize(request,{operation,jobId:id??null});insist(typeof actor==='string' && actor.length>0,'unauthorized',401);
|
|
24
26
|
if(!['GET','HEAD'].includes(request.method))insist(request.headers.get('Origin')===appOrigin,'origin_refused',403);
|
|
25
27
|
if(id)insist(await service.actor(id)===actor,'job_not_found',404);
|
|
28
|
+
authorizedActor=actor;
|
|
26
29
|
insist(['exports','imports'].includes(collection) && route.length<=4,'route_not_found',404);
|
|
27
30
|
const api=service[collection];
|
|
28
31
|
if(collection==='imports' && request.method==='GET' && !id)return json({active:await api.active(actor)});
|
|
@@ -38,6 +41,6 @@ export async function createPortabilityHandler({db,buckets,policy,appOrigin,base
|
|
|
38
41
|
if(collection==='imports' && request.method==='PUT' && id && action==='parts' && /^(0|[1-9]\d*)$/.test(index??''))return json(await api.uploadPart(id,Number(index),await boundedJson(request,maxBodyBytes)));
|
|
39
42
|
if(collection==='imports' && request.method==='POST' && id && action==='finalize' && !index)return json(await api.finalize(id,await boundedJson(request,maxBodyBytes)));
|
|
40
43
|
throw new PortabilityError('route_not_found',404);
|
|
41
|
-
}catch(e){
|
|
44
|
+
}catch(e){const failure=httpFailure(e,{actor:authorizedActor});return json(failure.body,failure.status);}
|
|
42
45
|
};
|
|
43
46
|
}
|