@runeditrun/rer 0.2.80 → 0.2.82
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -0
- package/cli/deploy-needs.mjs +4 -2
- package/cli/export.mjs +1 -1
- package/cli/ledger-refusal.mjs +18 -0
- package/cli/portability-migrations.mjs +30 -11
- package/cli/portability-worker.mjs +3 -1
- package/cli/portability.mjs +2 -0
- package/cli/release.mjs +4 -4
- package/cli/site-hosting.mjs +90 -0
- package/cli/site-render.mjs +3 -2
- package/cli/site.mjs +5 -2
- package/edition-authoring/base-copies.mjs +12 -1
- package/edition-authoring/copy-checkout.mjs +152 -1
- package/edition-authoring/files.mjs +1 -1
- package/edition-authoring/runtime-dependencies.mjs +108 -0
- package/global/CHANGELOG.md +25 -0
- package/global/README.md +12 -0
- package/modules/demo/README.md +8 -0
- package/modules/demo/navigation.mjs +131 -15
- package/modules/dependencies/auth.mjs +2 -2
- package/modules/deploy-kit/owner.mjs +8 -1
- package/modules/import-kit/src/d1/apply-effects.ts +33 -0
- package/modules/import-kit/src/d1/apply.ts +38 -10
- package/modules/import-kit/src/door/ADOPTING.md +452 -0
- package/modules/import-kit/src/door/README.md +553 -0
- package/modules/import-kit/src/door/admission.ts +45 -0
- package/modules/import-kit/src/door/archive.ts +129 -0
- package/modules/import-kit/src/door/authority.ts +107 -0
- package/modules/import-kit/src/door/failures.ts +70 -0
- package/modules/import-kit/src/door/grants.ts +162 -0
- package/modules/import-kit/src/door/handler.ts +90 -0
- package/modules/import-kit/src/door/index.ts +20 -0
- package/modules/import-kit/src/door/intake/csv.ts +74 -0
- package/modules/import-kit/src/door/intake/json.ts +182 -0
- package/modules/import-kit/src/door/intake/projection.ts +61 -0
- package/modules/import-kit/src/door/intake/types.ts +28 -0
- package/modules/import-kit/src/door/intake-policy.ts +38 -0
- package/modules/import-kit/src/door/intake-upload.ts +91 -0
- package/modules/import-kit/src/door/inventory.ts +68 -0
- package/modules/import-kit/src/door/jobs.ts +134 -0
- package/modules/import-kit/src/door/limits.ts +16 -0
- package/modules/import-kit/src/door/operator.ts +41 -0
- package/modules/import-kit/src/door/planning.ts +227 -0
- package/modules/import-kit/src/door/protocol.ts +38 -0
- package/modules/import-kit/src/door/refusals.ts +51 -0
- package/modules/import-kit/src/door/schema.ts +101 -0
- package/modules/import-kit/src/door/store.ts +152 -0
- package/modules/import-kit/src/door/types.ts +100 -0
- package/modules/import-kit/src/door/uploads.ts +98 -0
- package/modules/import-kit/src/provenance/numeric-credentials.ts +40 -0
- package/modules/import-kit/src/readers/README.md +139 -0
- package/modules/import-kit/src/readers/api-policy.ts +67 -0
- package/modules/import-kit/src/readers/api.ts +86 -0
- package/modules/import-kit/src/readers/bytes.ts +73 -0
- package/modules/import-kit/src/readers/errors.ts +33 -0
- package/modules/import-kit/src/readers/file.ts +74 -0
- package/modules/import-kit/src/readers/index.ts +3 -0
- package/modules/import-kit/src/readers/json.ts +29 -0
- package/modules/import-kit/src/readers/numeric-lexemes.ts +104 -0
- package/modules/import-kit/src/readers/original.ts +31 -0
- package/modules/import-kit/src/readers/records.ts +105 -0
- package/modules/import-kit/src/types.ts +3 -1
- package/modules/operator-auth/README.md +92 -3
- package/modules/operator-auth/accounts/admin.mjs +52 -20
- package/modules/operator-auth/accounts/audit.mjs +24 -0
- package/modules/operator-auth/accounts/business-team.mjs +91 -0
- package/modules/operator-auth/accounts/domain-audit.mjs +24 -0
- package/modules/operator-auth/accounts/native.mjs +3 -1
- package/modules/operator-auth/accounts/settings-page.mjs +15 -2
- package/modules/operator-auth/accounts/status.mjs +7 -2
- package/modules/operator-auth/accounts/store.mjs +32 -6
- package/modules/operator-auth/business-settings.mjs +40 -0
- package/modules/operator-auth/identity/email-link/index.mjs +1 -1
- package/modules/operator-auth/identity/oidc/admin-audit.mjs +31 -0
- package/modules/operator-auth/identity/oidc/index.mjs +12 -5
- package/modules/operator-auth/identity/oidc/store.mjs +2 -0
- package/modules/operator-auth/index.d.mts +40 -1
- package/modules/operator-auth/routes.mjs +11 -6
- package/modules/operator-auth/service.mjs +8 -3
- package/modules/operator-auth/signin/index.mjs +1 -1
- package/modules/portability/README.md +82 -1
- package/modules/portability/export.mjs +6 -4
- package/modules/portability/format.mjs +5 -0
- package/modules/portability/index.d.mts +10 -2
- package/modules/portability/index.mjs +3 -2
- package/modules/portability/ledgers.mjs +27 -4
- package/modules/release/README.md +74 -2
- package/modules/release/sign.mjs +4 -2
- package/modules/release/signature.mjs +14 -7
- package/modules/release/signing-cli.mjs +2 -1
- package/modules/release/statement.mjs +2 -2
- package/modules/release/vectors/generate.mjs +1 -0
- package/modules/release/vectors/publisher-projects.mjs +133 -0
- package/modules/release/vectors/release-signature-vectors.json +5195 -0
- package/modules/release/verify.mjs +5 -1
- package/onboarding/schema/SETUP.schema.json +43 -1
- package/onboarding/schema/business-field-inputs.mjs +0 -1
- package/onboarding/schema/setup-validation.mjs +3 -0
- package/package.json +3 -2
|
@@ -0,0 +1,553 @@
|
|
|
1
|
+
# Shared D1 import door
|
|
2
|
+
|
|
3
|
+
To bring existing data into a seed, sign in as the owner and use its import
|
|
4
|
+
screen. First step: open this path on the destination app:
|
|
5
|
+
|
|
6
|
+
```text
|
|
7
|
+
/app/import
|
|
8
|
+
```
|
|
9
|
+
|
|
10
|
+
Upload a supported source export, review the plan and its differences, then
|
|
11
|
+
approve that exact plan to apply it. Reconcile the result against native data;
|
|
12
|
+
use Undo from the same history when needed. Base's reference accepts people JSON
|
|
13
|
+
here, not installation `.rer` archives. Editions adopt this same door
|
|
14
|
+
with their own supported readers and native adapters.
|
|
15
|
+
|
|
16
|
+
See [ADOPTING.md](ADOPTING.md) for the single-path adoption checklist, the
|
|
17
|
+
owner-session mount without owner-claim, and public HTTP conformance.
|
|
18
|
+
|
|
19
|
+
Import `createImportDoorHandler`, `importDoorSchema`,
|
|
20
|
+
`importDoorApplyFailureMigration`, `importDoorScopesMigration`,
|
|
21
|
+
`importDoorArchiveMigration`, `importDoorIntakeMigration`,
|
|
22
|
+
`writeImportDoorSettings`, `enableDoorAtFirstClaim`, `readImportDoorHistory`
|
|
23
|
+
and `verifyRerGrant` from `./index.ts`.
|
|
24
|
+
The handler returns `(request: Request) => Promise<Response | null>`; pass
|
|
25
|
+
unhandled requests to the seed's existing router. Only POST routes under
|
|
26
|
+
`/api/rer/import/` are supported. Nothing mounts a route automatically.
|
|
27
|
+
|
|
28
|
+
Construct the handler with these seed-owned options:
|
|
29
|
+
|
|
30
|
+
| Option | Meaning |
|
|
31
|
+
| --- | --- |
|
|
32
|
+
| `database`, `targets` | D1 database and its real `D1TargetAdapter`. |
|
|
33
|
+
| `claim` | The same server-owned `OwnerClaimConfig` and trusted key ring used by owner-claim. |
|
|
34
|
+
| `installationId` | Optional permanent door key for settings, lineage and history; defaults to `claim.installationId`. Use the owner-session mount's key when RER's claim id differs. |
|
|
35
|
+
| `ownerPrincipalId` | Actual owner principal used by the seed's ordinary import history and undo context. |
|
|
36
|
+
| `destination`, `mapping` | Product destination and reviewed `ImportMapping`. |
|
|
37
|
+
| `format`, `parse` | `'json'` or `'ndjson'`, and the product's parser of source records. With declared intake, sanitized custody is always read as NDJSON. |
|
|
38
|
+
| `intake` | Optional server-owned `{formats, allow}` declaration for dropping non-allowlisted fields before custody. See below; no callback is accepted. |
|
|
39
|
+
| `source` | Instead of `parse`/`mapping`, for several sources: `source(namespace)` returns that namespace's `{parse, mapping, ownerPrincipalId?, intake?}` from the seed's server-owned allow-list, or `null`. It runs after grant verification and before any write; `null` is refused with 403 `unknown_import_source`. |
|
|
40
|
+
| `limits` | Optional reader limits; grant quotas still apply. |
|
|
41
|
+
| `reversalPolicy` | Optional override for new jobs. Without it, declaring `targets.preparePreservedUndo` selects `'preserve-conflicts'`; adapters without that method stay `'strict'`. Preserve keeps native edits on undo and requires that method. Existing jobs keep their creation policy. See ADOPTING.md. |
|
|
42
|
+
|
|
43
|
+
A fresh installation applies the statements in `importDoorSchema: string[]`
|
|
44
|
+
in a new product-owned numbered migration after the D1 import-kit schemas.
|
|
45
|
+
Preserve existing applied migration names. Copy the import-kit runtime and its
|
|
46
|
+
sibling owner-claim and D1 transport dependencies with their relative topology
|
|
47
|
+
intact.
|
|
48
|
+
|
|
49
|
+
A seed that installed an earlier door upgrades by applying the door
|
|
50
|
+
migrations once each, in this order, each in its own new numbered migration,
|
|
51
|
+
never by re-applying `importDoorSchema` (its first statement refuses an
|
|
52
|
+
earlier door's plan table, so re-applying it fails the deploy):
|
|
53
|
+
|
|
54
|
+
1. `importDoorApplyFailureMigration`, in a single atomic `database.batch()`
|
|
55
|
+
with foreign keys enabled. It preserves every plan id, chunk and approval,
|
|
56
|
+
removes the old one-plan-per-bundle uniqueness constraint and adds the
|
|
57
|
+
`ik_door_apply_failures` table (see "Catch-up and recovery").
|
|
58
|
+
2. `importDoorScopesMigration`, the additive `ik_door_scopes` table.
|
|
59
|
+
3. `importDoorArchiveMigration`, the additive `ik_door_archives` and
|
|
60
|
+
`ik_door_archive_chunks` tables (see "Archives").
|
|
61
|
+
|
|
62
|
+
An installed adapter that declares intake also applies the optional
|
|
63
|
+
`importDoorIntakeMigration` once in its own new numbered migration. It adds
|
|
64
|
+
nullable `intake_json` to `ik_door_uploads`; fresh `importDoorSchema` already
|
|
65
|
+
includes it. An existing installation without an intake declaration needs no
|
|
66
|
+
intake migration and keeps its existing upload behavior.
|
|
67
|
+
|
|
68
|
+
Until the first two have run, an enabled door answers every route with 503
|
|
69
|
+
`import_door_schema_outdated`; a disabled grant door still answers 404. The
|
|
70
|
+
owner-session mount (`createImportOperatorHandler` in `operator.ts`, which
|
|
71
|
+
exports the same schema and migrations) has no opt-in and answers 503 too.
|
|
72
|
+
Until the third has run, only `archive` and `restore-check` answer that 503.
|
|
73
|
+
|
|
74
|
+
`ik_door_scopes` binds each logical namespace to the product destination on
|
|
75
|
+
the first uploaded chunk, before a plan/job exists. Old applied namespaces remain scoped through their kit job id
|
|
76
|
+
or the recoverable `door:<plan id>` operation key. Rows the previous door wrote
|
|
77
|
+
in a namespace with neither link (an unplanned upload, a plan never applied,
|
|
78
|
+
or rows a rolled-back previous Worker keeps writing) were never applied by any
|
|
79
|
+
destination, so every destination's erasure withdraws them. No seed migration
|
|
80
|
+
has to bind them.
|
|
81
|
+
|
|
82
|
+
Import-kit custody retirement withdraws the record content in scope whole, in
|
|
83
|
+
the same D1 batch as native erasure, the restore checkpoint and the journal:
|
|
84
|
+
uploaded chunks, upload headers, saved snapshot bodies and archive bytes,
|
|
85
|
+
including unapplied uploads. The journal counts the withdrawn rows. Withdrawing door content fences
|
|
86
|
+
the erasure, so a values-only erasure that touches it requires the restore
|
|
87
|
+
checkpoint (`IMPORT_RETIREMENT_CHECKPOINT_REQUIRED` without one). Plans,
|
|
88
|
+
approvals, namespace heads, namespace bindings, apply-failure facts (a plan
|
|
89
|
+
id and an allowlisted reason code) and archive headers hold no record content and stay, so the seed's history still lists a surviving import "via Business",
|
|
90
|
+
and `status` and `undo` keep working through its kit job. Where a withdrawn
|
|
91
|
+
snapshot is needed (`reconcile`, catch-up `plan`, an `apply` that has not
|
|
92
|
+
completed) the door answers 410 `snapshot_withdrawn` with `details.planId`.
|
|
93
|
+
Door work already in flight is fenced against the erasure: every snapshot
|
|
94
|
+
batch and the batch completing a plan require the source upload, a catch-up's
|
|
95
|
+
previous snapshot and the chunks before it, and every kit batch an `apply`
|
|
96
|
+
writes requires the plan's snapshot. A `plan` or `apply` racing an erasure
|
|
97
|
+
answers the same 410 and never writes withdrawn content back; replaying `apply`
|
|
98
|
+
on a completed import still answers its job. Every door batch, and every kit
|
|
99
|
+
batch a door request runs, also asserts that the destination's erasure journal
|
|
100
|
+
count is unchanged since the request was admitted, so any other request an
|
|
101
|
+
erasure overtakes (an upload chunk, say) answers 410 `erased_during_request`
|
|
102
|
+
and leaves nothing behind. Owner-session requests run under the same fence
|
|
103
|
+
and answer the same 410s. `plan` refuses a bundle holding an erased record
|
|
104
|
+
with 409 `source_erased` and deletes that upload, whatever else would refuse
|
|
105
|
+
that plan first: a withdrawn snapshot, a pending import, interrupted staging,
|
|
106
|
+
the catch-up shape or a bound. Any other bundle keeps that refusal and stays.
|
|
107
|
+
A catch-up keeps each unchanged record as a zero-effect entry in its own kit
|
|
108
|
+
job. Erasing a target a door import wrote severs the same source record in every
|
|
109
|
+
kit job of that door namespace, and withdraws the originals that hold it. A
|
|
110
|
+
resumable erasure severs those copies in the bounded steps that read their jobs.
|
|
111
|
+
The same steps erase each earlier import of the target that a later catch-up
|
|
112
|
+
superseded or that was undone, oldest first, bound to the erased target like the
|
|
113
|
+
copies. Every step, and the final step, fits one request in statements (1,000,
|
|
114
|
+
or the caller's lower limit) and in bytes (one 32 MiB D1 call); one that would
|
|
115
|
+
not writes nothing and is retried with fewer jobs, then with pages of one job's
|
|
116
|
+
entries. The final step reads the target's live import and the jobs no step has
|
|
117
|
+
read since they last changed, and runs the native statements. So a lineage of
|
|
118
|
+
any length, however often the person changed, erases as long as each job (a step
|
|
119
|
+
rewrites its plan whole) fits one such request, and the live import's job fits
|
|
120
|
+
one together with the native statements; otherwise the erasure answers
|
|
121
|
+
`IMPORT_D1_REQUEST_LIMIT` and writes nothing. Earlier imports outside the live
|
|
122
|
+
import's door namespace, or superseded under a `preserve-conflicts` job, wait
|
|
123
|
+
for the final step. A step or an erasure fails closed if a catch-up joins that
|
|
124
|
+
namespace while it reads; a retry then erases it too.
|
|
125
|
+
Later imports start a new namespace, or a new baseline after `undo`.
|
|
126
|
+
Enablement, key revocations and grant replay protection survive.
|
|
127
|
+
|
|
128
|
+
The grant door starts disabled and returns 404 until explicitly enabled. From the
|
|
129
|
+
seed's server-only owner control, call
|
|
130
|
+
`writeImportDoorSettings(database, installationId, { enabled, revokedKids? },
|
|
131
|
+
ownerPredicate)`. The `D1Sql` predicate must check the existing owner session,
|
|
132
|
+
or provisioning authority under the owner's approved plan, in the same database.
|
|
133
|
+
No public door route enables itself. Revoked key ids persist across settings
|
|
134
|
+
writes; removal is unsupported. Rotate to a new `kid` to grant access again.
|
|
135
|
+
|
|
136
|
+
An install RER provisions for a Business plan the owner approved with the door
|
|
137
|
+
opted in carries `RER_IMPORT_DOOR=on` alongside its `RER_OWNER_CLAIM_ISSUER`,
|
|
138
|
+
`RER_OWNER_CLAIM_INSTALLATION_ID`, `RER_OWNER_CLAIM_APP_ORIGIN` and
|
|
139
|
+
`RER_OWNER_CLAIM_PUBLIC_KEYS` trust, whatever its claim mode (bootstrap or
|
|
140
|
+
signed). The recommended consent path is owner opt-in: grants stay closed
|
|
141
|
+
under `RER_IMPORT_DOOR=on` until the owner turns Business imports on at Move
|
|
142
|
+
in, through the settings action above. A seed may instead open the door at the
|
|
143
|
+
first owner claim: when it sees exactly `on`, it calls
|
|
144
|
+
`enableDoorAtFirstClaim(database, installationId, ownerPredicate)` from the
|
|
145
|
+
path that completes its first owner claim and again from every later owner path
|
|
146
|
+
while the variable is `on`: the claim's already-initialized answer, and owner
|
|
147
|
+
sign-in or session start. The call takes the door key and a predicate that holds
|
|
148
|
+
once that owner exists. One batch checks the predicate before and after and
|
|
149
|
+
creates the settings row enabled only when the installation has no settings row
|
|
150
|
+
yet. So it never overrides the owner: a door the owner turned off stays off,
|
|
151
|
+
revoked keys stay revoked, and a repeated call changes nothing. The claim and
|
|
152
|
+
this batch are separate commits and the claim cannot complete twice, so if the
|
|
153
|
+
Worker or D1 fails between them, the next owner path's call is what opens the
|
|
154
|
+
door; without it the door stays closed until the owner turns it on in settings.
|
|
155
|
+
Before an
|
|
156
|
+
owner exists it writes nothing and throws 403 `owner_required`. Self-host
|
|
157
|
+
`rer deploy` never sets `RER_IMPORT_DOOR`, so a self-hosted door stays off until
|
|
158
|
+
its owner opts in.
|
|
159
|
+
|
|
160
|
+
Call `readImportDoorHistory(database, installationId, ownerPredicate)` to show
|
|
161
|
+
approval metadata joined to real kit jobs. The caption is
|
|
162
|
+
"approved via Business by <owner> at <time>, fingerprint <x>" for a job without
|
|
163
|
+
an apply failure. A failed apply is captioned "approved, then failed (<reason>)
|
|
164
|
+
via Business by …"; `state`, `error` and the durable `failureCode` accompany it.
|
|
165
|
+
Keep the ordinary
|
|
166
|
+
seed history and undo flow, using the configured owner principal and its current
|
|
167
|
+
owner-session predicate. Existing operator job binding remains unchanged.
|
|
168
|
+
|
|
169
|
+
## Grants
|
|
170
|
+
|
|
171
|
+
Each request carries `Authorization: Bearer <compact JWS>`. The exact protected
|
|
172
|
+
header is `{ "alg": "ES256", "typ": "rer-import-grant+jwt", "kid": "..." }`.
|
|
173
|
+
Use the existing owner-claim trusted key ring; the door does not supply a signer.
|
|
174
|
+
The required flat payload fields are:
|
|
175
|
+
|
|
176
|
+
| Fields | Constraint |
|
|
177
|
+
| --- | --- |
|
|
178
|
+
| `v`, `iss`, `aud` | Version `1` and the configured owner-claim issuer/audience. |
|
|
179
|
+
| `installationId`, `email` | Exact installation and owner identity. |
|
|
180
|
+
| `jti` | UUID nonce, durably single-use. |
|
|
181
|
+
| `iat`, `exp` | Issued/expiry Unix seconds; lifetime at most 300 seconds, future issue allowance at most 30 seconds. |
|
|
182
|
+
| `action` | Exactly the requested route suffix. |
|
|
183
|
+
| `namespace` | Logical import-tool namespace. |
|
|
184
|
+
| `maxBytes`, `maxChunks`, `maxRecords` | Positive quotas capped at 300,000,000 bytes, 1,024 chunks and 1,000,000 records. |
|
|
185
|
+
|
|
186
|
+
Unknown keys are rejected. Apply additionally requires `fingerprint` (64 hex
|
|
187
|
+
characters), `approvalRecordId` and `approvedAt` (canonical ISO timestamp).
|
|
188
|
+
Plan grants carry no fingerprint. Optional `catchup: 'bounded'` is accepted only
|
|
189
|
+
for plan/apply; bounded plan and apply grants carry the same standing owner
|
|
190
|
+
`approvalRecordId` and `approvedAt`. The apply body `reviewedFingerprint` must
|
|
191
|
+
match both the signed fingerprint and the exact retained plan.
|
|
192
|
+
|
|
193
|
+
Expired, replayed, forged, wrong-installation and wrong-fingerprint grants fail
|
|
194
|
+
closed. Every request needs a fresh nonce, including each chunk, upload inventory
|
|
195
|
+
read, status call and retry. Current enablement, key revocation and expiry are
|
|
196
|
+
checked inside kit mutation batches, so a previously accepted request does not
|
|
197
|
+
grant continuing authority after revocation or expiry.
|
|
198
|
+
|
|
199
|
+
### RER-signed reads outside the door
|
|
200
|
+
|
|
201
|
+
`verifyRerGrant(request, claimConfig, action)` lets a seed accept an RER-signed
|
|
202
|
+
request on its own route outside `/api/rer/import/`, such as Uptime's
|
|
203
|
+
`POST /api/rer/runner-status` (action `runner-status`). Pass the same
|
|
204
|
+
server-owned claim configuration as the grant mount. The grant uses the same
|
|
205
|
+
protected header and key ring; its payload is exactly
|
|
206
|
+
`{v, iss, aud, installationId, email, jti, iat, exp, action}`, under the same
|
|
207
|
+
issuer, audience, installation, owner, nonce-format and 300-second rules, with
|
|
208
|
+
`action` equal to the route's action. `action` is lowercase letters, digits and
|
|
209
|
+
hyphens, and never a door action. A door grant, an unknown or missing member,
|
|
210
|
+
another action, a forged signature or an expired grant answers 403
|
|
211
|
+
`invalid_grant`; a missing bearer 401 `grant_required`; a request not addressed
|
|
212
|
+
to the claim's app origin 403 `grant_binding_mismatch`; a bad configuration or
|
|
213
|
+
action 503 `invalid_configuration`. It returns `{grant, kid}`. It records no
|
|
214
|
+
nonce, so serve only reads whose repeat within the grant's life discloses
|
|
215
|
+
nothing more, and check `kid` against the seed's revoked keys
|
|
216
|
+
(`ik_door_revoked_keys` for the door key) where the route needs revocation.
|
|
217
|
+
|
|
218
|
+
## Requests
|
|
219
|
+
|
|
220
|
+
Bodies are JSON except declared-intake upload streams described below. Quotas
|
|
221
|
+
are admission limits, not a hosted capacity promise.
|
|
222
|
+
|
|
223
|
+
| POST route suffix | Body and result |
|
|
224
|
+
| --- | --- |
|
|
225
|
+
| `uploads` (no intake declaration) | `{ uploadId?, name, bundleSha256, totalBytes, totalChunks, index, chunkSha256, base64 }`; decoded chunks are exactly 64 KiB except the final chunk. `totalChunks` must equal `ceil(totalBytes / 65536)`. Omit `uploadId` on the first chunk to receive a server-generated UUID. Reuse it and the same bundle metadata for subsequent chunks. Both chunk and complete bundle SHA-256 are checked before reporting completion. |
|
|
226
|
+
| `uploads` (declared intake) | Full raw stream with `Content-Type: application/octet-stream`, `X-Import-Format: csv\|ndjson\|json` and `X-Import-Name: <safe file name>`. Returns custody hashes and intake metadata; see below. |
|
|
227
|
+
| `uploads` | `{ uploadId }` returns received-chunk inventory/progress. Without intake, an identical duplicate chunk is idempotent with a fresh grant. |
|
|
228
|
+
| `plan` | `{ uploadId, catchup?: boolean }` returns `planId`, `fingerprint`, server-derived counts, summary, rejected rows with reasons, `diagnostics` and differences including gone source ids. Planning makes no native target writes. |
|
|
229
|
+
| `apply` | `{ planId, reviewedFingerprint }` creates or resumes the real kit job in bounded batches. Repeat with a fresh valid grant to continue. A large source (a stored plan over 4 MiB, such as a multi-megabyte Notion export) is created in bounded staged steps within one request, and the answer may be the still `queued` job; an interrupted creation is never a job and the next `apply` resumes it from what was staged. |
|
|
230
|
+
| `status` | `{ planId }` reads current job progress. |
|
|
231
|
+
| `reconcile` | `{ planId }` derives source-id disposition/presence and native/provenance revision, value, attachment and count checks on the server. |
|
|
232
|
+
| `undo` | `{ planId }` reverses the newest snapshot's attributed effects in bounded kit work, preserving native edits. |
|
|
233
|
+
| `archive` | `{ uploadId, tool, takenAt }` keeps a fully uploaded bundle as the tool's archive; returns `archiveId`, `key`, `tool`, `takenAt`, `bundleSha256`, `bytes` and `counts`. Grant only. See "Archives". |
|
|
234
|
+
| `restore-check` | `{ archiveId }` parses the archive with the namespace's reader and mapping and writes nothing native; returns the archive's fields with `bundleSha256`, `bytes` and `counts` recomputed from its bytes, and `rejected`. Grant only. |
|
|
235
|
+
|
|
236
|
+
### Optional intake before custody
|
|
237
|
+
|
|
238
|
+
Declare intake when the seed's source contract forbids storing fields beyond a
|
|
239
|
+
fixed list. It runs before any persistent write of source content, including
|
|
240
|
+
upload chunks, originals and snapshots. This is a static, server-owned,
|
|
241
|
+
drop-only declaration: no callback, client sanitizer or client-supplied hash can
|
|
242
|
+
replace it. For example:
|
|
243
|
+
|
|
244
|
+
```ts
|
|
245
|
+
intake: {
|
|
246
|
+
formats: ['csv', 'ndjson', 'json'],
|
|
247
|
+
allow: {
|
|
248
|
+
csv: ['user_id', 'email', 'name', 'blocked'],
|
|
249
|
+
ndjson: ['/user_id', '/email', '/name', '/blocked'],
|
|
250
|
+
json: ['/id', '/status', '/profile/email', '/profile/login',
|
|
251
|
+
'/profile/displayName', '/profile/firstName', '/profile/lastName',
|
|
252
|
+
'/profile/department'],
|
|
253
|
+
},
|
|
254
|
+
}
|
|
255
|
+
```
|
|
256
|
+
|
|
257
|
+
CSV entries match exact, case-sensitive decoded column headers. JSON and NDJSON
|
|
258
|
+
entries are exact RFC 6901 pointers to scalar leaves, relative to each source
|
|
259
|
+
record. Nested parents survive only for declared descendants; an object or array
|
|
260
|
+
at an allowed leaf is refused. Undeclared fields are dropped. CSV decoded cell
|
|
261
|
+
strings are unchanged; JSON primitive values and numeric lexemes are unchanged.
|
|
262
|
+
Every accepted row is re-encoded as NDJSON, so syntax representation changes
|
|
263
|
+
without normalizing field names or values. The product parser receives this
|
|
264
|
+
sanitized NDJSON during planning, and the existing `credentials: 'remove'`
|
|
265
|
+
policy still applies then.
|
|
266
|
+
|
|
267
|
+
POST the complete raw file stream to the same
|
|
268
|
+
`/api/operator/import/uploads` or `/api/rer/import/uploads` route. Set
|
|
269
|
+
`Content-Type: application/octet-stream`, `X-Import-Format` to a declared
|
|
270
|
+
`csv`, `ndjson` or `json`, and `X-Import-Name` to a safe file name. Use the same
|
|
271
|
+
owner authentication, Origin/CSRF checks or action-bound grant and quotas as
|
|
272
|
+
other door requests. Unknown or undeclared formats fail closed. A declared
|
|
273
|
+
source refuses JSON chunk envelopes; JSON `{uploadId}` remains the progress
|
|
274
|
+
read. A source without `intake` keeps the original chunk protocol byte-for-byte.
|
|
275
|
+
|
|
276
|
+
The response retains `uploadId`, `bundleSha256`, `received` and `complete`, and
|
|
277
|
+
adds:
|
|
278
|
+
|
|
279
|
+
```ts
|
|
280
|
+
intake: {
|
|
281
|
+
sanitized: true,
|
|
282
|
+
allowlistSha256,
|
|
283
|
+
inputFormat, // 'csv', 'ndjson' or 'json'
|
|
284
|
+
custodyFormat: 'ndjson',
|
|
285
|
+
}
|
|
286
|
+
```
|
|
287
|
+
|
|
288
|
+
`bundleSha256` and chunk hashes describe the sanitized custody bytes, not the
|
|
289
|
+
raw request. `allowlistSha256` is server-derived from the canonical sorted
|
|
290
|
+
formats and each format's sorted allowlist. Intake metadata is stored on the
|
|
291
|
+
upload and in the job's `plan.source.intake`, covered by the plan fingerprint
|
|
292
|
+
and snapshot hash, and compared on plan, apply, reconcile, undo and replay.
|
|
293
|
+
Share the same declaration between the operator mount and the grant source
|
|
294
|
+
resolver; changing it cannot silently reinterpret retained custody.
|
|
295
|
+
|
|
296
|
+
Streaming enforces byte and record quotas plus `limits.maxRecordBytes`
|
|
297
|
+
(default 1 MiB), buffering one row and a fixed output chunk batch. UTF-8,
|
|
298
|
+
quoted CSV fields, lines, CRLF and BOM may cross arbitrary stream boundaries.
|
|
299
|
+
An invalid row refuses with its row number and a fixed reason, without logging
|
|
300
|
+
source values. Forbidden fields never reach persistent storage. On an invalid
|
|
301
|
+
row or interrupted request, earlier sanitized rows may already be stored;
|
|
302
|
+
include these partial uploads in erasure so their custody can be withdrawn.
|
|
303
|
+
|
|
304
|
+
CSV reports logical rows (the header is row 1), preserves quoted commas,
|
|
305
|
+
newlines and escaped quotes, and refuses duplicate decoded headers, mismatched
|
|
306
|
+
widths and invalid quoting. NDJSON reports physical lines and skips blank
|
|
307
|
+
lines. JSON incrementally reads a plain array of object rows, including Okta's
|
|
308
|
+
Users API list, and refuses duplicate object keys or malformed array grammar.
|
|
309
|
+
Arrays in undeclared fields are dropped; traversing an array in an allowed
|
|
310
|
+
pointer is refused because reconstructing a sparse array would add values.
|
|
311
|
+
A leading UTF-8 BOM is accepted once; UTF-8 decoding is strict. Empty CSV,
|
|
312
|
+
header-only CSV, empty NDJSON and an empty JSON array yield no records. An
|
|
313
|
+
empty JSON file is malformed. Refusals carry `details.row` and a fixed
|
|
314
|
+
`details.reason` such as `invalid_json`, `invalid_utf8`, `duplicate_header` or
|
|
315
|
+
`width_mismatch`; dropped values and parser exception text are never logged.
|
|
316
|
+
|
|
317
|
+
The exact-original-byte rule applies to sources without an intake declaration.
|
|
318
|
+
For declared intake, sanitized NDJSON is the original custody used by plans,
|
|
319
|
+
reconciliation and archives; the raw input is never an original artifact.
|
|
320
|
+
|
|
321
|
+
### Large sources
|
|
322
|
+
|
|
323
|
+
A large reviewed plan does not push a door request past D1's 32 MiB RPC
|
|
324
|
+
ceiling or its 1,000-statement budget: snapshot,
|
|
325
|
+
upload and original reads are paged; job creation stages the plan (and the
|
|
326
|
+
original it holds) a few MiB at a time and publishes the job in one small
|
|
327
|
+
transaction that copies the staged rows inside D1; and later batches fence the
|
|
328
|
+
plan's custody revision instead of resending it. No request carries a whole
|
|
329
|
+
plan or a whole original. See the kit README, "Large plans: staged creation and
|
|
330
|
+
revision fences". The seed's target adapter `planCodec`, when it has one, now
|
|
331
|
+
retains a door job whose plan holds no original bytes (for example, an upload
|
|
332
|
+
whose credential values were removed).
|
|
333
|
+
|
|
334
|
+
### Credentials and source refusals
|
|
335
|
+
|
|
336
|
+
`plan` reads the upload with the kit's `credentials: 'remove'`. A credential
|
|
337
|
+
field holding null, an empty string or a mask is not a credential and imports as
|
|
338
|
+
it is. A real value is removed from its record, which is kept: a credential-named
|
|
339
|
+
field is dropped: any key whose whole name parts are a credential word or
|
|
340
|
+
phrase (`apiKey`, `httpPassword`, `http_password`, `client_secret`, `X-Api-Key`,
|
|
341
|
+
`pushToken`, `integration_key`, `webhook_url`, `secret_key`, `x-functions-key`,
|
|
342
|
+
`Ocp-Apim-Subscription-Key`, `pwd`, `X-Auth` and the like; `Signature` and
|
|
343
|
+
`Session-Id` only as header names, so an agent's `signature` field stays; `auth`
|
|
344
|
+
and `credentials` only when they hold a credential, so `auth: {type: 'oauth2'}`
|
|
345
|
+
stays and `auth: {token}` loses its token); a plural
|
|
346
|
+
(`apiKeys: [...]`, `tokens: [...]`) when it holds text, so a count such as
|
|
347
|
+
`tokens: 1500` stays; the `value` of a `{name|key, value}` pair naming one
|
|
348
|
+
(`{name: 'Authorization', value}`), and the value of such a `[name, value]`
|
|
349
|
+
pair in a list of them (Sentry's header tuples). Booleans stay, and names
|
|
350
|
+
merely containing such letters (`keyword`, `monkey`, `author`, `signal`, `design`,
|
|
351
|
+
`primaryKey`) are not credential names, so a keyword monitor's `?keyword=up` URL
|
|
352
|
+
imports intact. A URL of any scheme, after leading whitespace, loses its
|
|
353
|
+
user name and password and its credential query and fragment parameters
|
|
354
|
+
(`api_key`, `access_token`, `sig` and the like), including the `removed@` and
|
|
355
|
+
`?api_key=removed` markers rer's vendor exports leave. Each removal is a
|
|
356
|
+
`CREDENTIAL_REMOVED` warning in `diagnostics` with the record's `kind`,
|
|
357
|
+
`sourceId`, the field pointer and its kind, never the value; a field holding the
|
|
358
|
+
`[removed]` marker rer's vendor exports leave is reported as removed before
|
|
359
|
+
upload. The plan never holds a removed value: the original bytes are withheld
|
|
360
|
+
(`ORIGINAL_UNAVAILABLE`), and the upload's chunks are deleted once the plan is
|
|
361
|
+
saved, keeping its header so the same plan can be fetched again. A plan refused
|
|
362
|
+
for any other reason (`catchup_required`, a catch-up bound, a pending import)
|
|
363
|
+
deletes them too. Planning such an upload again reads its records from a plan
|
|
364
|
+
of the same bytes the namespace still holds (one whose apply failed, or that
|
|
365
|
+
was undone); with none, `plan` answers 410 `upload_credentials_withheld` and
|
|
366
|
+
Business uploads the bundle again. These wider rules apply only to the door's
|
|
367
|
+
removal; the kit's default refuse mode, which seeds' local imports use, is
|
|
368
|
+
unchanged.
|
|
369
|
+
|
|
370
|
+
Every refusal from the reader or a bundle check is a named 4xx whose
|
|
371
|
+
`details.reason` is a fixed sentence and whose `details.row`/`details.pointer`
|
|
372
|
+
locate it; no source value is reflected:
|
|
373
|
+
|
|
374
|
+
| Error / status | Meaning |
|
|
375
|
+
| --- | --- |
|
|
376
|
+
| `source_credential` / 422 | A credential the door cannot remove: one a record repeats elsewhere, an encoded or nested credential URL, or a credential field the mapping proposes. `details.field` names its kind (`credential field "token"`, `URL user name or password`, `URL credential query parameter`, `repeated credential value`). The upload is deleted. |
|
|
377
|
+
| `source_invalid_or_credential_source_data` / 422 | A value the reader cannot inspect for credentials: a URL that cannot be parsed, or text encoded more than 32 layers deep. The upload is deleted. |
|
|
378
|
+
| `source_<reader or bundle code>` / 422 | For example `source_malformed_json`, `source_malformed_utf8`, `source_parser_failed`, `source_duplicate_source_conflict`, `source_duplicate_target`, `source_active_effect_forbidden`. |
|
|
379
|
+
| `source_byte_limit`, `source_record_limit`, `source_record_byte_limit`, `source_effect_limit`, `source_import_plan_too_large` / 413 | A source or plan bound. |
|
|
380
|
+
|
|
381
|
+
JSON bodies are limited to 100,000 bytes; intake streams use their source and
|
|
382
|
+
grant byte/record quotas instead. The 1,024-chunk bound limits one bundle to
|
|
383
|
+
64 MiB even if a grant's byte quota is larger. A smaller grant quota wins.
|
|
384
|
+
|
|
385
|
+
Adapters for large records must implement a compact `matchesMany`, because plan,
|
|
386
|
+
reconcile and catch-up fence the destination's whole native set in one 2 MiB D1
|
|
387
|
+
read, and per-row `matches` or `matchesParts` guards repeat every row in full. A
|
|
388
|
+
native set too large for that read answers 413 `import_door_too_large`, with
|
|
389
|
+
`details.targetKinds` and `details.reason` naming the target kind.
|
|
390
|
+
|
|
391
|
+
Every statement a door request runs, the door's own, the adapter's and each kit
|
|
392
|
+
call's, counts in one 1,000-statement budget (hosted D1's per-invocation limit
|
|
393
|
+
on Workers Paid). A request that would exceed it answers 413
|
|
394
|
+
`import_door_too_large` instead of a provider error. Reconcile and catch-up
|
|
395
|
+
planning take each target's current state from the native rows the persisted
|
|
396
|
+
read fenced under its atomic guard, and read every owning plan in one query, so
|
|
397
|
+
their statement count does not grow with the records imported. An adapter's
|
|
398
|
+
`list` must read its native set in a fixed number of queries, not one per row.
|
|
399
|
+
|
|
400
|
+
Uploaded bytes and retained originals remain in the seed; RER retains receipts
|
|
401
|
+
only. Parsing and mapping are product-owned and cannot send, enqueue or activate
|
|
402
|
+
access. An adopting seed supplies its native field sampling and integrated
|
|
403
|
+
acceptance evidence; this module has no external side effects.
|
|
404
|
+
|
|
405
|
+
## Archives
|
|
406
|
+
|
|
407
|
+
When Business switches off a tool, it uploads the tool's final export and calls
|
|
408
|
+
`archive` with the `uploadId`, the tool's id (`tool`: lowercase letters, digits,
|
|
409
|
+
`.`, `_` and `-`, at most 64, starting with a letter or digit) and when the
|
|
410
|
+
export was taken (`takenAt`, a canonical ISO timestamp). The door copies the
|
|
411
|
+
exact uploaded bytes into the seed's own D1 as the tool's archive, named
|
|
412
|
+
`business-archive/<tool>/<takenAt>` (`key`), and answers its `archiveId`, the
|
|
413
|
+
bundle's `bundleSha256` and `bytes`, and `counts`: the reader and mapping's
|
|
414
|
+
counts of the bundle (`input`, `unique`, `mapped`, `rejected`, `byKind` and the
|
|
415
|
+
rest), as a first plan of it would report them. `counts.input` counts every
|
|
416
|
+
record read, a repeated identical record included, and `counts.unique` each
|
|
417
|
+
source identity once: to check an archive against the catch-up of the same
|
|
418
|
+
export, compare `counts.unique` with the catch-up's `summary.sourceRecords`, or
|
|
419
|
+
`counts.input` with the catch-up's `counts.input`. Archiving writes nothing native. The archive
|
|
420
|
+
holds its own copy, so `plan`, a catch-up, a cancelled preparation or an `undo`
|
|
421
|
+
never deletes it, and the upload stays usable for planning. The same request
|
|
422
|
+
again answers the same archive; another `tool` or `takenAt` for an archived
|
|
423
|
+
upload answers 409 `archive_exists` with `details.archiveId`.
|
|
424
|
+
|
|
425
|
+
An archive keeps exact bytes, so it refuses what the door never keeps: a bundle
|
|
426
|
+
from which the reader had to remove a live credential value answers 422
|
|
427
|
+
`archive_holds_credentials`. The markers rer's vendor exports leave in place of
|
|
428
|
+
the values they strip (`[removed]` in a field, `removed@` in a URL, `=removed`
|
|
429
|
+
for a query or fragment parameter) hold no credential, so such a bundle
|
|
430
|
+
archives, although a plan of it still withholds the original; a bundle holding an erased record answers 409 `source_erased`
|
|
431
|
+
and the upload goes, as `plan` does; a reader refusal answers its named 4xx. An
|
|
432
|
+
upload whose bytes already went (planned with credentials removed, or refused)
|
|
433
|
+
answers 410 `upload_bytes_withheld`: upload it again and archive before
|
|
434
|
+
planning.
|
|
435
|
+
|
|
436
|
+
`restore-check` with `{archiveId}` reads the archive's bytes back, checks each
|
|
437
|
+
chunk and the bundle hash (409 `archive_custody_mismatch` otherwise), and parses
|
|
438
|
+
them with the namespace's reader and current mapping without writing anything
|
|
439
|
+
native. It answers the archive's fields with `bundleSha256`, `bytes` and
|
|
440
|
+
`counts` recomputed from the bytes, and `rejected` (`kind`, `sourceId`,
|
|
441
|
+
`reason`). An unknown archive answers 404 `archive_not_found`; one over the
|
|
442
|
+
grant's quotas 403 `archive_exceeds_grant_quota`.
|
|
443
|
+
|
|
444
|
+
A person's erasure withdraws every archive's bytes in its scope with the rest
|
|
445
|
+
of the door's held content, and its journal counts them under
|
|
446
|
+
`ik_door_archive_chunks`. The archive header (id, tool, time, file name, hash,
|
|
447
|
+
size and kind counts) stays, so `restore-check` then answers 410
|
|
448
|
+
`archive_withdrawn` with `details.archiveId` and `details.reason`. The upload
|
|
449
|
+
went with it; the tool's export, uploaded again, is refused while it still
|
|
450
|
+
holds the erased person (`source_erased`).
|
|
451
|
+
|
|
452
|
+
Both are grant actions with the grant rules above: no fingerprint, approval or
|
|
453
|
+
`catchup`, a fresh single-use grant per call, bound to its action, namespace and
|
|
454
|
+
quotas. The owner-session mount answers them 404.
|
|
455
|
+
|
|
456
|
+
## Catch-up and recovery
|
|
457
|
+
|
|
458
|
+
A logical namespace tracks a lineage of snapshots; each snapshot's kit namespace
|
|
459
|
+
derives from the logical namespace, the exact bundle SHA-256 and its plan id. Original source ids
|
|
460
|
+
remain intact. Unchanged records produce no effects but remain accounted for.
|
|
461
|
+
Replacements use the revision in original import provenance as
|
|
462
|
+
`expectedRevision`, and compare native values too, protecting edits even when
|
|
463
|
+
their revision was unchanged. Gone source records are reported without deletion.
|
|
464
|
+
|
|
465
|
+
Only one plan may remain pending per logical namespace until applied or undone.
|
|
466
|
+
Committed kit state recovers lineage after an interrupted call. Repeating a
|
|
467
|
+
bundle reuses its exact retained plan unless that plan's apply failed or its
|
|
468
|
+
import was undone or rolled back: then the same bundle plans afresh, so a file
|
|
469
|
+
can be imported again after undo. A catch-up (`catchup: true`) never reuses an
|
|
470
|
+
applied plan, the head or one before it: a bundle identical to one already
|
|
471
|
+
applied plans afresh against the head, so an unchanged source yields a catch-up
|
|
472
|
+
with no changes that a standing approval can apply. Repeating it before it is
|
|
473
|
+
applied reuses that catch-up plan. Use a fresh explicit namespace and review
|
|
474
|
+
for a different mapping. Undo snapshots newest first, including undo from the
|
|
475
|
+
seed's ordinary owner history. Undoing a snapshot while a newer one still holds
|
|
476
|
+
its records answers 409 `import_superseded`; a strict undo after an owner edit
|
|
477
|
+
answers 409 `native_edit_conflict`. Each has a `details.reason`.
|
|
478
|
+
|
|
479
|
+
With an adapter declaring `preparePreservedUndo`, new jobs instead preserve
|
|
480
|
+
conflicts by default: undo reverses untouched imported rows, keeps native edits
|
|
481
|
+
and deletions (and required dependencies), and completes with `job.state:
|
|
482
|
+
'undone'`, even if every row was retained. Its durable
|
|
483
|
+
`readD1ImportUndoReport` has outcome `compensated` when any disposition is
|
|
484
|
+
`retained`, identifies each preserved effect and reason, and reports `undone`
|
|
485
|
+
when everything reversed. Native owner history can page that report with its
|
|
486
|
+
normal kit authorization. A second door undo is idempotent. The policy is saved
|
|
487
|
+
with the job, never inferred again during resume, undo or replay. Adding the
|
|
488
|
+
method later leaves old strict jobs strict; removing it refuses further apply
|
|
489
|
+
batches and active undo on preserving jobs. Restore a compatible adapter to
|
|
490
|
+
undo previously applied batches. Explicit
|
|
491
|
+
`reversalPolicy` settings still override the automatic choice for new jobs.
|
|
492
|
+
For example, Orbit's native preserving adapter needs only this shared door
|
|
493
|
+
update, with no Orbit-specific vendored changes. Preserving apply validates and
|
|
494
|
+
checks the preparation for every pending batch in a read-only admission before
|
|
495
|
+
that batch publishes, including resumed apply. Apply still atomically fences
|
|
496
|
+
its own exact before-images and current authority. An invalid preparation
|
|
497
|
+
refuses the entire pending batch; at checkpoint zero no native rows are applied.
|
|
498
|
+
Earlier admitted batches remain available to Undo. A preparation may cover several small consecutive batches
|
|
499
|
+
within one request; every exact prospective effect must match that admission
|
|
500
|
+
before its own batch publishes. Admission never carries across requests. See [ADOPTING.md](ADOPTING.md).
|
|
501
|
+
|
|
502
|
+
When an erasure withdrew the live import's snapshot, nothing can be compared
|
|
503
|
+
with it: an owner-session `plan` retires it as the baseline and plans a new
|
|
504
|
+
one, while a Business catch-up still answers 410 `snapshot_withdrawn`. While an
|
|
505
|
+
earlier import of the source still holds a record the new plan writes, the kit
|
|
506
|
+
would refuse its apply, so `plan` answers 409 `earlier_import_holds_records`
|
|
507
|
+
instead, with `details.planIds` and a `details.reason` to undo the earlier
|
|
508
|
+
import first; nothing is retired. After that undo the same upload plans and
|
|
509
|
+
applies.
|
|
510
|
+
|
|
511
|
+
An interrupted private plan preparation reports
|
|
512
|
+
`plan_staging_interrupted_cancel_and_replan` with `details.planId`. Call `undo` with that `planId`
|
|
513
|
+
to cancel its private preparation, then plan the retained upload again. A
|
|
514
|
+
preparation with no kit job cancels without native effects. Once a job exists,
|
|
515
|
+
the same route uses the kit's guarded rollback/undo instead.
|
|
516
|
+
|
|
517
|
+
An exception during the target apply ends that job as `failed`. Its native
|
|
518
|
+
batch is atomic: no writes from the failed batch remain. The approval and a
|
|
519
|
+
separate failure fact remain for audit, including after an owner rollback.
|
|
520
|
+
Status returns `job.state`, `job.error` and `failureCode`; history exposes the
|
|
521
|
+
same reason. Provider messages and customer records are never stored as reasons.
|
|
522
|
+
|
|
523
|
+
| Apply reason / HTTP status | Meaning |
|
|
524
|
+
| --- | --- |
|
|
525
|
+
| `import_door_apply_constraint` / 409 | A database constraint refused the batch. Correct the data or native writer before proposing another plan. |
|
|
526
|
+
| `import_door_apply_transient` / 503 | The provider reported busy, locked, interrupted or unavailable service, including D1's retryable errors (storage or code-update resets, lost connections, overload, timeouts). A fresh reviewed attempt can be proposed after recovery. |
|
|
527
|
+
| `import_door_apply_failed` / 409 | Another apply error. Inspect the seed's adapter before proposing another plan. |
|
|
528
|
+
|
|
529
|
+
Every failed plan is terminal: further applies return
|
|
530
|
+
`import_door_plan_failed`, even with a new grant or approval. The consumed grant
|
|
531
|
+
nonce stays consumed. A retry needs a new plan, fingerprint and approval record;
|
|
532
|
+
the approval associated with the failure cannot approve a retry, including a
|
|
533
|
+
standing catch-up approval. When checkpoint is zero, `plan` can immediately
|
|
534
|
+
prepare the same retained upload again in the same logical namespace, including
|
|
535
|
+
one whose credential-bearing bytes went: its records come from the failed
|
|
536
|
+
plan's snapshot. If earlier
|
|
537
|
+
batches committed, first finish `undo` (guarded rollback) before replanning.
|
|
538
|
+
The old job, immutable snapshot, approval and failure fact remain readable.
|
|
539
|
+
|
|
540
|
+
An erasure that overtakes an apply is not an apply failure: no failure fact is
|
|
541
|
+
recorded, the job keeps its state for `undo`, and the apply answers the
|
|
542
|
+
erasure's 410 (`snapshot_withdrawn` or `erased_during_request`). Nor is a kit
|
|
543
|
+
size or request limit, which refuses before its batch writes: the apply
|
|
544
|
+
answers 413 `import_door_too_large` and the plan stays applicable.
|
|
545
|
+
|
|
546
|
+
Seeds that installed an earlier door apply `importDoorApplyFailureMigration`
|
|
547
|
+
and then `importDoorScopesMigration`, as set out at the top of this document.
|
|
548
|
+
|
|
549
|
+
Bounded catch-up requires an existing nonempty baseline and standing owner
|
|
550
|
+
approval. It refuses gone, conflicting, rejected or unmapped records. Additions
|
|
551
|
+
plus replacements divided by baseline source count must be strictly below 5%;
|
|
552
|
+
the server derives these counts. A refusal leaves Business to present a new
|
|
553
|
+
proposal for owner review. No caller-supplied totals relax this boundary.
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import type { ImportDoorOptions, ImportDoorRuntimeOptions, DoorRequest } from './types.ts';
|
|
2
|
+
import { runtimeOptions } from './handler.ts';
|
|
3
|
+
import { DoorError, verifyImportGrant, type DoorAction } from './grants.ts';
|
|
4
|
+
import { doorFence, erasures, sql, transact } from './store.ts';
|
|
5
|
+
import { importDoorSchemaCheck } from './schema.ts';
|
|
6
|
+
|
|
7
|
+
/** The storage key; grants are still verified against `claim.installationId`. */
|
|
8
|
+
export const doorKey = (options: ImportDoorOptions): string => options.installationId ?? options.claim.installationId;
|
|
9
|
+
/** Opt-in and schema state in one read. A door whose seed has not applied the
|
|
10
|
+
* door migrations must refuse loudly rather than misbehave. */
|
|
11
|
+
export async function enabled(options: ImportDoorOptions): Promise<boolean> {
|
|
12
|
+
const row = await options.database.prepare(`SELECT (SELECT enabled FROM ik_door_settings WHERE installation_id=?) AS enabled,(${importDoorSchemaCheck}) AS current`)
|
|
13
|
+
.bind(doorKey(options)).first();
|
|
14
|
+
if (row?.enabled !== 1) return false;
|
|
15
|
+
if (row.current !== 1) throw new DoorError('import_door_schema_outdated', 503);
|
|
16
|
+
return true;
|
|
17
|
+
}
|
|
18
|
+
/** This grant's parser and mapping: the mount's own, or the seed's allow-list entry
|
|
19
|
+
* for the verified namespace. Unlisted namespaces are refused before any write. */
|
|
20
|
+
async function source(options: ImportDoorOptions, namespace: string): Promise<ImportDoorRuntimeOptions> {
|
|
21
|
+
if (!options.source) return runtimeOptions(options);
|
|
22
|
+
const listed = await options.source(namespace);
|
|
23
|
+
if (!listed) throw new DoorError('unknown_import_source');
|
|
24
|
+
return runtimeOptions({...options, parse: listed.parse, mapping: listed.mapping, intake: listed.intake, ownerPrincipalId: listed.ownerPrincipalId ?? options.ownerPrincipalId});
|
|
25
|
+
}
|
|
26
|
+
export async function admit(options: ImportDoorOptions, request: Request, action: DoorAction): Promise<DoorRequest> {
|
|
27
|
+
const authorization = request.headers.get('Authorization');
|
|
28
|
+
if (!authorization?.startsWith('Bearer ')) throw new DoorError('grant_required', 401);
|
|
29
|
+
const {grant, kid} = await verifyImportGrant(authorization.slice(7), options.claim);
|
|
30
|
+
if (grant.action !== action || new URL(request.url).origin !== options.claim.appOrigin) throw new DoorError('grant_binding_mismatch');
|
|
31
|
+
const runtime = await source(options, grant.namespace);
|
|
32
|
+
const key = doorKey(options), epoch = await erasures(options.database, options.destination);
|
|
33
|
+
const fence = doorFence(key, grant.namespace, options.destination, epoch);
|
|
34
|
+
const basic = sql(`(${fence.sql}) AND EXISTS(SELECT 1 FROM ik_door_settings WHERE installation_id=? AND enabled=1)
|
|
35
|
+
AND NOT EXISTS(SELECT 1 FROM ik_door_revoked_keys WHERE installation_id=? AND kid=?)
|
|
36
|
+
AND unixepoch() < ?`, ...fence.params ?? [], key, key, kid, grant.exp);
|
|
37
|
+
try {
|
|
38
|
+
await transact(options.database, basic, [sql('INSERT INTO ik_door_grants(installation_id,jti,action,namespace,kid,expires) VALUES(?,?,?,?,?,?)', key, grant.jti, action, grant.namespace, kid, grant.exp)]);
|
|
39
|
+
} catch {
|
|
40
|
+
if (await erasures(options.database, options.destination) !== epoch) throw new DoorError('erased_during_request', 410);
|
|
41
|
+
throw new DoorError('grant_replayed_revoked_or_expired');
|
|
42
|
+
}
|
|
43
|
+
const authority = sql(`(${basic.sql}) AND EXISTS(SELECT 1 FROM ik_door_grants WHERE installation_id=? AND jti=? AND action=? AND namespace=? AND kid=? AND expires=?)`, ...basic.params ?? [], key, grant.jti, action, grant.namespace, kid, grant.exp);
|
|
44
|
+
return {options: runtime, access: {...grant, installationId: key}, authority, via:'grant', epoch, nonce: {jti: grant.jti, kid}};
|
|
45
|
+
}
|