@runeditrun/rer 0.2.80 → 0.2.82

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. package/README.md +12 -0
  2. package/cli/deploy-needs.mjs +4 -2
  3. package/cli/export.mjs +1 -1
  4. package/cli/ledger-refusal.mjs +18 -0
  5. package/cli/portability-migrations.mjs +30 -11
  6. package/cli/portability-worker.mjs +3 -1
  7. package/cli/portability.mjs +2 -0
  8. package/cli/release.mjs +4 -4
  9. package/cli/site-hosting.mjs +90 -0
  10. package/cli/site-render.mjs +3 -2
  11. package/cli/site.mjs +5 -2
  12. package/edition-authoring/base-copies.mjs +12 -1
  13. package/edition-authoring/copy-checkout.mjs +152 -1
  14. package/edition-authoring/files.mjs +1 -1
  15. package/edition-authoring/runtime-dependencies.mjs +108 -0
  16. package/global/CHANGELOG.md +25 -0
  17. package/global/README.md +12 -0
  18. package/modules/demo/README.md +8 -0
  19. package/modules/demo/navigation.mjs +131 -15
  20. package/modules/dependencies/auth.mjs +2 -2
  21. package/modules/deploy-kit/owner.mjs +8 -1
  22. package/modules/import-kit/src/d1/apply-effects.ts +33 -0
  23. package/modules/import-kit/src/d1/apply.ts +38 -10
  24. package/modules/import-kit/src/door/ADOPTING.md +452 -0
  25. package/modules/import-kit/src/door/README.md +553 -0
  26. package/modules/import-kit/src/door/admission.ts +45 -0
  27. package/modules/import-kit/src/door/archive.ts +129 -0
  28. package/modules/import-kit/src/door/authority.ts +107 -0
  29. package/modules/import-kit/src/door/failures.ts +70 -0
  30. package/modules/import-kit/src/door/grants.ts +162 -0
  31. package/modules/import-kit/src/door/handler.ts +90 -0
  32. package/modules/import-kit/src/door/index.ts +20 -0
  33. package/modules/import-kit/src/door/intake/csv.ts +74 -0
  34. package/modules/import-kit/src/door/intake/json.ts +182 -0
  35. package/modules/import-kit/src/door/intake/projection.ts +61 -0
  36. package/modules/import-kit/src/door/intake/types.ts +28 -0
  37. package/modules/import-kit/src/door/intake-policy.ts +38 -0
  38. package/modules/import-kit/src/door/intake-upload.ts +91 -0
  39. package/modules/import-kit/src/door/inventory.ts +68 -0
  40. package/modules/import-kit/src/door/jobs.ts +134 -0
  41. package/modules/import-kit/src/door/limits.ts +16 -0
  42. package/modules/import-kit/src/door/operator.ts +41 -0
  43. package/modules/import-kit/src/door/planning.ts +227 -0
  44. package/modules/import-kit/src/door/protocol.ts +38 -0
  45. package/modules/import-kit/src/door/refusals.ts +51 -0
  46. package/modules/import-kit/src/door/schema.ts +101 -0
  47. package/modules/import-kit/src/door/store.ts +152 -0
  48. package/modules/import-kit/src/door/types.ts +100 -0
  49. package/modules/import-kit/src/door/uploads.ts +98 -0
  50. package/modules/import-kit/src/provenance/numeric-credentials.ts +40 -0
  51. package/modules/import-kit/src/readers/README.md +139 -0
  52. package/modules/import-kit/src/readers/api-policy.ts +67 -0
  53. package/modules/import-kit/src/readers/api.ts +86 -0
  54. package/modules/import-kit/src/readers/bytes.ts +73 -0
  55. package/modules/import-kit/src/readers/errors.ts +33 -0
  56. package/modules/import-kit/src/readers/file.ts +74 -0
  57. package/modules/import-kit/src/readers/index.ts +3 -0
  58. package/modules/import-kit/src/readers/json.ts +29 -0
  59. package/modules/import-kit/src/readers/numeric-lexemes.ts +104 -0
  60. package/modules/import-kit/src/readers/original.ts +31 -0
  61. package/modules/import-kit/src/readers/records.ts +105 -0
  62. package/modules/import-kit/src/types.ts +3 -1
  63. package/modules/operator-auth/README.md +92 -3
  64. package/modules/operator-auth/accounts/admin.mjs +52 -20
  65. package/modules/operator-auth/accounts/audit.mjs +24 -0
  66. package/modules/operator-auth/accounts/business-team.mjs +91 -0
  67. package/modules/operator-auth/accounts/domain-audit.mjs +24 -0
  68. package/modules/operator-auth/accounts/native.mjs +3 -1
  69. package/modules/operator-auth/accounts/settings-page.mjs +15 -2
  70. package/modules/operator-auth/accounts/status.mjs +7 -2
  71. package/modules/operator-auth/accounts/store.mjs +32 -6
  72. package/modules/operator-auth/business-settings.mjs +40 -0
  73. package/modules/operator-auth/identity/email-link/index.mjs +1 -1
  74. package/modules/operator-auth/identity/oidc/admin-audit.mjs +31 -0
  75. package/modules/operator-auth/identity/oidc/index.mjs +12 -5
  76. package/modules/operator-auth/identity/oidc/store.mjs +2 -0
  77. package/modules/operator-auth/index.d.mts +40 -1
  78. package/modules/operator-auth/routes.mjs +11 -6
  79. package/modules/operator-auth/service.mjs +8 -3
  80. package/modules/operator-auth/signin/index.mjs +1 -1
  81. package/modules/portability/README.md +82 -1
  82. package/modules/portability/export.mjs +6 -4
  83. package/modules/portability/format.mjs +5 -0
  84. package/modules/portability/index.d.mts +10 -2
  85. package/modules/portability/index.mjs +3 -2
  86. package/modules/portability/ledgers.mjs +27 -4
  87. package/modules/release/README.md +74 -2
  88. package/modules/release/sign.mjs +4 -2
  89. package/modules/release/signature.mjs +14 -7
  90. package/modules/release/signing-cli.mjs +2 -1
  91. package/modules/release/statement.mjs +2 -2
  92. package/modules/release/vectors/generate.mjs +1 -0
  93. package/modules/release/vectors/publisher-projects.mjs +133 -0
  94. package/modules/release/vectors/release-signature-vectors.json +5195 -0
  95. package/modules/release/verify.mjs +5 -1
  96. package/onboarding/schema/SETUP.schema.json +43 -1
  97. package/onboarding/schema/business-field-inputs.mjs +0 -1
  98. package/onboarding/schema/setup-validation.mjs +3 -0
  99. package/package.json +3 -2
@@ -0,0 +1,553 @@
1
+ # Shared D1 import door
2
+
3
+ To bring existing data into a seed, sign in as the owner and use its import
4
+ screen. First step: open this path on the destination app:
5
+
6
+ ```text
7
+ /app/import
8
+ ```
9
+
10
+ Upload a supported source export, review the plan and its differences, then
11
+ approve that exact plan to apply it. Reconcile the result against native data;
12
+ use Undo from the same history when needed. Base's reference accepts people JSON
13
+ here, not installation `.rer` archives. Editions adopt this same door
14
+ with their own supported readers and native adapters.
15
+
16
+ See [ADOPTING.md](ADOPTING.md) for the single-path adoption checklist, the
17
+ owner-session mount without owner-claim, and public HTTP conformance.
18
+
19
+ Import `createImportDoorHandler`, `importDoorSchema`,
20
+ `importDoorApplyFailureMigration`, `importDoorScopesMigration`,
21
+ `importDoorArchiveMigration`, `importDoorIntakeMigration`,
22
+ `writeImportDoorSettings`, `enableDoorAtFirstClaim`, `readImportDoorHistory`
23
+ and `verifyRerGrant` from `./index.ts`.
24
+ The handler returns `(request: Request) => Promise<Response | null>`; pass
25
+ unhandled requests to the seed's existing router. Only POST routes under
26
+ `/api/rer/import/` are supported. Nothing mounts a route automatically.
27
+
28
+ Construct the handler with these seed-owned options:
29
+
30
+ | Option | Meaning |
31
+ | --- | --- |
32
+ | `database`, `targets` | D1 database and its real `D1TargetAdapter`. |
33
+ | `claim` | The same server-owned `OwnerClaimConfig` and trusted key ring used by owner-claim. |
34
+ | `installationId` | Optional permanent door key for settings, lineage and history; defaults to `claim.installationId`. Use the owner-session mount's key when RER's claim id differs. |
35
+ | `ownerPrincipalId` | Actual owner principal used by the seed's ordinary import history and undo context. |
36
+ | `destination`, `mapping` | Product destination and reviewed `ImportMapping`. |
37
+ | `format`, `parse` | `'json'` or `'ndjson'`, and the product's parser of source records. With declared intake, sanitized custody is always read as NDJSON. |
38
+ | `intake` | Optional server-owned `{formats, allow}` declaration for dropping non-allowlisted fields before custody. See below; no callback is accepted. |
39
+ | `source` | Instead of `parse`/`mapping`, for several sources: `source(namespace)` returns that namespace's `{parse, mapping, ownerPrincipalId?, intake?}` from the seed's server-owned allow-list, or `null`. It runs after grant verification and before any write; `null` is refused with 403 `unknown_import_source`. |
40
+ | `limits` | Optional reader limits; grant quotas still apply. |
41
+ | `reversalPolicy` | Optional override for new jobs. Without it, declaring `targets.preparePreservedUndo` selects `'preserve-conflicts'`; adapters without that method stay `'strict'`. Preserve keeps native edits on undo and requires that method. Existing jobs keep their creation policy. See ADOPTING.md. |
42
+
43
+ A fresh installation applies the statements in `importDoorSchema: string[]`
44
+ in a new product-owned numbered migration after the D1 import-kit schemas.
45
+ Preserve existing applied migration names. Copy the import-kit runtime and its
46
+ sibling owner-claim and D1 transport dependencies with their relative topology
47
+ intact.
48
+
49
+ A seed that installed an earlier door upgrades by applying the door
50
+ migrations once each, in this order, each in its own new numbered migration,
51
+ never by re-applying `importDoorSchema` (its first statement refuses an
52
+ earlier door's plan table, so re-applying it fails the deploy):
53
+
54
+ 1. `importDoorApplyFailureMigration`, in a single atomic `database.batch()`
55
+ with foreign keys enabled. It preserves every plan id, chunk and approval,
56
+ removes the old one-plan-per-bundle uniqueness constraint and adds the
57
+ `ik_door_apply_failures` table (see "Catch-up and recovery").
58
+ 2. `importDoorScopesMigration`, the additive `ik_door_scopes` table.
59
+ 3. `importDoorArchiveMigration`, the additive `ik_door_archives` and
60
+ `ik_door_archive_chunks` tables (see "Archives").
61
+
62
+ An installed adapter that declares intake also applies the optional
63
+ `importDoorIntakeMigration` once in its own new numbered migration. It adds
64
+ nullable `intake_json` to `ik_door_uploads`; fresh `importDoorSchema` already
65
+ includes it. An existing installation without an intake declaration needs no
66
+ intake migration and keeps its existing upload behavior.
67
+
68
+ Until the first two have run, an enabled door answers every route with 503
69
+ `import_door_schema_outdated`; a disabled grant door still answers 404. The
70
+ owner-session mount (`createImportOperatorHandler` in `operator.ts`, which
71
+ exports the same schema and migrations) has no opt-in and answers 503 too.
72
+ Until the third has run, only `archive` and `restore-check` answer that 503.
73
+
74
+ `ik_door_scopes` binds each logical namespace to the product destination on
75
+ the first uploaded chunk, before a plan/job exists. Old applied namespaces remain scoped through their kit job id
76
+ or the recoverable `door:<plan id>` operation key. Rows the previous door wrote
77
+ in a namespace with neither link (an unplanned upload, a plan never applied,
78
+ or rows a rolled-back previous Worker keeps writing) were never applied by any
79
+ destination, so every destination's erasure withdraws them. No seed migration
80
+ has to bind them.
81
+
82
+ Import-kit custody retirement withdraws the record content in scope whole, in
83
+ the same D1 batch as native erasure, the restore checkpoint and the journal:
84
+ uploaded chunks, upload headers, saved snapshot bodies and archive bytes,
85
+ including unapplied uploads. The journal counts the withdrawn rows. Withdrawing door content fences
86
+ the erasure, so a values-only erasure that touches it requires the restore
87
+ checkpoint (`IMPORT_RETIREMENT_CHECKPOINT_REQUIRED` without one). Plans,
88
+ approvals, namespace heads, namespace bindings, apply-failure facts (a plan
89
+ id and an allowlisted reason code) and archive headers hold no record content and stay, so the seed's history still lists a surviving import "via Business",
90
+ and `status` and `undo` keep working through its kit job. Where a withdrawn
91
+ snapshot is needed (`reconcile`, catch-up `plan`, an `apply` that has not
92
+ completed) the door answers 410 `snapshot_withdrawn` with `details.planId`.
93
+ Door work already in flight is fenced against the erasure: every snapshot
94
+ batch and the batch completing a plan require the source upload, a catch-up's
95
+ previous snapshot and the chunks before it, and every kit batch an `apply`
96
+ writes requires the plan's snapshot. A `plan` or `apply` racing an erasure
97
+ answers the same 410 and never writes withdrawn content back; replaying `apply`
98
+ on a completed import still answers its job. Every door batch, and every kit
99
+ batch a door request runs, also asserts that the destination's erasure journal
100
+ count is unchanged since the request was admitted, so any other request an
101
+ erasure overtakes (an upload chunk, say) answers 410 `erased_during_request`
102
+ and leaves nothing behind. Owner-session requests run under the same fence
103
+ and answer the same 410s. `plan` refuses a bundle holding an erased record
104
+ with 409 `source_erased` and deletes that upload, whatever else would refuse
105
+ that plan first: a withdrawn snapshot, a pending import, interrupted staging,
106
+ the catch-up shape or a bound. Any other bundle keeps that refusal and stays.
107
+ A catch-up keeps each unchanged record as a zero-effect entry in its own kit
108
+ job. Erasing a target a door import wrote severs the same source record in every
109
+ kit job of that door namespace, and withdraws the originals that hold it. A
110
+ resumable erasure severs those copies in the bounded steps that read their jobs.
111
+ The same steps erase each earlier import of the target that a later catch-up
112
+ superseded or that was undone, oldest first, bound to the erased target like the
113
+ copies. Every step, and the final step, fits one request in statements (1,000,
114
+ or the caller's lower limit) and in bytes (one 32 MiB D1 call); one that would
115
+ not writes nothing and is retried with fewer jobs, then with pages of one job's
116
+ entries. The final step reads the target's live import and the jobs no step has
117
+ read since they last changed, and runs the native statements. So a lineage of
118
+ any length, however often the person changed, erases as long as each job (a step
119
+ rewrites its plan whole) fits one such request, and the live import's job fits
120
+ one together with the native statements; otherwise the erasure answers
121
+ `IMPORT_D1_REQUEST_LIMIT` and writes nothing. Earlier imports outside the live
122
+ import's door namespace, or superseded under a `preserve-conflicts` job, wait
123
+ for the final step. A step or an erasure fails closed if a catch-up joins that
124
+ namespace while it reads; a retry then erases it too.
125
+ Later imports start a new namespace, or a new baseline after `undo`.
126
+ Enablement, key revocations and grant replay protection survive.
127
+
128
+ The grant door starts disabled and returns 404 until explicitly enabled. From the
129
+ seed's server-only owner control, call
130
+ `writeImportDoorSettings(database, installationId, { enabled, revokedKids? },
131
+ ownerPredicate)`. The `D1Sql` predicate must check the existing owner session,
132
+ or provisioning authority under the owner's approved plan, in the same database.
133
+ No public door route enables itself. Revoked key ids persist across settings
134
+ writes; removal is unsupported. Rotate to a new `kid` to grant access again.
135
+
136
+ An install RER provisions for a Business plan the owner approved with the door
137
+ opted in carries `RER_IMPORT_DOOR=on` alongside its `RER_OWNER_CLAIM_ISSUER`,
138
+ `RER_OWNER_CLAIM_INSTALLATION_ID`, `RER_OWNER_CLAIM_APP_ORIGIN` and
139
+ `RER_OWNER_CLAIM_PUBLIC_KEYS` trust, whatever its claim mode (bootstrap or
140
+ signed). The recommended consent path is owner opt-in: grants stay closed
141
+ under `RER_IMPORT_DOOR=on` until the owner turns Business imports on at Move
142
+ in, through the settings action above. A seed may instead open the door at the
143
+ first owner claim: when it sees exactly `on`, it calls
144
+ `enableDoorAtFirstClaim(database, installationId, ownerPredicate)` from the
145
+ path that completes its first owner claim and again from every later owner path
146
+ while the variable is `on`: the claim's already-initialized answer, and owner
147
+ sign-in or session start. The call takes the door key and a predicate that holds
148
+ once that owner exists. One batch checks the predicate before and after and
149
+ creates the settings row enabled only when the installation has no settings row
150
+ yet. So it never overrides the owner: a door the owner turned off stays off,
151
+ revoked keys stay revoked, and a repeated call changes nothing. The claim and
152
+ this batch are separate commits and the claim cannot complete twice, so if the
153
+ Worker or D1 fails between them, the next owner path's call is what opens the
154
+ door; without it the door stays closed until the owner turns it on in settings.
155
+ Before an
156
+ owner exists it writes nothing and throws 403 `owner_required`. Self-host
157
+ `rer deploy` never sets `RER_IMPORT_DOOR`, so a self-hosted door stays off until
158
+ its owner opts in.
159
+
160
+ Call `readImportDoorHistory(database, installationId, ownerPredicate)` to show
161
+ approval metadata joined to real kit jobs. The caption is
162
+ "approved via Business by <owner> at <time>, fingerprint <x>" for a job without
163
+ an apply failure. A failed apply is captioned "approved, then failed (<reason>)
164
+ via Business by …"; `state`, `error` and the durable `failureCode` accompany it.
165
+ Keep the ordinary
166
+ seed history and undo flow, using the configured owner principal and its current
167
+ owner-session predicate. Existing operator job binding remains unchanged.
168
+
169
+ ## Grants
170
+
171
+ Each request carries `Authorization: Bearer <compact JWS>`. The exact protected
172
+ header is `{ "alg": "ES256", "typ": "rer-import-grant+jwt", "kid": "..." }`.
173
+ Use the existing owner-claim trusted key ring; the door does not supply a signer.
174
+ The required flat payload fields are:
175
+
176
+ | Fields | Constraint |
177
+ | --- | --- |
178
+ | `v`, `iss`, `aud` | Version `1` and the configured owner-claim issuer/audience. |
179
+ | `installationId`, `email` | Exact installation and owner identity. |
180
+ | `jti` | UUID nonce, durably single-use. |
181
+ | `iat`, `exp` | Issued/expiry Unix seconds; lifetime at most 300 seconds, future issue allowance at most 30 seconds. |
182
+ | `action` | Exactly the requested route suffix. |
183
+ | `namespace` | Logical import-tool namespace. |
184
+ | `maxBytes`, `maxChunks`, `maxRecords` | Positive quotas capped at 300,000,000 bytes, 1,024 chunks and 1,000,000 records. |
185
+
186
+ Unknown keys are rejected. Apply additionally requires `fingerprint` (64 hex
187
+ characters), `approvalRecordId` and `approvedAt` (canonical ISO timestamp).
188
+ Plan grants carry no fingerprint. Optional `catchup: 'bounded'` is accepted only
189
+ for plan/apply; bounded plan and apply grants carry the same standing owner
190
+ `approvalRecordId` and `approvedAt`. The apply body `reviewedFingerprint` must
191
+ match both the signed fingerprint and the exact retained plan.
192
+
193
+ Expired, replayed, forged, wrong-installation and wrong-fingerprint grants fail
194
+ closed. Every request needs a fresh nonce, including each chunk, upload inventory
195
+ read, status call and retry. Current enablement, key revocation and expiry are
196
+ checked inside kit mutation batches, so a previously accepted request does not
197
+ grant continuing authority after revocation or expiry.
198
+
199
+ ### RER-signed reads outside the door
200
+
201
+ `verifyRerGrant(request, claimConfig, action)` lets a seed accept an RER-signed
202
+ request on its own route outside `/api/rer/import/`, such as Uptime's
203
+ `POST /api/rer/runner-status` (action `runner-status`). Pass the same
204
+ server-owned claim configuration as the grant mount. The grant uses the same
205
+ protected header and key ring; its payload is exactly
206
+ `{v, iss, aud, installationId, email, jti, iat, exp, action}`, under the same
207
+ issuer, audience, installation, owner, nonce-format and 300-second rules, with
208
+ `action` equal to the route's action. `action` is lowercase letters, digits and
209
+ hyphens, and never a door action. A door grant, an unknown or missing member,
210
+ another action, a forged signature or an expired grant answers 403
211
+ `invalid_grant`; a missing bearer 401 `grant_required`; a request not addressed
212
+ to the claim's app origin 403 `grant_binding_mismatch`; a bad configuration or
213
+ action 503 `invalid_configuration`. It returns `{grant, kid}`. It records no
214
+ nonce, so serve only reads whose repeat within the grant's life discloses
215
+ nothing more, and check `kid` against the seed's revoked keys
216
+ (`ik_door_revoked_keys` for the door key) where the route needs revocation.
217
+
218
+ ## Requests
219
+
220
+ Bodies are JSON except declared-intake upload streams described below. Quotas
221
+ are admission limits, not a hosted capacity promise.
222
+
223
+ | POST route suffix | Body and result |
224
+ | --- | --- |
225
+ | `uploads` (no intake declaration) | `{ uploadId?, name, bundleSha256, totalBytes, totalChunks, index, chunkSha256, base64 }`; decoded chunks are exactly 64 KiB except the final chunk. `totalChunks` must equal `ceil(totalBytes / 65536)`. Omit `uploadId` on the first chunk to receive a server-generated UUID. Reuse it and the same bundle metadata for subsequent chunks. Both chunk and complete bundle SHA-256 are checked before reporting completion. |
226
+ | `uploads` (declared intake) | Full raw stream with `Content-Type: application/octet-stream`, `X-Import-Format: csv\|ndjson\|json` and `X-Import-Name: <safe file name>`. Returns custody hashes and intake metadata; see below. |
227
+ | `uploads` | `{ uploadId }` returns received-chunk inventory/progress. Without intake, an identical duplicate chunk is idempotent with a fresh grant. |
228
+ | `plan` | `{ uploadId, catchup?: boolean }` returns `planId`, `fingerprint`, server-derived counts, summary, rejected rows with reasons, `diagnostics` and differences including gone source ids. Planning makes no native target writes. |
229
+ | `apply` | `{ planId, reviewedFingerprint }` creates or resumes the real kit job in bounded batches. Repeat with a fresh valid grant to continue. A large source (a stored plan over 4 MiB, such as a multi-megabyte Notion export) is created in bounded staged steps within one request, and the answer may be the still `queued` job; an interrupted creation is never a job and the next `apply` resumes it from what was staged. |
230
+ | `status` | `{ planId }` reads current job progress. |
231
+ | `reconcile` | `{ planId }` derives source-id disposition/presence and native/provenance revision, value, attachment and count checks on the server. |
232
+ | `undo` | `{ planId }` reverses the newest snapshot's attributed effects in bounded kit work, preserving native edits. |
233
+ | `archive` | `{ uploadId, tool, takenAt }` keeps a fully uploaded bundle as the tool's archive; returns `archiveId`, `key`, `tool`, `takenAt`, `bundleSha256`, `bytes` and `counts`. Grant only. See "Archives". |
234
+ | `restore-check` | `{ archiveId }` parses the archive with the namespace's reader and mapping and writes nothing native; returns the archive's fields with `bundleSha256`, `bytes` and `counts` recomputed from its bytes, and `rejected`. Grant only. |
235
+
236
+ ### Optional intake before custody
237
+
238
+ Declare intake when the seed's source contract forbids storing fields beyond a
239
+ fixed list. It runs before any persistent write of source content, including
240
+ upload chunks, originals and snapshots. This is a static, server-owned,
241
+ drop-only declaration: no callback, client sanitizer or client-supplied hash can
242
+ replace it. For example:
243
+
244
+ ```ts
245
+ intake: {
246
+ formats: ['csv', 'ndjson', 'json'],
247
+ allow: {
248
+ csv: ['user_id', 'email', 'name', 'blocked'],
249
+ ndjson: ['/user_id', '/email', '/name', '/blocked'],
250
+ json: ['/id', '/status', '/profile/email', '/profile/login',
251
+ '/profile/displayName', '/profile/firstName', '/profile/lastName',
252
+ '/profile/department'],
253
+ },
254
+ }
255
+ ```
256
+
257
+ CSV entries match exact, case-sensitive decoded column headers. JSON and NDJSON
258
+ entries are exact RFC 6901 pointers to scalar leaves, relative to each source
259
+ record. Nested parents survive only for declared descendants; an object or array
260
+ at an allowed leaf is refused. Undeclared fields are dropped. CSV decoded cell
261
+ strings are unchanged; JSON primitive values and numeric lexemes are unchanged.
262
+ Every accepted row is re-encoded as NDJSON, so syntax representation changes
263
+ without normalizing field names or values. The product parser receives this
264
+ sanitized NDJSON during planning, and the existing `credentials: 'remove'`
265
+ policy still applies then.
266
+
267
+ POST the complete raw file stream to the same
268
+ `/api/operator/import/uploads` or `/api/rer/import/uploads` route. Set
269
+ `Content-Type: application/octet-stream`, `X-Import-Format` to a declared
270
+ `csv`, `ndjson` or `json`, and `X-Import-Name` to a safe file name. Use the same
271
+ owner authentication, Origin/CSRF checks or action-bound grant and quotas as
272
+ other door requests. Unknown or undeclared formats fail closed. A declared
273
+ source refuses JSON chunk envelopes; JSON `{uploadId}` remains the progress
274
+ read. A source without `intake` keeps the original chunk protocol byte-for-byte.
275
+
276
+ The response retains `uploadId`, `bundleSha256`, `received` and `complete`, and
277
+ adds:
278
+
279
+ ```ts
280
+ intake: {
281
+ sanitized: true,
282
+ allowlistSha256,
283
+ inputFormat, // 'csv', 'ndjson' or 'json'
284
+ custodyFormat: 'ndjson',
285
+ }
286
+ ```
287
+
288
+ `bundleSha256` and chunk hashes describe the sanitized custody bytes, not the
289
+ raw request. `allowlistSha256` is server-derived from the canonical sorted
290
+ formats and each format's sorted allowlist. Intake metadata is stored on the
291
+ upload and in the job's `plan.source.intake`, covered by the plan fingerprint
292
+ and snapshot hash, and compared on plan, apply, reconcile, undo and replay.
293
+ Share the same declaration between the operator mount and the grant source
294
+ resolver; changing it cannot silently reinterpret retained custody.
295
+
296
+ Streaming enforces byte and record quotas plus `limits.maxRecordBytes`
297
+ (default 1 MiB), buffering one row and a fixed output chunk batch. UTF-8,
298
+ quoted CSV fields, lines, CRLF and BOM may cross arbitrary stream boundaries.
299
+ An invalid row refuses with its row number and a fixed reason, without logging
300
+ source values. Forbidden fields never reach persistent storage. On an invalid
301
+ row or interrupted request, earlier sanitized rows may already be stored;
302
+ include these partial uploads in erasure so their custody can be withdrawn.
303
+
304
+ CSV reports logical rows (the header is row 1), preserves quoted commas,
305
+ newlines and escaped quotes, and refuses duplicate decoded headers, mismatched
306
+ widths and invalid quoting. NDJSON reports physical lines and skips blank
307
+ lines. JSON incrementally reads a plain array of object rows, including Okta's
308
+ Users API list, and refuses duplicate object keys or malformed array grammar.
309
+ Arrays in undeclared fields are dropped; traversing an array in an allowed
310
+ pointer is refused because reconstructing a sparse array would add values.
311
+ A leading UTF-8 BOM is accepted once; UTF-8 decoding is strict. Empty CSV,
312
+ header-only CSV, empty NDJSON and an empty JSON array yield no records. An
313
+ empty JSON file is malformed. Refusals carry `details.row` and a fixed
314
+ `details.reason` such as `invalid_json`, `invalid_utf8`, `duplicate_header` or
315
+ `width_mismatch`; dropped values and parser exception text are never logged.
316
+
317
+ The exact-original-byte rule applies to sources without an intake declaration.
318
+ For declared intake, sanitized NDJSON is the original custody used by plans,
319
+ reconciliation and archives; the raw input is never an original artifact.
320
+
321
+ ### Large sources
322
+
323
+ A large reviewed plan does not push a door request past D1's 32 MiB RPC
324
+ ceiling or its 1,000-statement budget: snapshot,
325
+ upload and original reads are paged; job creation stages the plan (and the
326
+ original it holds) a few MiB at a time and publishes the job in one small
327
+ transaction that copies the staged rows inside D1; and later batches fence the
328
+ plan's custody revision instead of resending it. No request carries a whole
329
+ plan or a whole original. See the kit README, "Large plans: staged creation and
330
+ revision fences". The seed's target adapter `planCodec`, when it has one, now
331
+ retains a door job whose plan holds no original bytes (for example, an upload
332
+ whose credential values were removed).
333
+
334
+ ### Credentials and source refusals
335
+
336
+ `plan` reads the upload with the kit's `credentials: 'remove'`. A credential
337
+ field holding null, an empty string or a mask is not a credential and imports as
338
+ it is. A real value is removed from its record, which is kept: a credential-named
339
+ field is dropped: any key whose whole name parts are a credential word or
340
+ phrase (`apiKey`, `httpPassword`, `http_password`, `client_secret`, `X-Api-Key`,
341
+ `pushToken`, `integration_key`, `webhook_url`, `secret_key`, `x-functions-key`,
342
+ `Ocp-Apim-Subscription-Key`, `pwd`, `X-Auth` and the like; `Signature` and
343
+ `Session-Id` only as header names, so an agent's `signature` field stays; `auth`
344
+ and `credentials` only when they hold a credential, so `auth: {type: 'oauth2'}`
345
+ stays and `auth: {token}` loses its token); a plural
346
+ (`apiKeys: [...]`, `tokens: [...]`) when it holds text, so a count such as
347
+ `tokens: 1500` stays; the `value` of a `{name|key, value}` pair naming one
348
+ (`{name: 'Authorization', value}`), and the value of such a `[name, value]`
349
+ pair in a list of them (Sentry's header tuples). Booleans stay, and names
350
+ merely containing such letters (`keyword`, `monkey`, `author`, `signal`, `design`,
351
+ `primaryKey`) are not credential names, so a keyword monitor's `?keyword=up` URL
352
+ imports intact. A URL of any scheme, after leading whitespace, loses its
353
+ user name and password and its credential query and fragment parameters
354
+ (`api_key`, `access_token`, `sig` and the like), including the `removed@` and
355
+ `?api_key=removed` markers rer's vendor exports leave. Each removal is a
356
+ `CREDENTIAL_REMOVED` warning in `diagnostics` with the record's `kind`,
357
+ `sourceId`, the field pointer and its kind, never the value; a field holding the
358
+ `[removed]` marker rer's vendor exports leave is reported as removed before
359
+ upload. The plan never holds a removed value: the original bytes are withheld
360
+ (`ORIGINAL_UNAVAILABLE`), and the upload's chunks are deleted once the plan is
361
+ saved, keeping its header so the same plan can be fetched again. A plan refused
362
+ for any other reason (`catchup_required`, a catch-up bound, a pending import)
363
+ deletes them too. Planning such an upload again reads its records from a plan
364
+ of the same bytes the namespace still holds (one whose apply failed, or that
365
+ was undone); with none, `plan` answers 410 `upload_credentials_withheld` and
366
+ Business uploads the bundle again. These wider rules apply only to the door's
367
+ removal; the kit's default refuse mode, which seeds' local imports use, is
368
+ unchanged.
369
+
370
+ Every refusal from the reader or a bundle check is a named 4xx whose
371
+ `details.reason` is a fixed sentence and whose `details.row`/`details.pointer`
372
+ locate it; no source value is reflected:
373
+
374
+ | Error / status | Meaning |
375
+ | --- | --- |
376
+ | `source_credential` / 422 | A credential the door cannot remove: one a record repeats elsewhere, an encoded or nested credential URL, or a credential field the mapping proposes. `details.field` names its kind (`credential field "token"`, `URL user name or password`, `URL credential query parameter`, `repeated credential value`). The upload is deleted. |
377
+ | `source_invalid_or_credential_source_data` / 422 | A value the reader cannot inspect for credentials: a URL that cannot be parsed, or text encoded more than 32 layers deep. The upload is deleted. |
378
+ | `source_<reader or bundle code>` / 422 | For example `source_malformed_json`, `source_malformed_utf8`, `source_parser_failed`, `source_duplicate_source_conflict`, `source_duplicate_target`, `source_active_effect_forbidden`. |
379
+ | `source_byte_limit`, `source_record_limit`, `source_record_byte_limit`, `source_effect_limit`, `source_import_plan_too_large` / 413 | A source or plan bound. |
380
+
381
+ JSON bodies are limited to 100,000 bytes; intake streams use their source and
382
+ grant byte/record quotas instead. The 1,024-chunk bound limits one bundle to
383
+ 64 MiB even if a grant's byte quota is larger. A smaller grant quota wins.
384
+
385
+ Adapters for large records must implement a compact `matchesMany`, because plan,
386
+ reconcile and catch-up fence the destination's whole native set in one 2 MiB D1
387
+ read, and per-row `matches` or `matchesParts` guards repeat every row in full. A
388
+ native set too large for that read answers 413 `import_door_too_large`, with
389
+ `details.targetKinds` and `details.reason` naming the target kind.
390
+
391
+ Every statement a door request runs, the door's own, the adapter's and each kit
392
+ call's, counts in one 1,000-statement budget (hosted D1's per-invocation limit
393
+ on Workers Paid). A request that would exceed it answers 413
394
+ `import_door_too_large` instead of a provider error. Reconcile and catch-up
395
+ planning take each target's current state from the native rows the persisted
396
+ read fenced under its atomic guard, and read every owning plan in one query, so
397
+ their statement count does not grow with the records imported. An adapter's
398
+ `list` must read its native set in a fixed number of queries, not one per row.
399
+
400
+ Uploaded bytes and retained originals remain in the seed; RER retains receipts
401
+ only. Parsing and mapping are product-owned and cannot send, enqueue or activate
402
+ access. An adopting seed supplies its native field sampling and integrated
403
+ acceptance evidence; this module has no external side effects.
404
+
405
+ ## Archives
406
+
407
+ When Business switches off a tool, it uploads the tool's final export and calls
408
+ `archive` with the `uploadId`, the tool's id (`tool`: lowercase letters, digits,
409
+ `.`, `_` and `-`, at most 64, starting with a letter or digit) and when the
410
+ export was taken (`takenAt`, a canonical ISO timestamp). The door copies the
411
+ exact uploaded bytes into the seed's own D1 as the tool's archive, named
412
+ `business-archive/<tool>/<takenAt>` (`key`), and answers its `archiveId`, the
413
+ bundle's `bundleSha256` and `bytes`, and `counts`: the reader and mapping's
414
+ counts of the bundle (`input`, `unique`, `mapped`, `rejected`, `byKind` and the
415
+ rest), as a first plan of it would report them. `counts.input` counts every
416
+ record read, a repeated identical record included, and `counts.unique` each
417
+ source identity once: to check an archive against the catch-up of the same
418
+ export, compare `counts.unique` with the catch-up's `summary.sourceRecords`, or
419
+ `counts.input` with the catch-up's `counts.input`. Archiving writes nothing native. The archive
420
+ holds its own copy, so `plan`, a catch-up, a cancelled preparation or an `undo`
421
+ never deletes it, and the upload stays usable for planning. The same request
422
+ again answers the same archive; another `tool` or `takenAt` for an archived
423
+ upload answers 409 `archive_exists` with `details.archiveId`.
424
+
425
+ An archive keeps exact bytes, so it refuses what the door never keeps: a bundle
426
+ from which the reader had to remove a live credential value answers 422
427
+ `archive_holds_credentials`. The markers rer's vendor exports leave in place of
428
+ the values they strip (`[removed]` in a field, `removed@` in a URL, `=removed`
429
+ for a query or fragment parameter) hold no credential, so such a bundle
430
+ archives, although a plan of it still withholds the original; a bundle holding an erased record answers 409 `source_erased`
431
+ and the upload goes, as `plan` does; a reader refusal answers its named 4xx. An
432
+ upload whose bytes already went (planned with credentials removed, or refused)
433
+ answers 410 `upload_bytes_withheld`: upload it again and archive before
434
+ planning.
435
+
436
+ `restore-check` with `{archiveId}` reads the archive's bytes back, checks each
437
+ chunk and the bundle hash (409 `archive_custody_mismatch` otherwise), and parses
438
+ them with the namespace's reader and current mapping without writing anything
439
+ native. It answers the archive's fields with `bundleSha256`, `bytes` and
440
+ `counts` recomputed from the bytes, and `rejected` (`kind`, `sourceId`,
441
+ `reason`). An unknown archive answers 404 `archive_not_found`; one over the
442
+ grant's quotas 403 `archive_exceeds_grant_quota`.
443
+
444
+ A person's erasure withdraws every archive's bytes in its scope with the rest
445
+ of the door's held content, and its journal counts them under
446
+ `ik_door_archive_chunks`. The archive header (id, tool, time, file name, hash,
447
+ size and kind counts) stays, so `restore-check` then answers 410
448
+ `archive_withdrawn` with `details.archiveId` and `details.reason`. The upload
449
+ went with it; the tool's export, uploaded again, is refused while it still
450
+ holds the erased person (`source_erased`).
451
+
452
+ Both are grant actions with the grant rules above: no fingerprint, approval or
453
+ `catchup`, a fresh single-use grant per call, bound to its action, namespace and
454
+ quotas. The owner-session mount answers them 404.
455
+
456
+ ## Catch-up and recovery
457
+
458
+ A logical namespace tracks a lineage of snapshots; each snapshot's kit namespace
459
+ derives from the logical namespace, the exact bundle SHA-256 and its plan id. Original source ids
460
+ remain intact. Unchanged records produce no effects but remain accounted for.
461
+ Replacements use the revision in original import provenance as
462
+ `expectedRevision`, and compare native values too, protecting edits even when
463
+ their revision was unchanged. Gone source records are reported without deletion.
464
+
465
+ Only one plan may remain pending per logical namespace until applied or undone.
466
+ Committed kit state recovers lineage after an interrupted call. Repeating a
467
+ bundle reuses its exact retained plan unless that plan's apply failed or its
468
+ import was undone or rolled back: then the same bundle plans afresh, so a file
469
+ can be imported again after undo. A catch-up (`catchup: true`) never reuses an
470
+ applied plan, the head or one before it: a bundle identical to one already
471
+ applied plans afresh against the head, so an unchanged source yields a catch-up
472
+ with no changes that a standing approval can apply. Repeating it before it is
473
+ applied reuses that catch-up plan. Use a fresh explicit namespace and review
474
+ for a different mapping. Undo snapshots newest first, including undo from the
475
+ seed's ordinary owner history. Undoing a snapshot while a newer one still holds
476
+ its records answers 409 `import_superseded`; a strict undo after an owner edit
477
+ answers 409 `native_edit_conflict`. Each has a `details.reason`.
478
+
479
+ With an adapter declaring `preparePreservedUndo`, new jobs instead preserve
480
+ conflicts by default: undo reverses untouched imported rows, keeps native edits
481
+ and deletions (and required dependencies), and completes with `job.state:
482
+ 'undone'`, even if every row was retained. Its durable
483
+ `readD1ImportUndoReport` has outcome `compensated` when any disposition is
484
+ `retained`, identifies each preserved effect and reason, and reports `undone`
485
+ when everything reversed. Native owner history can page that report with its
486
+ normal kit authorization. A second door undo is idempotent. The policy is saved
487
+ with the job, never inferred again during resume, undo or replay. Adding the
488
+ method later leaves old strict jobs strict; removing it refuses further apply
489
+ batches and active undo on preserving jobs. Restore a compatible adapter to
490
+ undo previously applied batches. Explicit
491
+ `reversalPolicy` settings still override the automatic choice for new jobs.
492
+ For example, Orbit's native preserving adapter needs only this shared door
493
+ update, with no Orbit-specific vendored changes. Preserving apply validates and
494
+ checks the preparation for every pending batch in a read-only admission before
495
+ that batch publishes, including resumed apply. Apply still atomically fences
496
+ its own exact before-images and current authority. An invalid preparation
497
+ refuses the entire pending batch; at checkpoint zero no native rows are applied.
498
+ Earlier admitted batches remain available to Undo. A preparation may cover several small consecutive batches
499
+ within one request; every exact prospective effect must match that admission
500
+ before its own batch publishes. Admission never carries across requests. See [ADOPTING.md](ADOPTING.md).
501
+
502
+ When an erasure withdrew the live import's snapshot, nothing can be compared
503
+ with it: an owner-session `plan` retires it as the baseline and plans a new
504
+ one, while a Business catch-up still answers 410 `snapshot_withdrawn`. While an
505
+ earlier import of the source still holds a record the new plan writes, the kit
506
+ would refuse its apply, so `plan` answers 409 `earlier_import_holds_records`
507
+ instead, with `details.planIds` and a `details.reason` to undo the earlier
508
+ import first; nothing is retired. After that undo the same upload plans and
509
+ applies.
510
+
511
+ An interrupted private plan preparation reports
512
+ `plan_staging_interrupted_cancel_and_replan` with `details.planId`. Call `undo` with that `planId`
513
+ to cancel its private preparation, then plan the retained upload again. A
514
+ preparation with no kit job cancels without native effects. Once a job exists,
515
+ the same route uses the kit's guarded rollback/undo instead.
516
+
517
+ An exception during the target apply ends that job as `failed`. Its native
518
+ batch is atomic: no writes from the failed batch remain. The approval and a
519
+ separate failure fact remain for audit, including after an owner rollback.
520
+ Status returns `job.state`, `job.error` and `failureCode`; history exposes the
521
+ same reason. Provider messages and customer records are never stored as reasons.
522
+
523
+ | Apply reason / HTTP status | Meaning |
524
+ | --- | --- |
525
+ | `import_door_apply_constraint` / 409 | A database constraint refused the batch. Correct the data or native writer before proposing another plan. |
526
+ | `import_door_apply_transient` / 503 | The provider reported busy, locked, interrupted or unavailable service, including D1's retryable errors (storage or code-update resets, lost connections, overload, timeouts). A fresh reviewed attempt can be proposed after recovery. |
527
+ | `import_door_apply_failed` / 409 | Another apply error. Inspect the seed's adapter before proposing another plan. |
528
+
529
+ Every failed plan is terminal: further applies return
530
+ `import_door_plan_failed`, even with a new grant or approval. The consumed grant
531
+ nonce stays consumed. A retry needs a new plan, fingerprint and approval record;
532
+ the approval associated with the failure cannot approve a retry, including a
533
+ standing catch-up approval. When checkpoint is zero, `plan` can immediately
534
+ prepare the same retained upload again in the same logical namespace, including
535
+ one whose credential-bearing bytes went: its records come from the failed
536
+ plan's snapshot. If earlier
537
+ batches committed, first finish `undo` (guarded rollback) before replanning.
538
+ The old job, immutable snapshot, approval and failure fact remain readable.
539
+
540
+ An erasure that overtakes an apply is not an apply failure: no failure fact is
541
+ recorded, the job keeps its state for `undo`, and the apply answers the
542
+ erasure's 410 (`snapshot_withdrawn` or `erased_during_request`). Nor is a kit
543
+ size or request limit, which refuses before its batch writes: the apply
544
+ answers 413 `import_door_too_large` and the plan stays applicable.
545
+
546
+ Seeds that installed an earlier door apply `importDoorApplyFailureMigration`
547
+ and then `importDoorScopesMigration`, as set out at the top of this document.
548
+
549
+ Bounded catch-up requires an existing nonempty baseline and standing owner
550
+ approval. It refuses gone, conflicting, rejected or unmapped records. Additions
551
+ plus replacements divided by baseline source count must be strictly below 5%;
552
+ the server derives these counts. A refusal leaves Business to present a new
553
+ proposal for owner review. No caller-supplied totals relax this boundary.
@@ -0,0 +1,45 @@
1
+ import type { ImportDoorOptions, ImportDoorRuntimeOptions, DoorRequest } from './types.ts';
2
+ import { runtimeOptions } from './handler.ts';
3
+ import { DoorError, verifyImportGrant, type DoorAction } from './grants.ts';
4
+ import { doorFence, erasures, sql, transact } from './store.ts';
5
+ import { importDoorSchemaCheck } from './schema.ts';
6
+
7
+ /** The storage key; grants are still verified against `claim.installationId`. */
8
+ export const doorKey = (options: ImportDoorOptions): string => options.installationId ?? options.claim.installationId;
9
+ /** Opt-in and schema state in one read. A door whose seed has not applied the
10
+ * door migrations must refuse loudly rather than misbehave. */
11
+ export async function enabled(options: ImportDoorOptions): Promise<boolean> {
12
+ const row = await options.database.prepare(`SELECT (SELECT enabled FROM ik_door_settings WHERE installation_id=?) AS enabled,(${importDoorSchemaCheck}) AS current`)
13
+ .bind(doorKey(options)).first();
14
+ if (row?.enabled !== 1) return false;
15
+ if (row.current !== 1) throw new DoorError('import_door_schema_outdated', 503);
16
+ return true;
17
+ }
18
+ /** This grant's parser and mapping: the mount's own, or the seed's allow-list entry
19
+ * for the verified namespace. Unlisted namespaces are refused before any write. */
20
+ async function source(options: ImportDoorOptions, namespace: string): Promise<ImportDoorRuntimeOptions> {
21
+ if (!options.source) return runtimeOptions(options);
22
+ const listed = await options.source(namespace);
23
+ if (!listed) throw new DoorError('unknown_import_source');
24
+ return runtimeOptions({...options, parse: listed.parse, mapping: listed.mapping, intake: listed.intake, ownerPrincipalId: listed.ownerPrincipalId ?? options.ownerPrincipalId});
25
+ }
26
+ export async function admit(options: ImportDoorOptions, request: Request, action: DoorAction): Promise<DoorRequest> {
27
+ const authorization = request.headers.get('Authorization');
28
+ if (!authorization?.startsWith('Bearer ')) throw new DoorError('grant_required', 401);
29
+ const {grant, kid} = await verifyImportGrant(authorization.slice(7), options.claim);
30
+ if (grant.action !== action || new URL(request.url).origin !== options.claim.appOrigin) throw new DoorError('grant_binding_mismatch');
31
+ const runtime = await source(options, grant.namespace);
32
+ const key = doorKey(options), epoch = await erasures(options.database, options.destination);
33
+ const fence = doorFence(key, grant.namespace, options.destination, epoch);
34
+ const basic = sql(`(${fence.sql}) AND EXISTS(SELECT 1 FROM ik_door_settings WHERE installation_id=? AND enabled=1)
35
+ AND NOT EXISTS(SELECT 1 FROM ik_door_revoked_keys WHERE installation_id=? AND kid=?)
36
+ AND unixepoch() < ?`, ...fence.params ?? [], key, key, kid, grant.exp);
37
+ try {
38
+ await transact(options.database, basic, [sql('INSERT INTO ik_door_grants(installation_id,jti,action,namespace,kid,expires) VALUES(?,?,?,?,?,?)', key, grant.jti, action, grant.namespace, kid, grant.exp)]);
39
+ } catch {
40
+ if (await erasures(options.database, options.destination) !== epoch) throw new DoorError('erased_during_request', 410);
41
+ throw new DoorError('grant_replayed_revoked_or_expired');
42
+ }
43
+ const authority = sql(`(${basic.sql}) AND EXISTS(SELECT 1 FROM ik_door_grants WHERE installation_id=? AND jti=? AND action=? AND namespace=? AND kid=? AND expires=?)`, ...basic.params ?? [], key, grant.jti, action, grant.namespace, kid, grant.exp);
44
+ return {options: runtime, access: {...grant, installationId: key}, authority, via:'grant', epoch, nonce: {jti: grant.jti, kid}};
45
+ }