@runeditrun/rer 0.2.81 → 0.2.83
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -0
- package/cli/export.mjs +1 -1
- package/edition-authoring/base-copies.mjs +3 -1
- package/edition-authoring/copy-checkout.mjs +152 -1
- package/edition-authoring/files.mjs +1 -1
- package/edition-authoring/runtime-dependencies.mjs +1 -1
- package/global/CHANGELOG.md +25 -0
- package/global/README.md +12 -0
- package/modules/import-kit/src/door/ADOPTING.md +452 -0
- package/modules/import-kit/src/door/README.md +553 -0
- package/modules/import-kit/src/door/admission.ts +45 -0
- package/modules/import-kit/src/door/archive.ts +129 -0
- package/modules/import-kit/src/door/authority.ts +107 -0
- package/modules/import-kit/src/door/failures.ts +70 -0
- package/modules/import-kit/src/door/grants.ts +162 -0
- package/modules/import-kit/src/door/handler.ts +90 -0
- package/modules/import-kit/src/door/index.ts +20 -0
- package/modules/import-kit/src/door/intake/csv.ts +74 -0
- package/modules/import-kit/src/door/intake/json.ts +182 -0
- package/modules/import-kit/src/door/intake/projection.ts +61 -0
- package/modules/import-kit/src/door/intake/types.ts +28 -0
- package/modules/import-kit/src/door/intake-policy.ts +38 -0
- package/modules/import-kit/src/door/intake-upload.ts +91 -0
- package/modules/import-kit/src/door/inventory.ts +68 -0
- package/modules/import-kit/src/door/jobs.ts +134 -0
- package/modules/import-kit/src/door/limits.ts +16 -0
- package/modules/import-kit/src/door/operator.ts +41 -0
- package/modules/import-kit/src/door/planning.ts +227 -0
- package/modules/import-kit/src/door/protocol.ts +38 -0
- package/modules/import-kit/src/door/refusals.ts +51 -0
- package/modules/import-kit/src/door/schema.ts +101 -0
- package/modules/import-kit/src/door/store.ts +152 -0
- package/modules/import-kit/src/door/types.ts +100 -0
- package/modules/import-kit/src/door/uploads.ts +98 -0
- package/modules/import-kit/src/provenance/numeric-credentials.ts +40 -0
- package/modules/import-kit/src/readers/README.md +139 -0
- package/modules/import-kit/src/readers/api-policy.ts +67 -0
- package/modules/import-kit/src/readers/api.ts +86 -0
- package/modules/import-kit/src/readers/bytes.ts +73 -0
- package/modules/import-kit/src/readers/errors.ts +33 -0
- package/modules/import-kit/src/readers/file.ts +74 -0
- package/modules/import-kit/src/readers/index.ts +3 -0
- package/modules/import-kit/src/readers/json.ts +29 -0
- package/modules/import-kit/src/readers/numeric-lexemes.ts +104 -0
- package/modules/import-kit/src/readers/original.ts +31 -0
- package/modules/import-kit/src/readers/records.ts +105 -0
- package/package.json +3 -2
|
@@ -0,0 +1,452 @@
|
|
|
1
|
+
# Adopt one import pipeline
|
|
2
|
+
|
|
3
|
+
For an owner moving data, start at the seed's signed-in import screen:
|
|
4
|
+
|
|
5
|
+
```text
|
|
6
|
+
/app/import
|
|
7
|
+
```
|
|
8
|
+
|
|
9
|
+
In a Base checkout, the [default reference journey](../../../../default-seed/IMPORTING.md) uses
|
|
10
|
+
an exported people JSON file, review, apply, reconcile and undo. The
|
|
11
|
+
[local-run guide](../../../../export/RUN-LOCALLY.md) describes how to launch and
|
|
12
|
+
claim the local app. Installation `.rer` archive recovery is currently a
|
|
13
|
+
separate developer-only mechanism; it is not accepted by this door.
|
|
14
|
+
|
|
15
|
+
Installation backup/restore keeps its archive integrity and erasure fences. It
|
|
16
|
+
is separate from bringing records in; never claim an archive traverses the door.
|
|
17
|
+
|
|
18
|
+
The seed's signed-in move-in screen, `rer` and Business must share **upload →
|
|
19
|
+
plan → reviewed fingerprint → apply → reconcile → undo**. Keep source readers,
|
|
20
|
+
mappings and native target adapters; delete the old importer routes, separate
|
|
21
|
+
job creation/apply code and their callers. Do not leave a hidden second importer.
|
|
22
|
+
Owner history and undo must include jobs from either authority.
|
|
23
|
+
|
|
24
|
+
## Add to the seed
|
|
25
|
+
|
|
26
|
+
1. **Schema and copies.** Vendor `modules/import-kit/src`, its schema JSON in
|
|
27
|
+
`modules/import-kit/schema`, and sibling `./modules/d1-transport` with relative
|
|
28
|
+
paths intact; record copies using the
|
|
29
|
+
seed's existing Base provenance workflow. Never rename applied migrations.
|
|
30
|
+
Apply these in new numbered seed migrations, in this order,
|
|
31
|
+
skipping any the seed already applied:
|
|
32
|
+
- the D1 kit schemas, in the order `initializeD1ImportStorage` in
|
|
33
|
+
`modules/import-kit/src/d1/database.ts` installs them (all exported from
|
|
34
|
+
`modules/import-kit/src/d1/index.ts`): `d1Schema` (with
|
|
35
|
+
`preservedUndoEvolution`), `d1ErasureSchema`, `d1StreamOriginSchema` and
|
|
36
|
+
`d1LegacyStreamReceiptUpgrade`, `d1StagingSchema`, `d1NativeStagingSchema`,
|
|
37
|
+
`d1CustodyStateSchema`, `d1ReversalFenceSchema`, `d1PreparationSchema`,
|
|
38
|
+
`d1PreparationStepsSchema`, `d1RetirementSchema`,
|
|
39
|
+
`d1CustodyStateReplaceSchema` (Base's default seed ships it as
|
|
40
|
+
`default-seed/migrations/core/0026_import_custody_replace.sql`) and
|
|
41
|
+
`d1ErasureProgressSchema`, plus the
|
|
42
|
+
kit evolutions in `modules/import-kit/src/d1/README.md` that apply to the
|
|
43
|
+
seed;
|
|
44
|
+
- a seed without a door: `importDoorSchema` from
|
|
45
|
+
`modules/import-kit/src/door/schema.ts` (also exported by `index.ts` and
|
|
46
|
+
`operator.ts`);
|
|
47
|
+
- a seed that already installed an earlier door: never `importDoorSchema`
|
|
48
|
+
again (it refuses that schema at deploy), but
|
|
49
|
+
`importDoorApplyFailureMigration` as one atomic batch, then
|
|
50
|
+
`importDoorScopesMigration`. Until both are applied, the door answers 503
|
|
51
|
+
`import_door_schema_outdated`. Then `importDoorArchiveMigration`; until it
|
|
52
|
+
is applied, only `archive` and `restore-check` answer that 503.
|
|
53
|
+
- only when an installed adapter declares `intake`:
|
|
54
|
+
`importDoorIntakeMigration`, which adds nullable `intake_json` to
|
|
55
|
+
`ik_door_uploads`. Fresh `importDoorSchema` includes the column. Existing
|
|
56
|
+
undeclared installations need no intake migration or upload changes.
|
|
57
|
+
2. **Native adapter.** Supply the seed's real `D1TargetAdapter`, parser,
|
|
58
|
+
`ImportMapping`, destination and actual owner principal. Read/write/remove
|
|
59
|
+
guards must protect native revisions and values in the same D1 transaction.
|
|
60
|
+
Mapping only creates inert native records: importing cannot send mail, page
|
|
61
|
+
someone, activate monitors or grant access. Keep any activation as a separate
|
|
62
|
+
owner-approved product action. Declare unsupported records as rejections or
|
|
63
|
+
explained omissions; never fabricate success. `write` must also replace an
|
|
64
|
+
already-imported target for catch-up, `list` must read the native set in a
|
|
65
|
+
fixed number of queries, and large imports need a compact `matchesMany`
|
|
66
|
+
(see Pitfalls).
|
|
67
|
+
**Undo policy.** An adapter declaring `preparePreservedUndo` automatically
|
|
68
|
+
selects `preserve-conflicts` for each new door job, on both operator and
|
|
69
|
+
grant mounts. An adapter without that method keeps the kit's `strict`
|
|
70
|
+
policy. `reversalPolicy` remains an explicit override; requesting preserve
|
|
71
|
+
without the method refuses construction with 503 `invalid_configuration`.
|
|
72
|
+
No seed-specific change to the vendored door is needed. For example, Orbit's
|
|
73
|
+
native adapter declaring the method makes new door imports preserve native
|
|
74
|
+
edits after re-vendoring this door; this does not change earlier strict jobs.
|
|
75
|
+
|
|
76
|
+
The selected policy is persisted and integrity-bound at job creation.
|
|
77
|
+
Apply resume, status, undo and request replay use that stored policy, even
|
|
78
|
+
after an adapter upgrade or a different mount setting. Adding the method
|
|
79
|
+
never upgrades an earlier strict job. Removing it from an adapter needed by
|
|
80
|
+
an existing preserving job refuses further apply batches and active undo
|
|
81
|
+
before native effects; restore a compatible adapter to undo earlier batches.
|
|
82
|
+
Apply resumes still use the stored policy. Every preserving publication
|
|
83
|
+
needs a valid preparation; if refused, that plan needs a new plan and approval. Keep adapter ownership
|
|
84
|
+
projections compatible with previously recorded undo evidence.
|
|
85
|
+
|
|
86
|
+
Before every preserving apply publication, including resumed or replayed
|
|
87
|
+
applies with pending work, the kit validates the adapter's preparation for
|
|
88
|
+
that batch's prospective effects in a read-only admission. Invalid structure
|
|
89
|
+
or a false observation fence answers 409 `import_door_apply_failed`, with no
|
|
90
|
+
effects from the refused batch. At checkpoint zero no imported native rows
|
|
91
|
+
remain; a new plan and approval are required. Admission makes
|
|
92
|
+
no undo decision and does not authorize effects: apply retains its existing
|
|
93
|
+
exact before-image and current-authority guards in each write transaction.
|
|
94
|
+
A later failed batch leaves earlier committed progress available to undo.
|
|
95
|
+
For small consecutive batches in one request, preparation can admit a window
|
|
96
|
+
before any of its effects publish. Every batch checks its exact prospective
|
|
97
|
+
effects against that admission; changed effects or adapter methods require a
|
|
98
|
+
new preparation. Admission never carries across requests.
|
|
99
|
+
|
|
100
|
+
Preserving undo reverses unchanged independent rows and leaves native edits,
|
|
101
|
+
native deletions and their required dependencies untouched. The door returns
|
|
102
|
+
`job.state: 'undone'` when finished, including when every effect was retained;
|
|
103
|
+
it does not answer a strict native-edit 409 for those conflicts. The durable
|
|
104
|
+
kit `readD1ImportUndoReport` reports `outcome: 'compensated'` when anything
|
|
105
|
+
was retained, with `retained` dispositions and `edited`, `dependency` or
|
|
106
|
+
`superseded` reasons; otherwise its outcome is `undone`. Read report pages
|
|
107
|
+
through the seed's owner-authorized kit context for native history details.
|
|
108
|
+
Repeated undo is idempotent. Undoing a catch-up that kept an owner's edit
|
|
109
|
+
releases the layer beneath it, so the baseline can then undo too.
|
|
110
|
+
|
|
111
|
+
3. **Operator mount.** Import `createImportOperatorHandler` from
|
|
112
|
+
`modules/import-kit/src/door/operator.ts`. Supply the runtime options in
|
|
113
|
+
`types.ts`, plus server-owned `installationId`, `appOrigin`, `ownerEmail`,
|
|
114
|
+
`namespace`, and `authorize(request, action)`. That callback authenticates the
|
|
115
|
+
current owner with the seed's normal session/CSRF checks and returns a
|
|
116
|
+
`D1Sql` predicate checking that exact session and owner in the same database;
|
|
117
|
+
return `null` for other callers. The predicate is rechecked inside mutations.
|
|
118
|
+
Mount the handler before ordinary route rejection, including the host router
|
|
119
|
+
and body-size boundary; pass its `null` response to the next dispatcher.
|
|
120
|
+
`installationId` is the **permanent door key** for settings, lineage and
|
|
121
|
+
history: pick one stable server-owned value (the RER owner-claim installation
|
|
122
|
+
id when the seed already has it, otherwise a fixed seed value) and never
|
|
123
|
+
change it.
|
|
124
|
+
|
|
125
|
+
**Several import sources.** One namespace per source (for example one per
|
|
126
|
+
vendor). Keep **one** server-owned allow-list, `importSource(name)`, that
|
|
127
|
+
returns a fresh `{parse, mapping, ownerPrincipalId?, intake?}` for a listed
|
|
128
|
+
name and `null` otherwise; step 4 passes the same function to the grant mount. Build
|
|
129
|
+
the owner handler per request from it and refuse anything else before the
|
|
130
|
+
door runs. An entry may supply a mapping that needs that request's whole
|
|
131
|
+
parsed bundle, or a per-request owner principal. Construction is cheap and
|
|
132
|
+
stateless; plans and uploads are reachable only in their own namespace. The
|
|
133
|
+
optional `intake` is a static declaration of formats and fields per source, shared
|
|
134
|
+
unchanged with both mounts. Declare it when the source contract forbids
|
|
135
|
+
retaining other fields, so the door drops them before any persistent write,
|
|
136
|
+
rather than dropping them later in the product parser. See the
|
|
137
|
+
[Auth example below](#example-auth-source-contract).
|
|
138
|
+
The move-in screen sends the source on every call. Answer `/api/rer/import/*`
|
|
139
|
+
with 404 here: a configured grant mount (step 4) runs before this handler.
|
|
140
|
+
```ts
|
|
141
|
+
const sources = {pagerduty: pagerDutyImport, opsgenie: opsgenieImport}; // server-owned factories
|
|
142
|
+
export const importSource = (name: string) => Object.hasOwn(sources, name) ? sources[name]() : null;
|
|
143
|
+
export async function importDoor(request: Request, env: Env) {
|
|
144
|
+
const path = new URL(request.url).pathname;
|
|
145
|
+
if (path.startsWith('/api/rer/import/')) return Response.json({error: 'not_found'}, {status: 404});
|
|
146
|
+
if (!path.startsWith('/api/operator/import/')) return null;
|
|
147
|
+
const name = request.headers.get('X-Import-Source') ?? '', source = importSource(name);
|
|
148
|
+
if (!source) return Response.json({error: 'unknown_import_source'}, {status: 400});
|
|
149
|
+
return createImportOperatorHandler({...runtime(env), ...source, namespace: name})(request);
|
|
150
|
+
}
|
|
151
|
+
```
|
|
152
|
+
4. **Settings and grants.** The door is off until a settings row enables it;
|
|
153
|
+
the owner's settings action writes it with
|
|
154
|
+
`writeImportDoorSettings(db, installationId, {enabled}, ownerPredicate)`.
|
|
155
|
+
This flag controls Business grants, not the owner's own imports.
|
|
156
|
+
**Recommended consent path: owner opt-in.** On an install RER provisions for
|
|
157
|
+
a Business plan, `RER_IMPORT_DOOR` is `on`, and grants still stay closed
|
|
158
|
+
until the owner turns **Business imports** on at Move in (step 5's screen,
|
|
159
|
+
through that settings action). Never write an enabled row on the owner's
|
|
160
|
+
behalf. Until then every grant route answers 404 and the owner's own imports
|
|
161
|
+
work as usual.
|
|
162
|
+
`enableDoorAtFirstClaim(db, installationId, ownerPredicate)` remains
|
|
163
|
+
available for a seed that instead opens the door at the first owner claim
|
|
164
|
+
when the variable is exactly `on`: call it from the path that completes the
|
|
165
|
+
first owner claim (bootstrap or signed) **and** from every later owner path
|
|
166
|
+
while the variable is `on`: the claim's already-initialized answer, and
|
|
167
|
+
owner sign-in or session start. Pass the door key and a predicate that holds
|
|
168
|
+
once that owner exists. The claim and the enable are separate commits and
|
|
169
|
+
the claim cannot complete twice, so the later calls are what heal a failure
|
|
170
|
+
between them; without them the door stays closed until the owner turns it
|
|
171
|
+
on in settings. It enables the door only while no settings row exists, so
|
|
172
|
+
never write a disabled row before the first claim on such an install; it
|
|
173
|
+
never reopens a door the owner turned off.
|
|
174
|
+
Self-host `rer deploy` never sets the variable. Read the grant mount's claim
|
|
175
|
+
configuration from `RER_OWNER_CLAIM_ISSUER`, `RER_OWNER_CLAIM_INSTALLATION_ID`,
|
|
176
|
+
`RER_OWNER_CLAIM_APP_ORIGIN`, `RER_OWNER_CLAIM_PUBLIC_KEYS` and the owner
|
|
177
|
+
email; RER sets that trust on Business installs whatever the claim mode. If the seed
|
|
178
|
+
has owner-claim, additionally mount `createImportDoorHandler` from `index.ts`
|
|
179
|
+
with the **same** server-owned claim/key-ring configuration and
|
|
180
|
+
`installationId` set to the operator mount's door key (it defaults to
|
|
181
|
+
`claim.installationId`). With several sources, pass `source: importSource`
|
|
182
|
+
(step 3's allow-list) instead of `parse`/`mapping`: the door calls it with
|
|
183
|
+
each verified grant's namespace before writing anything, parses with that
|
|
184
|
+
entry, and refuses an unlisted namespace with 403 `unknown_import_source`.
|
|
185
|
+
Dispatch this grant handler **before** the operator
|
|
186
|
+
handler. Enable only through an owner
|
|
187
|
+
settings action or `enableDoorAtFirstClaim` under the owner's provisioning
|
|
188
|
+
approval. Keep durable key
|
|
189
|
+
revocations and show them in that settings row; re-enabling never unrevokes.
|
|
190
|
+
A seed route RER reads outside the door (Uptime's runner status) verifies
|
|
191
|
+
its request with `verifyRerGrant(request, claimConfig, action)` from
|
|
192
|
+
`index.ts`; see README.md.
|
|
193
|
+
5. **Move-in screen.** Call POST `/api/operator/import/{uploads,plan,apply,status,
|
|
194
|
+
reconcile,undo}` with the current owner cookie, exact `Origin` and normal
|
|
195
|
+
seed CSRF headers. Show the **Business imports** setting here (step 4): it
|
|
196
|
+
starts off, and the owner's switch is what opens the door to Business
|
|
197
|
+
grants. Bodies and results are in [README.md](README.md). Without declared
|
|
198
|
+
intake, upload original bytes through the existing JSON chunk protocol.
|
|
199
|
+
With intake, POST the full raw stream to the same uploads route with
|
|
200
|
+
`Content-Type: application/octet-stream`, `X-Import-Format` and `X-Import-Name`;
|
|
201
|
+
custody and response hashes then describe sanitized NDJSON. Do not send
|
|
202
|
+
JSON chunk envelopes or a client sanitizer/hash for that source. Show the
|
|
203
|
+
returned counts, rejections, differences and each
|
|
204
|
+
`CREDENTIAL_REMOVED` diagnostic (a credential value the door removed from that
|
|
205
|
+
record), and each named source refusal's `details.reason`, then
|
|
206
|
+
require the owner's review before posting `{planId, reviewedFingerprint}`.
|
|
207
|
+
Repeat bounded apply calls with that fingerprint until `job.state` is
|
|
208
|
+
`completed`; reconcile from native data and expose Undo. Never mint an import
|
|
209
|
+
grant for this screen. Use `readImportDoorHistory` with the current owner
|
|
210
|
+
predicate and the door key for approval metadata; local approvals are
|
|
211
|
+
labelled `approved by`, Business approvals `approved via Business by`.
|
|
212
|
+
6. **Erasure.** Include retained uploads/chunks, plan snapshots/chunks, archive
|
|
213
|
+
bytes (`ik_door_archive_chunks`, withdrawn whole) and headers, approvals,
|
|
214
|
+
namespace lineage and destination bindings (`ik_door_scopes`), apply-failure
|
|
215
|
+
facts (`ik_door_apply_failures`) and grant metadata in the seed's
|
|
216
|
+
erasure/custody inventory,
|
|
217
|
+
alongside ordinary kit originals/provenance. Withdraw an indivisible bundle
|
|
218
|
+
when erasing any contained subject; preserve unrelated native records. Include
|
|
219
|
+
partial sanitized uploads left by interrupted or invalid intake streams;
|
|
220
|
+
forbidden raw fields were never stored. Keep installation settings and
|
|
221
|
+
permanent revoked keys. Exercise **every erase path with door rows present**
|
|
222
|
+
(see Pitfalls), even with grants disabled.
|
|
223
|
+
Do not omit tables from retirement tallies to hide retained personal data.
|
|
224
|
+
7. **Tests.** Extend existing HTTP journeys within the seed's 200-test cap. Use
|
|
225
|
+
its actual migrated Worker, real sign-in, synthetic exports and public native
|
|
226
|
+
reads. Cover anonymous/non-owner and cross-origin refusal, no-write planning,
|
|
227
|
+
fingerprint refusal, apply/reconcile/undo, native edit protection, session
|
|
228
|
+
revocation during mutation, disabled/revoked grants, owner undo while Business
|
|
229
|
+
is off, grants staying closed under `RER_IMPORT_DOOR=on` until the owner
|
|
230
|
+
turns Business imports on (or, if the seed uses `enableDoorAtFirstClaim`,
|
|
231
|
+
the door opening at the first claim and staying closed without the
|
|
232
|
+
variable), and erasure of retained door custody. No test
|
|
233
|
+
skips or SQL bypasses.
|
|
234
|
+
|
|
235
|
+
**Without owner-claim:** vendor the operator runtime, omitting the three optional
|
|
236
|
+
grant-only files `door/index.ts`, `door/admission.ts` and `door/grants.ts` from
|
|
237
|
+
the source-copy inventory. Import `operator.ts` directly. This also lets a seed
|
|
238
|
+
typecheck its entire copied source tree without unresolved owner-claim imports.
|
|
239
|
+
Do not vendor owner-claim or operator-auth merely for imports: the operator
|
|
240
|
+
runtime has no dependency on either;
|
|
241
|
+
use the seed's existing owner authentication. All `/api/rer/import/*` requests
|
|
242
|
+
return 404 through that mount. No signer, dummy claim configuration or grant is
|
|
243
|
+
needed. Keep the `installationId` you chose: it stays the door key. To add
|
|
244
|
+
Business later, add those three grant files, `modules/owner-claim/verify.mjs`
|
|
245
|
+
and its dependency `modules/operator-auth/loopback.mjs`, then configure its
|
|
246
|
+
trusted ring with RER's claim installation id, pass your existing door key as
|
|
247
|
+
the grant mount's `installationId` (and, with several sources, your
|
|
248
|
+
`importSource` as its `source`), mount the grant handler first and explicitly
|
|
249
|
+
opt in. Earlier owner imports then stay in history, and Business catches up on
|
|
250
|
+
their lineage instead of re-creating them. The existing grant signature, nonce,
|
|
251
|
+
scope, expiry and approval rules do not change.
|
|
252
|
+
|
|
253
|
+
## Example: Auth source contract
|
|
254
|
+
|
|
255
|
+
Auth's [MOVE-IN contract at commit 35c2080](https://github.com/just-every/run-auth/blob/35c2080c07c0c17fc7fd1315f0899bd953f0fa2e/MOVE-IN.md)
|
|
256
|
+
keeps email, name, disabled state, provider id and review context. Credentials,
|
|
257
|
+
MFA, roles, sessions and policies do not move. Its Auth0 and Okta sources can
|
|
258
|
+
declare separate intake lists:
|
|
259
|
+
|
|
260
|
+
```ts
|
|
261
|
+
const auth0Intake = {
|
|
262
|
+
formats: ['csv', 'ndjson'],
|
|
263
|
+
allow: {
|
|
264
|
+
csv: ['user_id', 'email', 'name', 'blocked'],
|
|
265
|
+
ndjson: ['/user_id', '/email', '/name', '/blocked'],
|
|
266
|
+
},
|
|
267
|
+
} as const;
|
|
268
|
+
const oktaIntake = {
|
|
269
|
+
formats: ['json'],
|
|
270
|
+
allow: {
|
|
271
|
+
json: ['/id', '/status', '/profile/email', '/profile/login',
|
|
272
|
+
'/profile/displayName', '/profile/firstName', '/profile/lastName',
|
|
273
|
+
'/profile/department'],
|
|
274
|
+
},
|
|
275
|
+
} as const;
|
|
276
|
+
```
|
|
277
|
+
|
|
278
|
+
Auth0's CSV and NDJSON lists contain only those four fields. Okta's Users API
|
|
279
|
+
JSON array retains email with login as fallback, displayName with firstName
|
|
280
|
+
plus lastName as fallback, and department for review only. Those choices stay
|
|
281
|
+
in the product parser/mapping; intake only drops fields and preserves values.
|
|
282
|
+
|
|
283
|
+
Return each declaration with its source's `{parse, mapping, ownerPrincipalId?,
|
|
284
|
+
intake?}`. The operator mount spreads that source entry and the grant mount
|
|
285
|
+
uses the same `importSource` resolver, so both enforce the same list before
|
|
286
|
+
custody. This is a documentation example, not an Auth seed change. See the
|
|
287
|
+
[streaming protocol and scalar-leaf rules](README.md#optional-intake-before-custody).
|
|
288
|
+
|
|
289
|
+
## Pitfalls found in adoptions
|
|
290
|
+
|
|
291
|
+
- **Credential fields in exports.** Vendor exports carry credential-named
|
|
292
|
+
fields (UptimeRobot's `apiKey: null` on every monitor) and URLs with
|
|
293
|
+
`user:pass@`. The door reads every upload with the kit's `credentials:
|
|
294
|
+
'remove'`: null, empty, masked and boolean values pass, and a real value is
|
|
295
|
+
removed from its record, which is kept and noted. Names match by whole parts
|
|
296
|
+
(camelCase, `_`, `-`, digits), not letters: it covers compound names
|
|
297
|
+
(`httpPassword`, `client_secret`, `customHttpHeaders['X-Api-Key']`,
|
|
298
|
+
`secret_key`, `Ocp-Apim-Subscription-Key`, `X-Auth`, `pwd`), plurals holding
|
|
299
|
+
text (`apiKeys: [...]`), header pairs (`{name: 'Authorization', value}`) and
|
|
300
|
+
`[name, value]` header tuples, and URLs of any scheme, including the markers
|
|
301
|
+
rer's vendor exports leave (`removed@`, `?api_key=removed`). Look-alikes such
|
|
302
|
+
as UptimeRobot's `?keyword=up`, `author`, `signal` or `primaryKey` are not
|
|
303
|
+
credentials and import with the URL and fields intact, and neither is a
|
|
304
|
+
support export's agent `signature` or a chat's `session_id` (`Signature` and
|
|
305
|
+
`Session-Id` count only as header names), nor `auth: {type: 'oauth2'}` or
|
|
306
|
+
`credentials: []`. The plan
|
|
307
|
+
holds none, the original bytes are withheld and the upload's chunks are
|
|
308
|
+
deleted once the plan is saved, or once a plan of them is refused.
|
|
309
|
+
Do not drop such records or fields in the seed's parser to get past a refusal.
|
|
310
|
+
|
|
311
|
+
- **Export custody.** A seed whose export policy excludes import custody
|
|
312
|
+
(`excludeTables`) must also exclude `ik_door_%`; door uploads and plans hold
|
|
313
|
+
original source bytes for undeclared sources, or sanitized NDJSON custody
|
|
314
|
+
for declared intake sources.
|
|
315
|
+
- **Catch-up writes.** Catch-up replaces imported targets; the kit guards each
|
|
316
|
+
replacement with `expectedRevision` from the import's provenance and the full
|
|
317
|
+
before-image. `write` must upsert the target, replace the child rows it owns
|
|
318
|
+
and set its new revision in the same statements, without cascading deletes
|
|
319
|
+
into rows that reference it. An insert-only writer fails every catch-up apply
|
|
320
|
+
with 409 `import_door_apply_constraint`; that plan is then terminal, so fix
|
|
321
|
+
`write` and plan again for a fresh approval.
|
|
322
|
+
- **Large imports.** Reconcile and catch-up planning guard the destination's
|
|
323
|
+
whole native set in one read capped at 2 MiB. Without `matchesMany`, guards
|
|
324
|
+
that inline row values exceed that read (Lamp's 124-record Statuspage fixture
|
|
325
|
+
did), and the door answers 413 `import_door_too_large`, naming the target
|
|
326
|
+
kind. Implement a compact exact `matchesMany(rows, destination)`: counts per
|
|
327
|
+
kind, revision markers and an in-SQL digest of each entity's owned rows
|
|
328
|
+
compared with the hash of its listed value (Base's default seed `personSet` in
|
|
329
|
+
`default-seed/src/core/features/people/imports.server.ts` is one). A native
|
|
330
|
+
set still past that read gets the same 413. Every statement of one door
|
|
331
|
+
request also counts in one 1,000-statement budget, so `list` must read the
|
|
332
|
+
native set in a fixed number of queries, not one per row; past the budget the
|
|
333
|
+
door answers the same 413. Test reconcile and catch-up on the seed's largest
|
|
334
|
+
fixture.
|
|
335
|
+
- **Erasure.** Run every erase path (each public erasure route, account,
|
|
336
|
+
project or workspace deletion, retirement) with door rows present (an applied
|
|
337
|
+
import's uploads, plans and approvals), not only with the tables installed.
|
|
338
|
+
- **Door migrations.** Apply every mandatory door migration Base ships, each once, in
|
|
339
|
+
order, in its own new numbered seed migration: today
|
|
340
|
+
`importDoorApplyFailureMigration` (one atomic batch, foreign keys on), then
|
|
341
|
+
`importDoorScopesMigration`, then `importDoorArchiveMigration`. Apply optional
|
|
342
|
+
`importDoorIntakeMigration` only for installed adapters declaring intake.
|
|
343
|
+
Never re-apply an updated `importDoorSchema` to
|
|
344
|
+
an installed door: its `CREATE TABLE IF NOT EXISTS` statements would skip
|
|
345
|
+
changed tables, so its first statement refuses an earlier door's plan table
|
|
346
|
+
and the deploy fails.
|
|
347
|
+
|
|
348
|
+
## Real integration example: Spool
|
|
349
|
+
|
|
350
|
+
[Spool PR 9](https://github.com/just-every/run-spool/pull/9) demonstrates the
|
|
351
|
+
native integration: `migrations/core/0041_business_import_door.sql`,
|
|
352
|
+
`src/domain/sentry-import/door.server.ts`, `source.ts`, `mapping.ts`, `targets.ts`
|
|
353
|
+
and `context.ts`; settings/history in `business-routes.ts` and
|
|
354
|
+
`src/ui/workbench/BusinessImports.tsx`; custody in
|
|
355
|
+
`src/domain/erasure/business-custody.ts`; public HTTP proof in
|
|
356
|
+
`tests/business-import.mjs` using `tests/worker.mjs`, `tests/auth.mjs` and
|
|
357
|
+
`tests/archive.mjs`. These paths are in Spool. Its Sentry parser and adapter are
|
|
358
|
+
reused, imports remain historical, and project erasure withdraws duplicate door
|
|
359
|
+
custody. That PR still keeps `/api/imports/sentry`: finishing single-path adoption
|
|
360
|
+
requires replacing that caller with the operator mount and deleting the old
|
|
361
|
+
HTTP importer in `src/domain/sentry-import/routes.ts`.
|
|
362
|
+
Do not copy its older vendored door instead of current Base.
|
|
363
|
+
|
|
364
|
+
## Copyable public HTTP conformance
|
|
365
|
+
|
|
366
|
+
Copy the following into an existing seed test journey and call `checkDoor`.
|
|
367
|
+
It needs only Node built-ins. `request(action, body, review)` must return the
|
|
368
|
+
actual HTTP `Response`: use owner-cookie requests to the operator mount, or
|
|
369
|
+
mint a **fresh action-bound grant per call** for the grant mount (the apply
|
|
370
|
+
`review` argument supplies its signed fingerprint/approval). `nativeSnapshot`
|
|
371
|
+
reads a stable projection of native records through the seed's public API/export,
|
|
372
|
+
excluding import bookkeeping. Supply nonempty synthetic source bytes plus
|
|
373
|
+
`assertApplied(before, after, report)` and `assertUndone(before, after)` to check
|
|
374
|
+
real field values and the seed's undo policy. Those hooks must use public reads,
|
|
375
|
+
not SQL, mocks or precomputed results. The existing seed journey separately
|
|
376
|
+
covers settings, revocation, native edits and erasure as above.
|
|
377
|
+
|
|
378
|
+
The chunk-upload portion below is for a source **without** an intake
|
|
379
|
+
declaration. For declared intake, replace it with one authenticated streaming
|
|
380
|
+
upload using the headers in [README.md](README.md#optional-intake-before-custody).
|
|
381
|
+
Take `bundleSha256` and received sizes from that response/inventory and compare
|
|
382
|
+
those custody hashes across plan and reconcile; do not assert the raw input's
|
|
383
|
+
hash or size. Assert `intake.sanitized`, `inputFormat`, `custodyFormat` and the
|
|
384
|
+
server-derived `allowlistSha256`, and exercise forbidden-field exclusion,
|
|
385
|
+
invalid-row refusal and partial-upload erasure in that source's HTTP journey.
|
|
386
|
+
|
|
387
|
+
```js
|
|
388
|
+
import assert from 'node:assert/strict';
|
|
389
|
+
import { createHash, randomUUID } from 'node:crypto';
|
|
390
|
+
|
|
391
|
+
export async function checkDoor({request, bytes, name, nativeSnapshot,
|
|
392
|
+
assertApplied, assertUndone}) {
|
|
393
|
+
const hash = b => createHash('sha256').update(b).digest('hex');
|
|
394
|
+
const json = async (action, body, review, expected = 200) => {
|
|
395
|
+
const response = await request(action, body, review);
|
|
396
|
+
assert.equal(response.status, expected, `${action}: ${response.status}`);
|
|
397
|
+
return response.json();
|
|
398
|
+
};
|
|
399
|
+
const until = async (action, body, review, state) => {
|
|
400
|
+
for (let batch = 0; batch < 100; batch++) {
|
|
401
|
+
const result = await json(action, body, review);
|
|
402
|
+
if (result.job?.state === state) return result.job;
|
|
403
|
+
assert.ok(['queued','applying','undoing'].includes(result.job?.state));
|
|
404
|
+
}
|
|
405
|
+
assert.fail(`${action} did not finish`);
|
|
406
|
+
};
|
|
407
|
+
assert.ok(bytes.length > 0);
|
|
408
|
+
const before = await nativeSnapshot(), bundleSha256 = hash(bytes);
|
|
409
|
+
const totalChunks = Math.ceil(bytes.length / 65536);
|
|
410
|
+
let uploadId;
|
|
411
|
+
for (let index = 0; index < totalChunks; index++) {
|
|
412
|
+
const chunk = bytes.subarray(index * 65536, (index + 1) * 65536);
|
|
413
|
+
const uploaded = await json('uploads', {uploadId, name, bundleSha256,
|
|
414
|
+
totalBytes:bytes.length, totalChunks, index,
|
|
415
|
+
chunkSha256:hash(chunk), base64:chunk.toString('base64')});
|
|
416
|
+
uploadId = uploaded.uploadId;
|
|
417
|
+
}
|
|
418
|
+
const inventory = await json('uploads', {uploadId});
|
|
419
|
+
assert.equal(inventory.complete, true);
|
|
420
|
+
assert.equal(inventory.bundleSha256, bundleSha256);
|
|
421
|
+
assert.equal(inventory.received.reduce((n,c) => n + c.size, 0), bytes.length);
|
|
422
|
+
const plan = await json('plan', {uploadId});
|
|
423
|
+
assert.equal(plan.bundleSha256, bundleSha256);
|
|
424
|
+
assert.ok(plan.summary.sourceRecords > 0);
|
|
425
|
+
assert.deepEqual(await nativeSnapshot(), before);
|
|
426
|
+
const review = {fingerprint:plan.fingerprint,
|
|
427
|
+
approvalRecordId:randomUUID(), approvedAt:new Date().toISOString()};
|
|
428
|
+
const wrong = (plan.fingerprint[0] === '0' ? '1' : '0') + plan.fingerprint.slice(1);
|
|
429
|
+
await json('apply', {planId:plan.planId, reviewedFingerprint:wrong}, review, 403);
|
|
430
|
+
assert.deepEqual(await nativeSnapshot(), before);
|
|
431
|
+
const job = await until('apply', {planId:plan.planId,
|
|
432
|
+
reviewedFingerprint:plan.fingerprint}, review, 'completed');
|
|
433
|
+
const status = await json('status', {planId:plan.planId});
|
|
434
|
+
assert.equal(status.job.id, job.id);
|
|
435
|
+
assert.equal(status.job.state, 'completed');
|
|
436
|
+
const report = await json('reconcile', {planId:plan.planId});
|
|
437
|
+
assert.equal(report.ok, true);
|
|
438
|
+
assert.equal(report.fingerprint, plan.fingerprint);
|
|
439
|
+
assert.equal(report.bundleSha256, bundleSha256);
|
|
440
|
+
assert.equal(report.counts.sourceRecords, plan.summary.sourceRecords);
|
|
441
|
+
assert.equal(report.counts.present + report.counts.explained, report.counts.sourceRecords);
|
|
442
|
+
assert.equal(report.sourceIds.length, report.counts.sourceRecords);
|
|
443
|
+
await assertApplied(before, await nativeSnapshot(), report);
|
|
444
|
+
await until('undo', {planId:plan.planId}, undefined, 'undone');
|
|
445
|
+
await assertUndone(before, await nativeSnapshot());
|
|
446
|
+
}
|
|
447
|
+
```
|
|
448
|
+
|
|
449
|
+
For a concrete grant client and harness see Spool's `tests/business-import.mjs`;
|
|
450
|
+
its `grant`, `doorRequest`, `nativeSnapshot`, `startTestWorker` and `signIn`
|
|
451
|
+
provide those network edges. Run with `nice -n 15 node <existing-test-journey>`
|
|
452
|
+
after the seed's normal build. No `rer check` extension is required.
|