@runeditrun/rer 0.2.100 → 0.2.101
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -102
- package/cli/README.md +16 -89
- package/cli/app-address.mjs +2 -0
- package/cli/business/api.mjs +2 -1
- package/cli/business/choices.mjs +1 -1
- package/cli/business/command.mjs +36 -30
- package/cli/business/connect.mjs +1 -1
- package/cli/business/lease.mjs +0 -9
- package/cli/business/local-commands.mjs +1 -27
- package/cli/business/outbound.mjs +2 -14
- package/cli/business/questions.mjs +85 -48
- package/cli/business/service.mjs +1 -24
- package/cli/business/setup-command.mjs +33 -26
- package/cli/business/state.mjs +1 -1
- package/cli/business/supervisor.mjs +26 -91
- package/cli/companion/command.mjs +2 -0
- package/cli/demo-native.mjs +1 -1
- package/cli/parse.mjs +2 -1
- package/global/CHANGELOG.md +24 -0
- package/global/README.md +30 -102
- package/global/main.mjs +2 -0
- package/modules/container/adapters/d1-limits.mjs +2 -10
- package/modules/d1-transport/limits.d.mts +10 -0
- package/modules/d1-transport/limits.mjs +10 -0
- package/modules/demo-host/native/prepare.mjs +2 -1
- package/modules/import-kit/schema/importers.schema.json +104 -2
- package/modules/import-kit/src/d1/README.md +13 -0
- package/modules/import-kit/src/d1/apply-reads.ts +48 -0
- package/modules/import-kit/src/d1/apply.ts +21 -8
- package/modules/import-kit/src/d1/history.ts +2 -2
- package/modules/import-kit/src/d1/types.ts +11 -0
- package/modules/import-kit/src/door/ADOPTING.md +0 -37
- package/modules/import-kit/src/door/README.md +0 -66
- package/modules/import-kit/src/door/admission.ts +3 -13
- package/modules/import-kit/src/door/authority.ts +0 -1
- package/modules/import-kit/src/door/failures.ts +1 -1
- package/modules/import-kit/src/door/grants.ts +5 -19
- package/modules/import-kit/src/door/handler.ts +1 -12
- package/modules/import-kit/src/door/index.ts +0 -6
- package/modules/import-kit/src/door/planning.ts +19 -44
- package/modules/import-kit/src/door/protocol.ts +2 -4
- package/modules/import-kit/src/door/schema.ts +0 -2
- package/modules/import-kit/src/door/store.ts +2 -2
- package/modules/import-kit/src/door/types.ts +1 -24
- package/modules/import-kit/src/provenance/credentials.ts +14 -5
- package/modules/import-kit/src/status/evidence.ts +3 -1
- package/modules/import-kit/src/status/index.ts +17 -2
- package/modules/import-kit/src/status/schema.ts +8 -0
- package/modules/import-kit/src/status/types.ts +25 -4
- package/package.json +1 -1
- package/cli/business/archive-task.mjs +0 -32
- package/cli/business/bounded-waves.d.mts +0 -23
- package/cli/business/bounded-waves.mjs +0 -57
- package/cli/business/catchup-task.mjs +0 -49
- package/cli/business/export-inputs.mjs +0 -79
- package/cli/business/export-task.mjs +0 -24
- package/cli/business/legacy-package.mjs +0 -117
- package/cli/business/package-transfer.mjs +0 -127
- package/cli/business/setup-demo.mjs +0 -95
- package/cli/business/vendor-door.mjs +0 -303
- package/cli/business/vendors/README.md +0 -215
- package/cli/business/vendors/calendly.mjs +0 -80
- package/cli/business/vendors/common.mjs +0 -280
- package/cli/business/vendors/credential-names.mjs +0 -124
- package/cli/business/vendors/index.mjs +0 -96
- package/cli/business/vendors/pagerduty.mjs +0 -61
- package/cli/business/vendors/sentry.mjs +0 -85
- package/cli/business/vendors/sorry.mjs +0 -106
- package/cli/business/vendors/uptimerobot.mjs +0 -21
- package/cli/business/vendors/zendesk-attachments.mjs +0 -57
- package/cli/business/vendors/zendesk-encode.mjs +0 -133
- package/cli/business/vendors/zendesk-guide.mjs +0 -34
- package/cli/business/vendors/zendesk-package.mjs +0 -28
- package/cli/business/vendors/zendesk-pages.mjs +0 -61
- package/cli/business/vendors/zendesk-people.mjs +0 -7
- package/cli/business/vendors/zendesk-store.mjs +0 -154
- package/cli/business/vendors/zendesk-tickets.mjs +0 -21
- package/cli/business/vendors/zendesk-token.mjs +0 -31
- package/cli/business/vendors/zendesk-transport.mjs +0 -48
- package/cli/business/vendors/zendesk.mjs +0 -61
- package/modules/import-kit/src/door/package.ts +0 -121
- package/modules/import-kit/src/door/waves/README.md +0 -158
- package/modules/import-kit/src/door/waves/digests.ts +0 -38
- package/modules/import-kit/src/door/waves/inventory.ts +0 -84
- package/modules/import-kit/src/door/waves/journal.ts +0 -19
- package/modules/import-kit/src/door/waves/lifecycle.ts +0 -242
- package/modules/import-kit/src/door/waves/preparation.ts +0 -190
- package/modules/import-kit/src/door/waves/schema.ts +0 -73
- package/modules/import-kit/src/door/waves/store.ts +0 -112
- package/modules/import-kit/src/door/waves/types.ts +0 -76
|
@@ -484,40 +484,3 @@ For a concrete grant client and harness see Spool's `tests/business-import.mjs`;
|
|
|
484
484
|
its `grant`, `doorRequest`, `nativeSnapshot`, `startTestWorker` and `signIn`
|
|
485
485
|
provide those network edges. Run with `nice -n 15 node <existing-test-journey>`
|
|
486
486
|
after the seed's normal build. No `rer check` extension is required.
|
|
487
|
-
|
|
488
|
-
### Package-aware sources
|
|
489
|
-
|
|
490
|
-
Re-vendor the door together with its ordinary dependency closure, including
|
|
491
|
-
`door/package.ts`, `planning.ts`, `types.ts`, `handler.ts` and `admission.ts`.
|
|
492
|
-
There is no package schema or migration-ledger change: segments use existing
|
|
493
|
-
upload/chunk storage. Include the static `package` declaration in each source
|
|
494
|
-
allow-list result on both mounts. An undeclared namespace remains refused before
|
|
495
|
-
any grant row is saved. See [Logical packages](README.md#logical-packages-across-upload-grants)
|
|
496
|
-
for the stream/index contract and an accessor's bounds.
|
|
497
|
-
|
|
498
|
-
Provide `package.formats`, `maxBytes`, `maxSegments` and `read`. Keep readers
|
|
499
|
-
responsible for format semantics, metadata, per-table counts, attachment bindings
|
|
500
|
-
and explained omissions. Base verifies transport custody and aggregate record
|
|
501
|
-
authority; a reader must return the entire snapshot in one call. It must not
|
|
502
|
-
split a snapshot into independently authorized plans. The legacy CLI deliberately
|
|
503
|
-
declares `records.total: null`: it chooses no record units or waves. HappyToHelp
|
|
504
|
-
must supply D-CAP (package capacity), D-LEG (legacy/delta reader) and D-WAVES
|
|
505
|
-
(aggregate bounded-wave authority) before real oversized legacy imports.
|
|
506
|
-
|
|
507
|
-
The agent registers a locally prepared legacy database export with:
|
|
508
|
-
|
|
509
|
-
```sh
|
|
510
|
-
rer business legacy-package add DIR
|
|
511
|
-
rer business legacy-package status DIR
|
|
512
|
-
rer business legacy-package remove DIR
|
|
513
|
-
```
|
|
514
|
-
|
|
515
|
-
Registration verifies all declared file sizes and digests, row chunk/context
|
|
516
|
-
counts, per-table totals, object identities and attachment counts. JSON files pass
|
|
517
|
-
the outbound detector before upload; customer-original media are exempt. It
|
|
518
|
-
stores a private local registration and prints only counts and the manifest SHA.
|
|
519
|
-
A leased `happytohelp-legacy` export checks that registration again, pauses and
|
|
520
|
-
names the local package if it changed, then sends the exact package files through
|
|
521
|
-
the logical package protocol. It needs no vendor key and makes no request to a
|
|
522
|
-
live HappyToHelp account. Disconnect removes the registration and transfer
|
|
523
|
-
journal/spool; removing registration leaves the source directory intact.
|
|
@@ -552,69 +552,3 @@ approval. It refuses gone, conflicting, rejected or unmapped records. Additions
|
|
|
552
552
|
plus replacements divided by baseline source count must be strictly below 5%;
|
|
553
553
|
the server derives these counts. A refusal leaves Business to present a new
|
|
554
554
|
proposal for owner review. No caller-supplied totals relax this boundary.
|
|
555
|
-
|
|
556
|
-
## Logical packages across upload grants
|
|
557
|
-
|
|
558
|
-
A source may opt in to manifests with `format: "rer-door-package/1"` and
|
|
559
|
-
`package: {formats, maxBytes, maxSegments, read}` on its resolved source. The
|
|
560
|
-
same declaration belongs on the operator and grant mounts. Other sources keep
|
|
561
|
-
their existing readers and upload limits. No additional schema migration or
|
|
562
|
-
larger grant is required. The declaration cannot be combined with `intake`.
|
|
563
|
-
|
|
564
|
-
The sender uploads `rer-package.json` first, within one ordinary upload grant,
|
|
565
|
-
then calls `plan` for that manifest's upload id. The manifest has:
|
|
566
|
-
|
|
567
|
-
```json
|
|
568
|
-
{
|
|
569
|
-
"format": "rer-door-package/1",
|
|
570
|
-
"source": {"format": "example-snapshot-v1", "version": 1, "origin": "legacy-database", "exportedAt": null, "synthetic": true, "adapter": "example"},
|
|
571
|
-
"mode": "snapshot",
|
|
572
|
-
"stream": {"bytes": 0, "sha256": "<SHA-256 of the entire stream>"},
|
|
573
|
-
"segments": [{"bytes": 2097152, "sha256": "<segment SHA-256>"}],
|
|
574
|
-
"index": {"path": "index.ndjson", "bytes": 0, "sha256": "<index SHA-256>"},
|
|
575
|
-
"records": {"total": null, "byKind": {}},
|
|
576
|
-
"totals": {}
|
|
577
|
-
}
|
|
578
|
-
```
|
|
579
|
-
|
|
580
|
-
The zero sizes and digest placeholders above describe fields, not an accepted
|
|
581
|
-
manifest. `source.format` must be listed by the seed. Both the index and manifest
|
|
582
|
-
fit one grant. The stream starts with `index.ndjson`, then the source files in
|
|
583
|
-
index order, with no padding or encoding. Each index line is
|
|
584
|
-
`{path, bytes, sha256, offset}`; offsets start at zero immediately after the index.
|
|
585
|
-
Paths are relative, unique, and contain no empty, `.` or `..` components. Source
|
|
586
|
-
files may be record NDJSON, `blobs/<sha256>`, census/tombstone/Guide sidecars, or
|
|
587
|
-
all files of a legacy package byte-for-byte. All stream bytes belong to indexed
|
|
588
|
-
files. Nonfinal segments are exactly the grant's `maxBytes`; the final segment
|
|
589
|
-
may be shorter. Upload each as `rer-segment` in ordinary 64 KiB chunks.
|
|
590
|
-
|
|
591
|
-
Before any plan exists, an aware door returns 409 `package_segment_missing`
|
|
592
|
-
with `details.missing: [{bytes, sha256}]`. Missing, incomplete, altered, wrong-size
|
|
593
|
-
or foreign segments have that same response. Only complete uploads named
|
|
594
|
-
`rer-segment` in the same installation and namespace resolve. The sender stops
|
|
595
|
-
with `SEED_PACKAGE_UNSUPPORTED` if a fresh manifest probe answers otherwise;
|
|
596
|
-
it sends no segments. A resumed, previously accepted probe may find the complete
|
|
597
|
-
package and return its plan. The CLI's private journal saves upload ids before
|
|
598
|
-
requests, uses door inventories to skip durable chunks, and restarts an upload
|
|
599
|
-
whose inventory returns 404. Each request uses a fresh single-use grant.
|
|
600
|
-
|
|
601
|
-
The reader receives `{manifest, files, read(path, maxBytes?)}`. The accessor checks
|
|
602
|
-
segment custody, index layout and each requested file's digest, and enforces the
|
|
603
|
-
declared package bound, the source's `limits.maxBytes` file bound (64 MiB by
|
|
604
|
-
default), and any tighter requested file bound. Read record files
|
|
605
|
-
with a tighter bound where practical. Return the whole snapshot as inert parsed
|
|
606
|
-
records. The door counts the returned aggregate against the current plan grant's
|
|
607
|
-
`maxRecords` before mapping or saving a plan, including legacy packages whose
|
|
608
|
-
`records.total` is null. A numeric declared total and its `byKind` counts must match the returned records.
|
|
609
|
-
A package is one plan, with one lineage comparison and one manifest digest; it
|
|
610
|
-
never authorizes record waves or increases record authority per segment.
|
|
611
|
-
|
|
612
|
-
The package path records manifest/file digest metadata in the plan; it does not
|
|
613
|
-
claim those files as kit-retained originals. Seeds needing durable original
|
|
614
|
-
custody must provide that through their reader/product contract. Grant and D1
|
|
615
|
-
request limits still apply; an oversized read/plan is a refusal, never a partial
|
|
616
|
-
import. HappyToHelp adoption still needs its own package reader, capacity and
|
|
617
|
-
legacy delta semantics. Legacy snapshots above the unchanged 100,000-record
|
|
618
|
-
grant require a separate bounded-wave authority decision before real imports.
|
|
619
|
-
|
|
620
|
-
The opt-in [complete-parent bounded-wave protocol](waves/README.md) uses separately approved children and the existing D1 journal. Ordinary package planning keeps its aggregate refusal.
|
|
@@ -1,8 +1,7 @@
|
|
|
1
|
-
import {importDoorBoundedWavesCheck} from './waves/schema.ts';
|
|
2
1
|
import type { ImportDoorOptions, ImportDoorRuntime, DoorRequest } from './types.ts';
|
|
3
2
|
import { runtimeOptions } from './handler.ts';
|
|
4
3
|
import { DoorError, verifyImportGrant, type DoorAction } from './grants.ts';
|
|
5
|
-
import {
|
|
4
|
+
import { doorFence, erasures, sql, transact } from './store.ts';
|
|
6
5
|
import { importDoorSchemaCheck } from './schema.ts';
|
|
7
6
|
|
|
8
7
|
/** The storage key; grants are still verified against `claim.installationId`. */
|
|
@@ -22,7 +21,7 @@ async function source(options: ImportDoorOptions, namespace: string): Promise<Im
|
|
|
22
21
|
if (!options.source) return runtimeOptions(options);
|
|
23
22
|
const listed = await options.source(namespace);
|
|
24
23
|
if (!listed) throw new DoorError('unknown_import_source');
|
|
25
|
-
return runtimeOptions({...options, parse: listed.parse, mapping: listed.mapping, intake: listed.intake,
|
|
24
|
+
return runtimeOptions({...options, parse: listed.parse, mapping: listed.mapping, intake: listed.intake,
|
|
26
25
|
format: listed.format ?? options.format, readRaw: 'readRaw' in listed ? listed.readRaw : undefined,
|
|
27
26
|
ownerPrincipalId: listed.ownerPrincipalId ?? options.ownerPrincipalId});
|
|
28
27
|
}
|
|
@@ -44,14 +43,5 @@ export async function admit(options: ImportDoorOptions, request: Request, action
|
|
|
44
43
|
throw new DoorError('grant_replayed_revoked_or_expired');
|
|
45
44
|
}
|
|
46
45
|
const authority = sql(`(${basic.sql}) AND EXISTS(SELECT 1 FROM ik_door_grants WHERE installation_id=? AND jti=? AND action=? AND namespace=? AND kid=? AND expires=?)`, ...basic.params ?? [], key, grant.jti, action, grant.namespace, kid, grant.exp);
|
|
47
|
-
|
|
48
|
-
if ((await options.database.prepare(importDoorBoundedWavesCheck).first())?.current!==1) throw new DoorError('bounded_waves_schema_outdated',503);
|
|
49
|
-
if (!runtime.bounded) throw new DoorError('bounded_waves_not_adopted',409);
|
|
50
|
-
const external=runtime.bounded.authority(grant.bounded,action);
|
|
51
|
-
authority.sql=`(${authority.sql}) AND (${external.sql})`;
|
|
52
|
-
authority.params=[...authority.params ?? [],...external.params ?? []];
|
|
53
|
-
}
|
|
54
|
-
const admitted: DoorRequest = {options: runtime, access: {...grant, installationId: key}, authority, via:'grant', epoch, nonce: {jti: grant.jti, kid}};
|
|
55
|
-
if (grant.bounded) await authorized(admitted);
|
|
56
|
-
return admitted;
|
|
46
|
+
return {options: runtime, access: {...grant, installationId: key}, authority, via:'grant', epoch, nonce: {jti: grant.jti, kid}};
|
|
57
47
|
}
|
|
@@ -7,7 +7,6 @@ import { approvalCaption } from './failures.ts';
|
|
|
7
7
|
import { reserveReturn } from '../d1/budget.ts';
|
|
8
8
|
|
|
9
9
|
const actions: Record<DoorAction, ImportAction[]> = {
|
|
10
|
-
prepare: [], 'seal-parent': [], 'review-parent': [], 'apply-child': [], 'wave-status': [], 'finalize-parent': [], 'reverse-parent': [],
|
|
11
10
|
uploads: [], plan: ['plan', 'read'], apply: ['create', 'apply', 'read'],
|
|
12
11
|
status: ['read'], reconcile: ['read'], undo: ['read', 'undo', 'rollback'],
|
|
13
12
|
cancel: [],
|
|
@@ -14,7 +14,7 @@ const transient = new RegExp([
|
|
|
14
14
|
].join('|'),'i');
|
|
15
15
|
/** Only persist allowlisted classifications, never adapter/SQL messages that
|
|
16
16
|
* may contain customer records. Providers can wrap SQLite errors in causes. */
|
|
17
|
-
|
|
17
|
+
function reasonCode(error: unknown): string {
|
|
18
18
|
const seen = new Set<unknown>();
|
|
19
19
|
for (let cause = error; cause && typeof cause === 'object' && !seen.has(cause);) {
|
|
20
20
|
seen.add(cause);
|
|
@@ -9,8 +9,8 @@ import type { DoorGrant } from './protocol.ts';
|
|
|
9
9
|
function fail(): never { throw new DoorError('invalid_grant'); }
|
|
10
10
|
const object = (value: unknown): value is Record<string, unknown> => value !== null && typeof value === 'object' && !Array.isArray(value);
|
|
11
11
|
const requiredKeys = ['v', 'iss', 'aud', 'installationId', 'email', 'jti', 'iat', 'exp', 'action', 'namespace', 'maxBytes', 'maxChunks', 'maxRecords'];
|
|
12
|
-
const optionalKeys = ['fingerprint', 'approvalRecordId', 'approvedAt', 'catchup'
|
|
13
|
-
const actions: readonly string[] = ['uploads', 'plan', 'apply', 'status', 'reconcile', 'undo', 'cancel', 'archive', 'restore-check'
|
|
12
|
+
const optionalKeys = ['fingerprint', 'approvalRecordId', 'approvedAt', 'catchup'];
|
|
13
|
+
const actions: readonly string[] = ['uploads', 'plan', 'apply', 'status', 'reconcile', 'undo', 'cancel', 'archive', 'restore-check'];
|
|
14
14
|
|
|
15
15
|
function decode(segment: string): Uint8Array<ArrayBuffer> {
|
|
16
16
|
if (!/^[A-Za-z0-9_-]+$/.test(segment)) fail();
|
|
@@ -80,28 +80,14 @@ function validateGrant(value: Record<string, unknown>, config: Readonly<OwnerCla
|
|
|
80
80
|
|| !quota(value.maxBytes, 300_000_000) || !quota(value.maxChunks, 1024) || !quota(value.maxRecords, 1_000_000)) fail();
|
|
81
81
|
|
|
82
82
|
const catchup = Object.hasOwn(value, 'catchup');
|
|
83
|
-
if (catchup && (value.catchup !== 'bounded' || (value.action !== 'plan' && value.action !== 'apply'
|
|
83
|
+
if (catchup && (value.catchup !== 'bounded' || (value.action !== 'plan' && value.action !== 'apply'))) fail();
|
|
84
84
|
if (catchup && !approved(value)) fail();
|
|
85
|
-
|
|
86
|
-
const wave=waveActions.includes(value.action as string);
|
|
87
|
-
if (wave !== Object.hasOwn(value,'bounded')) fail();
|
|
88
|
-
if (wave) {
|
|
89
|
-
const b=value.bounded;
|
|
90
|
-
if (!object(b) || b.protocol!=='rer-bounded-waves/1' || !['migration','revision','policy'].every(k=>typeof b[k]==='string' && /^[A-Za-z0-9_-]{1,128}$/.test(b[k] as string))
|
|
91
|
-
|| Object.keys(b).some(k=>!['protocol','migration','revision','policy','parentFingerprint','childFingerprint','parentApprovalRecordId','parentApprovedAt'].includes(k))
|
|
92
|
-
|| ['parentFingerprint','childFingerprint'].some(k=>Object.hasOwn(b,k) && (typeof b[k]!=='string' || !/^[a-f0-9]{64}$/.test(b[k] as string)))) fail();
|
|
93
|
-
if (['apply-child','finalize-parent','reverse-parent'].includes(value.action as string)
|
|
94
|
-
&& (!b.parentFingerprint || typeof b.parentApprovalRecordId!=='string' || !b.parentApprovalRecordId.trim() || b.parentApprovalRecordId.length>256
|
|
95
|
-
|| typeof b.parentApprovedAt!=='string' || !Number.isFinite(Date.parse(b.parentApprovedAt)) || new Date(b.parentApprovedAt).toISOString()!==b.parentApprovedAt)) fail();
|
|
96
|
-
if (value.action==='apply-child' && (!b.childFingerprint || value.fingerprint!==b.childFingerprint)) fail();
|
|
97
|
-
if (value.action==='review-parent' && (value.fingerprint!==b.parentFingerprint || value.approvalRecordId!==b.parentApprovalRecordId || value.approvedAt!==b.parentApprovedAt)) fail();
|
|
98
|
-
}
|
|
99
|
-
if (value.action === 'apply' || ['review-parent','apply-child'].includes(value.action as string)) {
|
|
85
|
+
if (value.action === 'apply') {
|
|
100
86
|
if (typeof value.fingerprint !== 'string' || !/^[0-9a-f]{64}$/.test(value.fingerprint) || !approved(value)) fail();
|
|
101
87
|
} else {
|
|
102
88
|
if (Object.hasOwn(value, 'fingerprint')) fail();
|
|
103
89
|
const hasApproval = Object.hasOwn(value, 'approvalRecordId') || Object.hasOwn(value, 'approvedAt');
|
|
104
|
-
if (hasApproval && (
|
|
90
|
+
if (hasApproval && (value.action !== 'plan' || !catchup || !approved(value))) fail();
|
|
105
91
|
}
|
|
106
92
|
return Object.freeze(value) as unknown as Readonly<DoorGrant>;
|
|
107
93
|
}
|
|
@@ -11,10 +11,7 @@ import { archive, restoreCheck } from './archive.ts';
|
|
|
11
11
|
import { streamUpload } from './intake-upload.ts';
|
|
12
12
|
import { intakeIdentity } from './intake-policy.ts';
|
|
13
13
|
|
|
14
|
-
|
|
15
|
-
import { seal, reviewParent, applyChild, waveStatus, finalize, reverseParent } from './waves/lifecycle.ts';
|
|
16
|
-
|
|
17
|
-
const routes = {prepare,'seal-parent':seal,'review-parent':reviewParent,'apply-child':applyChild,'wave-status':waveStatus,'finalize-parent':finalize,'reverse-parent':reverseParent,uploads,plan,apply,status,reconcile,undo,cancel,archive,'restore-check':restoreCheck};
|
|
14
|
+
const routes = {uploads,plan,apply,status,reconcile,undo,cancel,archive,'restore-check':restoreCheck};
|
|
18
15
|
const headers = {'Cache-Control':'no-store','Content-Type':'application/json','X-Content-Type-Options':'nosniff'};
|
|
19
16
|
async function readInput(request: Request): Promise<Record<string,unknown>> {
|
|
20
17
|
if (request.headers.get('Content-Type')?.split(';')[0] !== 'application/json' || !request.body) throw new DoorError('json_body_required',400);
|
|
@@ -49,12 +46,6 @@ export function runtimeOptions<T extends RuntimeConfiguration>(options: T): T &
|
|
|
49
46
|
|| (options.format === 'raw' ? typeof options.readRaw !== 'function' || options.intake !== undefined
|
|
50
47
|
: typeof options.parse !== 'function')) throw new DoorError('invalid_configuration',503);
|
|
51
48
|
if (options.intake !== undefined) intakeIdentity(options.intake);
|
|
52
|
-
const pkg = options.package;
|
|
53
|
-
if (pkg && (options.intake || !Array.isArray(pkg.formats) || !pkg.formats.length || pkg.formats.some(f=>typeof f!=='string' || !f.trim())
|
|
54
|
-
|| !Number.isSafeInteger(pkg.maxBytes) || pkg.maxBytes < 1 || !Number.isSafeInteger(pkg.maxSegments) || pkg.maxSegments < 1 || typeof pkg.read !== 'function')) throw new DoorError('invalid_configuration',503);
|
|
55
|
-
if (options.bounded && (!pkg || options.bounded.protocol !== 'rer-bounded-waves/1'
|
|
56
|
-
|| options.bounded.batchSize!==undefined && (!Number.isSafeInteger(options.bounded.batchSize) || options.bounded.batchSize<1 || options.bounded.batchSize>20)
|
|
57
|
-
|| ['describe','page','native','authority'].some(key=>typeof (options.bounded as any)[key]!=='function'))) throw new DoorError('invalid_configuration',503);
|
|
58
49
|
return options as T & ImportDoorRuntime;
|
|
59
50
|
}
|
|
60
51
|
/** Server-side mount only. A null response belongs to another dispatcher. Every
|
|
@@ -87,8 +78,6 @@ export function createDoorHandler(prefix: string, database: D1Database,
|
|
|
87
78
|
const input = await readInput(request);
|
|
88
79
|
admitted = await review(admitted,input);
|
|
89
80
|
if (['status','reconcile','undo','cancel'].includes(action) && Object.keys(input).some(k=>k!=='planId')) throw new DoorError('invalid_input',400);
|
|
90
|
-
if (['seal-parent','review-parent','wave-status','finalize-parent','reverse-parent'].includes(action) && Object.keys(input).some(k=>k!=='parentId')
|
|
91
|
-
|| action==='apply-child' && Object.keys(input).some(k=>!['parentId','ordinal','reviewedFingerprint'].includes(k))) throw new DoorError('invalid_input',400);
|
|
92
81
|
const result = await routes[action](admitted,input);
|
|
93
82
|
await authorized(admitted);
|
|
94
83
|
return Response.json(result,{headers});
|
|
@@ -18,9 +18,3 @@ export function createImportDoorHandler(options: ImportDoorOptions): (request: R
|
|
|
18
18
|
return createDoorHandler('/api/rer/import/', options.database, (request, action, database) => admit({...options, database}, request, action),
|
|
19
19
|
undefined, async database => !!options.claim?.publicKeys && Object.keys(options.claim.publicKeys).length > 0 && await enabled({...options, database}));
|
|
20
20
|
}
|
|
21
|
-
|
|
22
|
-
export { importDoorBoundedWavesMigration } from './waves/schema.ts';
|
|
23
|
-
export type * from './waves/types.ts';
|
|
24
|
-
export { revokeImportDoorWaveApproval } from './waves/store.ts';
|
|
25
|
-
export { waveInventoryHash, inventoryMember } from './waves/digests.ts';
|
|
26
|
-
export type {WaveInventoryInput, WaveInventoryMember} from './waves/digests.ts';
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { packageUpload, readPackageSource } from './package.ts';
|
|
2
1
|
import { readRawSource } from './raw.ts';
|
|
3
2
|
import { randomUUID } from 'node:crypto';
|
|
4
3
|
import { canonicalJson, sha256 } from '../provenance/identity.ts';
|
|
@@ -25,36 +24,26 @@ export function preview(snapshot: DoorSnapshot, created: boolean) {
|
|
|
25
24
|
export async function plan(request: DoorRequest, input: Record<string, unknown>) {
|
|
26
25
|
if (Object.keys(input).some(k=>!['uploadId','catchup'].includes(k)) || (input.catchup !== undefined && typeof input.catchup !== 'boolean')) throw new DoorError('invalid_plan_input',400);
|
|
27
26
|
const upload = await uploadRow(request,text(input.uploadId,'upload_id'));
|
|
28
|
-
|
|
29
|
-
try {
|
|
30
|
-
if (upload.name === 'rer-package.json') await packageUpload(request,upload,rows=>segments.push(...rows));
|
|
31
|
-
return await planUpload(request,input,upload,segments);
|
|
32
|
-
}
|
|
27
|
+
try { return await planUpload(request,input,upload); }
|
|
33
28
|
catch (error) {
|
|
34
29
|
// A reader or bundle-check refusal answers a named 4xx, never a bare conflict.
|
|
35
30
|
const refused = error instanceof DoorError ? error : sourceRefusal(error);
|
|
36
31
|
// An upload holding a credential the reader cannot remove, or a value it
|
|
37
32
|
// cannot inspect for one, never plans, so it is not kept.
|
|
38
33
|
const unreadable = refused?.code === 'source_credential' || refused?.code === 'source_invalid_or_credential_source_data';
|
|
39
|
-
if (unreadable
|
|
40
|
-
if (unreadable || refused?.code === 'source_erased'
|
|
34
|
+
if (unreadable) await forget(request,upload);
|
|
35
|
+
if (unreadable || refused?.code === 'source_erased') throw refused;
|
|
41
36
|
// Whatever refuses this plan first (a pending import, interrupted staging, a
|
|
42
37
|
// withdrawn snapshot, the catch-up shape, a bound or the source itself), an
|
|
43
38
|
// upload holding an erased record is refused for that instead, and goes. A
|
|
44
39
|
// failing screen leaves the original refusal standing. No refused plan keeps
|
|
45
40
|
// credential values the reader removes: their uploaded bytes go.
|
|
46
|
-
const withheld = await screen(request,upload
|
|
47
|
-
|
|
48
|
-
await forget(request,upload,segments);
|
|
49
|
-
throw screened;
|
|
50
|
-
}
|
|
51
|
-
return false;
|
|
52
|
-
});
|
|
53
|
-
if (withheld) await dropChunks(request,upload,segments);
|
|
41
|
+
const withheld = await screen(request,upload).catch(screened => { if (refused && screened instanceof DoorError && screened.code === 'source_erased') throw screened; return false; });
|
|
42
|
+
if (withheld) await dropChunks(request,upload);
|
|
54
43
|
throw refused ?? error;
|
|
55
44
|
}
|
|
56
45
|
}
|
|
57
|
-
async function planUpload(request: DoorRequest, input: Record<string, unknown>, upload: UploadRow
|
|
46
|
+
async function planUpload(request: DoorRequest, input: Record<string, unknown>, upload: UploadRow) {
|
|
58
47
|
const current = await lineage(request);
|
|
59
48
|
// A plan whose apply failed, or whose import was undone or rolled back, has
|
|
60
49
|
// left the lineage: the same bundle plans afresh instead of reusing it. A
|
|
@@ -78,7 +67,7 @@ async function planUpload(request: DoorRequest, input: Record<string, unknown>,
|
|
|
78
67
|
if (snapshot.sourceCount > request.access.maxRecords) throw new DoorError('plan_exceeds_grant_quota');
|
|
79
68
|
if (snapshot.plan.mappingVersion !== request.options.mapping.version) throw new DoorError('snapshot_mapping_changed_use_new_namespace',409);
|
|
80
69
|
if (request.access.catchup === 'bounded' && (!snapshot.standingApproval || snapshot.standingApproval.id !== request.access.approvalRecordId || snapshot.standingApproval.at !== request.access.approvedAt)) throw new DoorError('standing_approval_mismatch');
|
|
81
|
-
await dropCredentialBytes(request,snapshot,upload
|
|
70
|
+
await dropCredentialBytes(request,snapshot,upload);
|
|
82
71
|
return preview(snapshot,false);
|
|
83
72
|
}
|
|
84
73
|
if (current.pending_id) throw new DoorError('namespace_has_pending_import',409);
|
|
@@ -96,7 +85,7 @@ async function planUpload(request: DoorRequest, input: Record<string, unknown>,
|
|
|
96
85
|
if (!previous && (input.catchup === true || request.access.catchup === 'bounded')) throw new DoorError('catchup_requires_baseline',409);
|
|
97
86
|
// A catch-up reads the destination's whole native set, which a large destination
|
|
98
87
|
// refuses (413); an upload holding an erased record is refused and goes first.
|
|
99
|
-
if (previous) await screen(request,upload
|
|
88
|
+
if (previous) await screen(request,upload);
|
|
100
89
|
const planId = randomUUID(), options = request.options, {kitNamespace,source,retainedFrom} = await readUpload(request,upload,planId);
|
|
101
90
|
const prior = new Map((previous?.records ?? []).map(r=>[recordKey(r),r]));
|
|
102
91
|
const revisions = new Map<string,{revision:string;matches:boolean}>();
|
|
@@ -146,10 +135,10 @@ async function planUpload(request: DoorRequest, input: Record<string, unknown>,
|
|
|
146
135
|
snapshot.records.push({...old,explanation:'Absent from source snapshot; retained in the seed. No deletion proposed.'});
|
|
147
136
|
}
|
|
148
137
|
if (request.access.catchup === 'bounded') assertBounded(snapshot,previous!);
|
|
149
|
-
await refuseErased(request,upload,snapshot.plan
|
|
138
|
+
await refuseErased(request,upload,snapshot.plan);
|
|
150
139
|
if (retired) await retire(request,retired,snapshot.plan);
|
|
151
140
|
await saveSnapshot(request,snapshot,upload,retainedFrom);
|
|
152
|
-
await dropCredentialBytes(request,snapshot,upload
|
|
141
|
+
await dropCredentialBytes(request,snapshot,upload);
|
|
153
142
|
return preview(snapshot,true);
|
|
154
143
|
}
|
|
155
144
|
/** Retire an erasure-withdrawn live import as the namespace's baseline. The kit
|
|
@@ -171,24 +160,11 @@ async function retire(request: DoorRequest, head: DoorPlanRow, plan: ImportPlan)
|
|
|
171
160
|
/** The reader removed credential values: the saved plan holds none and withholds
|
|
172
161
|
* the original, so the uploaded bytes that still hold them go once it is saved.
|
|
173
162
|
* The upload's header stays, so repeating this plan finds the saved one. */
|
|
174
|
-
async function dropCredentialBytes(request: DoorRequest, snapshot: DoorSnapshot, upload: UploadRow
|
|
175
|
-
if (withheldCredentials(snapshot.plan.source)) await dropChunks(request,upload
|
|
163
|
+
async function dropCredentialBytes(request: DoorRequest, snapshot: DoorSnapshot, upload: UploadRow): Promise<void> {
|
|
164
|
+
if (withheldCredentials(snapshot.plan.source)) await dropChunks(request,upload);
|
|
176
165
|
}
|
|
177
166
|
const withheldCredentials = (source: SourceBundle) => source.artifacts.some(a=>a.original?.status==='unavailable' && a.original.reason==='credentials-removed');
|
|
178
|
-
|
|
179
|
-
* upload id. Match all equivalent copies in the authorized scope; manifests
|
|
180
|
-
* sharing them must request missing segments again. Foreign scopes stay intact. */
|
|
181
|
-
function cleanup(request: DoorRequest, upload: UploadRow, segments: readonly UploadRow[], headers: boolean) {
|
|
182
|
-
const {installationId,namespace} = request.access;
|
|
183
|
-
const matches = [sql('id=? AND installation_id=? AND namespace=?',upload.id,installationId,namespace),
|
|
184
|
-
...segments.map(segment=>sql("installation_id=? AND namespace=? AND name='rer-segment' AND bundle_sha256=? AND total_bytes=?",installationId,namespace,segment.bundle_sha256,segment.total_bytes))];
|
|
185
|
-
return matches.flatMap(match=>[
|
|
186
|
-
sql(`DELETE FROM ik_door_chunks WHERE upload_id IN (SELECT id FROM ik_door_uploads WHERE ${match.sql})`,...match.params ?? []),
|
|
187
|
-
...(headers ? [sql(`DELETE FROM ik_door_uploads WHERE ${match.sql}`,...match.params ?? [])] : []),
|
|
188
|
-
]);
|
|
189
|
-
}
|
|
190
|
-
const dropChunks = (request: DoorRequest, upload: UploadRow, segments: readonly UploadRow[] = []) =>
|
|
191
|
-
transact(request.options.database,request.authority,cleanup(request,upload,segments,false));
|
|
167
|
+
const dropChunks = (request: DoorRequest, upload: UploadRow) => transact(request.options.database,request.authority,[sql('DELETE FROM ik_door_chunks WHERE upload_id=?',upload.id)]);
|
|
192
168
|
/** Each plan has its own kit namespace, so a replan after a failed apply never
|
|
193
169
|
* meets the failed job's kit namespace. */
|
|
194
170
|
async function readUpload(request: DoorRequest, upload: UploadRow, planId: string) {
|
|
@@ -198,7 +174,6 @@ async function readUpload(request: DoorRequest, upload: UploadRow, planId: strin
|
|
|
198
174
|
const retained = await retainedSource(request,upload);
|
|
199
175
|
if (retained) return {kitNamespace,retainedFrom:retained.planId,source:{...retained.source,records:retained.source.records.map(record=>({...record,namespace:kitNamespace}))}};
|
|
200
176
|
// Credential values are removed per record, never imported, and never refuse a record.
|
|
201
|
-
if (upload.name === 'rer-package.json') return {kitNamespace,source:await readPackageSource(request,upload,kitNamespace),retainedFrom:undefined};
|
|
202
177
|
const bytes = await bundleBytes(request,upload);
|
|
203
178
|
const limits = {...options.limits,maxBytes:Math.min(options.limits?.maxBytes ?? request.access.maxBytes,request.access.maxBytes),maxRecords:Math.min(options.limits?.maxRecords ?? request.access.maxRecords,request.access.maxRecords)};
|
|
204
179
|
const source = options.format === 'raw'
|
|
@@ -231,20 +206,20 @@ async function retainedSource(request: DoorRequest, upload: UploadRow): Promise<
|
|
|
231
206
|
}
|
|
232
207
|
/** The kit never stages an erased record, and neither does the door: an upload
|
|
233
208
|
* holding one can never be planned, so it goes too. The epoch fences a later erasure. */
|
|
234
|
-
export async function refuseErased(request: DoorRequest, upload: UploadRow, plan: ImportPlan
|
|
209
|
+
export async function refuseErased(request: DoorRequest, upload: UploadRow, plan: ImportPlan): Promise<void> {
|
|
235
210
|
const db = request.options.database;
|
|
236
211
|
for (const clear of erasedSourcePredicates(plan)) if ((await statement(db, sql(`SELECT ${clear.sql} AS clear`, ...clear.params ?? [])).first())?.clear !== 1) {
|
|
237
|
-
await forget(request,upload
|
|
212
|
+
await forget(request, upload);
|
|
238
213
|
throw new DoorError('source_erased', 409);
|
|
239
214
|
}
|
|
240
215
|
}
|
|
241
|
-
|
|
242
|
-
|
|
216
|
+
const forget = (request: DoorRequest, upload: UploadRow) => transact(request.options.database, request.authority,
|
|
217
|
+
[sql('DELETE FROM ik_door_chunks WHERE upload_id=?', upload.id), sql('DELETE FROM ik_door_uploads WHERE id=?', upload.id)]);
|
|
243
218
|
/** The upload planned on its own with the plain mapping: one holding an erased
|
|
244
219
|
* record is refused and goes. Answers whether the reader removed credential values. */
|
|
245
|
-
async function screen(request: DoorRequest, upload: UploadRow
|
|
220
|
+
async function screen(request: DoorRequest, upload: UploadRow): Promise<boolean> {
|
|
246
221
|
const id = randomUUID(), {kitNamespace,source} = await readUpload(request,upload,id), mapping = request.options.mapping;
|
|
247
|
-
await refuseErased(request,upload,await planD1Import(source,mapping,kitContext(request,{id,kitNamespace,plan:{mappingVersion:mapping.version} as ImportPlan}))
|
|
222
|
+
await refuseErased(request,upload,await planD1Import(source,mapping,kitContext(request,{id,kitNamespace,plan:{mappingVersion:mapping.version} as ImportPlan})));
|
|
248
223
|
return withheldCredentials(source);
|
|
249
224
|
}
|
|
250
225
|
export function assertBounded(snapshot: DoorSnapshot, previous: DoorSnapshot): void {
|
|
@@ -1,7 +1,6 @@
|
|
|
1
|
-
|
|
2
|
-
export type DoorAction = 'uploads' | 'plan' | 'apply' | 'status' | 'reconcile' | 'undo' | 'cancel' | 'archive' | 'restore-check' | 'prepare' | 'seal-parent' | 'review-parent' | 'apply-child' | 'wave-status' | 'finalize-parent' | 'reverse-parent';
|
|
1
|
+
export type DoorAction = 'uploads' | 'plan' | 'apply' | 'status' | 'reconcile' | 'undo' | 'cancel' | 'archive' | 'restore-check';
|
|
3
2
|
/** Grant-only actions: the owner-session mount answers them 404. */
|
|
4
|
-
export const GRANT_ONLY_ACTIONS: readonly DoorAction[] = ['cancel', 'archive', 'restore-check'
|
|
3
|
+
export const GRANT_ONLY_ACTIONS: readonly DoorAction[] = ['cancel', 'archive', 'restore-check'];
|
|
5
4
|
|
|
6
5
|
export interface DoorGrant {
|
|
7
6
|
v: 1;
|
|
@@ -21,7 +20,6 @@ export interface DoorGrant {
|
|
|
21
20
|
approvalRecordId?: string;
|
|
22
21
|
approvedAt?: string;
|
|
23
22
|
catchup?: 'bounded';
|
|
24
|
-
bounded?: WaveBinding;
|
|
25
23
|
}
|
|
26
24
|
|
|
27
25
|
export class DoorError extends Error {
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { importDoorBoundedWavesMigration } from './waves/schema.ts';
|
|
2
1
|
function planTables(suffix = ''): string[] {
|
|
3
2
|
return [
|
|
4
3
|
`CREATE TABLE IF NOT EXISTS ik_door_plans${suffix} (
|
|
@@ -99,5 +98,4 @@ export const importDoorSchema = [
|
|
|
99
98
|
...failureSchema,
|
|
100
99
|
...importDoorScopesMigration,
|
|
101
100
|
...importDoorArchiveMigration,
|
|
102
|
-
...importDoorBoundedWavesMigration,
|
|
103
101
|
];
|
|
@@ -8,8 +8,8 @@ import { importDoorSchemaCheck } from './schema.ts';
|
|
|
8
8
|
export const sql = (sql: string, ...params: unknown[]): D1Sql => ({ sql, params });
|
|
9
9
|
export const statement = (db: D1Database, query: D1Sql) => db.prepare(query.sql).bind(...query.params ?? []);
|
|
10
10
|
export const assertion = (query: D1Sql) => sql(`INSERT INTO ik_d1_assertions(ok) SELECT CASE WHEN (${query.sql}) THEN 1 ELSE 0 END`, ...query.params ?? []);
|
|
11
|
-
export async function transact(db: D1Database, authority: D1Sql, queries: D1Sql[]
|
|
12
|
-
await db.batch([assertion(authority), ...queries, assertion(authority), sql('DELETE FROM ik_d1_assertions')
|
|
11
|
+
export async function transact(db: D1Database, authority: D1Sql, queries: D1Sql[]): Promise<void> {
|
|
12
|
+
await db.batch([assertion(authority), ...queries, assertion(authority), sql('DELETE FROM ik_d1_assertions')].map(q => statement(db, q)));
|
|
13
13
|
}
|
|
14
14
|
export async function authorized(request: DoorRequest): Promise<void> {
|
|
15
15
|
const row = await statement(request.options.database, sql(`SELECT CASE WHEN (${request.authority.sql}) THEN 1 ELSE 0 END AS ok`, ...request.authority.params ?? [])).first();
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import type { D1Database, D1Sql, D1TargetAdapter } from '../d1/types.ts';
|
|
2
2
|
import type { D1ReversalPolicy } from '../d1/undo-types.ts';
|
|
3
3
|
import type { ImportMapping, ImportPlan, Json, ParsedRecord, ReaderLimits } from '../types.ts';
|
|
4
|
-
import type { ImportDoorBoundedSource } from './waves/types.ts';
|
|
5
4
|
import type { DoorGrant } from './protocol.ts';
|
|
6
5
|
import type { ImportDoorIntake } from './intake/types.ts';
|
|
7
6
|
export type { ImportDoorIntake, IntakeFormat } from './intake/types.ts';
|
|
@@ -13,30 +12,8 @@ export interface ImportDoorRawUpload {
|
|
|
13
12
|
readonly sha256: string;
|
|
14
13
|
readonly size: number;
|
|
15
14
|
}
|
|
16
|
-
export interface DoorPackageFile { readonly path: string; readonly bytes: number; readonly sha256: string; readonly offset: number }
|
|
17
|
-
export interface DoorPackageManifest {
|
|
18
|
-
format: 'rer-door-package/1'; source: {format: string; [key: string]: Json}; mode: string;
|
|
19
|
-
stream: {bytes: number; sha256: string}; segments: {bytes: number; sha256: string}[];
|
|
20
|
-
index: {path: 'index.ndjson'; bytes: number; sha256: string};
|
|
21
|
-
records: {total: number | null; byKind: Record<string, number>}; totals: Json;
|
|
22
|
-
}
|
|
23
|
-
export interface ImportDoorPackageUpload {
|
|
24
|
-
readonly manifest: Readonly<DoorPackageManifest>;
|
|
25
|
-
readonly files: readonly Readonly<DoorPackageFile>[];
|
|
26
|
-
/** A verified file, bounded by the declaration and the caller's optional tighter limit. */
|
|
27
|
-
read(path: string, maxBytes?: number): Promise<Uint8Array>;
|
|
28
|
-
}
|
|
29
|
-
export interface ImportDoorPackageSource {
|
|
30
|
-
formats: readonly string[];
|
|
31
|
-
maxBytes: number;
|
|
32
|
-
maxSegments: number;
|
|
33
|
-
/** Return the whole logical snapshot, never one plan per segment or record wave. */
|
|
34
|
-
read(upload: ImportDoorPackageUpload, limits: Readonly<Required<ReaderLimits>>): ParsedRecord[] | Promise<ParsedRecord[]>;
|
|
35
|
-
}
|
|
36
15
|
/** One decoded source; existing adapters may keep the mount's format. */
|
|
37
16
|
export interface ImportDoorSource {
|
|
38
|
-
package?: ImportDoorPackageSource;
|
|
39
|
-
bounded?: ImportDoorBoundedSource;
|
|
40
17
|
mapping: ImportMapping;
|
|
41
18
|
parse(value: Json, index: number): ParsedRecord | ParsedRecord[];
|
|
42
19
|
ownerPrincipalId?: string;
|
|
@@ -95,7 +72,7 @@ export type ImportDoorOptions = ImportDoorMountOptions & (
|
|
|
95
72
|
parse?: undefined; mapping?: undefined; readRaw?: undefined;
|
|
96
73
|
});
|
|
97
74
|
export type DoorAccess = Pick<DoorGrant, 'action' | 'installationId' | 'email' | 'namespace'
|
|
98
|
-
| '
|
|
75
|
+
| 'maxBytes' | 'maxChunks' | 'maxRecords' | 'fingerprint' | 'approvalRecordId' | 'approvedAt' | 'catchup'>;
|
|
99
76
|
export interface ImportOperatorOptions extends ImportDoorRuntimeOptions {
|
|
100
77
|
installationId: string;
|
|
101
78
|
appOrigin: string;
|
|
@@ -142,6 +142,10 @@ function memberKey(key: string): ReturnType<typeof credentialKey> {
|
|
|
142
142
|
const name = credentialMember(decodedCredentialText(key));
|
|
143
143
|
return name === undefined ? undefined : { name, holds: value => !emptyCredential(value) };
|
|
144
144
|
}
|
|
145
|
+
/** The parsed URL, or undefined for text that looks like one but does not parse. */
|
|
146
|
+
function parseUrl(text: string): URL | undefined {
|
|
147
|
+
try { return new URL(text, 'https://credential-check.invalid'); } catch { return undefined; }
|
|
148
|
+
}
|
|
145
149
|
const looksLikeUrl = (value: string) => /^(?:https?:\/\/|\/|\?)/i.test(decodedCredentialText(value));
|
|
146
150
|
|
|
147
151
|
export function assertNoCredentials(value: unknown, policy?: PaginationCursorPolicy): void {
|
|
@@ -167,14 +171,19 @@ export function assertNoCredentials(value: unknown, policy?: PaginationCursorPol
|
|
|
167
171
|
} else if (typeof item === 'string') {
|
|
168
172
|
// Relative pagination links receive the same check as absolute links.
|
|
169
173
|
if (!looksLikeUrl(item)) return;
|
|
170
|
-
|
|
174
|
+
// Free text that merely starts like a URL (`http:// see below`) does not parse: it is prose,
|
|
175
|
+
// with no user name to find. Its raw text still goes through the query-key scans below.
|
|
176
|
+
const url = parseUrl(item);
|
|
171
177
|
const structure = decodedCredentialText(credentialUrlStructure(item, policy).split('#')[0]);
|
|
172
|
-
const decodedUrl =
|
|
173
|
-
if (url
|
|
174
|
-
|
|
178
|
+
const decodedUrl = parseUrl(structure);
|
|
179
|
+
if (url?.username || url?.password || decodedUrl?.username || decodedUrl?.password) throw new CredentialError('CREDENTIAL_IN_DURABLE_URL', { kind: 'url-userinfo' });
|
|
180
|
+
const rawQueryStart = item.indexOf('?'), rawFragmentStart = item.indexOf('#');
|
|
181
|
+
const search = url ? url.search.slice(1)
|
|
182
|
+
: rawQueryStart < 0 || (rawFragmentStart >= 0 && rawFragmentStart < rawQueryStart) ? '' : item.slice(rawQueryStart + 1, rawFragmentStart < 0 ? item.length : rawFragmentStart);
|
|
183
|
+
for (const pair of search.split('&')) {
|
|
175
184
|
if (credentialQueryKey(pair.split('=')[0], policy)) throw new CredentialError('CREDENTIAL_IN_DURABLE_URL', { kind: 'url-query' });
|
|
176
185
|
}
|
|
177
|
-
for (const child of url.searchParams.values()) visit(child);
|
|
186
|
+
if (url) for (const child of url.searchParams.values()) visit(child);
|
|
178
187
|
// Encoded delimiters outside opaque values must not hide credential keys.
|
|
179
188
|
// Any key introduced by decoding has no exact-wire cursor exemption.
|
|
180
189
|
// A decoded fragment delimiter must not truncate these query checks either.
|
|
@@ -8,6 +8,7 @@ import type {
|
|
|
8
8
|
ImporterRecord,
|
|
9
9
|
ImporterValidationIssue,
|
|
10
10
|
ImporterValidationResult,
|
|
11
|
+
MoveInImporterRecord,
|
|
11
12
|
VerifiedImporter,
|
|
12
13
|
} from './types.ts';
|
|
13
14
|
|
|
@@ -23,7 +24,7 @@ const strayPercent = /%(?![0-9A-Fa-f]{2})/;
|
|
|
23
24
|
const nonAscii = /[^\x00-\x7f]/;
|
|
24
25
|
|
|
25
26
|
export async function validateImporterEvidence(
|
|
26
|
-
records: ImporterRecord
|
|
27
|
+
records: Array<ImporterRecord | MoveInImporterRecord>,
|
|
27
28
|
evidence: ImporterEvidencePort | undefined,
|
|
28
29
|
issues: ImporterValidationIssue[],
|
|
29
30
|
): Promise<ImporterValidationResult['evidence']> {
|
|
@@ -37,6 +38,7 @@ export async function validateImporterEvidence(
|
|
|
37
38
|
for (let index = 0; index < records.length; index += 1) {
|
|
38
39
|
const record = records[index]!;
|
|
39
40
|
const path = `$.importers[${index}]`;
|
|
41
|
+
if (!('evidence' in record)) continue;
|
|
40
42
|
const reference = record.evidence as ImporterEvidenceReference;
|
|
41
43
|
const receipt = await resolveAndCheckArtifact(reference, evidence, `${path}.evidence`, issues, counters);
|
|
42
44
|
if (!receipt) continue;
|
|
@@ -6,6 +6,7 @@ import type {
|
|
|
6
6
|
ImporterValidationIssue,
|
|
7
7
|
ImporterValidationResult,
|
|
8
8
|
ImportersDocument,
|
|
9
|
+
MoveInImporterRecord,
|
|
9
10
|
} from './types.ts';
|
|
10
11
|
|
|
11
12
|
export type * from './types.ts';
|
|
@@ -22,7 +23,18 @@ export async function validateImporters(
|
|
|
22
23
|
const counters: ImporterValidationResult['evidence'] = { required: 0, resolved: 0, verifiedReceipts: 0 };
|
|
23
24
|
if (issues.length > 0) return result(issues, counters);
|
|
24
25
|
|
|
25
|
-
const
|
|
26
|
+
const { schemaVersion, importers } = document as ImportersDocument;
|
|
27
|
+
const records = importers as Array<ImporterRecord | MoveInImporterRecord>;
|
|
28
|
+
// Version 1 rows carry no Move in fields and version 2 rows all three; the schema allows either row shape in either file.
|
|
29
|
+
for (let index = 0; index < records.length; index += 1) {
|
|
30
|
+
if (Object.hasOwn(records[index]!, 'moveIn') !== (schemaVersion === 2)) {
|
|
31
|
+
issues.push({ code: 'SCHEMA_VERSION_ROW', path: `$.importers[${index}]`, message: schemaVersion === 2 ? 'Every version 2 row declares moveIn, accepts and export.' : 'Move in fields need schemaVersion 2.' });
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
if (schemaVersion === 1 && Object.hasOwn(document as object, 'format')) {
|
|
35
|
+
issues.push({ code: 'SCHEMA_VERSION_ROW', path: '$.format', message: 'The import format path needs schemaVersion 2.' });
|
|
36
|
+
}
|
|
37
|
+
if (issues.length > 0) return result(issues, counters);
|
|
26
38
|
const seen = new Map<string, number>();
|
|
27
39
|
for (let index = 0; index < records.length; index += 1) {
|
|
28
40
|
const record = records[index]!;
|
|
@@ -43,7 +55,10 @@ export async function validateImporters(
|
|
|
43
55
|
}
|
|
44
56
|
if (issues.length > 0) return result(issues, counters);
|
|
45
57
|
|
|
46
|
-
|
|
58
|
+
// A row with no status declares only its Move in page; it has no evidence to resolve.
|
|
59
|
+
const evidenced = records.some((record) => 'evidence' in record);
|
|
60
|
+
if (!evidenced) return result(issues, counters);
|
|
61
|
+
const evidenceCounters = await validateImporterEvidence(records, evidence, issues);
|
|
47
62
|
return result(issues, evidenceCounters);
|
|
48
63
|
}
|
|
49
64
|
|
|
@@ -13,6 +13,8 @@ interface JsonSchema {
|
|
|
13
13
|
pattern?: string;
|
|
14
14
|
minimum?: number;
|
|
15
15
|
minItems?: number;
|
|
16
|
+
maxItems?: number;
|
|
17
|
+
uniqueItems?: boolean;
|
|
16
18
|
minProperties?: number;
|
|
17
19
|
required?: string[];
|
|
18
20
|
properties?: Record<string, JsonSchema>;
|
|
@@ -91,6 +93,12 @@ function schemaIssues(value: unknown, schema: JsonSchema, root: JsonSchema, path
|
|
|
91
93
|
if (schema.minItems !== undefined && value.length < schema.minItems) {
|
|
92
94
|
issues.push({ code: 'SCHEMA_MIN_ITEMS', path, message: `Array must contain at least ${schema.minItems} item(s).` });
|
|
93
95
|
}
|
|
96
|
+
if (schema.maxItems !== undefined && value.length > schema.maxItems) {
|
|
97
|
+
issues.push({ code: 'SCHEMA_MAX_ITEMS', path, message: `Array must contain no more than ${schema.maxItems} item(s).` });
|
|
98
|
+
}
|
|
99
|
+
if (schema.uniqueItems && new Set(value.map(canonicalJson)).size !== value.length) {
|
|
100
|
+
issues.push({ code: 'SCHEMA_UNIQUE_ITEMS', path, message: 'Array items must be unique.' });
|
|
101
|
+
}
|
|
94
102
|
if (schema.items) {
|
|
95
103
|
for (let index = 0; index < value.length; index += 1) {
|
|
96
104
|
issues.push(...schemaIssues(value[index], schema.items, root, `${path}[${index}]`));
|