@runeditrun/rer 0.2.81 → 0.2.83

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/README.md +12 -0
  2. package/cli/export.mjs +1 -1
  3. package/edition-authoring/base-copies.mjs +3 -1
  4. package/edition-authoring/copy-checkout.mjs +152 -1
  5. package/edition-authoring/files.mjs +1 -1
  6. package/edition-authoring/runtime-dependencies.mjs +1 -1
  7. package/global/CHANGELOG.md +25 -0
  8. package/global/README.md +12 -0
  9. package/modules/import-kit/src/door/ADOPTING.md +452 -0
  10. package/modules/import-kit/src/door/README.md +553 -0
  11. package/modules/import-kit/src/door/admission.ts +45 -0
  12. package/modules/import-kit/src/door/archive.ts +129 -0
  13. package/modules/import-kit/src/door/authority.ts +107 -0
  14. package/modules/import-kit/src/door/failures.ts +70 -0
  15. package/modules/import-kit/src/door/grants.ts +162 -0
  16. package/modules/import-kit/src/door/handler.ts +90 -0
  17. package/modules/import-kit/src/door/index.ts +20 -0
  18. package/modules/import-kit/src/door/intake/csv.ts +74 -0
  19. package/modules/import-kit/src/door/intake/json.ts +182 -0
  20. package/modules/import-kit/src/door/intake/projection.ts +61 -0
  21. package/modules/import-kit/src/door/intake/types.ts +28 -0
  22. package/modules/import-kit/src/door/intake-policy.ts +38 -0
  23. package/modules/import-kit/src/door/intake-upload.ts +91 -0
  24. package/modules/import-kit/src/door/inventory.ts +68 -0
  25. package/modules/import-kit/src/door/jobs.ts +134 -0
  26. package/modules/import-kit/src/door/limits.ts +16 -0
  27. package/modules/import-kit/src/door/operator.ts +41 -0
  28. package/modules/import-kit/src/door/planning.ts +227 -0
  29. package/modules/import-kit/src/door/protocol.ts +38 -0
  30. package/modules/import-kit/src/door/refusals.ts +51 -0
  31. package/modules/import-kit/src/door/schema.ts +101 -0
  32. package/modules/import-kit/src/door/store.ts +152 -0
  33. package/modules/import-kit/src/door/types.ts +100 -0
  34. package/modules/import-kit/src/door/uploads.ts +98 -0
  35. package/modules/import-kit/src/provenance/numeric-credentials.ts +40 -0
  36. package/modules/import-kit/src/readers/README.md +139 -0
  37. package/modules/import-kit/src/readers/api-policy.ts +67 -0
  38. package/modules/import-kit/src/readers/api.ts +86 -0
  39. package/modules/import-kit/src/readers/bytes.ts +73 -0
  40. package/modules/import-kit/src/readers/errors.ts +33 -0
  41. package/modules/import-kit/src/readers/file.ts +74 -0
  42. package/modules/import-kit/src/readers/index.ts +3 -0
  43. package/modules/import-kit/src/readers/json.ts +29 -0
  44. package/modules/import-kit/src/readers/numeric-lexemes.ts +104 -0
  45. package/modules/import-kit/src/readers/original.ts +31 -0
  46. package/modules/import-kit/src/readers/records.ts +105 -0
  47. package/package.json +3 -2
@@ -0,0 +1,152 @@
1
+ import { canonicalJson, sha256 } from '../provenance/identity.ts';
2
+ import { planFingerprint } from '../plan/index.ts';
3
+ import type { D1Database, D1Sql } from '../d1/types.ts';
4
+ import type { DoorPlanRow, DoorRequest, DoorSnapshot } from './types.ts';
5
+ import { DoorError } from './protocol.ts';
6
+ import { importDoorSchemaCheck } from './schema.ts';
7
+
8
+ export const sql = (sql: string, ...params: unknown[]): D1Sql => ({ sql, params });
9
+ export const statement = (db: D1Database, query: D1Sql) => db.prepare(query.sql).bind(...query.params ?? []);
10
+ export const assertion = (query: D1Sql) => sql(`INSERT INTO ik_d1_assertions(ok) SELECT CASE WHEN (${query.sql}) THEN 1 ELSE 0 END`, ...query.params ?? []);
11
+ export async function transact(db: D1Database, authority: D1Sql, queries: D1Sql[]): Promise<void> {
12
+ await db.batch([assertion(authority), ...queries, assertion(authority), sql('DELETE FROM ik_d1_assertions')].map(q => statement(db, q)));
13
+ }
14
+ export async function authorized(request: DoorRequest): Promise<void> {
15
+ const row = await statement(request.options.database, sql(`SELECT CASE WHEN (${request.authority.sql}) THEN 1 ELSE 0 END AS ok`, ...request.authority.params ?? [])).first();
16
+ if (row?.ok !== 1) throw new DoorError(request.via === 'operator' ? 'owner_required' : 'grant_revoked_or_expired');
17
+ }
18
+ export async function planRow(request: DoorRequest, id: string): Promise<DoorPlanRow> {
19
+ const row = await request.options.database.prepare('SELECT * FROM ik_door_plans WHERE id=? AND installation_id=? AND namespace=? AND complete=1')
20
+ .bind(id, request.access.installationId, request.access.namespace).first<DoorPlanRow>();
21
+ if (!row) throw new DoorError('plan_not_found', 404);
22
+ return row;
23
+ }
24
+ /** A saved plan's snapshot. A person's erasure withdraws every chunk of it:
25
+ * the plan, its approval and its kit job stay, the reviewed records do not. */
26
+ /** Snapshot rows are at most 32,000 characters; a read returns a bounded page
27
+ * of them, never a whole multi-megabyte snapshot in one D1 result. */
28
+ const SNAPSHOT_READ_ROWS = 48;
29
+ export async function loadSnapshot(request: DoorRequest, row: DoorPlanRow): Promise<DoorSnapshot> {
30
+ const chunks: {ordinal: unknown; body: string}[] = [];
31
+ for (;;) {
32
+ const page = await request.options.database.prepare(`SELECT ordinal,body FROM ik_door_plan_chunks WHERE plan_id=? ORDER BY ordinal LIMIT ${SNAPSHOT_READ_ROWS} OFFSET ?`).bind(row.id, chunks.length).all<{ordinal: unknown; body: string}>();
33
+ chunks.push(...page.results);
34
+ if (page.results.length < SNAPSHOT_READ_ROWS) break;
35
+ }
36
+ if (!chunks.length) throw new DoorError('snapshot_withdrawn', 410, {planId: row.id});
37
+ // A torn read across pages fails the ordinals or the snapshot hash below.
38
+ if (chunks.some((c, i) => c.ordinal !== i)) throw new DoorError('plan_custody_mismatch', 409);
39
+ const snapshot = JSON.parse(chunks.map(c => c.body).join('')) as DoorSnapshot;
40
+ if (sha256(canonicalJson(snapshot)) !== row.snapshot_hash || snapshot.id !== row.id || snapshot.namespace !== row.namespace || snapshot.bundleSha256 !== row.bundle_sha256
41
+ || snapshot.previous !== row.previous_id || snapshot.plan.fingerprint !== row.fingerprint
42
+ || planFingerprint(snapshot.plan) !== row.fingerprint
43
+ || snapshot.plan.namespace !== snapshot.kitNamespace || snapshot.plan.destination !== request.options.destination
44
+ || snapshot.plan.principalId !== request.options.ownerPrincipalId) throw new DoorError('plan_custody_mismatch', 409);
45
+ return snapshot;
46
+ }
47
+ /** Recover indexing after a committed kit batch or an owner undo through the seed.
48
+ * Kit state remains authoritative. Never hide a partially applied pending import. */
49
+ export async function lineage(request: DoorRequest): Promise<{ head_id: string | null; pending_id: string | null }> {
50
+ const db = request.options.database, installation = request.access.installationId, namespace = request.access.namespace;
51
+ let current = await db.prepare('SELECT head_id,pending_id FROM ik_door_namespaces WHERE installation_id=? AND namespace=?').bind(installation, namespace).first<{head_id:string|null;pending_id:string|null}>();
52
+ if (!current) return { head_id: null, pending_id: null };
53
+ for (;;) {
54
+ const candidate = current.pending_id ?? current.head_id;
55
+ if (!candidate) return current;
56
+ const row = await db.prepare('SELECT p.previous_id,j.state,j.checkpoint FROM ik_door_plans p LEFT JOIN ik_jobs j ON j.id=p.job_id WHERE p.id=?').bind(candidate).first();
57
+ let next: {head_id:string|null;pending_id:string|null} = current;
58
+ if (current.pending_id && row?.state === 'completed') next = { head_id: candidate, pending_id: null };
59
+ else if (current.pending_id && row?.state === 'failed' && row.checkpoint === 0) next = { head_id: current.head_id, pending_id: null };
60
+ else if (row && ['undone', 'rolled_back'].includes(row.state)) next = { head_id: current.pending_id ? current.head_id : row.previous_id, pending_id: null };
61
+ if (next === current) return current;
62
+ await transact(db, request.authority, [sql('UPDATE ik_door_namespaces SET head_id=?,pending_id=? WHERE installation_id=? AND namespace=? AND head_id IS ? AND pending_id IS ?', next.head_id, next.pending_id, installation, namespace, current.head_id, current.pending_id)]);
63
+ current = await db.prepare('SELECT head_id,pending_id FROM ik_door_namespaces WHERE installation_id=? AND namespace=?').bind(installation, namespace).first<typeof next>();
64
+ if (!current) throw new DoorError('lineage_conflict', 409);
65
+ }
66
+ }
67
+ /** A saved plan's snapshot is still held. An erasure withdraws all of its chunks at once. */
68
+ export const snapshotHeld = (planId: string): D1Sql => sql('EXISTS(SELECT 1 FROM ik_door_plan_chunks WHERE plan_id=?)', planId);
69
+ /** Door work already in flight never writes back or applies content an erasure
70
+ * withdrew meanwhile: each of its batches asserts that what it builds on is still
71
+ * held. A batch refused because it is not answers 410; any other failure stands. */
72
+ export async function fenced<T>(request: DoorRequest, planId: string, held: D1Sql, work: () => Promise<T>): Promise<T> {
73
+ try { return await work(); }
74
+ catch (error) {
75
+ if (error instanceof DoorError) throw error;
76
+ const row = await statement(request.options.database, sql(`SELECT CASE WHEN (${held.sql}) THEN 1 ELSE 0 END AS ok`, ...held.params ?? [])).first();
77
+ if (row?.ok !== 1) throw new DoorError('snapshot_withdrawn', 410, {planId});
78
+ throw error;
79
+ }
80
+ }
81
+ /** JSON arrays of rows of at most ~1.9 MB each, the kit's own packing ceiling,
82
+ * each with the number of rows it holds. */
83
+ function packRows(rows: unknown[]): {json: string; count: number}[] {
84
+ const packed: {json: string; count: number}[] = []; let parts: string[] = [], size = 2;
85
+ const flush = () => { packed.push({json: `[${parts.join(',')}]`, count: parts.length}); parts = []; size = 2; };
86
+ for (const row of rows) {
87
+ const value = JSON.stringify(row), bytes = new TextEncoder().encode(value).length + 1;
88
+ if (parts.length && size + bytes > 1_900_000) flush();
89
+ parts.push(value); size += bytes;
90
+ }
91
+ if (parts.length) flush();
92
+ return packed;
93
+ }
94
+ /** `retainedFrom` names the plan whose records a re-plan read because the
95
+ * upload's credential-bearing bytes went: that plan's snapshot must stay held
96
+ * instead of the upload's chunks. */
97
+ export async function saveSnapshot(request: DoorRequest, snapshot: DoorSnapshot, upload: {id: string}, retainedFrom?: string): Promise<void> {
98
+ const db = request.options.database, g = request.access;
99
+ await transact(db, request.authority, [
100
+ sql('INSERT OR IGNORE INTO ik_door_namespaces(installation_id,namespace) VALUES(?,?)', g.installationId, g.namespace),
101
+ assertion(sql('EXISTS(SELECT 1 FROM ik_door_namespaces WHERE installation_id=? AND namespace=? AND head_id IS ? AND pending_id IS NULL)', g.installationId, g.namespace, snapshot.previous)),
102
+ sql('INSERT INTO ik_door_plans(id,installation_id,namespace,bundle_sha256,fingerprint,previous_id,snapshot_hash) VALUES(?,?,?,?,?,?,?)', snapshot.id, g.installationId, g.namespace, snapshot.bundleSha256, snapshot.plan.fingerprint, snapshot.previous, sha256(canonicalJson(snapshot))),
103
+ sql('UPDATE ik_door_namespaces SET pending_id=? WHERE installation_id=? AND namespace=?', snapshot.id, g.installationId, g.namespace),
104
+ ]);
105
+ // Private staging remains visibly pending until every immutable chunk is stored.
106
+ // Each statement inserts up to ~1.9 MB of 32,000-character rows, as the kit
107
+ // packs its own plan chunks, so storing a snapshot costs one statement per
108
+ // ~1.9 MB rather than per row; no domain effect occurs.
109
+ const body = canonicalJson(snapshot), rows: {ordinal: number; body: string}[] = [];
110
+ for (let offset = 0; offset < body.length;) {
111
+ let end = Math.min(offset + 32_000,body.length);
112
+ if (end < body.length && /[\uD800-\uDBFF]/.test(body[end-1])) end--;
113
+ rows.push({ordinal: rows.length, body: body.slice(offset,end)});
114
+ offset = end;
115
+ }
116
+ // Each batch, and the one that completes the plan, requires the whole source
117
+ // upload, a catch-up's previous snapshot and every chunk written before it.
118
+ const held = (written: number) => sql(`EXISTS(SELECT 1 FROM ik_door_uploads u WHERE u.id=? AND u.installation_id=? AND u.namespace=? AND u.bundle_sha256=?
119
+ AND (SELECT count(*) FROM ik_door_chunks c WHERE c.upload_id=u.id)=(CASE WHEN ? IS NULL THEN u.total_chunks ELSE 0 END))
120
+ AND (? IS NULL OR EXISTS(SELECT 1 FROM ik_door_plan_chunks WHERE plan_id=?))
121
+ AND (? IS NULL OR EXISTS(SELECT 1 FROM ik_door_plan_chunks WHERE plan_id=?))
122
+ AND (SELECT count(*) FROM ik_door_plan_chunks WHERE plan_id=?)=?`,
123
+ upload.id, g.installationId, g.namespace, snapshot.bundleSha256, retainedFrom ?? null, snapshot.previous, snapshot.previous, retainedFrom ?? null, retainedFrom ?? null, snapshot.id, written);
124
+ const batch = (written: number, queries: D1Sql[]) => fenced(request, snapshot.id, held(written), () => transact(db, request.authority, [assertion(held(written)), ...queries]));
125
+ let written = 0;
126
+ for (const packed of packRows(rows)) {
127
+ await batch(written, [sql("INSERT INTO ik_door_plan_chunks(plan_id,ordinal,body) SELECT ?,json_extract(value,'$.ordinal'),json_extract(value,'$.body') FROM json_each(?)", snapshot.id, packed.json)]);
128
+ written += packed.count;
129
+ }
130
+ await batch(written, [sql('UPDATE ik_door_plans SET complete=1 WHERE id=?', snapshot.id)]);
131
+ }
132
+ /** The owner-session mount has no opt-in, and refuses an outdated schema just as loudly. */
133
+ export async function schemaCurrent(database: D1Database): Promise<boolean> {
134
+ const row = await database.prepare(importDoorSchemaCheck).first();
135
+ if (row?.current !== 1) throw new DoorError('import_door_schema_outdated', 503);
136
+ return true;
137
+ }
138
+ export const erasures = async (database: D1Database, destination: string): Promise<number> =>
139
+ Number((await database.prepare('SELECT count(*) AS n FROM ik_d1_erasure_journal WHERE destination=?').bind(destination).first<{n: number}>())?.n ?? 0);
140
+ /** Whether an erasure in this destination committed after the request was admitted. */
141
+ export const erasedSince = async (request: DoorRequest): Promise<boolean> => await erasures(request.options.database, request.options.destination) !== request.epoch;
142
+ /** What every batch of a door request asserts, whoever admitted it: the
143
+ * destination's erasure count at admission (an erasure committed meanwhile
144
+ * wins), and that the logical namespace belongs to this destination alone. */
145
+ export function doorFence(key: string, namespace: string, destination: string, epoch: number): D1Sql {
146
+ return sql(`(SELECT count(*) FROM ik_d1_erasure_journal WHERE destination=?)=?
147
+ AND NOT EXISTS(SELECT 1 FROM ik_door_scopes WHERE installation_id=? AND namespace=? AND destination<>?)
148
+ AND NOT EXISTS(SELECT 1 FROM ik_door_plans p JOIN ik_jobs j
149
+ ON j.id=p.job_id OR (p.job_id IS NULL AND j.operation_key='door:'||p.id)
150
+ WHERE p.installation_id=? AND p.namespace=? AND j.destination<>?)`,
151
+ destination, epoch, key, namespace, destination, key, namespace, destination);
152
+ }
@@ -0,0 +1,100 @@
1
+ import type { D1Database, D1Sql, D1TargetAdapter } from '../d1/types.ts';
2
+ import type { D1ReversalPolicy } from '../d1/undo-types.ts';
3
+ import type { ImportMapping, ImportPlan, Json, ParsedRecord, ReaderLimits } from '../types.ts';
4
+ import type { DoorGrant } from './protocol.ts';
5
+ import type { ImportDoorIntake } from './intake/types.ts';
6
+ export type { ImportDoorIntake, IntakeFormat } from './intake/types.ts';
7
+
8
+ /** One import source: its parser and mapping, and optionally its own owner principal. */
9
+ export interface ImportDoorSource {
10
+ mapping: ImportMapping;
11
+ parse(value: Json, index: number): ParsedRecord | ParsedRecord[];
12
+ ownerPrincipalId?: string;
13
+ /** Project declared fields before the first persistent byte. Shared by both mounts. */
14
+ intake?: ImportDoorIntake;
15
+ }
16
+ export interface ImportDoorRuntimeOptions extends ImportDoorSource {
17
+ database: D1Database;
18
+ targets: D1TargetAdapter;
19
+ /** The seed's actual owner principal, so its ordinary import history/undo works. */
20
+ ownerPrincipalId: string;
21
+ destination: string;
22
+ format: 'json' | 'ndjson';
23
+ limits?: ReaderLimits;
24
+ /** Override the undo policy for new jobs. By default an adapter declaring
25
+ * `preparePreservedUndo` selects 'preserve-conflicts'; other adapters select
26
+ * 'strict'. Existing jobs always keep the policy they were created with. */
27
+ reversalPolicy?: D1ReversalPolicy;
28
+ }
29
+ /** Structurally identical to owner-claim's server-owned trust configuration. */
30
+ export interface ImportDoorClaimConfig {
31
+ issuer: string;
32
+ installationId: string;
33
+ appOrigin: string;
34
+ expectedEmail: string;
35
+ publicKeys: Record<string, JsonWebKey>;
36
+ }
37
+ interface ImportDoorMountOptions extends Omit<ImportDoorRuntimeOptions, 'parse' | 'mapping'> {
38
+ claim: ImportDoorClaimConfig;
39
+ /** Permanent door key for settings, lineage and history. Defaults to
40
+ * `claim.installationId`. Set it to the operator mount's `installationId`
41
+ * when RER assigns a different claim installation id later. */
42
+ installationId?: string;
43
+ }
44
+ /** A grant mount takes one source's `parse`/`mapping`, or a `source` resolver
45
+ * for a seed with several sources. */
46
+ export type ImportDoorOptions = ImportDoorMountOptions & (
47
+ | (Pick<ImportDoorRuntimeOptions, 'parse' | 'mapping'> & { source?: undefined })
48
+ | {
49
+ /** The seed's server-owned allow-list, shared with its owner-session mount.
50
+ * Called with the verified grant's namespace before any grant row is written;
51
+ * null refuses the grant with 403 `unknown_import_source`. */
52
+ source(namespace: string): ImportDoorSource | null | Promise<ImportDoorSource | null>;
53
+ parse?: undefined; mapping?: undefined;
54
+ });
55
+ export type DoorAccess = Pick<DoorGrant, 'action' | 'installationId' | 'email' | 'namespace'
56
+ | 'maxBytes' | 'maxChunks' | 'maxRecords' | 'fingerprint' | 'approvalRecordId' | 'approvedAt' | 'catchup'>;
57
+ export interface ImportOperatorOptions extends ImportDoorRuntimeOptions {
58
+ installationId: string;
59
+ appOrigin: string;
60
+ ownerEmail: string;
61
+ namespace: string;
62
+ /** Authenticate the owner and return a CURRENT same-database session predicate.
63
+ * Check the seed's normal CSRF policy here as well. Null denies access. */
64
+ authorize(request: Request, action: DoorAccess['action']): Promise<D1Sql | null>;
65
+ }
66
+ export interface DoorHistoryEntry {
67
+ planId: string; jobId: string; namespace: string; fingerprint: string; state: string;
68
+ owner: string; approvedAt: string; approvalRecordId: string; reviewer: string; caption: string;
69
+ /** The job's allowlisted failure reason and the durable apply-failure fact, when an apply failed. */
70
+ error: string | null; failureCode: string | null;
71
+ }
72
+ export interface DoorTarget { kind: string; id: string; ownerPlanId: string; sourceIdentity: string }
73
+ export interface DoorRecord {
74
+ kind: string; sourceId: string; sourceHash: string; targets: DoorTarget[];
75
+ explanation?: string;
76
+ }
77
+ export interface DoorDifferences {
78
+ additions: string[]; replacements: string[]; unchanged: string[];
79
+ gone: string[]; conflicts: string[];
80
+ }
81
+ export interface DoorSnapshot {
82
+ id: string; namespace: string; bundleSha256: string; kitNamespace: string;
83
+ previous: string | null; records: DoorRecord[]; sourceCount: number;
84
+ plan: ImportPlan; differences: DoorDifferences;
85
+ standingApproval: { id: string; at: string } | null;
86
+ }
87
+ export interface DoorPlanRow {
88
+ id: string; installation_id: string; namespace: string; bundle_sha256: string;
89
+ fingerprint: string; previous_id: string | null; job_id: string | null;
90
+ complete: number; snapshot_hash: string;
91
+ }
92
+ export interface DoorRequest {
93
+ options: ImportDoorRuntimeOptions; access: Readonly<DoorAccess>;
94
+ via: 'grant' | 'operator';
95
+ authority: D1Sql;
96
+ /** The destination's erasure journal rows when the request was admitted. */
97
+ epoch: number;
98
+ /** A grant request's accepted single-use nonce; owner-session requests have none. */
99
+ nonce?: { jti: string; kid: string };
100
+ }
@@ -0,0 +1,98 @@
1
+ import { Buffer } from 'node:buffer';
2
+ import { createHash, randomUUID } from 'node:crypto';
3
+ import { sha256 } from '../provenance/identity.ts';
4
+ import type { DoorRequest } from './types.ts';
5
+ import { DoorError } from './protocol.ts';
6
+ import { assertion, sql, transact } from './store.ts';
7
+ import { intakeIdentity, intakeState } from './intake-policy.ts';
8
+
9
+ export const CHUNK_BYTES = 65_536;
10
+ export const text = (value: unknown, name: string): string => {
11
+ if (typeof value !== 'string' || !value.trim() || value !== value.trim() || value.length > 256 || /[\x00-\x1f]/.test(value)) throw new DoorError('invalid_' + name, 400);
12
+ return value;
13
+ };
14
+ export const digest = (value: unknown): string => {
15
+ if (typeof value !== 'string' || !/^[a-f0-9]{64}$/.test(value)) throw new DoorError('invalid_sha256', 400);
16
+ return value;
17
+ };
18
+ export const integer = (value: unknown, min: number, max: number): number => {
19
+ if (!Number.isSafeInteger(value) || (value as number) < min || (value as number) > max) throw new DoorError('quota_or_chunk_limit', 400);
20
+ return value as number;
21
+ };
22
+ export interface UploadRow {id:string;installation_id:string;namespace:string;name:string;bundle_sha256:string;total_bytes:number;total_chunks:number;intake_json?:string|null}
23
+ export async function uploadRow(request: DoorRequest, id: string): Promise<UploadRow> {
24
+ const row = await request.options.database.prepare('SELECT * FROM ik_door_uploads WHERE id=? AND installation_id=? AND namespace=?').bind(id, request.access.installationId, request.access.namespace).first<UploadRow>();
25
+ if (!row) throw new DoorError('upload_not_found', 404);
26
+ const state = intakeState(row.intake_json);
27
+ if (!!state !== !!request.options.intake || (state && state.custody.allowlistSha256 !== intakeIdentity(request.options.intake!))) throw new DoorError('upload_intake_changed',409);
28
+ if (state && !state.complete) throw new DoorError('intake_upload_incomplete',409);
29
+ if (row.total_bytes > request.access.maxBytes || row.total_chunks > request.access.maxChunks) throw new DoorError('upload_exceeds_grant_quota');
30
+ return row;
31
+ }
32
+ export async function progress(request: DoorRequest, row: UploadRow) {
33
+ const chunks = await request.options.database.prepare('SELECT ordinal,sha256,size FROM ik_door_chunks WHERE upload_id=? ORDER BY ordinal').bind(row.id).all();
34
+ const bytes = chunks.results.reduce((n, c) => n + c.size, 0);
35
+ const complete = chunks.results.length === row.total_chunks && bytes === row.total_bytes;
36
+ const intake = intakeState(row.intake_json)?.custody;
37
+ if (complete) {
38
+ if (intake) await verifyIntakeBundle(request,row);
39
+ else await bundleBytes(request,row);
40
+ }
41
+ return {uploadId:row.id, bundleSha256:row.bundle_sha256, received:chunks.results, complete,...(intake ? {intake} : {})};
42
+ }
43
+ export async function uploads(request: DoorRequest, input: Record<string, unknown>) {
44
+ const id = input.uploadId === undefined ? randomUUID() : text(input.uploadId, 'upload_id');
45
+ if (Object.keys(input).length === 1 && input.uploadId) return progress(request, await uploadRow(request,id));
46
+ if (request.options.intake) throw new DoorError('intake_stream_required',415,{reason:'Send the complete upload stream as application/octet-stream with X-Import-Format and X-Import-Name.'});
47
+ if (Object.keys(input).some(k => !['uploadId','name','bundleSha256','totalBytes','totalChunks','index','chunkSha256','base64'].includes(k))) throw new DoorError('invalid_upload', 400);
48
+ const name = text(input.name, 'name'), hash = digest(input.bundleSha256);
49
+ const totalBytes = integer(input.totalBytes, 1, request.access.maxBytes), totalChunks = integer(input.totalChunks, 1, request.access.maxChunks);
50
+ if (totalChunks !== Math.ceil(totalBytes / CHUNK_BYTES)) throw new DoorError('invalid_chunk_layout', 400);
51
+ const index = integer(input.index, 0, totalChunks - 1), chunkHash = digest(input.chunkSha256);
52
+ if (typeof input.base64 !== 'string' || input.base64.length > Math.ceil(CHUNK_BYTES / 3) * 4) throw new DoorError('invalid_chunk', 400);
53
+ const bytes = Buffer.from(input.base64, 'base64');
54
+ if (bytes.toString('base64') !== input.base64 || bytes.length !== Math.min(CHUNK_BYTES, totalBytes - index * CHUNK_BYTES) || sha256(bytes) !== chunkHash) throw new DoorError('chunk_hash_or_size_mismatch', 400);
55
+ const g = request.access;
56
+ await transact(request.options.database, request.authority, [
57
+ sql('INSERT OR IGNORE INTO ik_door_scopes(installation_id,namespace,destination) VALUES(?,?,?)', g.installationId, g.namespace, request.options.destination),
58
+ assertion(sql('EXISTS(SELECT 1 FROM ik_door_scopes WHERE installation_id=? AND namespace=? AND destination=?)', g.installationId, g.namespace, request.options.destination)),
59
+ sql('INSERT OR IGNORE INTO ik_door_uploads(id,installation_id,namespace,name,bundle_sha256,total_bytes,total_chunks) VALUES(?,?,?,?,?,?,?)', id, g.installationId, g.namespace, name, hash, totalBytes, totalChunks),
60
+ assertion(sql('EXISTS(SELECT 1 FROM ik_door_uploads WHERE id=? AND installation_id=? AND namespace=? AND name=? AND bundle_sha256=? AND total_bytes=? AND total_chunks=?)', id, g.installationId, g.namespace, name, hash, totalBytes, totalChunks)),
61
+ sql('INSERT OR IGNORE INTO ik_door_chunks(upload_id,ordinal,sha256,size,base64) VALUES(?,?,?,?,?)', id, index, chunkHash, bytes.length, input.base64),
62
+ assertion(sql('EXISTS(SELECT 1 FROM ik_door_chunks WHERE upload_id=? AND ordinal=? AND sha256=? AND size=? AND base64=?)', id, index, chunkHash, bytes.length, input.base64)),
63
+ ]);
64
+ return progress(request,await uploadRow(request,id));
65
+ }
66
+ /** Upload chunks are 64 KiB each: a read returns a bounded page, never a whole bundle. */
67
+ const UPLOAD_READ_ROWS = 24;
68
+ /** Intake progress verifies custody without materializing the sanitized bundle. */
69
+ async function verifyIntakeBundle(request: DoorRequest, row: UploadRow): Promise<void> {
70
+ const hash = createHash('sha256'); let ordinal = 0, size = 0;
71
+ for (;;) {
72
+ const page = await request.options.database.prepare(`SELECT * FROM ik_door_chunks WHERE upload_id=? ORDER BY ordinal LIMIT ${UPLOAD_READ_ROWS} OFFSET ?`).bind(row.id,ordinal).all();
73
+ for (const chunk of page.results) {
74
+ const bytes = Buffer.from(chunk.base64,'base64');
75
+ if (chunk.ordinal !== ordinal++ || bytes.length !== chunk.size || sha256(bytes) !== chunk.sha256) throw new DoorError('upload_custody_mismatch',409);
76
+ size += bytes.length; hash.update(bytes);
77
+ }
78
+ if (page.results.length < UPLOAD_READ_ROWS) break;
79
+ }
80
+ if (ordinal !== row.total_chunks || size !== row.total_bytes || hash.digest('hex') !== row.bundle_sha256) throw new DoorError('bundle_hash_mismatch',409);
81
+ }
82
+ export async function bundleBytes(request: DoorRequest, row: UploadRow): Promise<Uint8Array> {
83
+ const chunks: Record<string, any>[] = [];
84
+ for (;;) {
85
+ const page = await request.options.database.prepare(`SELECT * FROM ik_door_chunks WHERE upload_id=? ORDER BY ordinal LIMIT ${UPLOAD_READ_ROWS} OFFSET ?`).bind(row.id, chunks.length).all();
86
+ chunks.push(...page.results);
87
+ if (page.results.length < UPLOAD_READ_ROWS) break;
88
+ }
89
+ if (chunks.length !== row.total_chunks) throw new DoorError('upload_incomplete', 409);
90
+ const values = chunks.map((c, i) => {
91
+ const bytes = Buffer.from(c.base64, 'base64');
92
+ if (c.ordinal !== i || bytes.length !== c.size || sha256(bytes) !== c.sha256) throw new DoorError('upload_custody_mismatch', 409);
93
+ return bytes;
94
+ });
95
+ const bytes = Buffer.concat(values);
96
+ if (bytes.length !== row.total_bytes || sha256(bytes) !== row.bundle_sha256) throw new DoorError('bundle_hash_mismatch', 409);
97
+ return bytes;
98
+ }
@@ -0,0 +1,40 @@
1
+ import { reflectedCredentialMatcher } from './credentials.ts';
2
+
3
+ const JSON_NUMBER = /^(-?)(0|[1-9]\d*)(?:\.(\d+))?(?:[eE]([+-]?\d+))?$/;
4
+
5
+ /** Exact decimal identity, without rounding the coefficient or expanding exponents. */
6
+ function decimalIdentity(text: string, maxExponentDigits = Infinity): string | undefined {
7
+ const parts = JSON_NUMBER.exec(text);
8
+ if (!parts) return;
9
+ const fraction = parts[3] ?? '', exponent = parts[4] ?? '0';
10
+ const digits = (parts[2] + fraction).replace(/^0+/, '');
11
+ if (!digits) return '0';
12
+ // A response exponent larger than any credential exponent plus its possible
13
+ // decimal-point adjustment cannot match. Avoid parsing an unbounded BigInt.
14
+ if (exponent.replace(/^[+-]?0*/, '').length > maxExponentDigits) return;
15
+ let end = digits.length;
16
+ while (digits[end - 1] === '0') end--;
17
+ const coefficient = digits.slice(0, end);
18
+ const scale = BigInt(exponent) + BigInt(digits.length - coefficient.length - fraction.length);
19
+ return `${parts[1]}${coefficient}e${scale}`;
20
+ }
21
+
22
+ /** Inspect received numeric text as well as values recoverable by JSON consumers. */
23
+ export function reflectedNumberCredentialMatcher(
24
+ secrets: readonly string[], reflectsText = reflectedCredentialMatcher(secrets),
25
+ ): (lexeme: string) => boolean {
26
+ if (!secrets.length) return () => false;
27
+ const identities = new Set(secrets.map(secret => decimalIdentity(secret)).filter(value => value !== undefined));
28
+ const exponentDigits = Math.max(0, ...Array.from(identities, value => value.split('e')[1]?.replace('-', '').length ?? 1));
29
+ return lexeme => {
30
+ if (reflectsText(lexeme)) return true;
31
+ const number = Number(lexeme);
32
+ if (Number.isFinite(number) && reflectsText(String(number))) return true;
33
+ if (!identities.size) return false;
34
+ // Normalization adjusts an exponent by at most the numeric text's length.
35
+ // One extra digit bounds the sum with the largest credential exponent.
36
+ const limit = Math.max(exponentDigits, String(lexeme.length).length) + 1;
37
+ const identity = decimalIdentity(lexeme, limit);
38
+ return identity !== undefined && identities.has(identity);
39
+ };
40
+ }
@@ -0,0 +1,139 @@
1
+ # Source readers
2
+
3
+ Import `readFileSource` and `readApiSource` from `index.ts`. Both either return a
4
+ complete `SourceBundle` or reject with `ReaderError` (`code`, safe `diagnostics`).
5
+ Errors deliberately omit thrown causes, response bodies, URL strings and headers.
6
+
7
+ File input is a `Uint8Array` or an async byte iterator (including Node file read
8
+ streams). UTF-8 is strict, with an optional initial BOM. JSON arrays call `parse`
9
+ once per element with its zero-based index; every other JSON root calls it once.
10
+ NDJSON calls it for each nonblank physical line, with the zero-based nonblank
11
+ index. Duplicate JSON object keys fail explicitly rather than discarding an earlier
12
+ field (including credential metadata). Locations use one-based rows; JSON arrays also have JSON pointers. A
13
+ parser may expand an object into multiple records and provide more exact pointers.
14
+ An empty parser result fails explicitly. Empty JSON arrays/NDJSON are valid empty
15
+ exports. Source kinds and IDs are nonempty strings. Duplicate IDs are retained
16
+ with diagnostics, including conflicting content; the core owns collapse/rejection.
17
+
18
+ Defaults are 64 MiB total source bytes, 100,000 records and 1 MiB per record.
19
+ Limits must be positive safe integers. Total bytes include every successful API
20
+ page. Record bytes bound serialized input objects (files) and complete parsed
21
+ records (both readers); NDJSON physical lines are bounded before parsing. JSON
22
+ and API pages are buffered up to the total-byte limit; NDJSON is parsed per line.
23
+ Streams are pulled serially and closed on early failure. The Promise-returning
24
+ frozen API retains the bounded completed record bundle in memory.
25
+
26
+ API `parsePage` receives the JSON document and a one-based page number. It must
27
+ report every source object, including unsupported objects as records for the
28
+ mapping's unmapped report. It declares the next URL (relative or absolute), or
29
+ null/undefined for completion. Empty pages are supported. As a generic reader,
30
+ the kit cannot infer whether a format adapter deliberately omitted a vendor field
31
+ or object; completeness inside an API envelope is the adapter's responsibility.
32
+ Pages have stable names `api-page-N`; request metadata is not added to the bundle.
33
+ Accepted cursor metadata present in response bytes remains in exact originals.
34
+
35
+ API readers may opt into `preserveNumberLexeme(pathSegments): boolean`. Each
36
+ numeric value is selected by its decoded path: object keys are strings, array
37
+ indices are numbers, and the root is `[]`. Only selected numeric primitives become
38
+ ordinary JSON strings containing the exact source token, including sign, trailing
39
+ zeros and exponent spelling. No arithmetic or normalization occurs. For example,
40
+ `path.length === 4 && path[0] === 'records' && typeof path[1] === 'number' &&
41
+ path[2] === 'fields' && typeof path[3] === 'string'` selects direct record cells.
42
+ Metadata, nested objects and array cells remain unchanged by that selector.
43
+
44
+ The additive third `parsePage` argument is an `ApiPageContext` with
45
+ `numericLexemes`, a frozen, null-prototype read-only object mapping JSON pointers
46
+ to selected tokens. Pointer segments escape `~` as `~0` and `/` as `~1`; the root
47
+ pointer is `''`. Context is page-local, contains no transport metadata or second
48
+ response body, and is empty when preservation is omitted. It distinguishes source
49
+ number `42` (selected string with provenance) from source string `"42"` (no entry).
50
+ Both the context and each policy path are frozen. Return a synchronous boolean;
51
+ invalid policies, non-boolean results and thrown exceptions fail closed without
52
+ exposing thrown causes. Policy failure uses `SOURCE_NUMBER_POLICY_FAILED`.
53
+
54
+ Syntax, duplicate keys and credentials in every decoded source key or string are
55
+ validated before either callback. Known authentication values reflected as boolean
56
+ or null scalars are rejected too. Numeric authentication reflections are checked
57
+ against exact received numeric text, equivalent decimal values and ordinary JSON
58
+ Number results, whether or not preservation is selected. Authentication values
59
+ themselves are never rounded for comparison. Selected overflow tokens such
60
+ as `1e309` can therefore reach product validation as strings; unselected nonfinite
61
+ numbers retain the ordinary rejection. The product owns numeric range, canonical
62
+ representation, type-conversion accounting and retention of any needed per-cell
63
+ numeric-origin provenance. Omission keeps ordinary JSON Number semantics. These
64
+ options do not change byte, record, request, cancellation or custody guarantees.
65
+
66
+ Every artifact hashes the exact successful response bytes and retains them as
67
+ `original: { status: "available", base64 }` using canonical base64. File originals
68
+ also preserve all bytes, including BOM, line endings, whitespace and parser-excluded
69
+ wrappers. Custody is copied in bounded 64 KiB blocks, so tiny or reused stream
70
+ buffers cannot lose original bytes or allocate one retained object per chunk.
71
+ Payload hashes bind canonical JSON. Whitespace changes therefore change artifact identity. HTTP error
72
+ and redirect bodies are discarded without parsing or buffering.
73
+
74
+ The transport receives a GET `Request` with manual redirects and omitted ambient
75
+ credentials, and **must honor these settings**. Fetch is a suitable transport.
76
+ The reader handles same-origin redirects itself, rejects automatically followed
77
+ redirects, cross-origin links, userinfo, fragments and credential-bearing query
78
+ parameters, and preserves all accepted pagination parameters and request headers.
79
+ The API reader never strips URL credentials. Tokens belong in the ephemeral `token`
80
+ or `headers` arguments. A conflicting Authorization header fails. Credential keys,
81
+ credential URLs and reflected credential values in source data fail, even if a
82
+ parser would omit them. Null, an empty string or a mask (`****`, `[redacted]`)
83
+ under a credential key is not a credential and passes. No token, response error
84
+ text or credential-bearing URL is returned in bundle metadata or diagnostics; a
85
+ credential refusal names only the field kind in its diagnostic `field`.
86
+
87
+ `readFileSource` takes `credentials: 'remove'` to keep records that hold a plain
88
+ credential value: the parser sees the value as read, then each record loses its
89
+ credential fields (any key whose whole name parts are a credential word or
90
+ phrase, such as `httpPassword`, `client_secret`, `X-Api-Key`, `secret_key`,
91
+ `Ocp-Apim-Subscription-Key`, `pwd` or `X-Auth`, never `keyword` or `author`; a plural such as
92
+ `apiKeys` or `tokens` when it holds text; the `value` of a `{name|key, value}`
93
+ pair naming one, and of such a `[name, value]` pair in a list of them; booleans
94
+ stay) and its URLs' (any scheme's)
95
+ user names, passwords and credential query and fragment parameters, with one
96
+ `CREDENTIAL_REMOVED` warning per removal naming the record,
97
+ the field pointer and its kind. The original bytes are then withheld
98
+ (`reason: 'credentials-removed'`). A record still repeating a removed value
99
+ fails with `CREDENTIAL_IN_SOURCE_DATA`; forms it cannot remove still fail as
100
+ above. A field holding `[removed]`, the marker rer's vendor exports leave, is
101
+ reported as removed before upload. A URL's `removed@` user name and a
102
+ `=removed` parameter, rer's URL markers, are removed and withhold the original
103
+ too (refuse mode refuses them), but are not live values: `readFileSourceRemovals`
104
+ answers the bundle with `liveCredentialsRemoved`, true only when a live value
105
+ went.
106
+
107
+ All requests, including redirects/retries, count toward `maxRequests` (default
108
+ 1000). Rate limits (429, or 403/503 with Retry-After) allow `maxRetries` (default 3)
109
+ per requested page/redirect target. Retry-After accepts seconds or HTTP dates;
110
+ missing 429 Retry-After uses 1s, 2s, 4s. A wait exceeding `maxRetryAfterMs` (default
111
+ 30s) fails instead of retrying before the server permits. Invalid Retry-After
112
+ fails. Cycles, exhausted requests/retries and later-page failure reject the entire
113
+ read. `signal` cancels requests, body reads and waits; reasons are never exposed.
114
+ The optional sleep adapter receives both delay and signal. No concurrent page
115
+ prefetch or unbounded error-response reads occur.
116
+
117
+ ### Explicit noncredential pagination cursors
118
+
119
+ Trusted adapters may select `paginationCursorPolicy: 'page-token-v1'` on
120
+ `readApiSource` or `readFileSource` when their source documents `page_token` as a
121
+ pagination cursor. Omission retains the strict default. This is a closed policy,
122
+ not an arbitrary allowlist: only the exact URL query spelling `page_token` is
123
+ exempt from the token-key prohibition. Encoded or case aliases, actual credential
124
+ keys, URL userinfo, cross-origin navigation/redirects, and reflected provided
125
+ header/token secrets remain rejected. No URL or original bytes are rewritten.
126
+ The exact wire query boundaries own the cursor value: encoded delimiters inside
127
+ it remain opaque data, including repeated encoding. Nested credential-bearing
128
+ URLs in that value and encoded credential parameters outside it still reject.
129
+ JSON pagination fields such as `next_page_token` and `previous_page_token` remain
130
+ ordinary noncredential metadata; API reflection checks still inspect their values.
131
+
132
+ The reader records the selection as `SourceBundle.paginationCursorPolicy`.
133
+ Adapters combining bundles must deliberately preserve that selection. Planning,
134
+ job creation and retained-original readback validate it and bind it in the reviewed
135
+ fingerprint. Never derive the selection from untrusted exported JSON. Saved files
136
+ have no in-memory authentication context, so adapters must only select this policy
137
+ for documented noncredential cursors; they cannot identify arbitrary secrets
138
+ hidden in otherwise ordinary values. Mapped target effects retain the strict
139
+ credential policy: the exception is for source custody and navigation.
@@ -0,0 +1,67 @@
1
+ import type { PaginationCursorPolicy } from '../types.ts';
2
+ import { credentialQueryKey, paginationPolicy } from '../provenance/credentials.ts';
3
+ import { setTimeout as delay } from 'node:timers/promises';
4
+ import { fail } from './errors.ts';
5
+ import { safeValue } from './records.ts';
6
+
7
+ export function boundedInteger(value: number | undefined, fallback: number, allowZero = false): number {
8
+ const result = value ?? fallback;
9
+ if (!Number.isSafeInteger(result) || result < (allowZero ? 0 : 1)) fail('INVALID_API_LIMIT');
10
+ return result;
11
+ }
12
+
13
+ /** Reject credentials instead of stripping them and changing pagination semantics. */
14
+ export function sourceUrl(value: string, base?: URL, secrets: readonly string[] = [], policy?: PaginationCursorPolicy): URL {
15
+ paginationPolicy(policy);
16
+ if (typeof value !== 'string' || !value.trim() || value !== value.trim()) fail('INVALID_SOURCE_URL');
17
+ let url: URL;
18
+ try { url = new URL(value, base); } catch { fail('INVALID_SOURCE_URL'); }
19
+ if (!['https:', 'http:'].includes(url.protocol) || url.username || url.password || url.hash) fail('UNSAFE_SOURCE_URL');
20
+ if (base && url.origin !== base.origin) fail('CROSS_ORIGIN_SOURCE_URL');
21
+ try {
22
+ for (const pair of url.search.slice(1).split('&')) {
23
+ if (credentialQueryKey(pair.split('=')[0], policy)) fail('CREDENTIAL_IN_SOURCE_URL');
24
+ }
25
+ } catch { fail('CREDENTIAL_IN_SOURCE_URL'); }
26
+ for (const item of url.searchParams.values()) safeValue(item, secrets, undefined, policy);
27
+ try { decodeURIComponent(url.href); } catch { fail('INVALID_SOURCE_URL'); }
28
+ safeValue(value, secrets, undefined, policy);
29
+ // Inspect the resolved wire URL too, without turning escaped value delimiters into syntax.
30
+ safeValue(url.href, secrets, undefined, policy);
31
+ return url;
32
+ }
33
+
34
+ export function requestHeaders(headers: Record<string, string> | undefined, token?: string): { headers: Headers; secrets: string[] } {
35
+ let result: Headers;
36
+ try { result = new Headers(headers); } catch { fail('INVALID_API_HEADERS'); }
37
+ const secrets: string[] = [];
38
+ for (const [key, value] of result) {
39
+ if (/authorization|cookie|token|secret|api[-_]?key/i.test(key)) {
40
+ if (value) secrets.push(value);
41
+ if (/authorization/i.test(key)) { const credential = value.replace(/^\S+\s+/, ''); if (credential) secrets.push(credential); }
42
+ }
43
+ }
44
+ if (token !== undefined) {
45
+ if (typeof token !== 'string' || !token || /\s/.test(token)) fail('INVALID_API_TOKEN');
46
+ if (result.has('authorization') && result.get('authorization') !== `Bearer ${token}`) fail('CONFLICTING_API_CREDENTIALS');
47
+ try { result.set('authorization', `Bearer ${token}`); } catch { fail('INVALID_API_TOKEN'); }
48
+ secrets.push(token);
49
+ }
50
+ return { headers: result, secrets };
51
+ }
52
+
53
+ export function retryDelay(header: string | null, retries: number, maximum: number): number {
54
+ let ms: number;
55
+ if (header === null) ms = 1000 * 2 ** retries;
56
+ else if (/^\d+(?:\.\d+)?$/.test(header.trim())) ms = Math.ceil(Number(header) * 1000);
57
+ else {
58
+ // Require an HTTP date, rather than Date.parse's permissive numeric/date shorthands.
59
+ if (!/^[A-Za-z]{3}, \d{2} [A-Za-z]{3} \d{4} \d{2}:\d{2}:\d{2} GMT$/.test(header)) fail('INVALID_RETRY_AFTER');
60
+ const time = Date.parse(header);
61
+ if (!Number.isFinite(time)) fail('INVALID_RETRY_AFTER');
62
+ ms = Math.max(0, time - Date.now());
63
+ }
64
+ if (!Number.isSafeInteger(ms) || ms > maximum) fail('RETRY_AFTER_LIMIT');
65
+ return ms;
66
+ }
67
+ export async function sleep(ms: number, signal?: AbortSignal): Promise<void> { await delay(ms, undefined, { signal }); }