@runeditrun/rer 0.2.81 → 0.2.82
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -0
- package/cli/export.mjs +1 -1
- package/edition-authoring/base-copies.mjs +3 -1
- package/edition-authoring/copy-checkout.mjs +152 -1
- package/edition-authoring/files.mjs +1 -1
- package/edition-authoring/runtime-dependencies.mjs +1 -1
- package/global/CHANGELOG.md +13 -0
- package/global/README.md +12 -0
- package/modules/import-kit/src/door/ADOPTING.md +452 -0
- package/modules/import-kit/src/door/README.md +553 -0
- package/modules/import-kit/src/door/admission.ts +45 -0
- package/modules/import-kit/src/door/archive.ts +129 -0
- package/modules/import-kit/src/door/authority.ts +107 -0
- package/modules/import-kit/src/door/failures.ts +70 -0
- package/modules/import-kit/src/door/grants.ts +162 -0
- package/modules/import-kit/src/door/handler.ts +90 -0
- package/modules/import-kit/src/door/index.ts +20 -0
- package/modules/import-kit/src/door/intake/csv.ts +74 -0
- package/modules/import-kit/src/door/intake/json.ts +182 -0
- package/modules/import-kit/src/door/intake/projection.ts +61 -0
- package/modules/import-kit/src/door/intake/types.ts +28 -0
- package/modules/import-kit/src/door/intake-policy.ts +38 -0
- package/modules/import-kit/src/door/intake-upload.ts +91 -0
- package/modules/import-kit/src/door/inventory.ts +68 -0
- package/modules/import-kit/src/door/jobs.ts +134 -0
- package/modules/import-kit/src/door/limits.ts +16 -0
- package/modules/import-kit/src/door/operator.ts +41 -0
- package/modules/import-kit/src/door/planning.ts +227 -0
- package/modules/import-kit/src/door/protocol.ts +38 -0
- package/modules/import-kit/src/door/refusals.ts +51 -0
- package/modules/import-kit/src/door/schema.ts +101 -0
- package/modules/import-kit/src/door/store.ts +152 -0
- package/modules/import-kit/src/door/types.ts +100 -0
- package/modules/import-kit/src/door/uploads.ts +98 -0
- package/modules/import-kit/src/provenance/numeric-credentials.ts +40 -0
- package/modules/import-kit/src/readers/README.md +139 -0
- package/modules/import-kit/src/readers/api-policy.ts +67 -0
- package/modules/import-kit/src/readers/api.ts +86 -0
- package/modules/import-kit/src/readers/bytes.ts +73 -0
- package/modules/import-kit/src/readers/errors.ts +33 -0
- package/modules/import-kit/src/readers/file.ts +74 -0
- package/modules/import-kit/src/readers/index.ts +3 -0
- package/modules/import-kit/src/readers/json.ts +29 -0
- package/modules/import-kit/src/readers/numeric-lexemes.ts +104 -0
- package/modules/import-kit/src/readers/original.ts +31 -0
- package/modules/import-kit/src/readers/records.ts +105 -0
- package/package.json +3 -2
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
import { canonicalJson, sha256 } from '../provenance/identity.ts';
|
|
2
|
+
import { planFingerprint } from '../plan/index.ts';
|
|
3
|
+
import type { D1Database, D1Sql } from '../d1/types.ts';
|
|
4
|
+
import type { DoorPlanRow, DoorRequest, DoorSnapshot } from './types.ts';
|
|
5
|
+
import { DoorError } from './protocol.ts';
|
|
6
|
+
import { importDoorSchemaCheck } from './schema.ts';
|
|
7
|
+
|
|
8
|
+
export const sql = (sql: string, ...params: unknown[]): D1Sql => ({ sql, params });
|
|
9
|
+
export const statement = (db: D1Database, query: D1Sql) => db.prepare(query.sql).bind(...query.params ?? []);
|
|
10
|
+
export const assertion = (query: D1Sql) => sql(`INSERT INTO ik_d1_assertions(ok) SELECT CASE WHEN (${query.sql}) THEN 1 ELSE 0 END`, ...query.params ?? []);
|
|
11
|
+
export async function transact(db: D1Database, authority: D1Sql, queries: D1Sql[]): Promise<void> {
|
|
12
|
+
await db.batch([assertion(authority), ...queries, assertion(authority), sql('DELETE FROM ik_d1_assertions')].map(q => statement(db, q)));
|
|
13
|
+
}
|
|
14
|
+
export async function authorized(request: DoorRequest): Promise<void> {
|
|
15
|
+
const row = await statement(request.options.database, sql(`SELECT CASE WHEN (${request.authority.sql}) THEN 1 ELSE 0 END AS ok`, ...request.authority.params ?? [])).first();
|
|
16
|
+
if (row?.ok !== 1) throw new DoorError(request.via === 'operator' ? 'owner_required' : 'grant_revoked_or_expired');
|
|
17
|
+
}
|
|
18
|
+
export async function planRow(request: DoorRequest, id: string): Promise<DoorPlanRow> {
|
|
19
|
+
const row = await request.options.database.prepare('SELECT * FROM ik_door_plans WHERE id=? AND installation_id=? AND namespace=? AND complete=1')
|
|
20
|
+
.bind(id, request.access.installationId, request.access.namespace).first<DoorPlanRow>();
|
|
21
|
+
if (!row) throw new DoorError('plan_not_found', 404);
|
|
22
|
+
return row;
|
|
23
|
+
}
|
|
24
|
+
/** A saved plan's snapshot. A person's erasure withdraws every chunk of it:
|
|
25
|
+
* the plan, its approval and its kit job stay, the reviewed records do not. */
|
|
26
|
+
/** Snapshot rows are at most 32,000 characters; a read returns a bounded page
|
|
27
|
+
* of them, never a whole multi-megabyte snapshot in one D1 result. */
|
|
28
|
+
const SNAPSHOT_READ_ROWS = 48;
|
|
29
|
+
export async function loadSnapshot(request: DoorRequest, row: DoorPlanRow): Promise<DoorSnapshot> {
|
|
30
|
+
const chunks: {ordinal: unknown; body: string}[] = [];
|
|
31
|
+
for (;;) {
|
|
32
|
+
const page = await request.options.database.prepare(`SELECT ordinal,body FROM ik_door_plan_chunks WHERE plan_id=? ORDER BY ordinal LIMIT ${SNAPSHOT_READ_ROWS} OFFSET ?`).bind(row.id, chunks.length).all<{ordinal: unknown; body: string}>();
|
|
33
|
+
chunks.push(...page.results);
|
|
34
|
+
if (page.results.length < SNAPSHOT_READ_ROWS) break;
|
|
35
|
+
}
|
|
36
|
+
if (!chunks.length) throw new DoorError('snapshot_withdrawn', 410, {planId: row.id});
|
|
37
|
+
// A torn read across pages fails the ordinals or the snapshot hash below.
|
|
38
|
+
if (chunks.some((c, i) => c.ordinal !== i)) throw new DoorError('plan_custody_mismatch', 409);
|
|
39
|
+
const snapshot = JSON.parse(chunks.map(c => c.body).join('')) as DoorSnapshot;
|
|
40
|
+
if (sha256(canonicalJson(snapshot)) !== row.snapshot_hash || snapshot.id !== row.id || snapshot.namespace !== row.namespace || snapshot.bundleSha256 !== row.bundle_sha256
|
|
41
|
+
|| snapshot.previous !== row.previous_id || snapshot.plan.fingerprint !== row.fingerprint
|
|
42
|
+
|| planFingerprint(snapshot.plan) !== row.fingerprint
|
|
43
|
+
|| snapshot.plan.namespace !== snapshot.kitNamespace || snapshot.plan.destination !== request.options.destination
|
|
44
|
+
|| snapshot.plan.principalId !== request.options.ownerPrincipalId) throw new DoorError('plan_custody_mismatch', 409);
|
|
45
|
+
return snapshot;
|
|
46
|
+
}
|
|
47
|
+
/** Recover indexing after a committed kit batch or an owner undo through the seed.
|
|
48
|
+
* Kit state remains authoritative. Never hide a partially applied pending import. */
|
|
49
|
+
export async function lineage(request: DoorRequest): Promise<{ head_id: string | null; pending_id: string | null }> {
|
|
50
|
+
const db = request.options.database, installation = request.access.installationId, namespace = request.access.namespace;
|
|
51
|
+
let current = await db.prepare('SELECT head_id,pending_id FROM ik_door_namespaces WHERE installation_id=? AND namespace=?').bind(installation, namespace).first<{head_id:string|null;pending_id:string|null}>();
|
|
52
|
+
if (!current) return { head_id: null, pending_id: null };
|
|
53
|
+
for (;;) {
|
|
54
|
+
const candidate = current.pending_id ?? current.head_id;
|
|
55
|
+
if (!candidate) return current;
|
|
56
|
+
const row = await db.prepare('SELECT p.previous_id,j.state,j.checkpoint FROM ik_door_plans p LEFT JOIN ik_jobs j ON j.id=p.job_id WHERE p.id=?').bind(candidate).first();
|
|
57
|
+
let next: {head_id:string|null;pending_id:string|null} = current;
|
|
58
|
+
if (current.pending_id && row?.state === 'completed') next = { head_id: candidate, pending_id: null };
|
|
59
|
+
else if (current.pending_id && row?.state === 'failed' && row.checkpoint === 0) next = { head_id: current.head_id, pending_id: null };
|
|
60
|
+
else if (row && ['undone', 'rolled_back'].includes(row.state)) next = { head_id: current.pending_id ? current.head_id : row.previous_id, pending_id: null };
|
|
61
|
+
if (next === current) return current;
|
|
62
|
+
await transact(db, request.authority, [sql('UPDATE ik_door_namespaces SET head_id=?,pending_id=? WHERE installation_id=? AND namespace=? AND head_id IS ? AND pending_id IS ?', next.head_id, next.pending_id, installation, namespace, current.head_id, current.pending_id)]);
|
|
63
|
+
current = await db.prepare('SELECT head_id,pending_id FROM ik_door_namespaces WHERE installation_id=? AND namespace=?').bind(installation, namespace).first<typeof next>();
|
|
64
|
+
if (!current) throw new DoorError('lineage_conflict', 409);
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
/** A saved plan's snapshot is still held. An erasure withdraws all of its chunks at once. */
|
|
68
|
+
export const snapshotHeld = (planId: string): D1Sql => sql('EXISTS(SELECT 1 FROM ik_door_plan_chunks WHERE plan_id=?)', planId);
|
|
69
|
+
/** Door work already in flight never writes back or applies content an erasure
|
|
70
|
+
* withdrew meanwhile: each of its batches asserts that what it builds on is still
|
|
71
|
+
* held. A batch refused because it is not answers 410; any other failure stands. */
|
|
72
|
+
export async function fenced<T>(request: DoorRequest, planId: string, held: D1Sql, work: () => Promise<T>): Promise<T> {
|
|
73
|
+
try { return await work(); }
|
|
74
|
+
catch (error) {
|
|
75
|
+
if (error instanceof DoorError) throw error;
|
|
76
|
+
const row = await statement(request.options.database, sql(`SELECT CASE WHEN (${held.sql}) THEN 1 ELSE 0 END AS ok`, ...held.params ?? [])).first();
|
|
77
|
+
if (row?.ok !== 1) throw new DoorError('snapshot_withdrawn', 410, {planId});
|
|
78
|
+
throw error;
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
/** JSON arrays of rows of at most ~1.9 MB each, the kit's own packing ceiling,
|
|
82
|
+
* each with the number of rows it holds. */
|
|
83
|
+
function packRows(rows: unknown[]): {json: string; count: number}[] {
|
|
84
|
+
const packed: {json: string; count: number}[] = []; let parts: string[] = [], size = 2;
|
|
85
|
+
const flush = () => { packed.push({json: `[${parts.join(',')}]`, count: parts.length}); parts = []; size = 2; };
|
|
86
|
+
for (const row of rows) {
|
|
87
|
+
const value = JSON.stringify(row), bytes = new TextEncoder().encode(value).length + 1;
|
|
88
|
+
if (parts.length && size + bytes > 1_900_000) flush();
|
|
89
|
+
parts.push(value); size += bytes;
|
|
90
|
+
}
|
|
91
|
+
if (parts.length) flush();
|
|
92
|
+
return packed;
|
|
93
|
+
}
|
|
94
|
+
/** `retainedFrom` names the plan whose records a re-plan read because the
|
|
95
|
+
* upload's credential-bearing bytes went: that plan's snapshot must stay held
|
|
96
|
+
* instead of the upload's chunks. */
|
|
97
|
+
export async function saveSnapshot(request: DoorRequest, snapshot: DoorSnapshot, upload: {id: string}, retainedFrom?: string): Promise<void> {
|
|
98
|
+
const db = request.options.database, g = request.access;
|
|
99
|
+
await transact(db, request.authority, [
|
|
100
|
+
sql('INSERT OR IGNORE INTO ik_door_namespaces(installation_id,namespace) VALUES(?,?)', g.installationId, g.namespace),
|
|
101
|
+
assertion(sql('EXISTS(SELECT 1 FROM ik_door_namespaces WHERE installation_id=? AND namespace=? AND head_id IS ? AND pending_id IS NULL)', g.installationId, g.namespace, snapshot.previous)),
|
|
102
|
+
sql('INSERT INTO ik_door_plans(id,installation_id,namespace,bundle_sha256,fingerprint,previous_id,snapshot_hash) VALUES(?,?,?,?,?,?,?)', snapshot.id, g.installationId, g.namespace, snapshot.bundleSha256, snapshot.plan.fingerprint, snapshot.previous, sha256(canonicalJson(snapshot))),
|
|
103
|
+
sql('UPDATE ik_door_namespaces SET pending_id=? WHERE installation_id=? AND namespace=?', snapshot.id, g.installationId, g.namespace),
|
|
104
|
+
]);
|
|
105
|
+
// Private staging remains visibly pending until every immutable chunk is stored.
|
|
106
|
+
// Each statement inserts up to ~1.9 MB of 32,000-character rows, as the kit
|
|
107
|
+
// packs its own plan chunks, so storing a snapshot costs one statement per
|
|
108
|
+
// ~1.9 MB rather than per row; no domain effect occurs.
|
|
109
|
+
const body = canonicalJson(snapshot), rows: {ordinal: number; body: string}[] = [];
|
|
110
|
+
for (let offset = 0; offset < body.length;) {
|
|
111
|
+
let end = Math.min(offset + 32_000,body.length);
|
|
112
|
+
if (end < body.length && /[\uD800-\uDBFF]/.test(body[end-1])) end--;
|
|
113
|
+
rows.push({ordinal: rows.length, body: body.slice(offset,end)});
|
|
114
|
+
offset = end;
|
|
115
|
+
}
|
|
116
|
+
// Each batch, and the one that completes the plan, requires the whole source
|
|
117
|
+
// upload, a catch-up's previous snapshot and every chunk written before it.
|
|
118
|
+
const held = (written: number) => sql(`EXISTS(SELECT 1 FROM ik_door_uploads u WHERE u.id=? AND u.installation_id=? AND u.namespace=? AND u.bundle_sha256=?
|
|
119
|
+
AND (SELECT count(*) FROM ik_door_chunks c WHERE c.upload_id=u.id)=(CASE WHEN ? IS NULL THEN u.total_chunks ELSE 0 END))
|
|
120
|
+
AND (? IS NULL OR EXISTS(SELECT 1 FROM ik_door_plan_chunks WHERE plan_id=?))
|
|
121
|
+
AND (? IS NULL OR EXISTS(SELECT 1 FROM ik_door_plan_chunks WHERE plan_id=?))
|
|
122
|
+
AND (SELECT count(*) FROM ik_door_plan_chunks WHERE plan_id=?)=?`,
|
|
123
|
+
upload.id, g.installationId, g.namespace, snapshot.bundleSha256, retainedFrom ?? null, snapshot.previous, snapshot.previous, retainedFrom ?? null, retainedFrom ?? null, snapshot.id, written);
|
|
124
|
+
const batch = (written: number, queries: D1Sql[]) => fenced(request, snapshot.id, held(written), () => transact(db, request.authority, [assertion(held(written)), ...queries]));
|
|
125
|
+
let written = 0;
|
|
126
|
+
for (const packed of packRows(rows)) {
|
|
127
|
+
await batch(written, [sql("INSERT INTO ik_door_plan_chunks(plan_id,ordinal,body) SELECT ?,json_extract(value,'$.ordinal'),json_extract(value,'$.body') FROM json_each(?)", snapshot.id, packed.json)]);
|
|
128
|
+
written += packed.count;
|
|
129
|
+
}
|
|
130
|
+
await batch(written, [sql('UPDATE ik_door_plans SET complete=1 WHERE id=?', snapshot.id)]);
|
|
131
|
+
}
|
|
132
|
+
/** The owner-session mount has no opt-in, and refuses an outdated schema just as loudly. */
|
|
133
|
+
export async function schemaCurrent(database: D1Database): Promise<boolean> {
|
|
134
|
+
const row = await database.prepare(importDoorSchemaCheck).first();
|
|
135
|
+
if (row?.current !== 1) throw new DoorError('import_door_schema_outdated', 503);
|
|
136
|
+
return true;
|
|
137
|
+
}
|
|
138
|
+
export const erasures = async (database: D1Database, destination: string): Promise<number> =>
|
|
139
|
+
Number((await database.prepare('SELECT count(*) AS n FROM ik_d1_erasure_journal WHERE destination=?').bind(destination).first<{n: number}>())?.n ?? 0);
|
|
140
|
+
/** Whether an erasure in this destination committed after the request was admitted. */
|
|
141
|
+
export const erasedSince = async (request: DoorRequest): Promise<boolean> => await erasures(request.options.database, request.options.destination) !== request.epoch;
|
|
142
|
+
/** What every batch of a door request asserts, whoever admitted it: the
|
|
143
|
+
* destination's erasure count at admission (an erasure committed meanwhile
|
|
144
|
+
* wins), and that the logical namespace belongs to this destination alone. */
|
|
145
|
+
export function doorFence(key: string, namespace: string, destination: string, epoch: number): D1Sql {
|
|
146
|
+
return sql(`(SELECT count(*) FROM ik_d1_erasure_journal WHERE destination=?)=?
|
|
147
|
+
AND NOT EXISTS(SELECT 1 FROM ik_door_scopes WHERE installation_id=? AND namespace=? AND destination<>?)
|
|
148
|
+
AND NOT EXISTS(SELECT 1 FROM ik_door_plans p JOIN ik_jobs j
|
|
149
|
+
ON j.id=p.job_id OR (p.job_id IS NULL AND j.operation_key='door:'||p.id)
|
|
150
|
+
WHERE p.installation_id=? AND p.namespace=? AND j.destination<>?)`,
|
|
151
|
+
destination, epoch, key, namespace, destination, key, namespace, destination);
|
|
152
|
+
}
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
import type { D1Database, D1Sql, D1TargetAdapter } from '../d1/types.ts';
|
|
2
|
+
import type { D1ReversalPolicy } from '../d1/undo-types.ts';
|
|
3
|
+
import type { ImportMapping, ImportPlan, Json, ParsedRecord, ReaderLimits } from '../types.ts';
|
|
4
|
+
import type { DoorGrant } from './protocol.ts';
|
|
5
|
+
import type { ImportDoorIntake } from './intake/types.ts';
|
|
6
|
+
export type { ImportDoorIntake, IntakeFormat } from './intake/types.ts';
|
|
7
|
+
|
|
8
|
+
/** One import source: its parser and mapping, and optionally its own owner principal. */
|
|
9
|
+
export interface ImportDoorSource {
|
|
10
|
+
mapping: ImportMapping;
|
|
11
|
+
parse(value: Json, index: number): ParsedRecord | ParsedRecord[];
|
|
12
|
+
ownerPrincipalId?: string;
|
|
13
|
+
/** Project declared fields before the first persistent byte. Shared by both mounts. */
|
|
14
|
+
intake?: ImportDoorIntake;
|
|
15
|
+
}
|
|
16
|
+
export interface ImportDoorRuntimeOptions extends ImportDoorSource {
|
|
17
|
+
database: D1Database;
|
|
18
|
+
targets: D1TargetAdapter;
|
|
19
|
+
/** The seed's actual owner principal, so its ordinary import history/undo works. */
|
|
20
|
+
ownerPrincipalId: string;
|
|
21
|
+
destination: string;
|
|
22
|
+
format: 'json' | 'ndjson';
|
|
23
|
+
limits?: ReaderLimits;
|
|
24
|
+
/** Override the undo policy for new jobs. By default an adapter declaring
|
|
25
|
+
* `preparePreservedUndo` selects 'preserve-conflicts'; other adapters select
|
|
26
|
+
* 'strict'. Existing jobs always keep the policy they were created with. */
|
|
27
|
+
reversalPolicy?: D1ReversalPolicy;
|
|
28
|
+
}
|
|
29
|
+
/** Structurally identical to owner-claim's server-owned trust configuration. */
|
|
30
|
+
export interface ImportDoorClaimConfig {
|
|
31
|
+
issuer: string;
|
|
32
|
+
installationId: string;
|
|
33
|
+
appOrigin: string;
|
|
34
|
+
expectedEmail: string;
|
|
35
|
+
publicKeys: Record<string, JsonWebKey>;
|
|
36
|
+
}
|
|
37
|
+
interface ImportDoorMountOptions extends Omit<ImportDoorRuntimeOptions, 'parse' | 'mapping'> {
|
|
38
|
+
claim: ImportDoorClaimConfig;
|
|
39
|
+
/** Permanent door key for settings, lineage and history. Defaults to
|
|
40
|
+
* `claim.installationId`. Set it to the operator mount's `installationId`
|
|
41
|
+
* when RER assigns a different claim installation id later. */
|
|
42
|
+
installationId?: string;
|
|
43
|
+
}
|
|
44
|
+
/** A grant mount takes one source's `parse`/`mapping`, or a `source` resolver
|
|
45
|
+
* for a seed with several sources. */
|
|
46
|
+
export type ImportDoorOptions = ImportDoorMountOptions & (
|
|
47
|
+
| (Pick<ImportDoorRuntimeOptions, 'parse' | 'mapping'> & { source?: undefined })
|
|
48
|
+
| {
|
|
49
|
+
/** The seed's server-owned allow-list, shared with its owner-session mount.
|
|
50
|
+
* Called with the verified grant's namespace before any grant row is written;
|
|
51
|
+
* null refuses the grant with 403 `unknown_import_source`. */
|
|
52
|
+
source(namespace: string): ImportDoorSource | null | Promise<ImportDoorSource | null>;
|
|
53
|
+
parse?: undefined; mapping?: undefined;
|
|
54
|
+
});
|
|
55
|
+
export type DoorAccess = Pick<DoorGrant, 'action' | 'installationId' | 'email' | 'namespace'
|
|
56
|
+
| 'maxBytes' | 'maxChunks' | 'maxRecords' | 'fingerprint' | 'approvalRecordId' | 'approvedAt' | 'catchup'>;
|
|
57
|
+
export interface ImportOperatorOptions extends ImportDoorRuntimeOptions {
|
|
58
|
+
installationId: string;
|
|
59
|
+
appOrigin: string;
|
|
60
|
+
ownerEmail: string;
|
|
61
|
+
namespace: string;
|
|
62
|
+
/** Authenticate the owner and return a CURRENT same-database session predicate.
|
|
63
|
+
* Check the seed's normal CSRF policy here as well. Null denies access. */
|
|
64
|
+
authorize(request: Request, action: DoorAccess['action']): Promise<D1Sql | null>;
|
|
65
|
+
}
|
|
66
|
+
export interface DoorHistoryEntry {
|
|
67
|
+
planId: string; jobId: string; namespace: string; fingerprint: string; state: string;
|
|
68
|
+
owner: string; approvedAt: string; approvalRecordId: string; reviewer: string; caption: string;
|
|
69
|
+
/** The job's allowlisted failure reason and the durable apply-failure fact, when an apply failed. */
|
|
70
|
+
error: string | null; failureCode: string | null;
|
|
71
|
+
}
|
|
72
|
+
export interface DoorTarget { kind: string; id: string; ownerPlanId: string; sourceIdentity: string }
|
|
73
|
+
export interface DoorRecord {
|
|
74
|
+
kind: string; sourceId: string; sourceHash: string; targets: DoorTarget[];
|
|
75
|
+
explanation?: string;
|
|
76
|
+
}
|
|
77
|
+
export interface DoorDifferences {
|
|
78
|
+
additions: string[]; replacements: string[]; unchanged: string[];
|
|
79
|
+
gone: string[]; conflicts: string[];
|
|
80
|
+
}
|
|
81
|
+
export interface DoorSnapshot {
|
|
82
|
+
id: string; namespace: string; bundleSha256: string; kitNamespace: string;
|
|
83
|
+
previous: string | null; records: DoorRecord[]; sourceCount: number;
|
|
84
|
+
plan: ImportPlan; differences: DoorDifferences;
|
|
85
|
+
standingApproval: { id: string; at: string } | null;
|
|
86
|
+
}
|
|
87
|
+
export interface DoorPlanRow {
|
|
88
|
+
id: string; installation_id: string; namespace: string; bundle_sha256: string;
|
|
89
|
+
fingerprint: string; previous_id: string | null; job_id: string | null;
|
|
90
|
+
complete: number; snapshot_hash: string;
|
|
91
|
+
}
|
|
92
|
+
export interface DoorRequest {
|
|
93
|
+
options: ImportDoorRuntimeOptions; access: Readonly<DoorAccess>;
|
|
94
|
+
via: 'grant' | 'operator';
|
|
95
|
+
authority: D1Sql;
|
|
96
|
+
/** The destination's erasure journal rows when the request was admitted. */
|
|
97
|
+
epoch: number;
|
|
98
|
+
/** A grant request's accepted single-use nonce; owner-session requests have none. */
|
|
99
|
+
nonce?: { jti: string; kid: string };
|
|
100
|
+
}
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
import { Buffer } from 'node:buffer';
|
|
2
|
+
import { createHash, randomUUID } from 'node:crypto';
|
|
3
|
+
import { sha256 } from '../provenance/identity.ts';
|
|
4
|
+
import type { DoorRequest } from './types.ts';
|
|
5
|
+
import { DoorError } from './protocol.ts';
|
|
6
|
+
import { assertion, sql, transact } from './store.ts';
|
|
7
|
+
import { intakeIdentity, intakeState } from './intake-policy.ts';
|
|
8
|
+
|
|
9
|
+
export const CHUNK_BYTES = 65_536;
|
|
10
|
+
export const text = (value: unknown, name: string): string => {
|
|
11
|
+
if (typeof value !== 'string' || !value.trim() || value !== value.trim() || value.length > 256 || /[\x00-\x1f]/.test(value)) throw new DoorError('invalid_' + name, 400);
|
|
12
|
+
return value;
|
|
13
|
+
};
|
|
14
|
+
export const digest = (value: unknown): string => {
|
|
15
|
+
if (typeof value !== 'string' || !/^[a-f0-9]{64}$/.test(value)) throw new DoorError('invalid_sha256', 400);
|
|
16
|
+
return value;
|
|
17
|
+
};
|
|
18
|
+
export const integer = (value: unknown, min: number, max: number): number => {
|
|
19
|
+
if (!Number.isSafeInteger(value) || (value as number) < min || (value as number) > max) throw new DoorError('quota_or_chunk_limit', 400);
|
|
20
|
+
return value as number;
|
|
21
|
+
};
|
|
22
|
+
export interface UploadRow {id:string;installation_id:string;namespace:string;name:string;bundle_sha256:string;total_bytes:number;total_chunks:number;intake_json?:string|null}
|
|
23
|
+
export async function uploadRow(request: DoorRequest, id: string): Promise<UploadRow> {
|
|
24
|
+
const row = await request.options.database.prepare('SELECT * FROM ik_door_uploads WHERE id=? AND installation_id=? AND namespace=?').bind(id, request.access.installationId, request.access.namespace).first<UploadRow>();
|
|
25
|
+
if (!row) throw new DoorError('upload_not_found', 404);
|
|
26
|
+
const state = intakeState(row.intake_json);
|
|
27
|
+
if (!!state !== !!request.options.intake || (state && state.custody.allowlistSha256 !== intakeIdentity(request.options.intake!))) throw new DoorError('upload_intake_changed',409);
|
|
28
|
+
if (state && !state.complete) throw new DoorError('intake_upload_incomplete',409);
|
|
29
|
+
if (row.total_bytes > request.access.maxBytes || row.total_chunks > request.access.maxChunks) throw new DoorError('upload_exceeds_grant_quota');
|
|
30
|
+
return row;
|
|
31
|
+
}
|
|
32
|
+
export async function progress(request: DoorRequest, row: UploadRow) {
|
|
33
|
+
const chunks = await request.options.database.prepare('SELECT ordinal,sha256,size FROM ik_door_chunks WHERE upload_id=? ORDER BY ordinal').bind(row.id).all();
|
|
34
|
+
const bytes = chunks.results.reduce((n, c) => n + c.size, 0);
|
|
35
|
+
const complete = chunks.results.length === row.total_chunks && bytes === row.total_bytes;
|
|
36
|
+
const intake = intakeState(row.intake_json)?.custody;
|
|
37
|
+
if (complete) {
|
|
38
|
+
if (intake) await verifyIntakeBundle(request,row);
|
|
39
|
+
else await bundleBytes(request,row);
|
|
40
|
+
}
|
|
41
|
+
return {uploadId:row.id, bundleSha256:row.bundle_sha256, received:chunks.results, complete,...(intake ? {intake} : {})};
|
|
42
|
+
}
|
|
43
|
+
export async function uploads(request: DoorRequest, input: Record<string, unknown>) {
|
|
44
|
+
const id = input.uploadId === undefined ? randomUUID() : text(input.uploadId, 'upload_id');
|
|
45
|
+
if (Object.keys(input).length === 1 && input.uploadId) return progress(request, await uploadRow(request,id));
|
|
46
|
+
if (request.options.intake) throw new DoorError('intake_stream_required',415,{reason:'Send the complete upload stream as application/octet-stream with X-Import-Format and X-Import-Name.'});
|
|
47
|
+
if (Object.keys(input).some(k => !['uploadId','name','bundleSha256','totalBytes','totalChunks','index','chunkSha256','base64'].includes(k))) throw new DoorError('invalid_upload', 400);
|
|
48
|
+
const name = text(input.name, 'name'), hash = digest(input.bundleSha256);
|
|
49
|
+
const totalBytes = integer(input.totalBytes, 1, request.access.maxBytes), totalChunks = integer(input.totalChunks, 1, request.access.maxChunks);
|
|
50
|
+
if (totalChunks !== Math.ceil(totalBytes / CHUNK_BYTES)) throw new DoorError('invalid_chunk_layout', 400);
|
|
51
|
+
const index = integer(input.index, 0, totalChunks - 1), chunkHash = digest(input.chunkSha256);
|
|
52
|
+
if (typeof input.base64 !== 'string' || input.base64.length > Math.ceil(CHUNK_BYTES / 3) * 4) throw new DoorError('invalid_chunk', 400);
|
|
53
|
+
const bytes = Buffer.from(input.base64, 'base64');
|
|
54
|
+
if (bytes.toString('base64') !== input.base64 || bytes.length !== Math.min(CHUNK_BYTES, totalBytes - index * CHUNK_BYTES) || sha256(bytes) !== chunkHash) throw new DoorError('chunk_hash_or_size_mismatch', 400);
|
|
55
|
+
const g = request.access;
|
|
56
|
+
await transact(request.options.database, request.authority, [
|
|
57
|
+
sql('INSERT OR IGNORE INTO ik_door_scopes(installation_id,namespace,destination) VALUES(?,?,?)', g.installationId, g.namespace, request.options.destination),
|
|
58
|
+
assertion(sql('EXISTS(SELECT 1 FROM ik_door_scopes WHERE installation_id=? AND namespace=? AND destination=?)', g.installationId, g.namespace, request.options.destination)),
|
|
59
|
+
sql('INSERT OR IGNORE INTO ik_door_uploads(id,installation_id,namespace,name,bundle_sha256,total_bytes,total_chunks) VALUES(?,?,?,?,?,?,?)', id, g.installationId, g.namespace, name, hash, totalBytes, totalChunks),
|
|
60
|
+
assertion(sql('EXISTS(SELECT 1 FROM ik_door_uploads WHERE id=? AND installation_id=? AND namespace=? AND name=? AND bundle_sha256=? AND total_bytes=? AND total_chunks=?)', id, g.installationId, g.namespace, name, hash, totalBytes, totalChunks)),
|
|
61
|
+
sql('INSERT OR IGNORE INTO ik_door_chunks(upload_id,ordinal,sha256,size,base64) VALUES(?,?,?,?,?)', id, index, chunkHash, bytes.length, input.base64),
|
|
62
|
+
assertion(sql('EXISTS(SELECT 1 FROM ik_door_chunks WHERE upload_id=? AND ordinal=? AND sha256=? AND size=? AND base64=?)', id, index, chunkHash, bytes.length, input.base64)),
|
|
63
|
+
]);
|
|
64
|
+
return progress(request,await uploadRow(request,id));
|
|
65
|
+
}
|
|
66
|
+
/** Upload chunks are 64 KiB each: a read returns a bounded page, never a whole bundle. */
|
|
67
|
+
const UPLOAD_READ_ROWS = 24;
|
|
68
|
+
/** Intake progress verifies custody without materializing the sanitized bundle. */
|
|
69
|
+
async function verifyIntakeBundle(request: DoorRequest, row: UploadRow): Promise<void> {
|
|
70
|
+
const hash = createHash('sha256'); let ordinal = 0, size = 0;
|
|
71
|
+
for (;;) {
|
|
72
|
+
const page = await request.options.database.prepare(`SELECT * FROM ik_door_chunks WHERE upload_id=? ORDER BY ordinal LIMIT ${UPLOAD_READ_ROWS} OFFSET ?`).bind(row.id,ordinal).all();
|
|
73
|
+
for (const chunk of page.results) {
|
|
74
|
+
const bytes = Buffer.from(chunk.base64,'base64');
|
|
75
|
+
if (chunk.ordinal !== ordinal++ || bytes.length !== chunk.size || sha256(bytes) !== chunk.sha256) throw new DoorError('upload_custody_mismatch',409);
|
|
76
|
+
size += bytes.length; hash.update(bytes);
|
|
77
|
+
}
|
|
78
|
+
if (page.results.length < UPLOAD_READ_ROWS) break;
|
|
79
|
+
}
|
|
80
|
+
if (ordinal !== row.total_chunks || size !== row.total_bytes || hash.digest('hex') !== row.bundle_sha256) throw new DoorError('bundle_hash_mismatch',409);
|
|
81
|
+
}
|
|
82
|
+
export async function bundleBytes(request: DoorRequest, row: UploadRow): Promise<Uint8Array> {
|
|
83
|
+
const chunks: Record<string, any>[] = [];
|
|
84
|
+
for (;;) {
|
|
85
|
+
const page = await request.options.database.prepare(`SELECT * FROM ik_door_chunks WHERE upload_id=? ORDER BY ordinal LIMIT ${UPLOAD_READ_ROWS} OFFSET ?`).bind(row.id, chunks.length).all();
|
|
86
|
+
chunks.push(...page.results);
|
|
87
|
+
if (page.results.length < UPLOAD_READ_ROWS) break;
|
|
88
|
+
}
|
|
89
|
+
if (chunks.length !== row.total_chunks) throw new DoorError('upload_incomplete', 409);
|
|
90
|
+
const values = chunks.map((c, i) => {
|
|
91
|
+
const bytes = Buffer.from(c.base64, 'base64');
|
|
92
|
+
if (c.ordinal !== i || bytes.length !== c.size || sha256(bytes) !== c.sha256) throw new DoorError('upload_custody_mismatch', 409);
|
|
93
|
+
return bytes;
|
|
94
|
+
});
|
|
95
|
+
const bytes = Buffer.concat(values);
|
|
96
|
+
if (bytes.length !== row.total_bytes || sha256(bytes) !== row.bundle_sha256) throw new DoorError('bundle_hash_mismatch', 409);
|
|
97
|
+
return bytes;
|
|
98
|
+
}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import { reflectedCredentialMatcher } from './credentials.ts';
|
|
2
|
+
|
|
3
|
+
const JSON_NUMBER = /^(-?)(0|[1-9]\d*)(?:\.(\d+))?(?:[eE]([+-]?\d+))?$/;
|
|
4
|
+
|
|
5
|
+
/** Exact decimal identity, without rounding the coefficient or expanding exponents. */
|
|
6
|
+
function decimalIdentity(text: string, maxExponentDigits = Infinity): string | undefined {
|
|
7
|
+
const parts = JSON_NUMBER.exec(text);
|
|
8
|
+
if (!parts) return;
|
|
9
|
+
const fraction = parts[3] ?? '', exponent = parts[4] ?? '0';
|
|
10
|
+
const digits = (parts[2] + fraction).replace(/^0+/, '');
|
|
11
|
+
if (!digits) return '0';
|
|
12
|
+
// A response exponent larger than any credential exponent plus its possible
|
|
13
|
+
// decimal-point adjustment cannot match. Avoid parsing an unbounded BigInt.
|
|
14
|
+
if (exponent.replace(/^[+-]?0*/, '').length > maxExponentDigits) return;
|
|
15
|
+
let end = digits.length;
|
|
16
|
+
while (digits[end - 1] === '0') end--;
|
|
17
|
+
const coefficient = digits.slice(0, end);
|
|
18
|
+
const scale = BigInt(exponent) + BigInt(digits.length - coefficient.length - fraction.length);
|
|
19
|
+
return `${parts[1]}${coefficient}e${scale}`;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/** Inspect received numeric text as well as values recoverable by JSON consumers. */
|
|
23
|
+
export function reflectedNumberCredentialMatcher(
|
|
24
|
+
secrets: readonly string[], reflectsText = reflectedCredentialMatcher(secrets),
|
|
25
|
+
): (lexeme: string) => boolean {
|
|
26
|
+
if (!secrets.length) return () => false;
|
|
27
|
+
const identities = new Set(secrets.map(secret => decimalIdentity(secret)).filter(value => value !== undefined));
|
|
28
|
+
const exponentDigits = Math.max(0, ...Array.from(identities, value => value.split('e')[1]?.replace('-', '').length ?? 1));
|
|
29
|
+
return lexeme => {
|
|
30
|
+
if (reflectsText(lexeme)) return true;
|
|
31
|
+
const number = Number(lexeme);
|
|
32
|
+
if (Number.isFinite(number) && reflectsText(String(number))) return true;
|
|
33
|
+
if (!identities.size) return false;
|
|
34
|
+
// Normalization adjusts an exponent by at most the numeric text's length.
|
|
35
|
+
// One extra digit bounds the sum with the largest credential exponent.
|
|
36
|
+
const limit = Math.max(exponentDigits, String(lexeme.length).length) + 1;
|
|
37
|
+
const identity = decimalIdentity(lexeme, limit);
|
|
38
|
+
return identity !== undefined && identities.has(identity);
|
|
39
|
+
};
|
|
40
|
+
}
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
# Source readers
|
|
2
|
+
|
|
3
|
+
Import `readFileSource` and `readApiSource` from `index.ts`. Both either return a
|
|
4
|
+
complete `SourceBundle` or reject with `ReaderError` (`code`, safe `diagnostics`).
|
|
5
|
+
Errors deliberately omit thrown causes, response bodies, URL strings and headers.
|
|
6
|
+
|
|
7
|
+
File input is a `Uint8Array` or an async byte iterator (including Node file read
|
|
8
|
+
streams). UTF-8 is strict, with an optional initial BOM. JSON arrays call `parse`
|
|
9
|
+
once per element with its zero-based index; every other JSON root calls it once.
|
|
10
|
+
NDJSON calls it for each nonblank physical line, with the zero-based nonblank
|
|
11
|
+
index. Duplicate JSON object keys fail explicitly rather than discarding an earlier
|
|
12
|
+
field (including credential metadata). Locations use one-based rows; JSON arrays also have JSON pointers. A
|
|
13
|
+
parser may expand an object into multiple records and provide more exact pointers.
|
|
14
|
+
An empty parser result fails explicitly. Empty JSON arrays/NDJSON are valid empty
|
|
15
|
+
exports. Source kinds and IDs are nonempty strings. Duplicate IDs are retained
|
|
16
|
+
with diagnostics, including conflicting content; the core owns collapse/rejection.
|
|
17
|
+
|
|
18
|
+
Defaults are 64 MiB total source bytes, 100,000 records and 1 MiB per record.
|
|
19
|
+
Limits must be positive safe integers. Total bytes include every successful API
|
|
20
|
+
page. Record bytes bound serialized input objects (files) and complete parsed
|
|
21
|
+
records (both readers); NDJSON physical lines are bounded before parsing. JSON
|
|
22
|
+
and API pages are buffered up to the total-byte limit; NDJSON is parsed per line.
|
|
23
|
+
Streams are pulled serially and closed on early failure. The Promise-returning
|
|
24
|
+
frozen API retains the bounded completed record bundle in memory.
|
|
25
|
+
|
|
26
|
+
API `parsePage` receives the JSON document and a one-based page number. It must
|
|
27
|
+
report every source object, including unsupported objects as records for the
|
|
28
|
+
mapping's unmapped report. It declares the next URL (relative or absolute), or
|
|
29
|
+
null/undefined for completion. Empty pages are supported. As a generic reader,
|
|
30
|
+
the kit cannot infer whether a format adapter deliberately omitted a vendor field
|
|
31
|
+
or object; completeness inside an API envelope is the adapter's responsibility.
|
|
32
|
+
Pages have stable names `api-page-N`; request metadata is not added to the bundle.
|
|
33
|
+
Accepted cursor metadata present in response bytes remains in exact originals.
|
|
34
|
+
|
|
35
|
+
API readers may opt into `preserveNumberLexeme(pathSegments): boolean`. Each
|
|
36
|
+
numeric value is selected by its decoded path: object keys are strings, array
|
|
37
|
+
indices are numbers, and the root is `[]`. Only selected numeric primitives become
|
|
38
|
+
ordinary JSON strings containing the exact source token, including sign, trailing
|
|
39
|
+
zeros and exponent spelling. No arithmetic or normalization occurs. For example,
|
|
40
|
+
`path.length === 4 && path[0] === 'records' && typeof path[1] === 'number' &&
|
|
41
|
+
path[2] === 'fields' && typeof path[3] === 'string'` selects direct record cells.
|
|
42
|
+
Metadata, nested objects and array cells remain unchanged by that selector.
|
|
43
|
+
|
|
44
|
+
The additive third `parsePage` argument is an `ApiPageContext` with
|
|
45
|
+
`numericLexemes`, a frozen, null-prototype read-only object mapping JSON pointers
|
|
46
|
+
to selected tokens. Pointer segments escape `~` as `~0` and `/` as `~1`; the root
|
|
47
|
+
pointer is `''`. Context is page-local, contains no transport metadata or second
|
|
48
|
+
response body, and is empty when preservation is omitted. It distinguishes source
|
|
49
|
+
number `42` (selected string with provenance) from source string `"42"` (no entry).
|
|
50
|
+
Both the context and each policy path are frozen. Return a synchronous boolean;
|
|
51
|
+
invalid policies, non-boolean results and thrown exceptions fail closed without
|
|
52
|
+
exposing thrown causes. Policy failure uses `SOURCE_NUMBER_POLICY_FAILED`.
|
|
53
|
+
|
|
54
|
+
Syntax, duplicate keys and credentials in every decoded source key or string are
|
|
55
|
+
validated before either callback. Known authentication values reflected as boolean
|
|
56
|
+
or null scalars are rejected too. Numeric authentication reflections are checked
|
|
57
|
+
against exact received numeric text, equivalent decimal values and ordinary JSON
|
|
58
|
+
Number results, whether or not preservation is selected. Authentication values
|
|
59
|
+
themselves are never rounded for comparison. Selected overflow tokens such
|
|
60
|
+
as `1e309` can therefore reach product validation as strings; unselected nonfinite
|
|
61
|
+
numbers retain the ordinary rejection. The product owns numeric range, canonical
|
|
62
|
+
representation, type-conversion accounting and retention of any needed per-cell
|
|
63
|
+
numeric-origin provenance. Omission keeps ordinary JSON Number semantics. These
|
|
64
|
+
options do not change byte, record, request, cancellation or custody guarantees.
|
|
65
|
+
|
|
66
|
+
Every artifact hashes the exact successful response bytes and retains them as
|
|
67
|
+
`original: { status: "available", base64 }` using canonical base64. File originals
|
|
68
|
+
also preserve all bytes, including BOM, line endings, whitespace and parser-excluded
|
|
69
|
+
wrappers. Custody is copied in bounded 64 KiB blocks, so tiny or reused stream
|
|
70
|
+
buffers cannot lose original bytes or allocate one retained object per chunk.
|
|
71
|
+
Payload hashes bind canonical JSON. Whitespace changes therefore change artifact identity. HTTP error
|
|
72
|
+
and redirect bodies are discarded without parsing or buffering.
|
|
73
|
+
|
|
74
|
+
The transport receives a GET `Request` with manual redirects and omitted ambient
|
|
75
|
+
credentials, and **must honor these settings**. Fetch is a suitable transport.
|
|
76
|
+
The reader handles same-origin redirects itself, rejects automatically followed
|
|
77
|
+
redirects, cross-origin links, userinfo, fragments and credential-bearing query
|
|
78
|
+
parameters, and preserves all accepted pagination parameters and request headers.
|
|
79
|
+
The API reader never strips URL credentials. Tokens belong in the ephemeral `token`
|
|
80
|
+
or `headers` arguments. A conflicting Authorization header fails. Credential keys,
|
|
81
|
+
credential URLs and reflected credential values in source data fail, even if a
|
|
82
|
+
parser would omit them. Null, an empty string or a mask (`****`, `[redacted]`)
|
|
83
|
+
under a credential key is not a credential and passes. No token, response error
|
|
84
|
+
text or credential-bearing URL is returned in bundle metadata or diagnostics; a
|
|
85
|
+
credential refusal names only the field kind in its diagnostic `field`.
|
|
86
|
+
|
|
87
|
+
`readFileSource` takes `credentials: 'remove'` to keep records that hold a plain
|
|
88
|
+
credential value: the parser sees the value as read, then each record loses its
|
|
89
|
+
credential fields (any key whose whole name parts are a credential word or
|
|
90
|
+
phrase, such as `httpPassword`, `client_secret`, `X-Api-Key`, `secret_key`,
|
|
91
|
+
`Ocp-Apim-Subscription-Key`, `pwd` or `X-Auth`, never `keyword` or `author`; a plural such as
|
|
92
|
+
`apiKeys` or `tokens` when it holds text; the `value` of a `{name|key, value}`
|
|
93
|
+
pair naming one, and of such a `[name, value]` pair in a list of them; booleans
|
|
94
|
+
stay) and its URLs' (any scheme's)
|
|
95
|
+
user names, passwords and credential query and fragment parameters, with one
|
|
96
|
+
`CREDENTIAL_REMOVED` warning per removal naming the record,
|
|
97
|
+
the field pointer and its kind. The original bytes are then withheld
|
|
98
|
+
(`reason: 'credentials-removed'`). A record still repeating a removed value
|
|
99
|
+
fails with `CREDENTIAL_IN_SOURCE_DATA`; forms it cannot remove still fail as
|
|
100
|
+
above. A field holding `[removed]`, the marker rer's vendor exports leave, is
|
|
101
|
+
reported as removed before upload. A URL's `removed@` user name and a
|
|
102
|
+
`=removed` parameter, rer's URL markers, are removed and withhold the original
|
|
103
|
+
too (refuse mode refuses them), but are not live values: `readFileSourceRemovals`
|
|
104
|
+
answers the bundle with `liveCredentialsRemoved`, true only when a live value
|
|
105
|
+
went.
|
|
106
|
+
|
|
107
|
+
All requests, including redirects/retries, count toward `maxRequests` (default
|
|
108
|
+
1000). Rate limits (429, or 403/503 with Retry-After) allow `maxRetries` (default 3)
|
|
109
|
+
per requested page/redirect target. Retry-After accepts seconds or HTTP dates;
|
|
110
|
+
missing 429 Retry-After uses 1s, 2s, 4s. A wait exceeding `maxRetryAfterMs` (default
|
|
111
|
+
30s) fails instead of retrying before the server permits. Invalid Retry-After
|
|
112
|
+
fails. Cycles, exhausted requests/retries and later-page failure reject the entire
|
|
113
|
+
read. `signal` cancels requests, body reads and waits; reasons are never exposed.
|
|
114
|
+
The optional sleep adapter receives both delay and signal. No concurrent page
|
|
115
|
+
prefetch or unbounded error-response reads occur.
|
|
116
|
+
|
|
117
|
+
### Explicit noncredential pagination cursors
|
|
118
|
+
|
|
119
|
+
Trusted adapters may select `paginationCursorPolicy: 'page-token-v1'` on
|
|
120
|
+
`readApiSource` or `readFileSource` when their source documents `page_token` as a
|
|
121
|
+
pagination cursor. Omission retains the strict default. This is a closed policy,
|
|
122
|
+
not an arbitrary allowlist: only the exact URL query spelling `page_token` is
|
|
123
|
+
exempt from the token-key prohibition. Encoded or case aliases, actual credential
|
|
124
|
+
keys, URL userinfo, cross-origin navigation/redirects, and reflected provided
|
|
125
|
+
header/token secrets remain rejected. No URL or original bytes are rewritten.
|
|
126
|
+
The exact wire query boundaries own the cursor value: encoded delimiters inside
|
|
127
|
+
it remain opaque data, including repeated encoding. Nested credential-bearing
|
|
128
|
+
URLs in that value and encoded credential parameters outside it still reject.
|
|
129
|
+
JSON pagination fields such as `next_page_token` and `previous_page_token` remain
|
|
130
|
+
ordinary noncredential metadata; API reflection checks still inspect their values.
|
|
131
|
+
|
|
132
|
+
The reader records the selection as `SourceBundle.paginationCursorPolicy`.
|
|
133
|
+
Adapters combining bundles must deliberately preserve that selection. Planning,
|
|
134
|
+
job creation and retained-original readback validate it and bind it in the reviewed
|
|
135
|
+
fingerprint. Never derive the selection from untrusted exported JSON. Saved files
|
|
136
|
+
have no in-memory authentication context, so adapters must only select this policy
|
|
137
|
+
for documented noncredential cursors; they cannot identify arbitrary secrets
|
|
138
|
+
hidden in otherwise ordinary values. Mapped target effects retain the strict
|
|
139
|
+
credential policy: the exception is for source custody and navigation.
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import type { PaginationCursorPolicy } from '../types.ts';
|
|
2
|
+
import { credentialQueryKey, paginationPolicy } from '../provenance/credentials.ts';
|
|
3
|
+
import { setTimeout as delay } from 'node:timers/promises';
|
|
4
|
+
import { fail } from './errors.ts';
|
|
5
|
+
import { safeValue } from './records.ts';
|
|
6
|
+
|
|
7
|
+
export function boundedInteger(value: number | undefined, fallback: number, allowZero = false): number {
|
|
8
|
+
const result = value ?? fallback;
|
|
9
|
+
if (!Number.isSafeInteger(result) || result < (allowZero ? 0 : 1)) fail('INVALID_API_LIMIT');
|
|
10
|
+
return result;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
/** Reject credentials instead of stripping them and changing pagination semantics. */
|
|
14
|
+
export function sourceUrl(value: string, base?: URL, secrets: readonly string[] = [], policy?: PaginationCursorPolicy): URL {
|
|
15
|
+
paginationPolicy(policy);
|
|
16
|
+
if (typeof value !== 'string' || !value.trim() || value !== value.trim()) fail('INVALID_SOURCE_URL');
|
|
17
|
+
let url: URL;
|
|
18
|
+
try { url = new URL(value, base); } catch { fail('INVALID_SOURCE_URL'); }
|
|
19
|
+
if (!['https:', 'http:'].includes(url.protocol) || url.username || url.password || url.hash) fail('UNSAFE_SOURCE_URL');
|
|
20
|
+
if (base && url.origin !== base.origin) fail('CROSS_ORIGIN_SOURCE_URL');
|
|
21
|
+
try {
|
|
22
|
+
for (const pair of url.search.slice(1).split('&')) {
|
|
23
|
+
if (credentialQueryKey(pair.split('=')[0], policy)) fail('CREDENTIAL_IN_SOURCE_URL');
|
|
24
|
+
}
|
|
25
|
+
} catch { fail('CREDENTIAL_IN_SOURCE_URL'); }
|
|
26
|
+
for (const item of url.searchParams.values()) safeValue(item, secrets, undefined, policy);
|
|
27
|
+
try { decodeURIComponent(url.href); } catch { fail('INVALID_SOURCE_URL'); }
|
|
28
|
+
safeValue(value, secrets, undefined, policy);
|
|
29
|
+
// Inspect the resolved wire URL too, without turning escaped value delimiters into syntax.
|
|
30
|
+
safeValue(url.href, secrets, undefined, policy);
|
|
31
|
+
return url;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export function requestHeaders(headers: Record<string, string> | undefined, token?: string): { headers: Headers; secrets: string[] } {
|
|
35
|
+
let result: Headers;
|
|
36
|
+
try { result = new Headers(headers); } catch { fail('INVALID_API_HEADERS'); }
|
|
37
|
+
const secrets: string[] = [];
|
|
38
|
+
for (const [key, value] of result) {
|
|
39
|
+
if (/authorization|cookie|token|secret|api[-_]?key/i.test(key)) {
|
|
40
|
+
if (value) secrets.push(value);
|
|
41
|
+
if (/authorization/i.test(key)) { const credential = value.replace(/^\S+\s+/, ''); if (credential) secrets.push(credential); }
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
if (token !== undefined) {
|
|
45
|
+
if (typeof token !== 'string' || !token || /\s/.test(token)) fail('INVALID_API_TOKEN');
|
|
46
|
+
if (result.has('authorization') && result.get('authorization') !== `Bearer ${token}`) fail('CONFLICTING_API_CREDENTIALS');
|
|
47
|
+
try { result.set('authorization', `Bearer ${token}`); } catch { fail('INVALID_API_TOKEN'); }
|
|
48
|
+
secrets.push(token);
|
|
49
|
+
}
|
|
50
|
+
return { headers: result, secrets };
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export function retryDelay(header: string | null, retries: number, maximum: number): number {
|
|
54
|
+
let ms: number;
|
|
55
|
+
if (header === null) ms = 1000 * 2 ** retries;
|
|
56
|
+
else if (/^\d+(?:\.\d+)?$/.test(header.trim())) ms = Math.ceil(Number(header) * 1000);
|
|
57
|
+
else {
|
|
58
|
+
// Require an HTTP date, rather than Date.parse's permissive numeric/date shorthands.
|
|
59
|
+
if (!/^[A-Za-z]{3}, \d{2} [A-Za-z]{3} \d{4} \d{2}:\d{2}:\d{2} GMT$/.test(header)) fail('INVALID_RETRY_AFTER');
|
|
60
|
+
const time = Date.parse(header);
|
|
61
|
+
if (!Number.isFinite(time)) fail('INVALID_RETRY_AFTER');
|
|
62
|
+
ms = Math.max(0, time - Date.now());
|
|
63
|
+
}
|
|
64
|
+
if (!Number.isSafeInteger(ms) || ms > maximum) fail('RETRY_AFTER_LIMIT');
|
|
65
|
+
return ms;
|
|
66
|
+
}
|
|
67
|
+
export async function sleep(ms: number, signal?: AbortSignal): Promise<void> { await delay(ms, undefined, { signal }); }
|