@runeditrun/rer 0.2.81 → 0.2.82

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/README.md +12 -0
  2. package/cli/export.mjs +1 -1
  3. package/edition-authoring/base-copies.mjs +3 -1
  4. package/edition-authoring/copy-checkout.mjs +152 -1
  5. package/edition-authoring/files.mjs +1 -1
  6. package/edition-authoring/runtime-dependencies.mjs +1 -1
  7. package/global/CHANGELOG.md +13 -0
  8. package/global/README.md +12 -0
  9. package/modules/import-kit/src/door/ADOPTING.md +452 -0
  10. package/modules/import-kit/src/door/README.md +553 -0
  11. package/modules/import-kit/src/door/admission.ts +45 -0
  12. package/modules/import-kit/src/door/archive.ts +129 -0
  13. package/modules/import-kit/src/door/authority.ts +107 -0
  14. package/modules/import-kit/src/door/failures.ts +70 -0
  15. package/modules/import-kit/src/door/grants.ts +162 -0
  16. package/modules/import-kit/src/door/handler.ts +90 -0
  17. package/modules/import-kit/src/door/index.ts +20 -0
  18. package/modules/import-kit/src/door/intake/csv.ts +74 -0
  19. package/modules/import-kit/src/door/intake/json.ts +182 -0
  20. package/modules/import-kit/src/door/intake/projection.ts +61 -0
  21. package/modules/import-kit/src/door/intake/types.ts +28 -0
  22. package/modules/import-kit/src/door/intake-policy.ts +38 -0
  23. package/modules/import-kit/src/door/intake-upload.ts +91 -0
  24. package/modules/import-kit/src/door/inventory.ts +68 -0
  25. package/modules/import-kit/src/door/jobs.ts +134 -0
  26. package/modules/import-kit/src/door/limits.ts +16 -0
  27. package/modules/import-kit/src/door/operator.ts +41 -0
  28. package/modules/import-kit/src/door/planning.ts +227 -0
  29. package/modules/import-kit/src/door/protocol.ts +38 -0
  30. package/modules/import-kit/src/door/refusals.ts +51 -0
  31. package/modules/import-kit/src/door/schema.ts +101 -0
  32. package/modules/import-kit/src/door/store.ts +152 -0
  33. package/modules/import-kit/src/door/types.ts +100 -0
  34. package/modules/import-kit/src/door/uploads.ts +98 -0
  35. package/modules/import-kit/src/provenance/numeric-credentials.ts +40 -0
  36. package/modules/import-kit/src/readers/README.md +139 -0
  37. package/modules/import-kit/src/readers/api-policy.ts +67 -0
  38. package/modules/import-kit/src/readers/api.ts +86 -0
  39. package/modules/import-kit/src/readers/bytes.ts +73 -0
  40. package/modules/import-kit/src/readers/errors.ts +33 -0
  41. package/modules/import-kit/src/readers/file.ts +74 -0
  42. package/modules/import-kit/src/readers/index.ts +3 -0
  43. package/modules/import-kit/src/readers/json.ts +29 -0
  44. package/modules/import-kit/src/readers/numeric-lexemes.ts +104 -0
  45. package/modules/import-kit/src/readers/original.ts +31 -0
  46. package/modules/import-kit/src/readers/records.ts +105 -0
  47. package/package.json +3 -2
@@ -0,0 +1,74 @@
1
+ import { malformed, tooLarge, utf8Length, type IntakeLimits, type IntakeParser } from './types.ts';
2
+
3
+ /** RFC 4180 cells, projected only after a complete row passes width checks. */
4
+ export class CsvIntake implements IntakeParser {
5
+ private state: 'start' | 'bare' | 'quoted' | 'closed' = 'start';
6
+ private cell = '';
7
+ private cells: string[] = [];
8
+ private headers: string[] | undefined;
9
+ private readonly allowed: Set<string>;
10
+ private size = 0;
11
+ private line = 1;
12
+ private count = 0;
13
+ private skipLf = false;
14
+ private readonly limits: IntakeLimits;
15
+ constructor(fields: readonly string[], limits: IntakeLimits) { this.limits = limits; this.allowed = new Set(fields); }
16
+ get row(): number { return this.line; }
17
+ push(char: string): string | undefined {
18
+ if (this.skipLf) {
19
+ this.skipLf = false;
20
+ if (char === '\n') return;
21
+ }
22
+ this.size += utf8Length(char);
23
+ if (this.size > this.limits.maxRecordBytes) tooLarge('intake_record_too_large', this.row);
24
+ if (this.state === 'quoted') {
25
+ if (char === '"') this.state = 'closed';
26
+ else this.cell += char;
27
+ return;
28
+ }
29
+ if (char === '"') {
30
+ if (this.state === 'start') this.state = 'quoted';
31
+ else if (this.state === 'closed') { this.cell += '"'; this.state = 'quoted'; }
32
+ else malformed(this.row, 'invalid_csv');
33
+ return;
34
+ }
35
+ if (char === ',') { this.endCell(); return; }
36
+ if (char === '\r' || char === '\n') {
37
+ this.skipLf = char === '\r';
38
+ const output = this.record();
39
+ this.line++;
40
+ return output;
41
+ }
42
+ if (this.state === 'closed') malformed(this.row, 'invalid_csv');
43
+ this.cell += char;
44
+ this.state = 'bare';
45
+ }
46
+ private endCell(): void {
47
+ this.cells.push(this.cell);
48
+ this.cell = ''; this.state = 'start';
49
+ }
50
+ private record(): string | undefined {
51
+ this.endCell();
52
+ const cells = this.cells;
53
+ this.cells = []; this.size = 0;
54
+ if (!this.headers) {
55
+ if (new Set(cells).size !== cells.length) malformed(this.row, 'duplicate_header');
56
+ this.headers = cells;
57
+ return;
58
+ }
59
+ if (cells.length !== this.headers.length) malformed(this.row, 'width_mismatch');
60
+ if (this.count >= this.limits.maxRecords) tooLarge('intake_too_many_records', this.row);
61
+ this.count++;
62
+ const output: string[] = [];
63
+ for (let index = 0; index < cells.length; index++) {
64
+ const key = this.headers[index];
65
+ if (this.allowed.has(key)) output.push(`${JSON.stringify(key)}:${JSON.stringify(cells[index])}`);
66
+ }
67
+ return `{${output.join(',')}}`;
68
+ }
69
+ finish(): string | undefined {
70
+ if (this.state === 'quoted') malformed(this.row, 'invalid_csv');
71
+ if (!this.size) return;
72
+ return this.record();
73
+ }
74
+ }
@@ -0,0 +1,182 @@
1
+ import { malformed, tooLarge, utf8Length, type IntakeLimits, type IntakeParser } from './types.ts';
2
+
3
+ interface Selection { leaf: boolean; children: Map<string, Selection> }
4
+ function selection(fields: readonly string[]): Selection {
5
+ const root: Selection = { leaf: false, children: new Map() };
6
+ for (const pointer of fields) {
7
+ let node = root;
8
+ for (const key of pointer.slice(1).split('/').map(part => part.replace(/~1/g, '/').replace(/~0/g, '~'))) {
9
+ let child = node.children.get(key);
10
+ if (!child) node.children.set(key, child = { leaf: false, children: new Map() });
11
+ node = child;
12
+ }
13
+ node.leaf = true;
14
+ }
15
+ return root;
16
+ }
17
+
18
+ /** Validate the entire record, including discarded subtrees. Retain scalar tokens
19
+ * verbatim so integers and decimal lexemes never pass through JS numbers. */
20
+ export function jsonProjection(fields: readonly string[]): (source: string, row: number) => string {
21
+ const selected = selection(fields);
22
+ return (source, row) => {
23
+ let at = 0;
24
+ const number = /-?(?:0|[1-9][0-9]*)(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?/y;
25
+ const fail = (reason = 'invalid_json'): never => malformed(row, reason);
26
+ const space = () => { while (/[\x20\t\r\n]/.test(source[at] ?? '') && at < source.length) at++; };
27
+ function string(): string {
28
+ const start = at++;
29
+ while (at < source.length) {
30
+ const char = source[at++];
31
+ if (char === '"') {
32
+ const token = source.slice(start, at);
33
+ try { JSON.parse(token); } catch { fail(); }
34
+ return token;
35
+ }
36
+ if (char === '\\') at++;
37
+ else if (char.charCodeAt(0) < 0x20) fail();
38
+ }
39
+ return fail();
40
+ }
41
+ function value(node: Selection | undefined, depth: number): string | undefined {
42
+ if (depth > 64) fail('max_depth');
43
+ space();
44
+ const first = source[at];
45
+ if (first === '{') {
46
+ if (node?.leaf) fail('leaf_not_scalar');
47
+ at++; space();
48
+ const keys = new Set<string>(), output: string[] = [];
49
+ if (source[at] !== '}') {
50
+ for (;;) {
51
+ if (source[at] !== '"') fail();
52
+ const token = string(), key = JSON.parse(token) as string;
53
+ if (keys.has(key)) fail('duplicate_key');
54
+ keys.add(key); space();
55
+ if (source[at++] !== ':') fail();
56
+ const child = value(node?.children.get(key), depth + 1);
57
+ if (child !== undefined) output.push(`${JSON.stringify(key)}:${child}`);
58
+ space();
59
+ if (source[at] === '}') break;
60
+ if (source[at++] !== ',') fail();
61
+ space();
62
+ }
63
+ }
64
+ at++;
65
+ return node && (output.length > 0 || depth === 0) ? `{${output.join(',')}}` : undefined;
66
+ }
67
+ if (first === '[') {
68
+ if (node?.leaf) fail('leaf_not_scalar');
69
+ if (node?.children.size) fail('unsupported_array_path');
70
+ at++; space();
71
+ if (source[at] !== ']') {
72
+ for (;;) {
73
+ value(undefined, depth + 1); space();
74
+ if (source[at] === ']') break;
75
+ if (source[at++] !== ',') fail();
76
+ }
77
+ }
78
+ at++;
79
+ return undefined;
80
+ }
81
+ const start = at;
82
+ if (first === '"') string();
83
+ else if (source.startsWith('true', at)) at += 4;
84
+ else if (source.startsWith('false', at)) at += 5;
85
+ else if (source.startsWith('null', at)) at += 4;
86
+ else {
87
+ number.lastIndex = at;
88
+ const match = number.exec(source);
89
+ if (!match) fail();
90
+ at = number.lastIndex;
91
+ }
92
+ return node?.leaf ? source.slice(start, at) : undefined;
93
+ }
94
+ space();
95
+ if (source[at] !== '{') fail('object_required');
96
+ const output = value(selected, 0)!;
97
+ space();
98
+ if (at !== source.length) fail();
99
+ return output;
100
+ };
101
+ }
102
+
103
+ /** A plain array of object records. Only the current element is retained. */
104
+ export class JsonIntake implements IntakeParser {
105
+ private state: 'start' | 'first' | 'next' | 'record' | 'separator' | 'done' = 'start';
106
+ private source = '';
107
+ private size = 0;
108
+ private depth = 0;
109
+ private quoted = false;
110
+ private escaped = false;
111
+ private count = 0;
112
+ private readonly project: ReturnType<typeof jsonProjection>;
113
+ private readonly limits: IntakeLimits;
114
+ constructor(fields: readonly string[], limits: IntakeLimits) { this.limits = limits; this.project = jsonProjection(fields); }
115
+ get row(): number { return this.count + 1; }
116
+ push(char: string): string | undefined {
117
+ if (this.state === 'record') {
118
+ this.size += utf8Length(char);
119
+ if (this.size > this.limits.maxRecordBytes) tooLarge('intake_record_too_large', this.row);
120
+ this.source += char;
121
+ if (this.quoted) {
122
+ if (this.escaped) this.escaped = false;
123
+ else if (char === '\\') this.escaped = true;
124
+ else if (char === '"') this.quoted = false;
125
+ } else if (char === '"') this.quoted = true;
126
+ else if (char === '{' || char === '[') {
127
+ if (++this.depth > 65) malformed(this.row, 'max_depth');
128
+ } else if (char === '}' || char === ']') {
129
+ if (--this.depth === 0) {
130
+ const output = this.project(this.source, this.row);
131
+ this.source = ''; this.size = 0; this.count++; this.state = 'separator';
132
+ return output;
133
+ }
134
+ }
135
+ return;
136
+ }
137
+ if (/[\x20\t\r\n]/.test(char)) return;
138
+ if (this.state === 'start' && char === '[') { this.state = 'first'; return; }
139
+ if ((this.state === 'first' || this.state === 'separator') && char === ']') { this.state = 'done'; return; }
140
+ if (this.state === 'separator' && char === ',') { this.state = 'next'; return; }
141
+ if ((this.state === 'first' || this.state === 'next') && char === '{') {
142
+ if (this.count >= this.limits.maxRecords) tooLarge('intake_too_many_records', this.row);
143
+ this.size = 1;
144
+ if (this.size > this.limits.maxRecordBytes) tooLarge('intake_record_too_large', this.row);
145
+ this.state = 'record'; this.source = char; this.depth = 1; return;
146
+ }
147
+ malformed(this.row, 'invalid_json');
148
+ }
149
+ finish(): undefined {
150
+ if (this.state !== 'done') malformed(this.row, 'invalid_json');
151
+ return undefined;
152
+ }
153
+ }
154
+
155
+ export class NdjsonIntake implements IntakeParser {
156
+ private source = '';
157
+ private size = 0;
158
+ private line = 1;
159
+ private count = 0;
160
+ private readonly project: ReturnType<typeof jsonProjection>;
161
+ private readonly limits: IntakeLimits;
162
+ constructor(fields: readonly string[], limits: IntakeLimits) { this.limits = limits; this.project = jsonProjection(fields); }
163
+ get row(): number { return this.line; }
164
+ push(char: string): string | undefined {
165
+ this.size += utf8Length(char);
166
+ if (this.size > this.limits.maxRecordBytes) tooLarge('intake_record_too_large', this.row);
167
+ if (char !== '\n') { this.source += char; return; }
168
+ const output = this.record();
169
+ this.line++;
170
+ return output;
171
+ }
172
+ private record(): string | undefined {
173
+ const source = this.source;
174
+ this.source = ''; this.size = 0;
175
+ if (/^[\x20\t\r]*$/.test(source)) return;
176
+ if (this.count >= this.limits.maxRecords) tooLarge('intake_too_many_records', this.row);
177
+ const output = this.project(source, this.row);
178
+ this.count++;
179
+ return output;
180
+ }
181
+ finish(): string | undefined { return this.record(); }
182
+ }
@@ -0,0 +1,61 @@
1
+ import { CsvIntake } from './csv.ts';
2
+ import { JsonIntake, NdjsonIntake } from './json.ts';
3
+ import { malformed, tooLarge, type IntakeFormat, type IntakeLimits } from './types.ts';
4
+ export type { IntakeFormat, ImportDoorIntake } from './types.ts';
5
+
6
+ /** Validate UTF-8 before decoding, retaining the exact valid prefix on errors so
7
+ * the parser reports the row containing the invalid sequence, even within a chunk. */
8
+ async function* decoded(bytes: AsyncIterable<Uint8Array>, limits: IntakeLimits, row: () => number): AsyncGenerator<string> {
9
+ const decoder = new TextDecoder('utf-8', { fatal: true, ignoreBOM: true });
10
+ let total = 0, remaining = 0, lower = 0x80, upper = 0xbf;
11
+ for await (const chunk of bytes) {
12
+ total += chunk.byteLength;
13
+ if (total > limits.maxBytes) tooLarge('intake_too_large', row());
14
+ for (let start = 0; start < chunk.length; start += 8192) {
15
+ const end = Math.min(start + 8192, chunk.length);
16
+ let at = start;
17
+ for (; at < end; at++) {
18
+ const byte = chunk[at];
19
+ if (remaining) {
20
+ if (byte < lower || byte > upper) break;
21
+ remaining--; lower = 0x80; upper = 0xbf;
22
+ } else if (byte <= 0x7f) continue;
23
+ else if (byte >= 0xc2 && byte <= 0xdf) remaining = 1;
24
+ else if (byte >= 0xe0 && byte <= 0xef) {
25
+ remaining = 2; lower = byte === 0xe0 ? 0xa0 : 0x80; upper = byte === 0xed ? 0x9f : 0xbf;
26
+ } else if (byte >= 0xf0 && byte <= 0xf4) {
27
+ remaining = 3; lower = byte === 0xf0 ? 0x90 : 0x80; upper = byte === 0xf4 ? 0x8f : 0xbf;
28
+ } else break;
29
+ }
30
+ const text = decoder.decode(chunk.subarray(start, at), { stream: true });
31
+ if (text) yield text;
32
+ if (at !== end) malformed(row(), 'invalid_utf8');
33
+ }
34
+ }
35
+ if (remaining) malformed(row(), 'invalid_utf8');
36
+ const text = decoder.decode();
37
+ if (text) yield text;
38
+ }
39
+
40
+ /** Yield only complete, validated allowlisted object rows. No source content is
41
+ * included in errors; the caller must discard any partial upload on refusal. */
42
+ export async function* sanitizeIntake(
43
+ bytes: AsyncIterable<Uint8Array>, format: IntakeFormat, fields: readonly string[], limits: IntakeLimits,
44
+ ): AsyncGenerator<Uint8Array> {
45
+ const parser = format === 'csv' ? new CsvIntake(fields, limits)
46
+ : format === 'ndjson' ? new NdjsonIntake(fields, limits) : new JsonIntake(fields, limits);
47
+ const encoder = new TextEncoder();
48
+ let first = true;
49
+ for await (const text of decoded(bytes, limits, () => parser.row)) {
50
+ for (const char of text) {
51
+ if (first) {
52
+ first = false;
53
+ if (char === '\ufeff') continue;
54
+ }
55
+ const output = parser.push(char);
56
+ if (output !== undefined) yield encoder.encode(`${output}\n`);
57
+ }
58
+ }
59
+ const output = parser.finish();
60
+ if (output !== undefined) yield encoder.encode(`${output}\n`);
61
+ }
@@ -0,0 +1,28 @@
1
+ import { DoorError } from '../protocol.ts';
2
+
3
+ export type IntakeFormat = 'csv' | 'ndjson' | 'json';
4
+ export interface ImportDoorIntake {
5
+ readonly formats: readonly IntakeFormat[];
6
+ readonly allow: Readonly<Partial<Record<IntakeFormat, readonly string[]>>>;
7
+ }
8
+ export interface IntakeLimits {
9
+ readonly maxBytes: number;
10
+ readonly maxRecordBytes: number;
11
+ readonly maxRecords: number;
12
+ }
13
+
14
+ export function malformed(row: number, reason: string): never {
15
+ throw new DoorError('intake_malformed_row', 422, { row, reason });
16
+ }
17
+ export function tooLarge(code: 'intake_too_large' | 'intake_record_too_large' | 'intake_too_many_records', row: number): never {
18
+ throw new DoorError(code, 413, { row });
19
+ }
20
+ export function utf8Length(char: string): number {
21
+ const point = char.codePointAt(0)!;
22
+ return point <= 0x7f ? 1 : point <= 0x7ff ? 2 : point <= 0xffff ? 3 : 4;
23
+ }
24
+ export interface IntakeParser {
25
+ readonly row: number;
26
+ push(char: string): string | undefined;
27
+ finish(): string | undefined;
28
+ }
@@ -0,0 +1,38 @@
1
+ import { canonicalJson, sha256 } from '../provenance/identity.ts';
2
+ import type { IntakeCustody } from '../types.ts';
3
+ import type { ImportDoorIntake, IntakeFormat } from './intake/types.ts';
4
+ import { DoorError } from './protocol.ts';
5
+
6
+ const formats: readonly string[] = ['csv', 'ndjson', 'json'];
7
+ /** Server configuration only: no executable sanitizer, default fields or normalization. */
8
+ export function intakeIdentity(intake: ImportDoorIntake): string {
9
+ if (!intake || Object.keys(intake).some(key => !['formats', 'allow'].includes(key))
10
+ || !Array.isArray(intake.formats) || !intake.formats.length || new Set(intake.formats).size !== intake.formats.length
11
+ || !intake.allow || typeof intake.allow !== 'object' || Array.isArray(intake.allow)
12
+ || Object.keys(intake.allow).some(format => !intake.formats.includes(format as IntakeFormat))) throw new DoorError('invalid_intake_configuration', 503);
13
+ const allow: Record<string, string[]> = {};
14
+ for (const format of intake.formats as readonly IntakeFormat[]) {
15
+ const fields = intake.allow[format];
16
+ if (!formats.includes(format) || !Array.isArray(fields) || new Set(fields).size !== fields.length
17
+ || fields.some(field => typeof field !== 'string' || !field.length || field.length > 1024
18
+ || (format !== 'csv' && (!field.startsWith('/') || /~(?![01])/.test(field))))) throw new DoorError('invalid_intake_configuration', 503);
19
+ allow[format] = [...fields].sort();
20
+ }
21
+ return sha256(canonicalJson({formats: [...intake.formats].sort(), allow}));
22
+ }
23
+ export function intakeCustody(intake: ImportDoorIntake, format: string | null): IntakeCustody {
24
+ const allowlistSha256 = intakeIdentity(intake);
25
+ if (!format || !intake.formats.includes(format as IntakeFormat)) throw new DoorError('intake_format_refused', 415,
26
+ {reason: 'Declare a supported X-Import-Format; an intake upload is never accepted as raw custody.'});
27
+ return {sanitized: true, allowlistSha256, inputFormat: format as IntakeFormat, custodyFormat: 'ndjson'};
28
+ }
29
+ export interface IntakeState {complete: boolean; custody: IntakeCustody}
30
+ export function intakeState(json: string | null | undefined): IntakeState | undefined {
31
+ if (json == null) return undefined;
32
+ let state: IntakeState;
33
+ try { state = JSON.parse(json); } catch { throw new DoorError('upload_custody_mismatch', 409); }
34
+ const c = state?.custody;
35
+ if (typeof state?.complete !== 'boolean' || !c || c.sanitized !== true || c.custodyFormat !== 'ndjson'
36
+ || !formats.includes(c.inputFormat) || !/^[a-f0-9]{64}$/.test(c.allowlistSha256)) throw new DoorError('upload_custody_mismatch', 409);
37
+ return state;
38
+ }
@@ -0,0 +1,91 @@
1
+ import { Buffer } from 'node:buffer';
2
+ import { createHash, randomUUID } from 'node:crypto';
3
+ import { sha256 } from '../provenance/identity.ts';
4
+ import type { DoorRequest } from './types.ts';
5
+ import { DoorError } from './protocol.ts';
6
+ import { assertion, sql, transact } from './store.ts';
7
+ import { CHUNK_BYTES, progress, text, uploadRow } from './uploads.ts';
8
+ import { intakeCustody } from './intake-policy.ts';
9
+ import { sanitizeIntake } from './intake/projection.ts';
10
+
11
+ /** Raw bytes live only in the request reader and a bounded parser row. The
12
+ * database receives projected complete rows, including on interruption/refusal. */
13
+ export async function streamUpload(request: DoorRequest, incoming: Request) {
14
+ const intake = request.options.intake;
15
+ if (!intake) throw new DoorError('chunk_upload_required', 415);
16
+ const custody = intakeCustody(intake, incoming.headers.get('X-Import-Format'));
17
+ const name = text(incoming.headers.get('X-Import-Name'), 'name');
18
+ if (!incoming.body) throw new DoorError('intake_body_required', 400);
19
+ const installed = await request.options.database.prepare("SELECT 1 AS ok FROM pragma_table_info('ik_door_uploads') WHERE name='intake_json'").first();
20
+ if (!installed) throw new DoorError('import_door_schema_outdated', 503, {reason: 'Apply importDoorIntakeMigration before declaring intake.'});
21
+ const id = randomUUID(), g = request.access, pending = JSON.stringify({complete: false, custody});
22
+ // The pending header cannot be planned. Its metadata and every subsequent
23
+ // write use the same destination/authority/erasure fence as chunk uploads.
24
+ await transact(request.options.database, request.authority, [
25
+ sql('INSERT OR IGNORE INTO ik_door_scopes(installation_id,namespace,destination) VALUES(?,?,?)', g.installationId, g.namespace, request.options.destination),
26
+ sql('INSERT INTO ik_door_uploads(id,installation_id,namespace,name,bundle_sha256,total_bytes,total_chunks,intake_json) VALUES(?,?,?,?,?,-1,-1,?)', id, g.installationId, g.namespace, name, '0'.repeat(64), pending),
27
+ ]);
28
+ const held = sql('EXISTS(SELECT 1 FROM ik_door_uploads WHERE id=? AND installation_id=? AND namespace=? AND intake_json=? AND total_bytes=-1)', id, g.installationId, g.namespace, pending);
29
+ let ordinal = 0, size = 0, buffered = Buffer.alloc(CHUNK_BYTES), used = 0;
30
+ const hash = createHash('sha256');
31
+ let rows: {ordinal: number; sha256: string; size: number; base64: string}[] = [];
32
+ const flush = async () => {
33
+ if (!rows.length) return;
34
+ await transact(request.options.database, request.authority, [assertion(held),
35
+ sql("INSERT INTO ik_door_chunks(upload_id,ordinal,sha256,size,base64) SELECT ?,json_extract(value,'$.ordinal'),json_extract(value,'$.sha256'),json_extract(value,'$.size'),json_extract(value,'$.base64') FROM json_each(?)", id, JSON.stringify(rows)),
36
+ ]);
37
+ rows = [];
38
+ };
39
+ const chunk = async () => {
40
+ if (ordinal >= g.maxChunks) throw new DoorError('intake_too_large', 413);
41
+ const bytes = buffered.subarray(0, used);
42
+ rows.push({ordinal: ordinal++, sha256: sha256(bytes), size: bytes.length, base64: bytes.toString('base64')});
43
+ buffered = Buffer.alloc(CHUNK_BYTES); used = 0;
44
+ // One bounded D1 value/batch, never an unbounded list of row statements.
45
+ if (rows.length === 16) await flush();
46
+ };
47
+ const reader = incoming.body.getReader();
48
+ async function* input() {
49
+ try {
50
+ for (;;) {
51
+ incoming.signal.throwIfAborted();
52
+ const {done, value} = await reader.read();
53
+ if (done) return;
54
+ // A host may deliver an arbitrarily large body segment. The parser
55
+ // receives bounded pieces regardless of that transport choice.
56
+ for (let offset = 0; offset < value.length; offset += CHUNK_BYTES) yield value.subarray(offset, offset + CHUNK_BYTES);
57
+ }
58
+ } finally { await reader.cancel(); reader.releaseLock(); }
59
+ }
60
+ try {
61
+ for await (const row of sanitizeIntake(input(), custody.inputFormat, intake.allow[custody.inputFormat]!, {
62
+ maxBytes: Math.min(request.options.limits?.maxBytes ?? g.maxBytes, g.maxBytes),
63
+ maxRecords: Math.min(request.options.limits?.maxRecords ?? g.maxRecords, g.maxRecords),
64
+ maxRecordBytes: request.options.limits?.maxRecordBytes ?? 1_048_576,
65
+ })) {
66
+ size += row.length;
67
+ if (size > g.maxBytes) throw new DoorError('intake_too_large', 413);
68
+ hash.update(row);
69
+ for (let offset = 0; offset < row.length;) {
70
+ const length = Math.min(CHUNK_BYTES - used, row.length - offset);
71
+ buffered.set(row.subarray(offset, offset + length), used); used += length; offset += length;
72
+ if (used === CHUNK_BYTES) await chunk();
73
+ }
74
+ }
75
+ if (used) await chunk();
76
+ await flush();
77
+ await transact(request.options.database, request.authority, [assertion(held),
78
+ sql('UPDATE ik_door_uploads SET bundle_sha256=?,total_bytes=?,total_chunks=?,intake_json=? WHERE id=?', hash.digest('hex'), size, ordinal, JSON.stringify({complete: true, custody}), id),
79
+ ]);
80
+ return progress(request, await uploadRow(request, id));
81
+ } catch (error) {
82
+ // No raw bytes reached SQLite, even its freed pages or rollback journal.
83
+ // If authority/erasure wins, the same fence refuses this cleanup too; any
84
+ // surviving partial custody is sanitized and remains visibly incomplete.
85
+ await transact(request.options.database, request.authority, [
86
+ sql('DELETE FROM ik_door_chunks WHERE upload_id=?', id),
87
+ sql('DELETE FROM ik_door_uploads WHERE id=?', id),
88
+ ]).catch(() => undefined);
89
+ throw error;
90
+ }
91
+ }
@@ -0,0 +1,68 @@
1
+ import { canonicalJson, sha256, attachmentHash } from '../provenance/identity.ts';
2
+ import type { ProvenanceRow, TargetRow } from '../types.ts';
3
+ import type { DoorPlanRow, DoorRecord, DoorRequest, DoorTarget } from './types.ts';
4
+ import { DoorError } from './protocol.ts';
5
+ import { readD1PersistedImport } from '../d1/index.ts';
6
+
7
+ export const recordKey = (r: {kind:string;sourceId:string}) => canonicalJson([r.kind,r.sourceId]);
8
+ export interface ImportedTarget { provenance: ProvenanceRow; matches: boolean }
9
+ /** Each distinct owning plan's job, in one read. */
10
+ async function ownerPlans(request: DoorRequest, ids: string[]): Promise<Map<string, Pick<DoorPlanRow,'id'|'job_id'>>> {
11
+ const {results} = await request.options.database.prepare('SELECT id,job_id FROM ik_door_plans WHERE id IN (SELECT value FROM json_each(?)) AND installation_id=? AND namespace=? AND complete=1')
12
+ .bind(JSON.stringify(ids), request.access.installationId, request.access.namespace).all<Pick<DoorPlanRow,'id'|'job_id'>>();
13
+ return new Map(results.map(row => [row.id, row]));
14
+ }
15
+ /** Product lineage only locates evidence. The kit's immutable plan/provenance
16
+ * owns the revision and full native snapshot; a current native revision is never
17
+ * promoted to permission to overwrite an edit. Every target's current state comes
18
+ * from the native rows the persisted read fenced under its atomic guard, so the
19
+ * statements stay fixed however many records the import holds. */
20
+ export async function importedTargets(request: DoorRequest, targets: DoorTarget[]): Promise<ImportedTarget[]> {
21
+ if (!targets.length) return [];
22
+ const owners = await ownerPlans(request, [...new Set(targets.map(t => t.ownerPlanId))]);
23
+ for (const target of targets) {
24
+ const owner = owners.get(target.ownerPlanId);
25
+ if (!owner) throw new DoorError('plan_not_found', 404);
26
+ if (!owner.job_id) throw new DoorError('import_lineage_incomplete', 409);
27
+ }
28
+ const destination = request.options.destination;
29
+ const persisted = await readD1PersistedImport({destination,namespace:request.access.namespace,mappingVersion:request.options.mapping.version,
30
+ storage:{database:request.options.database,targets:request.options.targets},
31
+ authorization:{principalId:request.options.ownerPrincipalId,predicate:action=>action==='read'?request.authority:{sql:'0'}}});
32
+ const live = new Map<string, ProvenanceRow>();
33
+ for (const p of persisted.provenance) {
34
+ const key = canonicalJson([p.jobId,p.sourceIdentity,p.targetKind,p.targetId]);
35
+ if (!p.reversed && p.destination === destination && !live.has(key)) live.set(key, p);
36
+ }
37
+ // A kind/id the native set holds twice is ambiguous and never matches.
38
+ const rows = new Map<string, TargetRow | null>();
39
+ for (const row of persisted.rows) if (row.destination === destination) {
40
+ const key = canonicalJson([row.targetKind,row.targetId]);
41
+ rows.set(key, rows.has(key) ? null : row);
42
+ }
43
+ return targets.map(target => {
44
+ const provenance = live.get(canonicalJson([owners.get(target.ownerPlanId)!.job_id,target.sourceIdentity,target.kind,target.id]));
45
+ if (!provenance) throw new DoorError('import_attribution_missing', 409);
46
+ const current = rows.get(canonicalJson([target.kind,target.id]));
47
+ let matches = false;
48
+ try { matches = !!current && current.revision === provenance.appliedRevision && sha256(canonicalJson(current.value)) === provenance.valueHash && attachmentHash(current.attachment) === provenance.attachmentHash; } catch { /* Corrupt native attachment is a conflict, never replacement authority. */ }
49
+ return {provenance,matches};
50
+ });
51
+ }
52
+ export async function inspectRecords(request: DoorRequest, records: DoorRecord[]) {
53
+ const result = [], found = await importedTargets(request, records.flatMap(record => record.targets));
54
+ let present = 0, explained = 0, targets = 0, next = 0;
55
+ const counts: Record<string, number> = Object.create(null);
56
+ for (const record of records) {
57
+ const conflicts: string[] = [];
58
+ for (const target of record.targets) {
59
+ if (!found[next++].matches) conflicts.push(canonicalJson([target.kind,target.id]));
60
+ else { targets++; counts[target.kind] = (counts[target.kind] ?? 0) + 1; }
61
+ }
62
+ if (!record.targets.length && !record.explanation) conflicts.push('no_native_target_or_explanation');
63
+ if (conflicts.length) result.push({kind:record.kind,sourceId:record.sourceId,status:'mismatched',conflicts});
64
+ else if (record.explanation) { explained++; result.push({kind:record.kind,sourceId:record.sourceId,status:'explained',reason:record.explanation}); }
65
+ else { present++; result.push({kind:record.kind,sourceId:record.sourceId,status:'present'}); }
66
+ }
67
+ return {ok: result.every(r=>r.status!=='mismatched'), sourceIds:result, counts:{sourceRecords:records.length,present,explained,nativeTargets:targets,byKind:counts}};
68
+ }