@runeditrun/rer 0.2.81 → 0.2.83
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -0
- package/cli/export.mjs +1 -1
- package/edition-authoring/base-copies.mjs +3 -1
- package/edition-authoring/copy-checkout.mjs +152 -1
- package/edition-authoring/files.mjs +1 -1
- package/edition-authoring/runtime-dependencies.mjs +1 -1
- package/global/CHANGELOG.md +25 -0
- package/global/README.md +12 -0
- package/modules/import-kit/src/door/ADOPTING.md +452 -0
- package/modules/import-kit/src/door/README.md +553 -0
- package/modules/import-kit/src/door/admission.ts +45 -0
- package/modules/import-kit/src/door/archive.ts +129 -0
- package/modules/import-kit/src/door/authority.ts +107 -0
- package/modules/import-kit/src/door/failures.ts +70 -0
- package/modules/import-kit/src/door/grants.ts +162 -0
- package/modules/import-kit/src/door/handler.ts +90 -0
- package/modules/import-kit/src/door/index.ts +20 -0
- package/modules/import-kit/src/door/intake/csv.ts +74 -0
- package/modules/import-kit/src/door/intake/json.ts +182 -0
- package/modules/import-kit/src/door/intake/projection.ts +61 -0
- package/modules/import-kit/src/door/intake/types.ts +28 -0
- package/modules/import-kit/src/door/intake-policy.ts +38 -0
- package/modules/import-kit/src/door/intake-upload.ts +91 -0
- package/modules/import-kit/src/door/inventory.ts +68 -0
- package/modules/import-kit/src/door/jobs.ts +134 -0
- package/modules/import-kit/src/door/limits.ts +16 -0
- package/modules/import-kit/src/door/operator.ts +41 -0
- package/modules/import-kit/src/door/planning.ts +227 -0
- package/modules/import-kit/src/door/protocol.ts +38 -0
- package/modules/import-kit/src/door/refusals.ts +51 -0
- package/modules/import-kit/src/door/schema.ts +101 -0
- package/modules/import-kit/src/door/store.ts +152 -0
- package/modules/import-kit/src/door/types.ts +100 -0
- package/modules/import-kit/src/door/uploads.ts +98 -0
- package/modules/import-kit/src/provenance/numeric-credentials.ts +40 -0
- package/modules/import-kit/src/readers/README.md +139 -0
- package/modules/import-kit/src/readers/api-policy.ts +67 -0
- package/modules/import-kit/src/readers/api.ts +86 -0
- package/modules/import-kit/src/readers/bytes.ts +73 -0
- package/modules/import-kit/src/readers/errors.ts +33 -0
- package/modules/import-kit/src/readers/file.ts +74 -0
- package/modules/import-kit/src/readers/index.ts +3 -0
- package/modules/import-kit/src/readers/json.ts +29 -0
- package/modules/import-kit/src/readers/numeric-lexemes.ts +104 -0
- package/modules/import-kit/src/readers/original.ts +31 -0
- package/modules/import-kit/src/readers/records.ts +105 -0
- package/package.json +3 -2
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import { parseApiJson } from './numeric-lexemes.ts';
|
|
2
|
+
import type { ApiSourceInput, ApiTransport, SourceBundle } from '../types.ts';
|
|
3
|
+
import { consumeBytes, discardBody, responseBytes, utf8Decoder } from './bytes.ts';
|
|
4
|
+
import { boundedInteger, requestHeaders, retryDelay, sleep, sourceUrl } from './api-policy.ts';
|
|
5
|
+
import { abortable, checkAbort, fail, ReaderError } from './errors.ts';
|
|
6
|
+
import { limitsFor, RecordCollector, readerPaginationPolicy } from './records.ts';
|
|
7
|
+
|
|
8
|
+
/** Fetches and consumes one bounded page at a time. Authentication stays ephemeral; exact response bytes are retained. */
|
|
9
|
+
export async function readApiSource(input: ApiSourceInput, transport: ApiTransport): Promise<SourceBundle> {
|
|
10
|
+
const policy = readerPaginationPolicy(input.paginationCursorPolicy);
|
|
11
|
+
checkAbort(input.signal);
|
|
12
|
+
const limits = limitsFor(input.limits);
|
|
13
|
+
const maxRequests = boundedInteger(input.maxRequests, 1000);
|
|
14
|
+
const maxRetries = boundedInteger(input.maxRetries, 3, true);
|
|
15
|
+
const maxRetryAfterMs = boundedInteger(input.maxRetryAfterMs, 30_000, true);
|
|
16
|
+
const { headers, secrets } = requestHeaders(input.headers, input.token);
|
|
17
|
+
const collector = new RecordCollector(input.namespace, limits, secrets, policy);
|
|
18
|
+
if (typeof input.parsePage !== 'function' || typeof transport !== 'function') fail('INVALID_API_PARSER_OR_TRANSPORT');
|
|
19
|
+
if (input.preserveNumberLexeme !== undefined && typeof input.preserveNumberLexeme !== 'function') fail('INVALID_API_NUMBER_POLICY');
|
|
20
|
+
const origin = sourceUrl(input.url, undefined, secrets, policy);
|
|
21
|
+
let url = origin, requests = 0, page = 1, retries = 0;
|
|
22
|
+
const visited = new Set<string>();
|
|
23
|
+
const budget = { used: 0, max: limits.maxBytes };
|
|
24
|
+
while (true) {
|
|
25
|
+
checkAbort(input.signal);
|
|
26
|
+
if (requests >= maxRequests) fail('SOURCE_REQUEST_LIMIT');
|
|
27
|
+
if (!retries) {
|
|
28
|
+
if (visited.has(url.href)) fail('SOURCE_PAGINATION_CYCLE');
|
|
29
|
+
visited.add(url.href);
|
|
30
|
+
}
|
|
31
|
+
const location = { artifact: `api-page-${page}`, page };
|
|
32
|
+
let response: Response;
|
|
33
|
+
try {
|
|
34
|
+
// A transport must honor manual redirects. No ambient cookies are permitted.
|
|
35
|
+
const request = new Request(url, { headers, signal: input.signal, redirect: 'manual', credentials: 'omit', cache: 'no-store' });
|
|
36
|
+
requests++;
|
|
37
|
+
const pending = Promise.resolve(transport(request));
|
|
38
|
+
void pending.then(value => { if (input.signal?.aborted && value instanceof Response) discardBody(value); }, () => {});
|
|
39
|
+
response = await abortable(pending, input.signal);
|
|
40
|
+
} catch { checkAbort(input.signal); fail('SOURCE_REQUEST_FAILED', location); }
|
|
41
|
+
if (!(response instanceof Response)) fail('INVALID_API_RESPONSE', location);
|
|
42
|
+
try {
|
|
43
|
+
checkAbort(input.signal);
|
|
44
|
+
if (response.redirected) fail('UNSAFE_TRANSPORT_REDIRECT', location);
|
|
45
|
+
if (response.url) sourceUrl(response.url, origin, secrets, policy);
|
|
46
|
+
if ([301, 302, 303, 307, 308].includes(response.status)) {
|
|
47
|
+
const next = response.headers.get('location');
|
|
48
|
+
if (!next) fail('MISSING_REDIRECT_LOCATION', location);
|
|
49
|
+
url = sourceUrl(next, url, secrets, policy);
|
|
50
|
+
retries = 0;
|
|
51
|
+
continue;
|
|
52
|
+
}
|
|
53
|
+
if (response.status === 429 || ([403, 503].includes(response.status) && response.headers.has('retry-after'))) {
|
|
54
|
+
if (retries >= maxRetries) fail('SOURCE_RATE_LIMIT', location);
|
|
55
|
+
const ms = retryDelay(response.headers.get('retry-after'), retries, maxRetryAfterMs);
|
|
56
|
+
discardBody(response);
|
|
57
|
+
try { await abortable(Promise.resolve((input.sleep ?? sleep)(ms, input.signal)), input.signal); }
|
|
58
|
+
catch { checkAbort(input.signal); fail('SOURCE_WAIT_FAILED', location); }
|
|
59
|
+
retries++;
|
|
60
|
+
continue;
|
|
61
|
+
}
|
|
62
|
+
if (!response.ok) fail('SOURCE_HTTP_ERROR', location);
|
|
63
|
+
const length = response.headers.get('content-length');
|
|
64
|
+
if (length && /^\d+$/.test(length) && Number(length) > budget.max - budget.used) fail('SOURCE_BYTE_LIMIT', location);
|
|
65
|
+
const decoder = utf8Decoder(location);
|
|
66
|
+
let text = '';
|
|
67
|
+
const artifact = await consumeBytes(responseBytes(response, input.signal), budget, location, chunk => { text += decoder.decode(chunk); }, input.signal);
|
|
68
|
+
text += decoder.decode(undefined, true);
|
|
69
|
+
const { value, context } = parseApiJson(text, location, secrets, input);
|
|
70
|
+
checkAbort(input.signal);
|
|
71
|
+
let parsed: ReturnType<ApiSourceInput['parsePage']>;
|
|
72
|
+
try { parsed = input.parsePage(value, page, context); } catch { fail('SOURCE_PARSER_FAILED', location); }
|
|
73
|
+
if (!parsed || typeof parsed !== 'object' || !Array.isArray(parsed.records)) fail('INVALID_PARSED_PAGE', location);
|
|
74
|
+
if (parsed.next !== undefined && parsed.next !== null && (typeof parsed.next !== 'string' || !parsed.next)) fail('INVALID_NEXT_PAGE', location);
|
|
75
|
+
collector.add(parsed.records, location, artifact.sha256);
|
|
76
|
+
collector.bundle.artifacts.push({ name: location.artifact, ...artifact });
|
|
77
|
+
checkAbort(input.signal);
|
|
78
|
+
if (parsed.next === undefined || parsed.next === null) return collector.bundle;
|
|
79
|
+
url = sourceUrl(parsed.next, url, secrets, policy);
|
|
80
|
+
retries = 0; page++;
|
|
81
|
+
} catch (error) {
|
|
82
|
+
if (error instanceof ReaderError) throw error;
|
|
83
|
+
checkAbort(input.signal); fail('SOURCE_READ_FAILED', location);
|
|
84
|
+
} finally { discardBody(response); }
|
|
85
|
+
}
|
|
86
|
+
}
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
import { OriginalBytes } from './original.ts';
|
|
3
|
+
import type { SourceLocation } from '../types.ts';
|
|
4
|
+
import { abortable, checkAbort, fail } from './errors.ts';
|
|
5
|
+
|
|
6
|
+
export interface ByteBudget { used: number; max: number }
|
|
7
|
+
/** One pull at a time; closes the iterator on decoding, parsing, size or cancellation failure. */
|
|
8
|
+
export async function consumeBytes(
|
|
9
|
+
bytes: Uint8Array | AsyncIterable<Uint8Array>, budget: ByteBudget, location: SourceLocation,
|
|
10
|
+
consume: (chunk: Uint8Array) => void, signal?: AbortSignal,
|
|
11
|
+
): Promise<{ sha256: string; size: number; original: { status: 'available'; base64: string } }> {
|
|
12
|
+
const hash = createHash('sha256');
|
|
13
|
+
const original = new OriginalBytes();
|
|
14
|
+
let size = 0;
|
|
15
|
+
const accept = (chunk: Uint8Array): void => {
|
|
16
|
+
checkAbort(signal);
|
|
17
|
+
if (!(chunk instanceof Uint8Array)) fail('INVALID_SOURCE_CHUNK', location);
|
|
18
|
+
if (chunk.byteLength > budget.max - budget.used) fail('SOURCE_BYTE_LIMIT', location);
|
|
19
|
+
budget.used += chunk.byteLength; size += chunk.byteLength;
|
|
20
|
+
hash.update(chunk); original.append(chunk); consume(chunk);
|
|
21
|
+
};
|
|
22
|
+
if (bytes instanceof Uint8Array) accept(bytes);
|
|
23
|
+
else {
|
|
24
|
+
let iterator: AsyncIterator<Uint8Array>;
|
|
25
|
+
try { iterator = bytes[Symbol.asyncIterator](); } catch { fail('INVALID_SOURCE_STREAM', location); }
|
|
26
|
+
let complete = false;
|
|
27
|
+
try {
|
|
28
|
+
while (true) {
|
|
29
|
+
checkAbort(signal);
|
|
30
|
+
let result: IteratorResult<Uint8Array>;
|
|
31
|
+
try { result = await abortable(Promise.resolve(iterator.next()), signal); }
|
|
32
|
+
catch { checkAbort(signal); fail('SOURCE_READ_FAILED', location); }
|
|
33
|
+
if (result.done) { complete = true; break; }
|
|
34
|
+
accept(result.value);
|
|
35
|
+
}
|
|
36
|
+
} finally {
|
|
37
|
+
if (!complete && iterator.return) {
|
|
38
|
+
// Do not let an uncooperative producer delay cancellation or mask the real error.
|
|
39
|
+
try { void Promise.resolve(iterator.return()).catch(() => {}); } catch {}
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
return { sha256: hash.digest('hex'), size, original: original.finish() };
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export function utf8Decoder(location: SourceLocation): { decode(chunk?: Uint8Array, final?: boolean): string } {
|
|
47
|
+
const decoder = new TextDecoder('utf-8', { fatal: true });
|
|
48
|
+
return { decode(chunk, final = false) {
|
|
49
|
+
try { return decoder.decode(chunk, { stream: !final }); }
|
|
50
|
+
catch { fail('MALFORMED_UTF8', location); }
|
|
51
|
+
} };
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/** Cancels a web body on early exit, including a read that is still pending. */
|
|
55
|
+
export async function* responseBytes(response: Response, signal?: AbortSignal): AsyncGenerator<Uint8Array> {
|
|
56
|
+
if (!response.body) return;
|
|
57
|
+
const reader = response.body.getReader();
|
|
58
|
+
let complete = false;
|
|
59
|
+
try {
|
|
60
|
+
while (true) {
|
|
61
|
+
checkAbort(signal);
|
|
62
|
+
const result = await abortable(reader.read(), signal);
|
|
63
|
+
if (result.done) { complete = true; break; }
|
|
64
|
+
yield result.value;
|
|
65
|
+
}
|
|
66
|
+
} finally {
|
|
67
|
+
if (!complete) { try { await reader.cancel(); } catch {} }
|
|
68
|
+
reader.releaseLock();
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
export function discardBody(response: Response): void {
|
|
72
|
+
try { void response.body?.cancel().catch(() => {}); } catch {}
|
|
73
|
+
}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import type { Diagnostic, SourceLocation } from '../types.ts';
|
|
2
|
+
|
|
3
|
+
/** Safe, structured diagnostics: never include input bytes, URLs or thrown causes. */
|
|
4
|
+
export class ReaderError extends Error {
|
|
5
|
+
readonly code: string;
|
|
6
|
+
readonly diagnostics: Diagnostic[];
|
|
7
|
+
/** The kind of field a credential refusal concerns (never its value). */
|
|
8
|
+
readonly field?: string;
|
|
9
|
+
constructor(code: string, location?: SourceLocation, field?: string) {
|
|
10
|
+
super(code);
|
|
11
|
+
this.name = 'ReaderError';
|
|
12
|
+
this.code = code;
|
|
13
|
+
if (field !== undefined) this.field = field;
|
|
14
|
+
this.diagnostics = [{ severity: 'error', code, message: code, ...(location ? { location } : {}), ...(field !== undefined ? { field } : {}) }];
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
export function fail(code: string, location?: SourceLocation, field?: string): never { throw new ReaderError(code, location, field); }
|
|
18
|
+
export function checkAbort(signal?: AbortSignal): void { if (signal?.aborted) fail('SOURCE_CANCELLED'); }
|
|
19
|
+
|
|
20
|
+
/** Observe cancellation even when a supplied transport/sleep does not cooperate. */
|
|
21
|
+
export async function abortable<T>(work: Promise<T>, signal?: AbortSignal): Promise<T> {
|
|
22
|
+
// Creating work can synchronously abort the signal. Observe its rejection even
|
|
23
|
+
// when cancellation has already won, so abandoned reads cannot reject unhandled.
|
|
24
|
+
if (signal?.aborted) { void work.catch(() => {}); fail('SOURCE_CANCELLED'); }
|
|
25
|
+
if (!signal) return work;
|
|
26
|
+
let onAbort: () => void = () => {};
|
|
27
|
+
const aborted = new Promise<never>((_, reject) => {
|
|
28
|
+
onAbort = () => reject(new ReaderError('SOURCE_CANCELLED'));
|
|
29
|
+
signal.addEventListener('abort', onAbort, { once: true });
|
|
30
|
+
});
|
|
31
|
+
try { return await Promise.race([work, aborted]); }
|
|
32
|
+
finally { signal.removeEventListener('abort', onAbort); }
|
|
33
|
+
}
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import { Buffer } from 'node:buffer';
|
|
2
|
+
import { parseJson } from './json.ts';
|
|
3
|
+
import type { FileSourceInput, Json, ParsedRecord, ReaderLimits, SourceBundle, SourceLocation } from '../types.ts';
|
|
4
|
+
import { consumeBytes, utf8Decoder } from './bytes.ts';
|
|
5
|
+
import { fail } from './errors.ts';
|
|
6
|
+
import { limitsFor, RecordCollector, readerCredentialPolicy, readerPaginationPolicy, safeValue, textId } from './records.ts';
|
|
7
|
+
import { liveCredentialRemoved, removeCredentials, withholdsOriginal } from '../provenance/credentials.ts';
|
|
8
|
+
|
|
9
|
+
/** JSON arrays are visited per element; an object/scalar is one input. NDJSON uses physical rows. */
|
|
10
|
+
export async function readFileSource(input: FileSourceInput, limits?: ReaderLimits): Promise<SourceBundle> {
|
|
11
|
+
return (await readFileSourceRemovals(input, limits)).source;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
/** readFileSource, also saying whether remove mode took out a live credential
|
|
15
|
+
* value, as opposed to only an exporter's markers (whose original bytes are
|
|
16
|
+
* withheld too, but hold no credential). */
|
|
17
|
+
export async function readFileSourceRemovals(input: FileSourceInput, limits?: ReaderLimits): Promise<{ source: SourceBundle; liveCredentialsRemoved: boolean }> {
|
|
18
|
+
const policy = readerPaginationPolicy(input.paginationCursorPolicy);
|
|
19
|
+
const remove = readerCredentialPolicy(input.credentials) === 'remove';
|
|
20
|
+
const bounded = limitsFor(limits);
|
|
21
|
+
textId(input.name); safeValue(input.name, [], undefined, policy);
|
|
22
|
+
if (input.format !== 'json' && input.format !== 'ndjson') fail('INVALID_SOURCE_FORMAT');
|
|
23
|
+
if (typeof input.parse !== 'function') fail('INVALID_SOURCE_PARSER');
|
|
24
|
+
const collector = new RecordCollector(input.namespace, bounded, [], policy);
|
|
25
|
+
const location = { artifact: input.name };
|
|
26
|
+
const decoder = utf8Decoder(location);
|
|
27
|
+
let withheld = false, live = false;
|
|
28
|
+
const parse = (value: Json, index: number, at: SourceLocation): void => {
|
|
29
|
+
// Removing credentials: the source value must hold none once its plain ones
|
|
30
|
+
// go, and its records may repeat none of them. The parser sees the value as
|
|
31
|
+
// read, so each record reports its own removals.
|
|
32
|
+
// A value too deeply encoded to inspect is refused by name, as refuse mode does.
|
|
33
|
+
let clean: ReturnType<typeof removeCredentials> | undefined;
|
|
34
|
+
try { clean = remove ? removeCredentials(value, policy) : undefined; } catch { fail('INVALID_OR_CREDENTIAL_SOURCE_DATA', at); }
|
|
35
|
+
if (clean?.removed.some(withholdsOriginal)) withheld = true;
|
|
36
|
+
if (clean?.removed.some(liveCredentialRemoved)) live = true;
|
|
37
|
+
const encoded = safeValue(clean ? clean.value : value, [], at, policy);
|
|
38
|
+
if (Buffer.byteLength(encoded) > bounded.maxRecordBytes) fail('SOURCE_RECORD_BYTE_LIMIT', at);
|
|
39
|
+
let result: ParsedRecord | ParsedRecord[];
|
|
40
|
+
try { result = input.parse(value, index); } catch { fail('SOURCE_PARSER_FAILED', at); }
|
|
41
|
+
const records = Array.isArray(result) ? result : [result];
|
|
42
|
+
if (!records.length) fail('SOURCE_OBJECT_DROPPED', at);
|
|
43
|
+
collector.add(records, at, '', clean && { secrets: clean.secrets });
|
|
44
|
+
};
|
|
45
|
+
let pending = '', row = 0, index = 0;
|
|
46
|
+
const line = (text: string): void => {
|
|
47
|
+
row++;
|
|
48
|
+
const at = { ...location, row };
|
|
49
|
+
if (!text.trim()) return;
|
|
50
|
+
if (Buffer.byteLength(text) > bounded.maxRecordBytes) fail('SOURCE_RECORD_BYTE_LIMIT', at);
|
|
51
|
+
parse(parseJson(text, at), index++, at);
|
|
52
|
+
};
|
|
53
|
+
const decoded = (text: string): void => {
|
|
54
|
+
pending += text;
|
|
55
|
+
if (input.format === 'ndjson') {
|
|
56
|
+
let start = 0, end: number;
|
|
57
|
+
while ((end = pending.indexOf('\n', start)) >= 0) { line(pending.slice(start, end)); start = end + 1; }
|
|
58
|
+
pending = pending.slice(start);
|
|
59
|
+
if (Buffer.byteLength(pending) > bounded.maxRecordBytes) fail('SOURCE_RECORD_BYTE_LIMIT', { ...location, row: row + 1 });
|
|
60
|
+
}
|
|
61
|
+
};
|
|
62
|
+
const artifact = await consumeBytes(input.bytes, { used: 0, max: bounded.maxBytes }, location, chunk => decoded(decoder.decode(chunk)));
|
|
63
|
+
decoded(decoder.decode(undefined, true));
|
|
64
|
+
if (input.format === 'ndjson') { if (pending.length) line(pending); }
|
|
65
|
+
else {
|
|
66
|
+
const root = parseJson(pending, location);
|
|
67
|
+
if (Array.isArray(root)) root.forEach((value, position) => parse(value, position, { ...location, row: position + 1, pointer: `/${position}` }));
|
|
68
|
+
else parse(root, 0, { ...location, row: 1, pointer: '' });
|
|
69
|
+
}
|
|
70
|
+
// Original bytes that held a removed credential value are never retained.
|
|
71
|
+
collector.bundle.artifacts.push({ name: input.name, ...artifact, ...(withheld || collector.credentialsRemoved ? { original: { status: 'unavailable' as const, reason: 'credentials-removed' as const } } : {}) });
|
|
72
|
+
for (const record of collector.bundle.records) record.artifactHash = artifact.sha256;
|
|
73
|
+
return { source: collector.bundle, liveCredentialsRemoved: live || collector.liveCredentialsRemoved };
|
|
74
|
+
}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import type { Json, SourceLocation } from '../types.ts';
|
|
2
|
+
import { fail } from './errors.ts';
|
|
3
|
+
|
|
4
|
+
/** JSON.parse validates syntax; this pass rejects fields its last-key-wins policy would discard. */
|
|
5
|
+
export function parseJson(text: string, location: SourceLocation): Json {
|
|
6
|
+
let value: Json;
|
|
7
|
+
try { value = JSON.parse(text) as Json; } catch { fail('MALFORMED_JSON', location); }
|
|
8
|
+
const stack: ({ keys: Set<string>; expectingKey: boolean } | null)[] = [];
|
|
9
|
+
for (let index = 0; index < text.length; index++) {
|
|
10
|
+
const character = text[index];
|
|
11
|
+
if (character === '{') stack.push({ keys: new Set(), expectingKey: true });
|
|
12
|
+
else if (character === '[') stack.push(null);
|
|
13
|
+
else if (character === '}' || character === ']') stack.pop();
|
|
14
|
+
else if (character === ':' || character === ',') {
|
|
15
|
+
const object = stack.at(-1);
|
|
16
|
+
if (object) object.expectingKey = character === ',';
|
|
17
|
+
} else if (character === '"') {
|
|
18
|
+
const start = index++;
|
|
19
|
+
while (text[index] !== '"') { if (text[index] === '\\') index++; index++; }
|
|
20
|
+
const object = stack.at(-1);
|
|
21
|
+
if (object?.expectingKey) {
|
|
22
|
+
const key = JSON.parse(text.slice(start, index + 1)) as string;
|
|
23
|
+
if (object.keys.has(key)) fail('DUPLICATE_JSON_KEY', location);
|
|
24
|
+
object.keys.add(key);
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
return value;
|
|
29
|
+
}
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
import type { ApiPageContext, ApiSourceInput, Json, JsonPathSegment, SourceLocation } from '../types.ts';
|
|
2
|
+
import { checkAbort, fail } from './errors.ts';
|
|
3
|
+
import { parseJson } from './json.ts';
|
|
4
|
+
import { safeValue } from './records.ts';
|
|
5
|
+
import { reflectedNumberCredentialMatcher } from '../provenance/numeric-credentials.ts';
|
|
6
|
+
|
|
7
|
+
interface NumberToken { start: number; end: number; path: JsonPathSegment[] }
|
|
8
|
+
export const EMPTY_API_PAGE_CONTEXT: ApiPageContext = Object.freeze({ numericLexemes: Object.freeze(Object.create(null)) });
|
|
9
|
+
|
|
10
|
+
/** Walk validated JSON syntax, never string contents, to locate numeric value tokens. */
|
|
11
|
+
function numberTokens(text: string): NumberToken[] {
|
|
12
|
+
const tokens: NumberToken[] = [];
|
|
13
|
+
let index = 0;
|
|
14
|
+
const whitespace = () => { while (' \t\r\n'.includes(text[index]) && index < text.length) index++; };
|
|
15
|
+
const stringEnd = () => {
|
|
16
|
+
index++; // opening quote
|
|
17
|
+
while (text[index] !== '"') { if (text[index] === '\\') index++; index++; }
|
|
18
|
+
index++;
|
|
19
|
+
};
|
|
20
|
+
const visit = (path: JsonPathSegment[]): void => {
|
|
21
|
+
whitespace();
|
|
22
|
+
const character = text[index];
|
|
23
|
+
if (character === '"') { stringEnd(); return; }
|
|
24
|
+
if (character === '{' || character === '[') {
|
|
25
|
+
const object = character === '{', close = object ? '}' : ']';
|
|
26
|
+
index++; whitespace();
|
|
27
|
+
let arrayIndex = 0;
|
|
28
|
+
while (text[index] !== close) {
|
|
29
|
+
let key: JsonPathSegment = arrayIndex++;
|
|
30
|
+
if (object) {
|
|
31
|
+
const start = index;
|
|
32
|
+
stringEnd();
|
|
33
|
+
key = JSON.parse(text.slice(start, index)) as string;
|
|
34
|
+
whitespace(); index++; // colon
|
|
35
|
+
}
|
|
36
|
+
visit([...path, key]); whitespace();
|
|
37
|
+
if (text[index] !== ',') break;
|
|
38
|
+
index++; whitespace();
|
|
39
|
+
}
|
|
40
|
+
index++; return;
|
|
41
|
+
}
|
|
42
|
+
const start = index;
|
|
43
|
+
while (index < text.length && !' \t\r\n,]}'.includes(text[index])) index++;
|
|
44
|
+
if (character === '-' || (character >= '0' && character <= '9')) tokens.push({ start, end: index, path });
|
|
45
|
+
};
|
|
46
|
+
visit([]);
|
|
47
|
+
return tokens;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function withStrings(text: string, tokens: readonly NumberToken[]): Json {
|
|
51
|
+
const parts: string[] = [];
|
|
52
|
+
let offset = 0;
|
|
53
|
+
for (const token of tokens) {
|
|
54
|
+
parts.push(text.slice(offset, token.start), '"', text.slice(token.start, token.end), '"');
|
|
55
|
+
offset = token.end;
|
|
56
|
+
}
|
|
57
|
+
parts.push(text.slice(offset));
|
|
58
|
+
return JSON.parse(parts.join('')) as Json;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/** Security validation precedes both callbacks; original bytes remain owned by the reader. */
|
|
62
|
+
export function parseApiJson(text: string, location: SourceLocation, secrets: readonly string[], input: ApiSourceInput): { value: Json; context: ApiPageContext } {
|
|
63
|
+
const original = parseJson(text, location); // syntax and duplicate-key validation, including escaped keys
|
|
64
|
+
const preserve = input.preserveNumberLexeme;
|
|
65
|
+
const tokens = preserve !== undefined || secrets.length ? numberTokens(text) : [];
|
|
66
|
+
const reflectsNumber = reflectedNumberCredentialMatcher(secrets);
|
|
67
|
+
// The original lexeme can retain digits lost by Number rounding, even when the
|
|
68
|
+
// adapter does not select numeric preservation or excludes this field entirely.
|
|
69
|
+
for (const token of tokens) {
|
|
70
|
+
if (reflectsNumber(text.slice(token.start, token.end))) fail('CREDENTIAL_IN_SOURCE_DATA', location);
|
|
71
|
+
}
|
|
72
|
+
if (preserve === undefined) {
|
|
73
|
+
safeValue(original, secrets, location, input.paginationCursorPolicy);
|
|
74
|
+
return { value: original, context: EMPTY_API_PAGE_CONTEXT };
|
|
75
|
+
}
|
|
76
|
+
// Validate decoded keys/strings and exact numeric text before exposing even paths.
|
|
77
|
+
// Strings permit valid overflowing JSON numbers here; unselected numbers must still
|
|
78
|
+
// pass the ordinary finite-value check below.
|
|
79
|
+
safeValue(withStrings(text, tokens), secrets, location, input.paginationCursorPolicy);
|
|
80
|
+
const selected: NumberToken[] = [];
|
|
81
|
+
const numericLexemes: Record<string, string> = Object.create(null);
|
|
82
|
+
for (const token of tokens) {
|
|
83
|
+
checkAbort(input.signal);
|
|
84
|
+
let keep: unknown;
|
|
85
|
+
try { keep = preserve(Object.freeze([...token.path])); }
|
|
86
|
+
catch { fail('SOURCE_NUMBER_POLICY_FAILED', location); }
|
|
87
|
+
if (typeof keep !== 'boolean') {
|
|
88
|
+
// Invalid async policies must not leak a rejected promise's private cause.
|
|
89
|
+
void Promise.resolve(keep).catch(() => {});
|
|
90
|
+
fail('SOURCE_NUMBER_POLICY_FAILED', location);
|
|
91
|
+
}
|
|
92
|
+
checkAbort(input.signal);
|
|
93
|
+
if (keep) {
|
|
94
|
+
selected.push(token);
|
|
95
|
+
const pointer = token.path.map(part => '/' + String(part).replaceAll('~', '~0').replaceAll('/', '~1')).join('');
|
|
96
|
+
numericLexemes[pointer] = text.slice(token.start, token.end);
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
const value = withStrings(text, selected);
|
|
100
|
+
const context = Object.freeze({ numericLexemes: Object.freeze(numericLexemes) });
|
|
101
|
+
safeValue(value, secrets, location, input.paginationCursorPolicy);
|
|
102
|
+
safeValue(context, secrets, location, input.paginationCursorPolicy);
|
|
103
|
+
return { value, context };
|
|
104
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { Buffer } from 'node:buffer';
|
|
2
|
+
/** Retain exact bytes in fixed blocks; one-byte stream chunks cannot create an unbounded object list. */
|
|
3
|
+
export class OriginalBytes {
|
|
4
|
+
private readonly blocks: Buffer[] = [];
|
|
5
|
+
private current: Buffer | undefined;
|
|
6
|
+
private offset = 0;
|
|
7
|
+
private size = 0;
|
|
8
|
+
append(bytes: Uint8Array): void {
|
|
9
|
+
let inputOffset = 0;
|
|
10
|
+
while (inputOffset < bytes.byteLength) {
|
|
11
|
+
let current = this.current;
|
|
12
|
+
if (!current || this.offset === current.byteLength) {
|
|
13
|
+
// The explicit type keeps this narrowing sound in projects whose global Buffer is overridden
|
|
14
|
+
// by Worker typings (their `declare const Buffer: any` makes allocUnsafe return any).
|
|
15
|
+
const fresh: Buffer = Buffer.allocUnsafe(64 * 1024);
|
|
16
|
+
this.current = fresh;
|
|
17
|
+
this.blocks.push(fresh);
|
|
18
|
+
this.offset = 0;
|
|
19
|
+
current = fresh;
|
|
20
|
+
}
|
|
21
|
+
const length = Math.min(bytes.byteLength - inputOffset, current.byteLength - this.offset);
|
|
22
|
+
current.set(bytes.subarray(inputOffset, inputOffset + length), this.offset);
|
|
23
|
+
this.offset += length; inputOffset += length; this.size += length;
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
finish(): { status: 'available'; base64: string } {
|
|
27
|
+
// Limit the final view: unused bytes in allocUnsafe blocks must never be exposed.
|
|
28
|
+
const filled = this.blocks.map((block, index) => index === this.blocks.length - 1 ? block.subarray(0, this.offset) : block);
|
|
29
|
+
return { status: 'available', base64: Buffer.concat(filled, this.size).toString('base64') };
|
|
30
|
+
}
|
|
31
|
+
}
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
import { Buffer } from 'node:buffer';
|
|
2
|
+
import type { Json, PaginationCursorPolicy, ParsedRecord, ReaderLimits, SourceBundle, SourceCredentialPolicy, SourceLocation } from '../types.ts';
|
|
3
|
+
import { assertNoCredentials, canonicalJson, sha256 } from '../provenance/identity.ts';
|
|
4
|
+
import { CredentialError, credentialFieldText, liveCredentialRemoved, paginationPolicy, reflectedCredentialMatcher, removeCredentials, withholdsOriginal, type RemovedCredential } from '../provenance/credentials.ts';
|
|
5
|
+
import { reflectedNumberCredentialMatcher } from '../provenance/numeric-credentials.ts';
|
|
6
|
+
import { fail } from './errors.ts';
|
|
7
|
+
|
|
8
|
+
export function readerPaginationPolicy(value: unknown): PaginationCursorPolicy | undefined {
|
|
9
|
+
try { return paginationPolicy(value); } catch { fail('INVALID_PAGINATION_CURSOR_POLICY'); }
|
|
10
|
+
}
|
|
11
|
+
export function readerCredentialPolicy(value: unknown): SourceCredentialPolicy {
|
|
12
|
+
if (value !== undefined && value !== 'refuse' && value !== 'remove') fail('INVALID_CREDENTIAL_POLICY');
|
|
13
|
+
return value ?? 'refuse';
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export const DEFAULT_LIMITS = { maxBytes: 64 * 1024 * 1024, maxRecords: 100_000, maxRecordBytes: 1024 * 1024 };
|
|
17
|
+
export function limitsFor(input: ReaderLimits = {}): Required<ReaderLimits> {
|
|
18
|
+
const limits = { ...DEFAULT_LIMITS, ...input };
|
|
19
|
+
for (const value of Object.values(limits)) if (!Number.isSafeInteger(value) || value < 1) fail('INVALID_READER_LIMIT');
|
|
20
|
+
return limits;
|
|
21
|
+
}
|
|
22
|
+
export function textId(value: unknown): asserts value is string {
|
|
23
|
+
if (typeof value !== 'string' || !value.trim() || value.length > 4096) fail('INVALID_SOURCE_ID');
|
|
24
|
+
}
|
|
25
|
+
export function safeValue(value: unknown, secrets: readonly string[] = [], location?: SourceLocation, policy?: PaginationCursorPolicy): string {
|
|
26
|
+
let json: string;
|
|
27
|
+
try {
|
|
28
|
+
json = canonicalJson(value);
|
|
29
|
+
assertNoCredentials(value, policy);
|
|
30
|
+
} catch (error) { fail('INVALID_OR_CREDENTIAL_SOURCE_DATA', location, error instanceof CredentialError ? credentialFieldText(error.field) : undefined); }
|
|
31
|
+
// Inspect scalar values; JSON escaping and numeric coercion cannot hide authentication.
|
|
32
|
+
const reflectsCredential = reflectedCredentialMatcher(secrets);
|
|
33
|
+
const reflectsNumber = reflectedNumberCredentialMatcher(secrets, reflectsCredential);
|
|
34
|
+
const reflected = credentialFieldText({ kind: 'reflected' });
|
|
35
|
+
const check = (item: unknown): void => {
|
|
36
|
+
if (typeof item === 'string' && reflectsCredential(item)) fail('CREDENTIAL_IN_SOURCE_DATA', location, reflected);
|
|
37
|
+
if (typeof item === 'number' && reflectsNumber(String(item))) fail('CREDENTIAL_IN_SOURCE_DATA', location, reflected);
|
|
38
|
+
if ((item === null || typeof item === 'boolean') && reflectsCredential(String(item))) fail('CREDENTIAL_IN_SOURCE_DATA', location, reflected);
|
|
39
|
+
if (item && typeof item === 'object') for (const [key, child] of Object.entries(item)) { check(key); check(child); }
|
|
40
|
+
};
|
|
41
|
+
check(value);
|
|
42
|
+
return json;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export class RecordCollector {
|
|
46
|
+
readonly bundle: SourceBundle = { records: [], artifacts: [], diagnostics: [] };
|
|
47
|
+
readonly seen = new Map<string, string>();
|
|
48
|
+
readonly namespace: string;
|
|
49
|
+
readonly limits: Required<ReaderLimits>;
|
|
50
|
+
readonly secrets: readonly string[];
|
|
51
|
+
readonly paginationCursorPolicy?: PaginationCursorPolicy;
|
|
52
|
+
/** A record lost a credential value, so the source's original bytes are withheld. */
|
|
53
|
+
credentialsRemoved = false;
|
|
54
|
+
/** A record lost a live credential value, not only an exporter's marker. */
|
|
55
|
+
liveCredentialsRemoved = false;
|
|
56
|
+
constructor(namespace: string, limits: Required<ReaderLimits>, secrets: readonly string[] = [], paginationCursorPolicy?: PaginationCursorPolicy) {
|
|
57
|
+
paginationPolicy(paginationCursorPolicy);
|
|
58
|
+
this.paginationCursorPolicy = paginationCursorPolicy;
|
|
59
|
+
if (paginationCursorPolicy) this.bundle.paginationCursorPolicy = paginationCursorPolicy;
|
|
60
|
+
textId(namespace); safeValue(namespace, secrets, undefined, paginationCursorPolicy);
|
|
61
|
+
this.namespace = namespace; this.limits = limits; this.secrets = secrets;
|
|
62
|
+
}
|
|
63
|
+
/** With `removal`, each record's payload loses its plain credential values
|
|
64
|
+
* (reported per record), and no record may repeat a value removed from it or
|
|
65
|
+
* from the source value it came from (`removal.secrets`). */
|
|
66
|
+
add(parsed: ParsedRecord[], location: SourceLocation, artifactHash: string, removal?: { secrets: readonly string[] }): void {
|
|
67
|
+
if (!Array.isArray(parsed)) fail('INVALID_PARSED_RECORDS', location);
|
|
68
|
+
if (this.bundle.records.length + parsed.length > this.limits.maxRecords) fail('SOURCE_RECORD_LIMIT', location);
|
|
69
|
+
for (let record of parsed) {
|
|
70
|
+
if (!record || typeof record !== 'object') fail('INVALID_PARSED_RECORD', location);
|
|
71
|
+
try { textId(record.kind); textId(record.sourceId); } catch { fail('INVALID_PARSED_RECORD_ID', location); }
|
|
72
|
+
let secrets = this.secrets, removed: RemovedCredential[] = [];
|
|
73
|
+
if (removal) {
|
|
74
|
+
let clean: ReturnType<typeof removeCredentials>;
|
|
75
|
+
try { clean = removeCredentials(record.payload, this.paginationCursorPolicy); } catch { fail('INVALID_OR_CREDENTIAL_SOURCE_DATA', location); }
|
|
76
|
+
record = { ...record, payload: clean.value };
|
|
77
|
+
secrets = [...secrets, ...removal.secrets, ...clean.secrets];
|
|
78
|
+
removed = clean.removed;
|
|
79
|
+
if (removed.some(withholdsOriginal)) this.credentialsRemoved = true;
|
|
80
|
+
if (removed.some(liveCredentialRemoved)) this.liveCredentialsRemoved = true;
|
|
81
|
+
}
|
|
82
|
+
const encoded = safeValue(record, secrets, location, this.paginationCursorPolicy);
|
|
83
|
+
if (Buffer.byteLength(encoded) > this.limits.maxRecordBytes) fail('SOURCE_RECORD_BYTE_LIMIT', location);
|
|
84
|
+
if (record.pointer !== undefined && (typeof record.pointer !== 'string' || (record.pointer !== '' && !record.pointer.startsWith('/')))) fail('INVALID_SOURCE_POINTER', location);
|
|
85
|
+
// Detach callback-owned objects; later callback mutations cannot alter earlier records.
|
|
86
|
+
const payload: Json = JSON.parse(safeValue(record.payload, secrets, location, this.paginationCursorPolicy));
|
|
87
|
+
const payloadHash = sha256(canonicalJson(payload));
|
|
88
|
+
const actualLocation = { ...location, ...(record.pointer !== undefined ? { pointer: record.pointer } : {}) };
|
|
89
|
+
const key = canonicalJson([record.kind, record.sourceId]);
|
|
90
|
+
if (this.seen.has(key)) {
|
|
91
|
+
const conflict = this.seen.get(key) !== payloadHash;
|
|
92
|
+
this.bundle.diagnostics.push({ severity: conflict ? 'error' : 'warning', code: conflict ? 'DUPLICATE_SOURCE_CONFLICT' : 'DUPLICATE_SOURCE', message: conflict ? 'Source identity has different content.' : 'Source identity repeats identical content.', kind: record.kind, sourceId: record.sourceId, location: actualLocation });
|
|
93
|
+
} else this.seen.set(key, payloadHash);
|
|
94
|
+
this.bundle.records.push({ namespace: this.namespace, kind: record.kind, sourceId: record.sourceId, payload, payloadHash, artifactHash, location: actualLocation });
|
|
95
|
+
for (const item of removed) this.bundle.diagnostics.push({ severity: 'warning', code: 'CREDENTIAL_REMOVED', message: removalMessage(item), kind: record.kind, sourceId: record.sourceId, location: actualLocation, field: item.path });
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
function removalMessage(item: RemovedCredential): string {
|
|
101
|
+
const kept = 'the record is kept without it.';
|
|
102
|
+
if (item.field.kind === 'url-userinfo') return 'A user name and password were removed from a URL; the record is kept.';
|
|
103
|
+
if (item.field.kind === 'url-query') return 'A credential query parameter was removed from a URL; the record is kept.';
|
|
104
|
+
return `The value of ${credentialFieldText(item.field)} was removed${item.beforeUpload ? ' before upload' : ''}; ${kept}`;
|
|
105
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@runeditrun/rer",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.83",
|
|
4
4
|
"description": "The rer command: install, set up, run and launch Run Edit Run checkouts.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -93,7 +93,8 @@
|
|
|
93
93
|
"./modules/owner-claim": "./modules/owner-claim/index.mjs",
|
|
94
94
|
"./modules/dependencies": "./modules/dependencies/index.mjs",
|
|
95
95
|
"./internal/*": "./*",
|
|
96
|
-
"./package.json": "./package.json"
|
|
96
|
+
"./package.json": "./package.json",
|
|
97
|
+
"./modules/import-kit/door": "./modules/import-kit/src/door/index.ts"
|
|
97
98
|
},
|
|
98
99
|
"engines": {
|
|
99
100
|
"node": ">=22"
|