@tenphi/akno-core 0.1.1 → 0.3.0

package/dist/ingest/fetch.d.ts

@@ -1,19 +1,10 @@
 /**
  * `ingest` pulls documents from a file, a folder, or a URL. This is the URL.
  *
- * Downloading is the one thing `ingest` does that reaches outside the machine, so the
- * limits are deliberate rather than defensive-by-habit:
- *
- * - **http and https only.** `file://` would make `ingest({url})` a way to read any
- *   path on the machine through an interface that looks like it fetches the web, and
- *   `data:` would smuggle bytes past the size cap.
- * - **Redirects are followed** by `fetch`, which caps the chain itself; a redirect is
- *   normal and an infinite one is already handled below us.
- * - **The size cap applies to what arrives, not to what the server claims.** A
- *   `Content-Length` is a promise, and the stream is what actually lands on disk.
- * - **The filename comes from the response, then the URL, then the content type.** A
- *   URL path is often `/download` or `/`, which says nothing — and a name adding nothing is
- *   exactly what naming-from-content replaces.
+ * A URL is an instruction to make a network connection, so validation is performed on
+ * resolved destinations rather than on spelling alone. Every DNS answer must be public,
+ * the request is pinned to one of those validated answers, and every redirect starts the
+ * same process again. That closes both redirect-to-private and DNS-rebinding paths.
  */
 export interface Fetched {
     /** A temp file. The caller ingests it and is responsible for cleaning up. */
@@ -24,12 +15,24 @@ export interface Fetched {
     bytes: number;
     finalUrl: string;
 }
+interface ResolvedAddress {
+    address: string;
+    family: 4 | 6;
+}
 export interface FetchOptions {
     url: string;
     maxBytes: number;
     timeoutMs?: number;
+    /** Exact origins explicitly trusted to reach non-public destinations. */
+    trustedOrigins?: string[];
+    /** Test seam; production uses the operating system resolver. */
+    resolve?: (hostname: string) => Promise<ResolvedAddress[]>;
+    maxRedirects?: number;
 }
 export declare function fetchDocument(options: FetchOptions): Promise<Fetched>;
 /** The temp directory a fetch created. Removed once the bytes have been stored. */
 export declare function cleanupFetch(fetched: Fetched): Promise<void>;
+/** Exposed for deterministic policy tests without making a network request. */
+export declare function assertSafeUrlDestination(raw: string, options?: Pick<FetchOptions, 'trustedOrigins' | 'resolve'>): Promise<void>;
+export {};
 //# sourceMappingURL=fetch.d.ts.map

package/dist/ingest/fetch.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"fetch.d.ts","sourceRoot":"","sources":["../../src/ingest/fetch.ts"],"names":[],"mappings":"AAKA;;;;;;;;;;;;;;;;GAgBG;AAEH,MAAM,WAAW,OAAO;IACtB,6EAA6E;IAC7E,IAAI,EAAE,MAAM,CAAC;IACb,uDAAuD;IACvD,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,YAAY;IAC3B,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,wBAAsB,aAAa,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,OAAO,CAAC,CA4D3E;AAED,mFAAmF;AACnF,wBAAsB,YAAY,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAElE"}
+{"version":3,"file":"fetch.d.ts","sourceRoot":"","sources":["../../src/ingest/fetch.ts"],"names":[],"mappings":"AASA;;;;;;;GAOG;AAEH,MAAM,WAAW,OAAO;IACtB,6EAA6E;IAC7E,IAAI,EAAE,MAAM,CAAC;IACb,uDAAuD;IACvD,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,UAAU,eAAe;IACvB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC;CACf;AAED,MAAM,WAAW,YAAY;IAC3B,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,yEAAyE;IACzE,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,gEAAgE;IAChE,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC,eAAe,EAAE,CAAC,CAAC;IAC3D,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAKD,wBAAsB,aAAa,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,OAAO,CAAC,CAkE3E;AAED,mFAAmF;AACnF,wBAAsB,YAAY,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAElE;AAED,+EAA+E;AAC/E,wBAAsB,wBAAwB,CAC5C,GAAG,EAAE,MAAM,EACX,OAAO,GAAE,IAAI,CAAC,YAAY,EAAE,gBAAgB,GAAG,SAAS,CAAM,GAC7D,OAAO,CAAC,IAAI,CAAC,CAIf"}

package/dist/ingest/fetch.js

@@ -1,88 +1,222 @@
+import dns from 'node:dns/promises';
+import http, {} from 'node:http';
+import https from 'node:https';
+import { BlockList, isIP } from 'node:net';
 import fsp from 'node:fs/promises';
 import os from 'node:os';
 import path from 'node:path';
 import { AknoError } from '@tenphi/akno-protocol';
+const DENIED_DESTINATIONS = deniedDestinations();
+const REDIRECTS = new Set([301, 302, 303, 307, 308]);
 export async function fetchDocument(options) {
-    const url = parseUrl(options.url);
+    const timeoutMs = options.timeoutMs ?? 60_000;
+    const trusted = new Set((options.trustedOrigins ?? []).map(normalizeTrustedOrigin));
+    const resolve = options.resolve ?? resolveHost;
+    let url = parseUrl(options.url);
+    let redirects = 0;
     let response;
-    try {
-        response = await fetch(url, {
-            redirect: 'follow',
-            signal: AbortSignal.timeout(options.timeoutMs ?? 60_000),
-            headers: { accept: '*/*' },
-        });
-    }
-    catch (err) {
-        const timedOut = err instanceof Error && (err.name === 'TimeoutError' || err.name === 'AbortError');
-        throw new AknoError('unavailable', timedOut
-            ? `fetching ${url.href} timed out`
-            : `could not fetch ${url.href}: ${err instanceof Error ? err.message : String(err)}`);
-    }
-    if (!response.ok) {
-        throw new AknoError('unavailable', `${url.href} returned ${response.status} ${response.statusText}`);
+    while (true) {
+        const addresses = await permittedAddresses(url, trusted, resolve);
+        response = await request(url, addresses, timeoutMs);
+        const status = response.statusCode ?? 0;
+        if (!REDIRECTS.has(status))
+            break;
+        const location = header(response.headers, 'location');
+        response.resume();
+        if (!location)
+            throw new AknoError('unavailable', 'URL redirect returned no location');
+        if (redirects++ >= (options.maxRedirects ?? 5)) {
+            throw new AknoError('unavailable', 'URL redirect limit exceeded');
+        }
+        url = parseUrl(new URL(location, url).href);
     }
-    if (!response.body) {
-        throw new AknoError('unavailable', `${url.href} returned no body`);
+    const status = response.statusCode ?? 0;
+    if (status < 200 || status >= 300) {
+        response.resume();
+        throw new AknoError('unavailable', `URL returned ${status}${response.statusMessage ? ` ${response.statusMessage}` : ''}`);
     }
-    const contentType = (response.headers.get('content-type') ?? '').split(';')[0].trim() || null;
-    const originalName = filenameFor(response, url, contentType);
+    const contentType = (header(response.headers, 'content-type') ?? '').split(';')[0].trim() || null;
+    const originalName = filenameFor(response.headers, url, contentType);
     const directory = await fsp.mkdtemp(path.join(os.tmpdir(), 'akno-fetch-'));
     const target = path.join(directory, originalName);
-    const handle = await fsp.open(target, 'w');
     let written = 0;
+    let handle = null;
     try {
-        for await (const chunk of response.body) {
-            written += chunk.byteLength;
-            // Enforced against the bytes that actually arrive: `Content-Length` is a claim,
-            // and a server willing to lie about the size is exactly the one to guard against.
+        handle = await fsp.open(target, 'w');
+        for await (const chunk of response) {
+            const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
+            written += bytes.byteLength;
+            // Enforced against the bytes that actually arrive: Content-Length is only a claim.
             if (written > options.maxBytes) {
-                await handle.close();
-                await fsp.rm(directory, { recursive: true, force: true });
-                throw new AknoError('invalid', `${url.href} is larger than the configured limit of ${Math.round(options.maxBytes / 1_048_576)} MB`);
+                throw new AknoError('invalid', `download is larger than the configured limit of ${Math.round(options.maxBytes / 1_048_576)} MB`);
             }
-            await handle.write(chunk);
+            await handle.write(bytes);
         }
     }
+    catch (error) {
+        await fsp.rm(directory, { recursive: true, force: true });
+        throw networkFailure(error);
+    }
     finally {
-        await handle.close().catch(() => { });
+        if (handle)
+            await handle.close().catch(() => { });
     }
     if (written === 0) {
         await fsp.rm(directory, { recursive: true, force: true });
-        throw new AknoError('unavailable', `${url.href} returned an empty body`);
+        throw new AknoError('unavailable', 'URL returned an empty body');
     }
-    return { path: target, originalName, contentType, bytes: written, finalUrl: response.url || url.href };
+    return { path: target, originalName, contentType, bytes: written, finalUrl: url.href };
 }
 /** The temp directory a fetch created. Removed once the bytes have been stored. */
 export async function cleanupFetch(fetched) {
     await fsp.rm(path.dirname(fetched.path), { recursive: true, force: true });
 }
+/** Exposed for deterministic policy tests without making a network request. */
+export async function assertSafeUrlDestination(raw, options = {}) {
+    const url = parseUrl(raw);
+    const trusted = new Set((options.trustedOrigins ?? []).map(normalizeTrustedOrigin));
+    await permittedAddresses(url, trusted, options.resolve ?? resolveHost);
+}
+async function permittedAddresses(url, trusted, resolve) {
+    const hostname = normalizeHost(url.hostname);
+    let addresses;
+    try {
+        addresses = isIP(hostname)
+            ? [{ address: hostname, family: isIP(hostname) }]
+            : await resolve(hostname);
+    }
+    catch {
+        throw new AknoError('unavailable', 'could not resolve URL host');
+    }
+    if (addresses.length === 0)
+        throw new AknoError('unavailable', 'URL host resolved to no addresses');
+    if (!trusted.has(url.origin) && addresses.some((entry) => !isPublicAddress(entry))) {
+        throw new AknoError('forbidden', 'URL destination is not allowed by network policy', {
+            reason: 'network_destination_denied',
+        });
+    }
+    return addresses;
+}
+async function resolveHost(hostname) {
+    const answers = await dns.lookup(hostname, { all: true, verbatim: true });
+    return answers.map((answer) => ({ address: answer.address, family: answer.family }));
+}
+function isPublicAddress(entry) {
+    return !DENIED_DESTINATIONS.check(entry.address, entry.family === 4 ? 'ipv4' : 'ipv6');
+}
+function deniedDestinations() {
+    const list = new BlockList();
+    for (const [network, prefix] of [
+        ['0.0.0.0', 8],
+        ['10.0.0.0', 8],
+        ['100.64.0.0', 10],
+        ['127.0.0.0', 8],
+        ['169.254.0.0', 16],
+        ['172.16.0.0', 12],
+        ['192.0.0.0', 24],
+        ['192.0.2.0', 24],
+        ['192.168.0.0', 16],
+        ['192.88.99.0', 24],
+        ['198.18.0.0', 15],
+        ['198.51.100.0', 24],
+        ['203.0.113.0', 24],
+        ['224.0.0.0', 4],
+        ['240.0.0.0', 4],
+    ]) {
+        list.addSubnet(network, prefix, 'ipv4');
+    }
+    for (const [network, prefix] of [
+        ['::', 128],
+        ['::1', 128],
+        ['::', 96],
+        ['::ffff:0:0', 96],
+        ['::ffff:0:0:0', 96],
+        ['64:ff9b::', 96],
+        ['64:ff9b:1::', 48],
+        ['100::', 64],
+        ['100:0:0:1::', 64],
+        ['fc00::', 7],
+        ['fe80::', 10],
+        ['fec0::', 10],
+        ['ff00::', 8],
+        ['2001::', 23],
+        ['2001:db8::', 32],
+        ['2002::', 16],
+        ['3fff::', 20],
+        ['5f00::', 16],
+    ]) {
+        list.addSubnet(network, prefix, 'ipv6');
+    }
+    return list;
+}
+function request(url, addresses, timeoutMs) {
+    const transport = url.protocol === 'https:' ? https : http;
+    const lookup = (_hostname, lookupOptions, callback) => {
+        if (lookupOptions.all) {
+            callback(null, addresses);
+            return;
+        }
+        const selected = addresses[0];
+        callback(null, selected.address, selected.family);
+    };
+    return new Promise((resolve, reject) => {
+        const req = transport.get(url, {
+            signal: AbortSignal.timeout(timeoutMs),
+            headers: { accept: '*/*' },
+            lookup,
+        }, resolve);
+        req.on('error', (error) => reject(networkFailure(error)));
+    });
+}
 function parseUrl(raw) {
     let url;
     try {
         url = new URL(raw);
     }
     catch {
-        throw new AknoError('invalid', `not a usable URL: ${raw}`);
+        throw new AknoError('invalid', 'not a usable URL');
     }
     if (url.protocol !== 'http:' && url.protocol !== 'https:') {
         throw new AknoError('invalid', `only http and https URLs are fetched, not ${url.protocol} — pass a local path instead`);
     }
+    if (url.username || url.password) {
+        throw new AknoError('invalid', 'credentials are not allowed in an ingest URL');
+    }
     return url;
 }
-/**
- * `Content-Disposition`, then the URL's last path segment, then an extension guessed
- * from the content type. Every one of these can be useless, which is fine: the page is named
- * from the content anyway, and a useless name is what triggers the rename.
- */
-function filenameFor(response, url, contentType) {
-    const disposition = response.headers.get('content-disposition');
+function normalizeHost(value) {
+    return value
+        .replace(/^\[|\]$/g, '')
+        .replace(/\.$/, '')
+        .toLowerCase();
+}
+function normalizeTrustedOrigin(value) {
+    const url = parseUrl(value);
+    if (url.pathname !== '/' || url.search || url.hash) {
+        throw new AknoError('invalid', 'a trusted URL origin may contain only scheme, host, and port');
+    }
+    return url.origin;
+}
+function networkFailure(error) {
+    if (error instanceof AknoError)
+        return error;
+    const name = error instanceof Error ? error.name : '';
+    return new AknoError('unavailable', name === 'AbortError' || name === 'TimeoutError' ? 'fetching URL timed out' : 'could not fetch URL');
+}
+function header(headers, name) {
+    const value = headers[name];
+    if (Array.isArray(value))
+        return value[0] ?? null;
+    return value ?? null;
+}
+/** Content-Disposition, then URL path, then an extension guessed from content type. */
+function filenameFor(headers, url, contentType) {
+    const disposition = header(headers, 'content-disposition');
     const candidates = [];
     if (disposition) {
         const star = /filename\*=(?:UTF-8'')?([^;]+)/i.exec(disposition);
         const plain = /filename="?([^";]+)"?/i.exec(disposition);
         const raw = star?.[1] ?? plain?.[1];
-        // `path.basename` is what stops `../../../etc/passwd` being a path: a server chooses
-        // this header, so it is untrusted input that becomes a filename.
         if (raw)
             candidates.push(sanitize(path.basename(safeDecode(raw.trim()))));
     }
@@ -91,18 +225,10 @@ function filenameFor(response, url, contentType) {
     for (const candidate of candidates) {
         if (candidate.length === 0 || candidate === '.' || candidate === '..')
             continue;
-        // The extractor dispatches on the extension, so a name without one is unusable even
-        // though it reads fine. Servers send `filename="report"` often enough that guessing
-        // from the content type is the difference between working and skipping.
         return path.extname(candidate) ? candidate : `${candidate}${extension}`;
     }
     return `download${extension}`;
 }
-/**
- * A server chooses these bytes, so they are untrusted input on their way to becoming a
- * filename: separators and control characters are dropped. Spaces survive, because a
- * filename with spaces is ordinary and the rename handles presentation anyway.
- */
 function sanitize(name) {
     return [...name]
         .filter((char) => char !== '/' && char !== '\\' && char.codePointAt(0) >= 0x20)
@@ -135,8 +261,6 @@ function extensionFor(contentType) {
         'application/msword': '.doc',
         'application/vnd.openxmlformats-officedocument.wordprocessingml.document': '.docx',
     };
-    // No extension at all is honest: the extractor dispatches on it and will say it has
-    // none, which is better than claiming `.bin` and looking like a decision.
     return contentType ? (table[contentType] ?? '') : '';
 }
 //# sourceMappingURL=fetch.js.map

package/dist/ingest/fetch.js.map

@@ -1 +1 @@
-{"version":3,"file":"fetch.js","sourceRoot":"","sources":["../../src/ingest/fetch.ts"],"names":[],"mappings":"AAAA,OAAO,GAAG,MAAM,kBAAkB,CAAC;AACnC,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAoClD,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,OAAqB;IACvD,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAElC,IAAI,QAAkB,CAAC;IACvB,IAAI,CAAC;QACH,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;YAC1B,QAAQ,EAAE,QAAQ;YAClB,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,IAAI,MAAM,CAAC;YACxD,OAAO,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE;SAC3B,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,QAAQ,GAAG,GAAG,YAAY,KAAK,IAAI,CAAC,GAAG,CAAC,IAAI,KAAK,cAAc,IAAI,GAAG,CAAC,IAAI,KAAK,YAAY,CAAC,CAAC;QACpG,MAAM,IAAI,SAAS,CACjB,aAAa,EACb,QAAQ;YACN,CAAC,CAAC,YAAY,GAAG,CAAC,IAAI,YAAY;YAClC,CAAC,CAAC,mBAAmB,GAAG,CAAC,IAAI,KAAK,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CACvF,CAAC;IACJ,CAAC;IAED,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,MAAM,IAAI,SAAS,CAAC,aAAa,EAAE,GAAG,GAAG,CAAC,IAAI,aAAa,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;IACvG,CAAC;IACD,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;QACnB,MAAM,IAAI,SAAS,CAAC,aAAa,EAAE,GAAG,GAAG,CAAC,IAAI,mBAAmB,CAAC,CAAC;IACrE,CAAC;IAED,MAAM,WAAW,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC,IAAI,EAAE,IAAI,IAAI,CAAC;IAC/F,MAAM,YAAY,GAAG,WAAW,CAAC,QAAQ,EAAE,GAAG,EAAE,WAAW,CAAC,CAAC;IAE7D,MAAM,SAAS,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,aAAa,CAAC,CAAC,CAAC;IAC3E,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;IAClD,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAE3C,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,CAAC;QACH,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,QAAQ,CAAC,IAA4C,EAAE,CAAC;YAChF,OAAO,IAAI,KAAK,CAAC,UAAU,CAAC;YAC5B,gFAAgF;YAChF,kFAAkF;YAClF,IAAI,OAAO,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;gBAC/B,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;gBACrB,MAAM,GAAG,CAAC,EAAE,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;gBAC1D,MAAM,IAAI,SAAS,CACjB,SAAS,EACT,GAAG,GAAG,CAAC,IAAI,2CAA2C,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,GAAG,SAAS,CAAC,KAAK,CACpG,CAAC;YACJ,CAAC;YACD,MAAM,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;YAAS,CAAC;QACT,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IACvC,CAAC;IAED,IAAI,OAAO,KAAK,CAAC,EAAE,CAAC;QAClB,MAAM,GAAG,CAAC,EAAE,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1D,MAAM,IAAI,SAAS,CAAC,aAAa,EAAE,GAAG,GAAG,CAAC,IAAI,yBAAyB,CAAC,CAAC;IAC3E,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,WAAW,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;AACzG,CAAC;AAED,mFAAmF;AACnF,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,OAAgB;IACjD,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AAC7E,CAAC;AAED,SAAS,QAAQ,CAAC,GAAW;IAC3B,IAAI,GAAQ,CAAC;IACb,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IACrB,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,SAAS,CAAC,SAAS,EAAE,qBAAqB,GAAG,EAAE,CAAC,CAAC;IAC7D,CAAC;IACD,IAAI,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC1D,MAAM,IAAI,SAAS,CACjB,SAAS,EACT,6CAA6C,GAAG,CAAC,QAAQ,8BAA8B,CACxF,CAAC;IACJ,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;GAIG;AACH,SAAS,WAAW,CAAC,QAAkB,EAAE,GAAQ,EAAE,WAA0B;IAC3E,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;IAChE,MAAM,UAAU,GAAa,EAAE,CAAC;IAEhC,IAAI,WAAW,EAAE,CAAC;QAChB,MAAM,IAAI,GAAG,iCAAiC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACjE,MAAM,KAAK,GAAG,wBAAwB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACzD,MAAM,GAAG,GAAG,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC;QACpC,qFAAqF;QACrF,iEAAiE;QACjE,IAAI,GAAG;YAAE,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5E,CAAC;IACD,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;IAEnE,MAAM,SAAS,GAAG,YAAY,CAAC,WAAW,CAAC,CAAC;IAC5C,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,IAAI,SAAS,KAAK,GAAG,IAAI,SAAS,KAAK,IAAI;YAAE,SAAS;QAChF,oFAAoF;QACpF,oFAAoF;QACpF,wEAAwE;QACxE,OAAO,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,SAAS,GAAG,SAAS,EAAE,CAAC;IAC1E,CAAC;IAED,OAAO,WAAW,SAAS,EAAE,CAAC;AAChC,CAAC;AAED;;;;GAIG;AACH,SAAS,QAAQ,CAAC,IAAY;IAC5B,OAAO,CAAC,GAAG,IAAI,CAAC;SACb,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,GAAG,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,CAAC,WAAW,CAAC,CAAC,CAAE,IAAI,IAAI,CAAC;SAC/E,IAAI,CAAC,EAAE,CAAC;SACR,IAAI,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,UAAU,CAAC,KAAa;IAC/B,IAAI,CAAC;QACH,OAAO,kBAAkB,CAAC,KAAK,CAAC,CAAC;IACnC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,SAAS,YAAY,CAAC,WAA0B;IAC9C,MAAM,KAAK,GAA2B;QACpC,iBAAiB,EAAE,MAAM;QACzB,YAAY,EAAE,MAAM;QACpB,eAAe,EAAE,KAAK;QACtB,WAAW,EAAE,OAAO;QACpB,UAAU,EAAE,MAAM;QAClB,kBAAkB,EAAE,OAAO;QAC3B,WAAW,EAAE,MAAM;QACnB,YAAY,EAAE,MAAM;QACpB,YAAY,EAAE,OAAO;QACrB,YAAY,EAAE,OAAO;QACrB,WAAW,EAAE,MAAM;QACnB,YAAY,EAAE,OAAO;QACrB,iBAAiB,EAAE,MAAM;QACzB,oBAAoB,EAAE,MAAM;QAC5B,yEAAyE,EAAE,OAAO;KACnF,CAAC;IACF,oFAAoF;IACpF,0EAA0E;IAC1E,OAAO,WAAW,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;AACvD,CAAC"}
+{"version":3,"file":"fetch.js","sourceRoot":"","sources":["../../src/ingest/fetch.ts"],"names":[],"mappings":"AAAA,OAAO,GAAG,MAAM,mBAAmB,CAAC;AACpC,OAAO,IAAI,EAAE,EAAkD,MAAM,WAAW,CAAC;AACjF,OAAO,KAAK,MAAM,YAAY,CAAC;AAC/B,OAAO,EAAE,SAAS,EAAE,IAAI,EAAuB,MAAM,UAAU,CAAC;AAChE,OAAO,GAAG,MAAM,kBAAkB,CAAC;AACnC,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAqClD,MAAM,mBAAmB,GAAG,kBAAkB,EAAE,CAAC;AACjD,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAErD,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,OAAqB;IACvD,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,MAAM,CAAC;IAC9C,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,CAAC,OAAO,CAAC,cAAc,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC,CAAC;IACpF,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,WAAW,CAAC;IAC/C,IAAI,GAAG,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAChC,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,QAAyB,CAAC;IAE9B,OAAO,IAAI,EAAE,CAAC;QACZ,MAAM,SAAS,GAAG,MAAM,kBAAkB,CAAC,GAAG,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;QAClE,QAAQ,GAAG,MAAM,OAAO,CAAC,GAAG,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;QACpD,MAAM,MAAM,GAAG,QAAQ,CAAC,UAAU,IAAI,CAAC,CAAC;QACxC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC;YAAE,MAAM;QAElC,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;QACtD,QAAQ,CAAC,MAAM,EAAE,CAAC;QAClB,IAAI,CAAC,QAAQ;YAAE,MAAM,IAAI,SAAS,CAAC,aAAa,EAAE,mCAAmC,CAAC,CAAC;QACvF,IAAI,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,YAAY,IAAI,CAAC,CAAC,EAAE,CAAC;YAC/C,MAAM,IAAI,SAAS,CAAC,aAAa,EAAE,6BAA6B,CAAC,CAAC;QACpE,CAAC;QACD,GAAG,GAAG,QAAQ,CAAC,IAAI,GAAG,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;IAC9C,CAAC;IAED,MAAM,MAAM,GAAG,QAAQ,CAAC,UAAU,IAAI,CAAC,CAAC;IACxC,IAAI,MAAM,GAAG,GAAG,IAAI,MAAM,IAAI,GAAG,EAAE,CAAC;QAClC,QAAQ,CAAC,MAAM,EAAE,CAAC;QAClB,MAAM,IAAI,SAAS,CACjB,aAAa,EACb,gBAAgB,MAAM,GAAG,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CACtF,CAAC;IACJ,CAAC;IAED,MAAM,WAAW,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,cAAc,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC,IAAI,EAAE,IAAI,IAAI,CAAC;IACnG,MAAM,YAAY,GAAG,WAAW,CAAC,QAAQ,CAAC,OAAO,EAAE,GAAG,EAAE,WAAW,CAAC,CAAC;IACrE,MAAM,SAAS,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,aAAa,CAAC,CAAC,CAAC;IAC3E,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;IAElD,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,MAAM,GAAgD,IAAI,CAAC;IAC/D,IAAI,CAAC;QACH,MAAM,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QACrC,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;YACnC,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAmB,CAAC,CAAC;YAChF,OAAO,IAAI,KAAK,CAAC,UAAU,CAAC;YAC5B,mFAAmF;YACnF,IAAI,OAAO,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;gBAC/B,MAAM,IAAI,SAAS,CACjB,SAAS,EACT,mDAAmD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,GAAG,SAAS,CAAC,KAAK,CACjG,CAAC;YACJ,CAAC;YACD,MAAM,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,GAAG,CAAC,EAAE,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1D,MAAM,cAAc,CAAC,KAAK,CAAC,CAAC;IAC9B,CAAC;YAAS,CAAC;QACT,IAAI,MAAM;YAAE,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IACnD,CAAC;IAED,IAAI,OAAO,KAAK,CAAC,EAAE,CAAC;QAClB,MAAM,GAAG,CAAC,EAAE,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1D,MAAM,IAAI,SAAS,CAAC,aAAa,EAAE,4BAA4B,CAAC,CAAC;IACnE,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,WAAW,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,CAAC,IAAI,EAAE,CAAC;AACzF,CAAC;AAED,mFAAmF;AACnF,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,OAAgB;IACjD,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AAC7E,CAAC;AAED,+EAA+E;AAC/E,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAC5C,GAAW,EACX,UAA4D,EAAE;IAE9D,MAAM,GAAG,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC;IAC1B,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,CAAC,OAAO,CAAC,cAAc,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC,CAAC;IACpF,MAAM,kBAAkB,CAAC,GAAG,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,WAAW,CAAC,CAAC;AACzE,CAAC;AAED,KAAK,UAAU,kBAAkB,CAC/B,GAAQ,EACR,OAAoB,EACpB,OAAyD;IAEzD,MAAM,QAAQ,GAAG,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC7C,IAAI,SAA4B,CAAC;IACjC,IAAI,CAAC;QACH,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC;YACxB,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAU,EAAE,CAAC;YAC1D,CAAC,CAAC,MAAM,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC9B,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,SAAS,CAAC,aAAa,EAAE,4BAA4B,CAAC,CAAC;IACnE,CAAC;IACD,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,aAAa,EAAE,mCAAmC,CAAC,CAAC;IAEpG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;QACnF,MAAM,IAAI,SAAS,CAAC,WAAW,EAAE,kDAAkD,EAAE;YACnF,MAAM,EAAE,4BAA4B;SACrC,CAAC,CAAC;IACL,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,KAAK,UAAU,WAAW,CAAC,QAAgB;IACzC,MAAM,OAAO,GAAG,MAAM,GAAG,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;IAC1E,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,MAAe,EAAE,CAAC,CAAC,CAAC;AAChG,CAAC;AAED,SAAS,eAAe,CAAC,KAAsB;IAC7C,OAAO,CAAC,mBAAmB,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;AACzF,CAAC;AAED,SAAS,kBAAkB;IACzB,MAAM,IAAI,GAAG,IAAI,SAAS,EAAE,CAAC;IAC7B,KAAK,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,IAAI;QAC9B,CAAC,SAAS,EAAE,CAAC,CAAC;QACd,CAAC,UAAU,EAAE,CAAC,CAAC;QACf,CAAC,YAAY,EAAE,EAAE,CAAC;QAClB,CAAC,WAAW,EAAE,CAAC,CAAC;QAChB,CAAC,aAAa,EAAE,EAAE,CAAC;QACnB,CAAC,YAAY,EAAE,EAAE,CAAC;QAClB,CAAC,WAAW,EAAE,EAAE,CAAC;QACjB,CAAC,WAAW,EAAE,EAAE,CAAC;QACjB,CAAC,aAAa,EAAE,EAAE,CAAC;QACnB,CAAC,aAAa,EAAE,EAAE,CAAC;QACnB,CAAC,YAAY,EAAE,EAAE,CAAC;QAClB,CAAC,cAAc,EAAE,EAAE,CAAC;QACpB,CAAC,aAAa,EAAE,EAAE,CAAC;QACnB,CAAC,WAAW,EAAE,CAAC,CAAC;QAChB,CAAC,WAAW,EAAE,CAAC,CAAC;KACR,EAAE,CAAC;QACX,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IAC1C,CAAC;IACD,KAAK,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,IAAI;QAC9B,CAAC,IAAI,EAAE,GAAG,CAAC;QACX,CAAC,KAAK,EAAE,GAAG,CAAC;QACZ,CAAC,IAAI,EAAE,EAAE,CAAC;QACV,CAAC,YAAY,EAAE,EAAE,CAAC;QAClB,CAAC,cAAc,EAAE,EAAE,CAAC;QACpB,CAAC,WAAW,EAAE,EAAE,CAAC;QACjB,CAAC,aAAa,EAAE,EAAE,CAAC;QACnB,CAAC,OAAO,EAAE,EAAE,CAAC;QACb,CAAC,aAAa,EAAE,EAAE,CAAC;QACnB,CAAC,QAAQ,EAAE,CAAC,CAAC;QACb,CAAC,QAAQ,EAAE,EAAE,CAAC;QACd,CAAC,QAAQ,EAAE,EAAE,CAAC;QACd,CAAC,QAAQ,EAAE,CAAC,CAAC;QACb,CAAC,QAAQ,EAAE,EAAE,CAAC;QACd,CAAC,YAAY,EAAE,EAAE,CAAC;QAClB,CAAC,QAAQ,EAAE,EAAE,CAAC;QACd,CAAC,QAAQ,EAAE,EAAE,CAAC;QACd,CAAC,QAAQ,EAAE,EAAE,CAAC;KACN,EAAE,CAAC;QACX,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IAC1C,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,OAAO,CAAC,GAAQ,EAAE,SAA4B,EAAE,SAAiB;IACxE,MAAM,SAAS,GAAG,GAAG,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;IAC3D,MAAM,MAAM,GAAmB,CAAC,SAAS,EAAE,aAAa,EAAE,QAAQ,EAAE,EAAE;QACpE,IAAI,aAAa,CAAC,GAAG,EAAE,CAAC;YACtB,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;YAC1B,OAAO;QACT,CAAC;QACD,MAAM,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAE,CAAC;QAC/B,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;IACpD,CAAC,CAAC;IACF,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,MAAM,GAAG,GAAG,SAAS,CAAC,GAAG,CACvB,GAAG,EACH;YACE,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC;YACtC,OAAO,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE;YAC1B,MAAM;SACP,EACD,OAAO,CACR,CAAC;QACF,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAC5D,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,QAAQ,CAAC,GAAW;IAC3B,IAAI,GAAQ,CAAC;IACb,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IACrB,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,SAAS,CAAC,SAAS,EAAE,kBAAkB,CAAC,CAAC;IACrD,CAAC;IACD,IAAI,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC1D,MAAM,IAAI,SAAS,CACjB,SAAS,EACT,6CAA6C,GAAG,CAAC,QAAQ,8BAA8B,CACxF,CAAC;IACJ,CAAC;IACD,IAAI,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,QAAQ,EAAE,CAAC;QACjC,MAAM,IAAI,SAAS,CAAC,SAAS,EAAE,8CAA8C,CAAC,CAAC;IACjF,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,aAAa,CAAC,KAAa;IAClC,OAAO,KAAK;SACT,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC;SACvB,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC;SAClB,WAAW,EAAE,CAAC;AACnB,CAAC;AAED,SAAS,sBAAsB,CAAC,KAAa;IAC3C,MAAM,GAAG,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC5B,IAAI,GAAG,CAAC,QAAQ,KAAK,GAAG,IAAI,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;QACnD,MAAM,IAAI,SAAS,CAAC,SAAS,EAAE,8DAA8D,CAAC,CAAC;IACjG,CAAC;IACD,OAAO,GAAG,CAAC,MAAM,CAAC;AACpB,CAAC;AAED,SAAS,cAAc,CAAC,KAAc;IACpC,IAAI,KAAK,YAAY,SAAS;QAAE,OAAO,KAAK,CAAC;IAC7C,MAAM,IAAI,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;IACtD,OAAO,IAAI,SAAS,CAClB,aAAa,EACb,IAAI,KAAK,YAAY,IAAI,IAAI,KAAK,cAAc,CAAC,CAAC,CAAC,wBAAwB,CAAC,CAAC,CAAC,qBAAqB,CACpG,CAAC;AACJ,CAAC;AAED,SAAS,MAAM,CAAC,OAA4B,EAAE,IAAY;IACxD,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5B,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;IAClD,OAAO,KAAK,IAAI,IAAI,CAAC;AACvB,CAAC;AAED,uFAAuF;AACvF,SAAS,WAAW,CAAC,OAA4B,EAAE,GAAQ,EAAE,WAA0B;IACrF,MAAM,WAAW,GAAG,MAAM,CAAC,OAAO,EAAE,qBAAqB,CAAC,CAAC;IAC3D,MAAM,UAAU,GAAa,EAAE,CAAC;IAEhC,IAAI,WAAW,EAAE,CAAC;QAChB,MAAM,IAAI,GAAG,iCAAiC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACjE,MAAM,KAAK,GAAG,wBAAwB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACzD,MAAM,GAAG,GAAG,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC;QACpC,IAAI,GAAG;YAAE,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5E,CAAC;IACD,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;IAEnE,MAAM,SAAS,GAAG,YAAY,CAAC,WAAW,CAAC,CAAC;IAC5C,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,IAAI,SAAS,KAAK,GAAG,IAAI,SAAS,KAAK,IAAI;YAAE,SAAS;QAChF,OAAO,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,SAAS,GAAG,SAAS,EAAE,CAAC;IAC1E,CAAC;IACD,OAAO,WAAW,SAAS,EAAE,CAAC;AAChC,CAAC;AAED,SAAS,QAAQ,CAAC,IAAY;IAC5B,OAAO,CAAC,GAAG,IAAI,CAAC;SACb,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,GAAG,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,CAAC,WAAW,CAAC,CAAC,CAAE,IAAI,IAAI,CAAC;SAC/E,IAAI,CAAC,EAAE,CAAC;SACR,IAAI,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,UAAU,CAAC,KAAa;IAC/B,IAAI,CAAC;QACH,OAAO,kBAAkB,CAAC,KAAK,CAAC,CAAC;IACnC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,SAAS,YAAY,CAAC,WAA0B;IAC9C,MAAM,KAAK,GAA2B;QACpC,iBAAiB,EAAE,MAAM;QACzB,YAAY,EAAE,MAAM;QACpB,eAAe,EAAE,KAAK;QACtB,WAAW,EAAE,OAAO;QACpB,UAAU,EAAE,MAAM;QAClB,kBAAkB,EAAE,OAAO;QAC3B,WAAW,EAAE,MAAM;QACnB,YAAY,EAAE,MAAM;QACpB,YAAY,EAAE,OAAO;QACrB,YAAY,EAAE,OAAO;QACrB,WAAW,EAAE,MAAM;QACnB,YAAY,EAAE,OAAO;QACrB,iBAAiB,EAAE,MAAM;QACzB,oBAAoB,EAAE,MAAM;QAC5B,yEAAyE,EAAE,OAAO;KACnF,CAAC;IACF,OAAO,WAAW,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;AACvD,CAAC"}

package/dist/kb/frontmatter.d.ts

@@ -21,6 +21,21 @@ export interface Frontmatter {
     /** 1-based line number of the first body line. Line addressing hangs off this. */
     bodyLine: number;
 }
+/**
+ * Render a generated YAML string without letting its contents become YAML syntax.
+ * JSON string syntax is a YAML 1.2 string syntax too, and gives every troublesome
+ * scalar — newlines, comments, brackets, dates, booleans and the empty string — one
+ * unambiguous representation.
+ */
+export declare function serializeYamlString(value: unknown, field?: string): string;
+/** Render an inline YAML sequence whose members are guaranteed to stay strings. */
+export declare function serializeYamlStringArray(value: unknown, field?: string): string;
+/**
+ * Append missing strings to one existing top-level sequence while leaving every existing byte alone.
+ * The YAML parser supplies the exact sequence boundary; new members are serialized safely and
+ * appended in the sequence's existing block or flow style.
+ */
+export declare function mergeTopLevelStringArray(content: string, key: string, additions: unknown): string | null;
 export declare function parseFrontmatter(content: string): Frontmatter;
 /** `tags: [a, b]`, `tags:\n  - a`, and `tags: a` all mean the same thing here. */
 export declare function readTags(data: Record<string, unknown>): string[];

package/dist/kb/frontmatter.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"frontmatter.d.ts","sourceRoot":"","sources":["../../src/kb/frontmatter.ts"],"names":[],"mappings":"AAEA;;;;;;;;;;GAUG;AAEH,MAAM,WAAW,WAAW;IAC1B,0EAA0E;IAC1E,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC9B,iEAAiE;IACjE,GAAG,EAAE,MAAM,CAAC;IACZ,sDAAsD;IACtD,OAAO,EAAE,OAAO,CAAC;IACjB,8DAA8D;IAC9D,UAAU,EAAE,MAAM,CAAC;IACnB,kFAAkF;IAClF,QAAQ,EAAE,MAAM,CAAC;CAClB;AAID,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,MAAM,GAAG,WAAW,CAoC7D;AAQD,kFAAkF;AAClF,wBAAgB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,EAAE,CAUhE;AAED,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAGpF;AAED;;;;;GAKG;AACH,wBAAgB,qBAAqB,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAkBhG;AAED;;;;GAIG;AACH,wBAAgB,6BAA6B,CAC3C,OAAO,EAAE,MAAM,EACf,IAAI,EAAE,MAAM,EAAE,EACd,IAAI,EAAE,MAAM,EACZ,EAAE,EAAE,MAAM,GACT,MAAM,GAAG,IAAI,CA0Cf;AAED;;;;;GAKG;AACH,wBAAgB,MAAM,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,MAAM,CAO1D;AAED;;;;;;;GAOG;AACH,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,CAYzE;AAED;;;;;;;GAOG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,MAAM,GAAG,IAAI,CA0DjF;AAED,+EAA+E;AAC/E,MAAM,WAAW,mBAAmB;IAClC,gEAAgE;IAChE,IAAI,EAAE,MAAM,CAAC;IACb,0CAA0C;IAC1C,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAC/B;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,mBAAmB,GAAG,IAAI,CAU5E"}
+{"version":3,"file":"frontmatter.d.ts","sourceRoot":"","sources":["../../src/kb/frontmatter.ts"],"names":[],"mappings":"AAGA;;;;;;;;;;GAUG;AAEH,MAAM,WAAW,WAAW;IAC1B,0EAA0E;IAC1E,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC9B,iEAAiE;IACjE,GAAG,EAAE,MAAM,CAAC;IACZ,sDAAsD;IACtD,OAAO,EAAE,OAAO,CAAC;IACjB,8DAA8D;IAC9D,UAAU,EAAE,MAAM,CAAC;IACnB,kFAAkF;IAClF,QAAQ,EAAE,MAAM,CAAC;CAClB;AAID;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,OAAO,EAAE,KAAK,SAAsB,GAAG,MAAM,CAQvF;AAED,mFAAmF;AACnF,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,OAAO,EAAE,KAAK,SAAsB,GAAG,MAAM,CAQ5F;AAED;;;;GAIG;AACH,wBAAgB,wBAAwB,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,GAAG,MAAM,GAAG,IAAI,CAsDxG;AAED,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,MAAM,GAAG,WAAW,CAoC7D;AAQD,kFAAkF;AAClF,wBAAgB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,EAAE,CAUhE;AAED,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAGpF;AAED;;;;;GAKG;AACH,wBAAgB,qBAAqB,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAkBhG;AAED;;;;GAIG;AACH,wBAAgB,6BAA6B,CAC3C,OAAO,EAAE,MAAM,EACf,IAAI,EAAE,MAAM,EAAE,EACd,IAAI,EAAE,MAAM,EACZ,EAAE,EAAE,MAAM,GACT,MAAM,GAAG,IAAI,CA6Cf;AAED;;;;;GAKG;AACH,wBAAgB,MAAM,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,MAAM,CAO1D;AAED;;;;;;;GAOG;AACH,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,CAYzE;AAED;;;;;;;GAOG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,MAAM,GAAG,IAAI,CA+DjF;AAED,+EAA+E;AAC/E,MAAM,WAAW,mBAAmB;IAClC,gEAAgE;IAChE,IAAI,EAAE,MAAM,CAAC;IACb,0CAA0C;IAC1C,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAC/B;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,mBAAmB,GAAG,IAAI,CAU5E"}

package/dist/kb/frontmatter.js

@@ -1,5 +1,96 @@
+import { AknoError } from '@tenphi/akno-protocol';
 import YAML, { isMap, isPair, isScalar, isSeq } from 'yaml';
 const FENCE = /^---\r?\n/;
+/**
+ * Render a generated YAML string without letting its contents become YAML syntax.
+ * JSON string syntax is a YAML 1.2 string syntax too, and gives every troublesome
+ * scalar — newlines, comments, brackets, dates, booleans and the empty string — one
+ * unambiguous representation.
+ */
+export function serializeYamlString(value, field = 'frontmatter value') {
+    if (typeof value !== 'string') {
+        throw new AknoError('invalid', `${field} must be a string`, {
+            reason: 'invalid_frontmatter_value',
+            field,
+        });
+    }
+    return JSON.stringify(value);
+}
+/** Render an inline YAML sequence whose members are guaranteed to stay strings. */
+export function serializeYamlStringArray(value, field = 'frontmatter value') {
+    if (!Array.isArray(value) || value.some((entry) => typeof entry !== 'string')) {
+        throw new AknoError('invalid', `${field} must be an array of strings`, {
+            reason: 'invalid_frontmatter_value',
+            field,
+        });
+    }
+    return `[${value.map((entry) => serializeYamlString(entry, field)).join(', ')}]`;
+}
+/**
+ * Append missing strings to one existing top-level sequence while leaving every existing byte alone.
+ * The YAML parser supplies the exact sequence boundary; new members are serialized safely and
+ * appended in the sequence's existing block or flow style.
+ */
+export function mergeTopLevelStringArray(content, key, additions) {
+    // Validate even when the key is absent or malformed. Callers should never silently ignore a
+    // value they would have been unable to serialize safely.
+    serializeYamlStringArray(additions, key);
+    const values = additions;
+    const fm = parseFrontmatter(content);
+    if (!fm.present)
+        return null;
+    const document = YAML.parseDocument(fm.raw, { keepSourceTokens: true });
+    if (document.errors.length > 0 || !isMap(document.contents))
+        return null;
+    const matches = document.contents.items.filter((item) => isPair(item) && isScalar(item.key) && item.key.value === key);
+    if (matches.length !== 1)
+        return null;
+    const sequence = matches[0].value;
+    if (!isSeq(sequence) || !sequence.range)
+        return null;
+    if (sequence.items.some((item) => !isScalar(item) || typeof item.value !== 'string'))
+        return null;
+    const existing = sequence.items.map((item) => item.value);
+    const missing = [...new Set(values)].filter((value) => !existing.includes(value));
+    if (missing.length === 0)
+        return content;
+    const [start, end] = sequence.range;
+    if (start < 0 || end < start || end > fm.raw.length)
+        return null;
+    const original = fm.raw.slice(start, end);
+    let replacement;
+    if (sequence.flow) {
+        const close = original.lastIndexOf(']');
+        if (close < 0)
+            return null;
+        // A multiline flow sequence keeps its closing bracket on a separate line. Insert before
+        // that trailing whitespace; putting the comma immediately before `]` starts a new mapping
+        // key on the closing line instead of extending the sequence.
+        const beforeClose = original.slice(0, close);
+        const trailingWhitespace = /\s*$/.exec(beforeClose)?.[0] ?? '';
+        const insertionPoint = beforeClose.length - trailingWhitespace.length;
+        replacement =
+            beforeClose.slice(0, insertionPoint) +
+                `${existing.length > 0 ? ', ' : ''}${missing.map((value) => serializeYamlString(value, key)).join(', ')}` +
+                trailingWhitespace +
+                original.slice(close);
+    }
+    else {
+        const newline = fm.raw.includes('\r\n') ? '\r\n' : '\n';
+        const trailingNewline = original.endsWith(newline) ? newline : '';
+        const lineStart = fm.raw.lastIndexOf('\n', start - 1) + 1;
+        const indent = fm.raw.slice(lineStart, start);
+        if (!/^[ \t]*$/.test(indent))
+            return null;
+        replacement =
+            original.slice(0, original.length - trailingNewline.length) +
+                newline +
+                missing.map((value) => `${indent}- ${serializeYamlString(value, key)}`).join(newline) +
+                trailingNewline;
+    }
+    const rawStart = content.indexOf('\n') + 1;
+    return content.slice(0, rawStart + start) + replacement + content.slice(rawStart + end);
+}
 export function parseFrontmatter(content) {
     if (!FENCE.test(content)) {
         return { data: {}, raw: '', present: false, bodyOffset: 0, bodyLine: 1 };
@@ -84,7 +175,7 @@ export function replaceTopLevelString(content, key, value) {
     if (start < 0 || end < start || end > fm.raw.length)
         return null;
     const rawStart = content.indexOf('\n') + 1;
-    return content.slice(0, rawStart + start) + JSON.stringify(value) + content.slice(rawStart + end);
+    return content.slice(0, rawStart + start) + serializeYamlString(value, key) + content.slice(rawStart + end);
 }
 /**
  * Replace every exact string in one existing nested YAML sequence while preserving all other bytes.
@@ -126,7 +217,10 @@ export function replaceNestedStringArrayValue(content, keys, from, to) {
             for (const [start, end] of ranges) {
                 if (start < 0 || end < start || end > fm.raw.length)
                     return null;
-                result = result.slice(0, rawStart + start) + JSON.stringify(to) + result.slice(rawStart + end);
+                result =
+                    result.slice(0, rawStart + start) +
+                        serializeYamlString(to, keys.join('.')) +
+                        result.slice(rawStart + end);
             }
             return result;
         }
@@ -145,11 +239,11 @@ export function replaceNestedStringArrayValue(content, keys, from, to) {
 export function withId(content, id) {
     const fm = parseFrontmatter(content);
     if (!fm.present) {
-        return `---\nid: ${id}\n---\n\n${content}`;
+        return `---\nid: ${serializeYamlString(id, 'id')}\n---\n\n${content}`;
     }
     if (typeof fm.data.id === 'string' && fm.data.id.length > 0)
         return content;
-    return spliceAfterFence(content, [`id: ${id}`]);
+    return spliceAfterFence(content, [`id: ${serializeYamlString(id, 'id')}`]);
 }
 /**
  * Inserts lines immediately after the opening fence of an existing block, leaving
@@ -194,12 +288,14 @@ export function withAknoAliases(content, aliases) {
     const existingValue = record?.aliases;
     if (existingValue !== undefined && !Array.isArray(existingValue))
         return null;
-    const existing = (existingValue ?? []).filter((value) => typeof value === 'string');
+    if (existingValue?.some((value) => typeof value !== 'string'))
+        return null;
+    const existing = (existingValue ?? []);
     const wanted = [...new Set([...existing, ...additions])];
-    if (wanted.length === existing.length)
+    if (additions.every((value) => existing.includes(value)))
         return content;
     const missing = wanted.filter((value) => !existing.includes(value));
-    const rendered = wanted.map((value) => `    - ${JSON.stringify(value)}`);
+    const rendered = wanted.map((value) => `    - ${serializeYamlString(value, 'akno.aliases')}`);
     if (!fm.present || !record) {
         return spliceAfterFence(content, ['akno:', '  aliases:', ...rendered]);
     }
@@ -226,14 +322,14 @@ export function withAknoAliases(content, aliases) {
         const match = /^  aliases:[ \t]*(.*)$/.exec(lines[aliasesIndex]);
         const tail = match?.[1]?.trim() ?? '';
         if (tail.startsWith('[') && tail.endsWith(']')) {
-            lines[aliasesIndex] = `  aliases: ${JSON.stringify(wanted)}`;
+            lines[aliasesIndex] = `  aliases: ${serializeYamlStringArray(wanted, 'akno.aliases')}`;
         }
         else if (tail.length === 0) {
             let insertAt = aliasesIndex + 1;
             while (insertAt < blockEnd && (/^[ \t]{4,}/.test(lines[insertAt]) || !lines[insertAt].trim())) {
                 insertAt++;
             }
-            lines.splice(insertAt, 0, ...missing.map((value) => `    - ${JSON.stringify(value)}`));
+            lines.splice(insertAt, 0, ...missing.map((value) => `    - ${serializeYamlString(value, 'akno.aliases')}`));
         }
         else {
             return null;