@tenphi/akno-core 0.2.0 → 0.4.0

package/dist/ingest/extractor-contract.js

@@ -0,0 +1,26 @@
+export function selectDocumentExtractorBackend(platform, backends) {
+    if (platform === 'darwin')
+        return backends.macos;
+    if (platform === 'linux')
+        return backends.linux;
+    return unsupportedBackend(platform);
+}
+function unsupportedBackend(platform) {
+    return {
+        name: 'unsupported',
+        extract: async () => ({
+            text: '',
+            pages: null,
+            ocr: false,
+            confidence: null,
+            provenance: { backend: 'unsupported', tool: null },
+            text_from: 'none',
+            degradation: {
+                kind: 'unsupported-platform',
+                message: `document extraction is not supported on ${platform}`,
+                platform,
+            },
+        }),
+    };
+}
+//# sourceMappingURL=extractor-contract.js.map

package/dist/ingest/extractor-contract.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"extractor-contract.js","sourceRoot":"","sources":["../../src/ingest/extractor-contract.ts"],"names":[],"mappings":"AAyFA,MAAM,UAAU,8BAA8B,CAC5C,QAAgB,EAChB,QAA2B;IAE3B,IAAI,QAAQ,KAAK,QAAQ;QAAE,OAAO,QAAQ,CAAC,KAAK,CAAC;IACjD,IAAI,QAAQ,KAAK,OAAO;QAAE,OAAO,QAAQ,CAAC,KAAK,CAAC;IAChD,OAAO,kBAAkB,CAAC,QAAQ,CAAC,CAAC;AACtC,CAAC;AAED,SAAS,kBAAkB,CAAC,QAAgB;IAC1C,OAAO;QACL,IAAI,EAAE,aAAa;QACnB,OAAO,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC;YACpB,IAAI,EAAE,EAAE;YACR,KAAK,EAAE,IAAI;YACX,GAAG,EAAE,KAAK;YACV,UAAU,EAAE,IAAI;YAChB,UAAU,EAAE,EAAE,OAAO,EAAE,aAAa,EAAE,IAAI,EAAE,IAAI,EAAE;YAClD,SAAS,EAAE,MAAM;YACjB,WAAW,EAAE;gBACX,IAAI,EAAE,sBAAsB;gBAC5B,OAAO,EAAE,2CAA2C,QAAQ,EAAE;gBAC9D,QAAQ;aACT;SACF,CAAC;KACH,CAAC;AACJ,CAAC"}

package/dist/ingest/fetch.d.ts

@@ -1,19 +1,10 @@
 /**
  * `ingest` pulls documents from a file, a folder, or a URL. This is the URL.
  *
- * Downloading is the one thing `ingest` does that reaches outside the machine, so the
- * limits are deliberate rather than defensive-by-habit:
- *
- * - **http and https only.** `file://` would make `ingest({url})` a way to read any
- *   path on the machine through an interface that looks like it fetches the web, and
- *   `data:` would smuggle bytes past the size cap.
- * - **Redirects are followed** by `fetch`, which caps the chain itself; a redirect is
- *   normal and an infinite one is already handled below us.
- * - **The size cap applies to what arrives, not to what the server claims.** A
- *   `Content-Length` is a promise, and the stream is what actually lands on disk.
- * - **The filename comes from the response, then the URL, then the content type.** A
- *   URL path is often `/download` or `/`, which says nothing — and a name adding nothing is
- *   exactly what naming-from-content replaces.
+ * A URL is an instruction to make a network connection, so validation is performed on
+ * resolved destinations rather than on spelling alone. Every DNS answer must be public,
+ * the request is pinned to one of those validated answers, and every redirect starts the
+ * same process again. That closes both redirect-to-private and DNS-rebinding paths.
  */
 export interface Fetched {
     /** A temp file. The caller ingests it and is responsible for cleaning up. */
@@ -24,12 +15,24 @@ export interface Fetched {
     bytes: number;
     finalUrl: string;
 }
+interface ResolvedAddress {
+    address: string;
+    family: 4 | 6;
+}
 export interface FetchOptions {
     url: string;
     maxBytes: number;
     timeoutMs?: number;
+    /** Exact origins explicitly trusted to reach non-public destinations. */
+    trustedOrigins?: string[];
+    /** Test seam; production uses the operating system resolver. */
+    resolve?: (hostname: string) => Promise<ResolvedAddress[]>;
+    maxRedirects?: number;
 }
 export declare function fetchDocument(options: FetchOptions): Promise<Fetched>;
 /** The temp directory a fetch created. Removed once the bytes have been stored. */
 export declare function cleanupFetch(fetched: Fetched): Promise<void>;
+/** Exposed for deterministic policy tests without making a network request. */
+export declare function assertSafeUrlDestination(raw: string, options?: Pick<FetchOptions, 'trustedOrigins' | 'resolve'>): Promise<void>;
+export {};
 //# sourceMappingURL=fetch.d.ts.map

package/dist/ingest/fetch.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"fetch.d.ts","sourceRoot":"","sources":["../../src/ingest/fetch.ts"],"names":[],"mappings":"AAKA;;;;;;;;;;;;;;;;GAgBG;AAEH,MAAM,WAAW,OAAO;IACtB,6EAA6E;IAC7E,IAAI,EAAE,MAAM,CAAC;IACb,uDAAuD;IACvD,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,YAAY;IAC3B,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,wBAAsB,aAAa,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,OAAO,CAAC,CA4D3E;AAED,mFAAmF;AACnF,wBAAsB,YAAY,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAElE"}
+{"version":3,"file":"fetch.d.ts","sourceRoot":"","sources":["../../src/ingest/fetch.ts"],"names":[],"mappings":"AASA;;;;;;;GAOG;AAEH,MAAM,WAAW,OAAO;IACtB,6EAA6E;IAC7E,IAAI,EAAE,MAAM,CAAC;IACb,uDAAuD;IACvD,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,UAAU,eAAe;IACvB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC;CACf;AAED,MAAM,WAAW,YAAY;IAC3B,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,yEAAyE;IACzE,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,gEAAgE;IAChE,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC,eAAe,EAAE,CAAC,CAAC;IAC3D,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAKD,wBAAsB,aAAa,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,OAAO,CAAC,CAkE3E;AAED,mFAAmF;AACnF,wBAAsB,YAAY,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAElE;AAED,+EAA+E;AAC/E,wBAAsB,wBAAwB,CAC5C,GAAG,EAAE,MAAM,EACX,OAAO,GAAE,IAAI,CAAC,YAAY,EAAE,gBAAgB,GAAG,SAAS,CAAM,GAC7D,OAAO,CAAC,IAAI,CAAC,CAIf"}

package/dist/ingest/fetch.js

@@ -1,88 +1,222 @@
+import dns from 'node:dns/promises';
+import http, {} from 'node:http';
+import https from 'node:https';
+import { BlockList, isIP } from 'node:net';
 import fsp from 'node:fs/promises';
 import os from 'node:os';
 import path from 'node:path';
 import { AknoError } from '@tenphi/akno-protocol';
+const DENIED_DESTINATIONS = deniedDestinations();
+const REDIRECTS = new Set([301, 302, 303, 307, 308]);
 export async function fetchDocument(options) {
-    const url = parseUrl(options.url);
+    const timeoutMs = options.timeoutMs ?? 60_000;
+    const trusted = new Set((options.trustedOrigins ?? []).map(normalizeTrustedOrigin));
+    const resolve = options.resolve ?? resolveHost;
+    let url = parseUrl(options.url);
+    let redirects = 0;
     let response;
-    try {
-        response = await fetch(url, {
-            redirect: 'follow',
-            signal: AbortSignal.timeout(options.timeoutMs ?? 60_000),
-            headers: { accept: '*/*' },
-        });
-    }
-    catch (err) {
-        const timedOut = err instanceof Error && (err.name === 'TimeoutError' || err.name === 'AbortError');
-        throw new AknoError('unavailable', timedOut
-            ? `fetching ${url.href} timed out`
-            : `could not fetch ${url.href}: ${err instanceof Error ? err.message : String(err)}`);
-    }
-    if (!response.ok) {
-        throw new AknoError('unavailable', `${url.href} returned ${response.status} ${response.statusText}`);
+    while (true) {
+        const addresses = await permittedAddresses(url, trusted, resolve);
+        response = await request(url, addresses, timeoutMs);
+        const status = response.statusCode ?? 0;
+        if (!REDIRECTS.has(status))
+            break;
+        const location = header(response.headers, 'location');
+        response.resume();
+        if (!location)
+            throw new AknoError('unavailable', 'URL redirect returned no location');
+        if (redirects++ >= (options.maxRedirects ?? 5)) {
+            throw new AknoError('unavailable', 'URL redirect limit exceeded');
+        }
+        url = parseUrl(new URL(location, url).href);
     }
-    if (!response.body) {
-        throw new AknoError('unavailable', `${url.href} returned no body`);
+    const status = response.statusCode ?? 0;
+    if (status < 200 || status >= 300) {
+        response.resume();
+        throw new AknoError('unavailable', `URL returned ${status}${response.statusMessage ? ` ${response.statusMessage}` : ''}`);
     }
-    const contentType = (response.headers.get('content-type') ?? '').split(';')[0].trim() || null;
-    const originalName = filenameFor(response, url, contentType);
+    const contentType = (header(response.headers, 'content-type') ?? '').split(';')[0].trim() || null;
+    const originalName = filenameFor(response.headers, url, contentType);
     const directory = await fsp.mkdtemp(path.join(os.tmpdir(), 'akno-fetch-'));
     const target = path.join(directory, originalName);
-    const handle = await fsp.open(target, 'w');
     let written = 0;
+    let handle = null;
     try {
-        for await (const chunk of response.body) {
-            written += chunk.byteLength;
-            // Enforced against the bytes that actually arrive: `Content-Length` is a claim,
-            // and a server willing to lie about the size is exactly the one to guard against.
+        handle = await fsp.open(target, 'w');
+        for await (const chunk of response) {
+            const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
+            written += bytes.byteLength;
+            // Enforced against the bytes that actually arrive: Content-Length is only a claim.
             if (written > options.maxBytes) {
-                await handle.close();
-                await fsp.rm(directory, { recursive: true, force: true });
-                throw new AknoError('invalid', `${url.href} is larger than the configured limit of ${Math.round(options.maxBytes / 1_048_576)} MB`);
+                throw new AknoError('invalid', `download is larger than the configured limit of ${Math.round(options.maxBytes / 1_048_576)} MB`);
             }
-            await handle.write(chunk);
+            await handle.write(bytes);
         }
     }
+    catch (error) {
+        await fsp.rm(directory, { recursive: true, force: true });
+        throw networkFailure(error);
+    }
     finally {
-        await handle.close().catch(() => { });
+        if (handle)
+            await handle.close().catch(() => { });
     }
     if (written === 0) {
         await fsp.rm(directory, { recursive: true, force: true });
-        throw new AknoError('unavailable', `${url.href} returned an empty body`);
+        throw new AknoError('unavailable', 'URL returned an empty body');
     }
-    return { path: target, originalName, contentType, bytes: written, finalUrl: response.url || url.href };
+    return { path: target, originalName, contentType, bytes: written, finalUrl: url.href };
 }
 /** The temp directory a fetch created. Removed once the bytes have been stored. */
 export async function cleanupFetch(fetched) {
     await fsp.rm(path.dirname(fetched.path), { recursive: true, force: true });
 }
+/** Exposed for deterministic policy tests without making a network request. */
+export async function assertSafeUrlDestination(raw, options = {}) {
+    const url = parseUrl(raw);
+    const trusted = new Set((options.trustedOrigins ?? []).map(normalizeTrustedOrigin));
+    await permittedAddresses(url, trusted, options.resolve ?? resolveHost);
+}
+async function permittedAddresses(url, trusted, resolve) {
+    const hostname = normalizeHost(url.hostname);
+    let addresses;
+    try {
+        addresses = isIP(hostname)
+            ? [{ address: hostname, family: isIP(hostname) }]
+            : await resolve(hostname);
+    }
+    catch {
+        throw new AknoError('unavailable', 'could not resolve URL host');
+    }
+    if (addresses.length === 0)
+        throw new AknoError('unavailable', 'URL host resolved to no addresses');
+    if (!trusted.has(url.origin) && addresses.some((entry) => !isPublicAddress(entry))) {
+        throw new AknoError('forbidden', 'URL destination is not allowed by network policy', {
+            reason: 'network_destination_denied',
+        });
+    }
+    return addresses;
+}
+async function resolveHost(hostname) {
+    const answers = await dns.lookup(hostname, { all: true, verbatim: true });
+    return answers.map((answer) => ({ address: answer.address, family: answer.family }));
+}
+function isPublicAddress(entry) {
+    return !DENIED_DESTINATIONS.check(entry.address, entry.family === 4 ? 'ipv4' : 'ipv6');
+}
+function deniedDestinations() {
+    const list = new BlockList();
+    for (const [network, prefix] of [
+        ['0.0.0.0', 8],
+        ['10.0.0.0', 8],
+        ['100.64.0.0', 10],
+        ['127.0.0.0', 8],
+        ['169.254.0.0', 16],
+        ['172.16.0.0', 12],
+        ['192.0.0.0', 24],
+        ['192.0.2.0', 24],
+        ['192.168.0.0', 16],
+        ['192.88.99.0', 24],
+        ['198.18.0.0', 15],
+        ['198.51.100.0', 24],
+        ['203.0.113.0', 24],
+        ['224.0.0.0', 4],
+        ['240.0.0.0', 4],
+    ]) {
+        list.addSubnet(network, prefix, 'ipv4');
+    }
+    for (const [network, prefix] of [
+        ['::', 128],
+        ['::1', 128],
+        ['::', 96],
+        ['::ffff:0:0', 96],
+        ['::ffff:0:0:0', 96],
+        ['64:ff9b::', 96],
+        ['64:ff9b:1::', 48],
+        ['100::', 64],
+        ['100:0:0:1::', 64],
+        ['fc00::', 7],
+        ['fe80::', 10],
+        ['fec0::', 10],
+        ['ff00::', 8],
+        ['2001::', 23],
+        ['2001:db8::', 32],
+        ['2002::', 16],
+        ['3fff::', 20],
+        ['5f00::', 16],
+    ]) {
+        list.addSubnet(network, prefix, 'ipv6');
+    }
+    return list;
+}
+function request(url, addresses, timeoutMs) {
+    const transport = url.protocol === 'https:' ? https : http;
+    const lookup = (_hostname, lookupOptions, callback) => {
+        if (lookupOptions.all) {
+            callback(null, addresses);
+            return;
+        }
+        const selected = addresses[0];
+        callback(null, selected.address, selected.family);
+    };
+    return new Promise((resolve, reject) => {
+        const req = transport.get(url, {
+            signal: AbortSignal.timeout(timeoutMs),
+            headers: { accept: '*/*' },
+            lookup,
+        }, resolve);
+        req.on('error', (error) => reject(networkFailure(error)));
+    });
+}
 function parseUrl(raw) {
     let url;
     try {
         url = new URL(raw);
     }
     catch {
-        throw new AknoError('invalid', `not a usable URL: ${raw}`);
+        throw new AknoError('invalid', 'not a usable URL');
     }
     if (url.protocol !== 'http:' && url.protocol !== 'https:') {
         throw new AknoError('invalid', `only http and https URLs are fetched, not ${url.protocol} — pass a local path instead`);
     }
+    if (url.username || url.password) {
+        throw new AknoError('invalid', 'credentials are not allowed in an ingest URL');
+    }
     return url;
 }
-/**
- * `Content-Disposition`, then the URL's last path segment, then an extension guessed
- * from the content type. Every one of these can be useless, which is fine: the page is named
- * from the content anyway, and a useless name is what triggers the rename.
- */
-function filenameFor(response, url, contentType) {
-    const disposition = response.headers.get('content-disposition');
+function normalizeHost(value) {
+    return value
+        .replace(/^\[|\]$/g, '')
+        .replace(/\.$/, '')
+        .toLowerCase();
+}
+function normalizeTrustedOrigin(value) {
+    const url = parseUrl(value);
+    if (url.pathname !== '/' || url.search || url.hash) {
+        throw new AknoError('invalid', 'a trusted URL origin may contain only scheme, host, and port');
+    }
+    return url.origin;
+}
+function networkFailure(error) {
+    if (error instanceof AknoError)
+        return error;
+    const name = error instanceof Error ? error.name : '';
+    return new AknoError('unavailable', name === 'AbortError' || name === 'TimeoutError' ? 'fetching URL timed out' : 'could not fetch URL');
+}
+function header(headers, name) {
+    const value = headers[name];
+    if (Array.isArray(value))
+        return value[0] ?? null;
+    return value ?? null;
+}
+/** Content-Disposition, then URL path, then an extension guessed from content type. */
+function filenameFor(headers, url, contentType) {
+    const disposition = header(headers, 'content-disposition');
     const candidates = [];
     if (disposition) {
         const star = /filename\*=(?:UTF-8'')?([^;]+)/i.exec(disposition);
         const plain = /filename="?([^";]+)"?/i.exec(disposition);
         const raw = star?.[1] ?? plain?.[1];
-        // `path.basename` is what stops `../../../etc/passwd` being a path: a server chooses
-        // this header, so it is untrusted input that becomes a filename.
         if (raw)
             candidates.push(sanitize(path.basename(safeDecode(raw.trim()))));
     }
@@ -91,18 +225,10 @@ function filenameFor(response, url, contentType) {
     for (const candidate of candidates) {
         if (candidate.length === 0 || candidate === '.' || candidate === '..')
             continue;
-        // The extractor dispatches on the extension, so a name without one is unusable even
-        // though it reads fine. Servers send `filename="report"` often enough that guessing
-        // from the content type is the difference between working and skipping.
         return path.extname(candidate) ? candidate : `${candidate}${extension}`;
     }
     return `download${extension}`;
 }
-/**
- * A server chooses these bytes, so they are untrusted input on their way to becoming a
- * filename: separators and control characters are dropped. Spaces survive, because a
- * filename with spaces is ordinary and the rename handles presentation anyway.
- */
 function sanitize(name) {
     return [...name]
         .filter((char) => char !== '/' && char !== '\\' && char.codePointAt(0) >= 0x20)
@@ -135,8 +261,6 @@ function extensionFor(contentType) {
         'application/msword': '.doc',
         'application/vnd.openxmlformats-officedocument.wordprocessingml.document': '.docx',
     };
-    // No extension at all is honest: the extractor dispatches on it and will say it has
-    // none, which is better than claiming `.bin` and looking like a decision.
     return contentType ? (table[contentType] ?? '') : '';
 }
 //# sourceMappingURL=fetch.js.map

package/dist/ingest/fetch.js.map

@@ -1 +1 @@
-{"version":3,"file":"fetch.js","sourceRoot":"","sources":["../../src/ingest/fetch.ts"],"names":[],"mappings":"AAAA,OAAO,GAAG,MAAM,kBAAkB,CAAC;AACnC,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAoClD,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,OAAqB;IACvD,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAElC,IAAI,QAAkB,CAAC;IACvB,IAAI,CAAC;QACH,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;YAC1B,QAAQ,EAAE,QAAQ;YAClB,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,IAAI,MAAM,CAAC;YACxD,OAAO,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE;SAC3B,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,QAAQ,GAAG,GAAG,YAAY,KAAK,IAAI,CAAC,GAAG,CAAC,IAAI,KAAK,cAAc,IAAI,GAAG,CAAC,IAAI,KAAK,YAAY,CAAC,CAAC;QACpG,MAAM,IAAI,SAAS,CACjB,aAAa,EACb,QAAQ;YACN,CAAC,CAAC,YAAY,GAAG,CAAC,IAAI,YAAY;YAClC,CAAC,CAAC,mBAAmB,GAAG,CAAC,IAAI,KAAK,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CACvF,CAAC;IACJ,CAAC;IAED,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,MAAM,IAAI,SAAS,CAAC,aAAa,EAAE,GAAG,GAAG,CAAC,IAAI,aAAa,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;IACvG,CAAC;IACD,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;QACnB,MAAM,IAAI,SAAS,CAAC,aAAa,EAAE,GAAG,GAAG,CAAC,IAAI,mBAAmB,CAAC,CAAC;IACrE,CAAC;IAED,MAAM,WAAW,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC,IAAI,EAAE,IAAI,IAAI,CAAC;IAC/F,MAAM,YAAY,GAAG,WAAW,CAAC,QAAQ,EAAE,GAAG,EAAE,WAAW,CAAC,CAAC;IAE7D,MAAM,SAAS,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,aAAa,CAAC,CAAC,CAAC;IAC3E,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;IAClD,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAE3C,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,CAAC;QACH,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,QAAQ,CAAC,IAA4C,EAAE,CAAC;YAChF,OAAO,IAAI,KAAK,CAAC,UAAU,CAAC;YAC5B,gFAAgF;YAChF,kFAAkF;YAClF,IAAI,OAAO,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;gBAC/B,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;gBACrB,MAAM,GAAG,CAAC,EAAE,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;gBAC1D,MAAM,IAAI,SAAS,CACjB,SAAS,EACT,GAAG,GAAG,CAAC,IAAI,2CAA2C,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,GAAG,SAAS,CAAC,KAAK,CACpG,CAAC;YACJ,CAAC;YACD,MAAM,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;YAAS,CAAC;QACT,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IACvC,CAAC;IAED,IAAI,OAAO,KAAK,CAAC,EAAE,CAAC;QAClB,MAAM,GAAG,CAAC,EAAE,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1D,MAAM,IAAI,SAAS,CAAC,aAAa,EAAE,GAAG,GAAG,CAAC,IAAI,yBAAyB,CAAC,CAAC;IAC3E,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,WAAW,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;AACzG,CAAC;AAED,mFAAmF;AACnF,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,OAAgB;IACjD,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AAC7E,CAAC;AAED,SAAS,QAAQ,CAAC,GAAW;IAC3B,IAAI,GAAQ,CAAC;IACb,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IACrB,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,SAAS,CAAC,SAAS,EAAE,qBAAqB,GAAG,EAAE,CAAC,CAAC;IAC7D,CAAC;IACD,IAAI,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC1D,MAAM,IAAI,SAAS,CACjB,SAAS,EACT,6CAA6C,GAAG,CAAC,QAAQ,8BAA8B,CACxF,CAAC;IACJ,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;GAIG;AACH,SAAS,WAAW,CAAC,QAAkB,EAAE,GAAQ,EAAE,WAA0B;IAC3E,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;IAChE,MAAM,UAAU,GAAa,EAAE,CAAC;IAEhC,IAAI,WAAW,EAAE,CAAC;QAChB,MAAM,IAAI,GAAG,iCAAiC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACjE,MAAM,KAAK,GAAG,wBAAwB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACzD,MAAM,GAAG,GAAG,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC;QACpC,qFAAqF;QACrF,iEAAiE;QACjE,IAAI,GAAG;YAAE,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5E,CAAC;IACD,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;IAEnE,MAAM,SAAS,GAAG,YAAY,CAAC,WAAW,CAAC,CAAC;IAC5C,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,IAAI,SAAS,KAAK,GAAG,IAAI,SAAS,KAAK,IAAI;YAAE,SAAS;QAChF,oFAAoF;QACpF,oFAAoF;QACpF,wEAAwE;QACxE,OAAO,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,SAAS,GAAG,SAAS,EAAE,CAAC;IAC1E,CAAC;IAED,OAAO,WAAW,SAAS,EAAE,CAAC;AAChC,CAAC;AAED;;;;GAIG;AACH,SAAS,QAAQ,CAAC,IAAY;IAC5B,OAAO,CAAC,GAAG,IAAI,CAAC;SACb,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,GAAG,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,CAAC,WAAW,CAAC,CAAC,CAAE,IAAI,IAAI,CAAC;SAC/E,IAAI,CAAC,EAAE,CAAC;SACR,IAAI,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,UAAU,CAAC,KAAa;IAC/B,IAAI,CAAC;QACH,OAAO,kBAAkB,CAAC,KAAK,CAAC,CAAC;IACnC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,SAAS,YAAY,CAAC,WAA0B;IAC9C,MAAM,KAAK,GAA2B;QACpC,iBAAiB,EAAE,MAAM;QACzB,YAAY,EAAE,MAAM;QACpB,eAAe,EAAE,KAAK;QACtB,WAAW,EAAE,OAAO;QACpB,UAAU,EAAE,MAAM;QAClB,kBAAkB,EAAE,OAAO;QAC3B,WAAW,EAAE,MAAM;QACnB,YAAY,EAAE,MAAM;QACpB,YAAY,EAAE,OAAO;QACrB,YAAY,EAAE,OAAO;QACrB,WAAW,EAAE,MAAM;QACnB,YAAY,EAAE,OAAO;QACrB,iBAAiB,EAAE,MAAM;QACzB,oBAAoB,EAAE,MAAM;QAC5B,yEAAyE,EAAE,OAAO;KACnF,CAAC;IACF,oFAAoF;IACpF,0EAA0E;IAC1E,OAAO,WAAW,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;AACvD,CAAC"}
+{"version":3,"file":"fetch.js","sourceRoot":"","sources":["../../src/ingest/fetch.ts"],"names":[],"mappings":"AAAA,OAAO,GAAG,MAAM,mBAAmB,CAAC;AACpC,OAAO,IAAI,EAAE,EAAkD,MAAM,WAAW,CAAC;AACjF,OAAO,KAAK,MAAM,YAAY,CAAC;AAC/B,OAAO,EAAE,SAAS,EAAE,IAAI,EAAuB,MAAM,UAAU,CAAC;AAChE,OAAO,GAAG,MAAM,kBAAkB,CAAC;AACnC,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAqClD,MAAM,mBAAmB,GAAG,kBAAkB,EAAE,CAAC;AACjD,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAErD,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,OAAqB;IACvD,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,MAAM,CAAC;IAC9C,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,CAAC,OAAO,CAAC,cAAc,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC,CAAC;IACpF,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,WAAW,CAAC;IAC/C,IAAI,GAAG,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAChC,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,QAAyB,CAAC;IAE9B,OAAO,IAAI,EAAE,CAAC;QACZ,MAAM,SAAS,GAAG,MAAM,kBAAkB,CAAC,GAAG,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;QAClE,QAAQ,GAAG,MAAM,OAAO,CAAC,GAAG,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;QACpD,MAAM,MAAM,GAAG,QAAQ,CAAC,UAAU,IAAI,CAAC,CAAC;QACxC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC;YAAE,MAAM;QAElC,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;QACtD,QAAQ,CAAC,MAAM,EAAE,CAAC;QAClB,IAAI,CAAC,QAAQ;YAAE,MAAM,IAAI,SAAS,CAAC,aAAa,EAAE,mCAAmC,CAAC,CAAC;QACvF,IAAI,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,YAAY,IAAI,CAAC,CAAC,EAAE,CAAC;YAC/C,MAAM,IAAI,SAAS,CAAC,aAAa,EAAE,6BAA6B,CAAC,CAAC;QACpE,CAAC;QACD,GAAG,GAAG,QAAQ,CAAC,IAAI,GAAG,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;IAC9C,CAAC;IAED,MAAM,MAAM,GAAG,QAAQ,CAAC,UAAU,IAAI,CAAC,CAAC;IACxC,IAAI,MAAM,GAAG,GAAG,IAAI,MAAM,IAAI,GAAG,EAAE,CAAC;QAClC,QAAQ,CAAC,MAAM,EAAE,CAAC;QAClB,MAAM,IAAI,SAAS,CACjB,aAAa,EACb,gBAAgB,MAAM,GAAG,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CACtF,CAAC;IACJ,CAAC;IAED,MAAM,WAAW,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,cAAc,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC,IAAI,EAAE,IAAI,IAAI,CAAC;IACnG,MAAM,YAAY,GAAG,WAAW,CAAC,QAAQ,CAAC,OAAO,EAAE,GAAG,EAAE,WAAW,CAAC,CAAC;IACrE,MAAM,SAAS,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,aAAa,CAAC,CAAC,CAAC;IAC3E,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;IAElD,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,MAAM,GAAgD,IAAI,CAAC;IAC/D,IAAI,CAAC;QACH,MAAM,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QACrC,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;YACnC,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAmB,CAAC,CAAC;YAChF,OAAO,IAAI,KAAK,CAAC,UAAU,CAAC;YAC5B,mFAAmF;YACnF,IAAI,OAAO,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;gBAC/B,MAAM,IAAI,SAAS,CACjB,SAAS,EACT,mDAAmD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,GAAG,SAAS,CAAC,KAAK,CACjG,CAAC;YACJ,CAAC;YACD,MAAM,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,GAAG,CAAC,EAAE,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1D,MAAM,cAAc,CAAC,KAAK,CAAC,CAAC;IAC9B,CAAC;YAAS,CAAC;QACT,IAAI,MAAM;YAAE,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IACnD,CAAC;IAED,IAAI,OAAO,KAAK,CAAC,EAAE,CAAC;QAClB,MAAM,GAAG,CAAC,EAAE,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1D,MAAM,IAAI,SAAS,CAAC,aAAa,EAAE,4BAA4B,CAAC,CAAC;IACnE,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,WAAW,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,CAAC,IAAI,EAAE,CAAC;AACzF,CAAC;AAED,mFAAmF;AACnF,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,OAAgB;IACjD,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AAC7E,CAAC;AAED,+EAA+E;AAC/E,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAC5C,GAAW,EACX,UAA4D,EAAE;IAE9D,MAAM,GAAG,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC;IAC1B,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,CAAC,OAAO,CAAC,cAAc,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC,CAAC;IACpF,MAAM,kBAAkB,CAAC,GAAG,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,WAAW,CAAC,CAAC;AACzE,CAAC;AAED,KAAK,UAAU,kBAAkB,CAC/B,GAAQ,EACR,OAAoB,EACpB,OAAyD;IAEzD,MAAM,QAAQ,GAAG,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC7C,IAAI,SAA4B,CAAC;IACjC,IAAI,CAAC;QACH,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC;YACxB,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAU,EAAE,CAAC;YAC1D,CAAC,CAAC,MAAM,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC9B,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,SAAS,CAAC,aAAa,EAAE,4BAA4B,CAAC,CAAC;IACnE,CAAC;IACD,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,aAAa,EAAE,mCAAmC,CAAC,CAAC;IAEpG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;QACnF,MAAM,IAAI,SAAS,CAAC,WAAW,EAAE,kDAAkD,EAAE;YACnF,MAAM,EAAE,4BAA4B;SACrC,CAAC,CAAC;IACL,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,KAAK,UAAU,WAAW,CAAC,QAAgB;IACzC,MAAM,OAAO,GAAG,MAAM,GAAG,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;IAC1E,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,MAAe,EAAE,CAAC,CAAC,CAAC;AAChG,CAAC;AAED,SAAS,eAAe,CAAC,KAAsB;IAC7C,OAAO,CAAC,mBAAmB,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;AACzF,CAAC;AAED,SAAS,kBAAkB;IACzB,MAAM,IAAI,GAAG,IAAI,SAAS,EAAE,CAAC;IAC7B,KAAK,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,IAAI;QAC9B,CAAC,SAAS,EAAE,CAAC,CAAC;QACd,CAAC,UAAU,EAAE,CAAC,CAAC;QACf,CAAC,YAAY,EAAE,EAAE,CAAC;QAClB,CAAC,WAAW,EAAE,CAAC,CAAC;QAChB,CAAC,aAAa,EAAE,EAAE,CAAC;QACnB,CAAC,YAAY,EAAE,EAAE,CAAC;QAClB,CAAC,WAAW,EAAE,EAAE,CAAC;QACjB,CAAC,WAAW,EAAE,EAAE,CAAC;QACjB,CAAC,aAAa,EAAE,EAAE,CAAC;QACnB,CAAC,aAAa,EAAE,EAAE,CAAC;QACnB,CAAC,YAAY,EAAE,EAAE,CAAC;QAClB,CAAC,cAAc,EAAE,EAAE,CAAC;QACpB,CAAC,aAAa,EAAE,EAAE,CAAC;QACnB,CAAC,WAAW,EAAE,CAAC,CAAC;QAChB,CAAC,WAAW,EAAE,CAAC,CAAC;KACR,EAAE,CAAC;QACX,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IAC1C,CAAC;IACD,KAAK,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,IAAI;QAC9B,CAAC,IAAI,EAAE,GAAG,CAAC;QACX,CAAC,KAAK,EAAE,GAAG,CAAC;QACZ,CAAC,IAAI,EAAE,EAAE,CAAC;QACV,CAAC,YAAY,EAAE,EAAE,CAAC;QAClB,CAAC,cAAc,EAAE,EAAE,CAAC;QACpB,CAAC,WAAW,EAAE,EAAE,CAAC;QACjB,CAAC,aAAa,EAAE,EAAE,CAAC;QACnB,CAAC,OAAO,EAAE,EAAE,CAAC;QACb,CAAC,aAAa,EAAE,EAAE,CAAC;QACnB,CAAC,QAAQ,EAAE,CAAC,CAAC;QACb,CAAC,QAAQ,EAAE,EAAE,CAAC;QACd,CAAC,QAAQ,EAAE,EAAE,CAAC;QACd,CAAC,QAAQ,EAAE,CAAC,CAAC;QACb,CAAC,QAAQ,EAAE,EAAE,CAAC;QACd,CAAC,YAAY,EAAE,EAAE,CAAC;QAClB,CAAC,QAAQ,EAAE,EAAE,CAAC;QACd,CAAC,QAAQ,EAAE,EAAE,CAAC;QACd,CAAC,QAAQ,EAAE,EAAE,CAAC;KACN,EAAE,CAAC;QACX,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IAC1C,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,OAAO,CAAC,GAAQ,EAAE,SAA4B,EAAE,SAAiB;IACxE,MAAM,SAAS,GAAG,GAAG,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;IAC3D,MAAM,MAAM,GAAmB,CAAC,SAAS,EAAE,aAAa,EAAE,QAAQ,EAAE,EAAE;QACpE,IAAI,aAAa,CAAC,GAAG,EAAE,CAAC;YACtB,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;YAC1B,OAAO;QACT,CAAC;QACD,MAAM,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAE,CAAC;QAC/B,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;IACpD,CAAC,CAAC;IACF,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,MAAM,GAAG,GAAG,SAAS,CAAC,GAAG,CACvB,GAAG,EACH;YACE,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC;YACtC,OAAO,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE;YAC1B,MAAM;SACP,EACD,OAAO,CACR,CAAC;QACF,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAC5D,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,QAAQ,CAAC,GAAW;IAC3B,IAAI,GAAQ,CAAC;IACb,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IACrB,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,SAAS,CAAC,SAAS,EAAE,kBAAkB,CAAC,CAAC;IACrD,CAAC;IACD,IAAI,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC1D,MAAM,IAAI,SAAS,CACjB,SAAS,EACT,6CAA6C,GAAG,CAAC,QAAQ,8BAA8B,CACxF,CAAC;IACJ,CAAC;IACD,IAAI,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,QAAQ,EAAE,CAAC;QACjC,MAAM,IAAI,SAAS,CAAC,SAAS,EAAE,8CAA8C,CAAC,CAAC;IACjF,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,aAAa,CAAC,KAAa;IAClC,OAAO,KAAK;SACT,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC;SACvB,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC;SAClB,WAAW,EAAE,CAAC;AACnB,CAAC;AAED,SAAS,sBAAsB,CAAC,KAAa;IAC3C,MAAM,GAAG,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC5B,IAAI,GAAG,CAAC,QAAQ,KAAK,GAAG,IAAI,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;QACnD,MAAM,IAAI,SAAS,CAAC,SAAS,EAAE,8DAA8D,CAAC,CAAC;IACjG,CAAC;IACD,OAAO,GAAG,CAAC,MAAM,CAAC;AACpB,CAAC;AAED,SAAS,cAAc,CAAC,KAAc;IACpC,IAAI,KAAK,YAAY,SAAS;QAAE,OAAO,KAAK,CAAC;IAC7C,MAAM,IAAI,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;IACtD,OAAO,IAAI,SAAS,CAClB,aAAa,EACb,IAAI,KAAK,YAAY,IAAI,IAAI,KAAK,cAAc,CAAC,CAAC,CAAC,wBAAwB,CAAC,CAAC,CAAC,qBAAqB,CACpG,CAAC;AACJ,CAAC;AAED,SAAS,MAAM,CAAC,OAA4B,EAAE,IAAY;IACxD,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5B,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;IAClD,OAAO,KAAK,IAAI,IAAI,CAAC;AACvB,CAAC;AAED,uFAAuF;AACvF,SAAS,WAAW,CAAC,OAA4B,EAAE,GAAQ,EAAE,WAA0B;IACrF,MAAM,WAAW,GAAG,MAAM,CAAC,OAAO,EAAE,qBAAqB,CAAC,CAAC;IAC3D,MAAM,UAAU,GAAa,EAAE,CAAC;IAEhC,IAAI,WAAW,EAAE,CAAC;QAChB,MAAM,IAAI,GAAG,iCAAiC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACjE,MAAM,KAAK,GAAG,wBAAwB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACzD,MAAM,GAAG,GAAG,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC;QACpC,IAAI,GAAG;YAAE,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5E,CAAC;IACD,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;IAEnE,MAAM,SAAS,GAAG,YAAY,CAAC,WAAW,CAAC,CAAC;IAC5C,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,IAAI,SAAS,KAAK,GAAG,IAAI,SAAS,KAAK,IAAI;YAAE,SAAS;QAChF,OAAO,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,SAAS,GAAG,SAAS,EAAE,CAAC;IAC1E,CAAC;IACD,OAAO,WAAW,SAAS,EAAE,CAAC;AAChC,CAAC;AAED,SAAS,QAAQ,CAAC,IAAY;IAC5B,OAAO,CAAC,GAAG,IAAI,CAAC;SACb,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,GAAG,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,CAAC,WAAW,CAAC,CAAC,CAAE,IAAI,IAAI,CAAC;SAC/E,IAAI,CAAC,EAAE,CAAC;SACR,IAAI,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,UAAU,CAAC,KAAa;IAC/B,IAAI,CAAC;QACH,OAAO,kBAAkB,CAAC,KAAK,CAAC,CAAC;IACnC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,SAAS,YAAY,CAAC,WAA0B;IAC9C,MAAM,KAAK,GAA2B;QACpC,iBAAiB,EAAE,MAAM;QACzB,YAAY,EAAE,MAAM;QACpB,eAAe,EAAE,KAAK;QACtB,WAAW,EAAE,OAAO;QACpB,UAAU,EAAE,MAAM;QAClB,kBAAkB,EAAE,OAAO;QAC3B,WAAW,EAAE,MAAM;QACnB,YAAY,EAAE,MAAM;QACpB,YAAY,EAAE,OAAO;QACrB,YAAY,EAAE,OAAO;QACrB,WAAW,EAAE,MAAM;QACnB,YAAY,EAAE,OAAO;QACrB,iBAAiB,EAAE,MAAM;QACzB,oBAAoB,EAAE,MAAM;QAC5B,yEAAyE,EAAE,OAAO;KACnF,CAAC;IACF,OAAO,WAAW,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;AACvD,CAAC"}

package/dist/ingest/linux-extractor.d.ts

@@ -0,0 +1,32 @@
+import type { DocumentExtractorBackend } from './extractor-contract.ts';
+type LinuxExecuteFile = (command: string, args: string[], options: {
+    maxBuffer: number;
+    timeout: number;
+    killSignal: NodeJS.Signals;
+}) => Promise<{
+    stdout: string;
+    stderr: string;
+}>;
+export interface LinuxCommandRunner {
+    available(command: string): Promise<boolean>;
+    makeTempDir(): Promise<string>;
+    readText(file: string): Promise<string>;
+    removeTempDir(directory: string): Promise<void>;
+    run(command: string, args: string[]): Promise<{
+        stdout: string;
+    }>;
+}
+export declare function createNodeLinuxCommandRunner(execute?: LinuxExecuteFile): LinuxCommandRunner;
+export interface LinuxExtractionCapabilities {
+    backend: 'linux-native';
+    libreoffice: boolean;
+    pdfinfo: boolean;
+    pdftotext: boolean;
+    pdftoppm: boolean;
+    tesseract: boolean;
+    note: string | null;
+}
+export declare function linuxExtractionCapabilities(commands: LinuxCommandRunner): Promise<LinuxExtractionCapabilities>;
+export declare function createLinuxDocumentExtractor(commands: LinuxCommandRunner): DocumentExtractorBackend;
+export {};
+//# sourceMappingURL=linux-extractor.d.ts.map

package/dist/ingest/linux-extractor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"linux-extractor.d.ts","sourceRoot":"","sources":["../../src/ingest/linux-extractor.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAEV,wBAAwB,EAEzB,MAAM,yBAAyB,CAAC;AAOjC,KAAK,gBAAgB,GAAG,CACtB,OAAO,EAAE,MAAM,EACf,IAAI,EAAE,MAAM,EAAE,EACd,OAAO,EAAE;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAC,OAAO,CAAA;CAAE,KACxE,OAAO,CAAC;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC,CAAC;AAOjD,MAAM,WAAW,kBAAkB;IACjC,SAAS,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAC7C,WAAW,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC;IAC/B,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACxC,aAAa,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAChD,GAAG,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;QAAE,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACnE;AAED,wBAAgB,4BAA4B,CAAC,OAAO,GAAE,gBAA8B,GAAG,kBAAkB,CAexG;AAkBD,MAAM,WAAW,2BAA2B;IAC1C,OAAO,EAAE,cAAc,CAAC;IACxB,WAAW,EAAE,OAAO,CAAC;IACrB,OAAO,EAAE,OAAO,CAAC;IACjB,SAAS,EAAE,OAAO,CAAC;IACnB,QAAQ,EAAE,OAAO,CAAC;IAClB,SAAS,EAAE,OAAO,CAAC;IACnB,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;CACrB;AAED,wBAAsB,2BAA2B,CAC/C,QAAQ,EAAE,kBAAkB,GAC3B,OAAO,CAAC,2BAA2B,CAAC,CAgBtC;AAED,wBAAgB,4BAA4B,CAAC,QAAQ,EAAE,kBAAkB,GAAG,wBAAwB,CAcnG"}