@humain/terminal 0.0.3-beta.3 → 0.0.4-beta.2

package/dist/humain/web-access/vendor/data-uri-sanitize.js

@@ -0,0 +1,373 @@
+import { Buffer } from "node:buffer";
+import { createHash } from "node:crypto";
+// Replaces inline RFC 2397 data: URIs in extracted text with explicit bounded
+// omission markers so base64 payloads never reach model-visible tool results,
+// the fetch cache, or session persistence. Carved out of the original
+// persistence-safety layer; only the text-sanitization path is retained.
+const MAX_DATA_URI_HEADER_CHARS = 1024;
+const MAX_MIME_CHARS = 127;
+const MAX_MARKER_SOURCE_CHARS = 180;
+function sha256Bytes(value) {
+    return createHash("sha256").update(value).digest("hex");
+}
+function asciiLowerCode(code) {
+    return code >= 65 && code <= 90 ? code + 32 : code;
+}
+function matchesDataScheme(text, index) {
+    return index + 5 <= text.length
+        && asciiLowerCode(text.charCodeAt(index)) === 100
+        && asciiLowerCode(text.charCodeAt(index + 1)) === 97
+        && asciiLowerCode(text.charCodeAt(index + 2)) === 116
+        && asciiLowerCode(text.charCodeAt(index + 3)) === 97
+        && text.charCodeAt(index + 4) === 58;
+}
+function isSchemeBoundary(text, index) {
+    if (index === 0)
+        return true;
+    const code = text.charCodeAt(index - 1);
+    const alphaNumeric = (code >= 48 && code <= 57)
+        || (code >= 65 && code <= 90)
+        || (code >= 97 && code <= 122);
+    return !alphaNumeric && code !== 43 && code !== 45 && code !== 46;
+}
+function findNextDataScheme(text, from) {
+    for (let i = from; i + 5 <= text.length; i++) {
+        if (matchesDataScheme(text, i) && isSchemeBoundary(text, i))
+            return i;
+    }
+    return -1;
+}
+function dataUriEnclosure(text, start) {
+    if (start === 0)
+        return null;
+    const immediateIndex = start - 1;
+    const immediate = text[immediateIndex];
+    if (immediate === "\"" || immediate === "'" || immediate === "`") {
+        const beforeQuote = immediateIndex > 0 ? text[immediateIndex - 1] : "";
+        const likelyOpeningQuote = immediateIndex === 0
+            || beforeQuote === "="
+            || beforeQuote === "("
+            || beforeQuote === "["
+            || beforeQuote === "{"
+            || beforeQuote === ":"
+            || beforeQuote === ","
+            || /\s/.test(beforeQuote);
+        if (likelyOpeningQuote)
+            return { kind: "quote", close: immediate };
+    }
+    let openerIndex = immediateIndex;
+    while (openerIndex >= 0 && (text[openerIndex] === " " || text[openerIndex] === "\t"))
+        openerIndex--;
+    const opener = text[openerIndex];
+    if (opener === "(")
+        return { kind: "parentheses" };
+    if (opener === "<")
+        return { kind: "angle" };
+    return null;
+}
+function isBareDataUriTerminator(ch) {
+    const code = ch.charCodeAt(0);
+    return code <= 32
+        || code === 127
+        || ch === "\""
+        || ch === "`"
+        || ch === "<"
+        || ch === ">";
+}
+/** Scan one candidate exactly once. Enclosures consume malformed whitespace and
+ * balanced inner parentheses so invalid payload suffixes cannot escape. */
+function scanDataUriCandidate(text, start) {
+    const enclosure = dataUriEnclosure(text, start);
+    let comma = -1;
+    let depth = enclosure?.kind === "parentheses" ? 1 : 0;
+    let index = start + 5;
+    for (; index < text.length; index++) {
+        const ch = text[index];
+        if (enclosure?.kind === "quote") {
+            if (ch === enclosure.close)
+                break;
+        }
+        else if (enclosure?.kind === "parentheses") {
+            if (ch === "(") {
+                depth++;
+            }
+            else if (ch === ")") {
+                depth--;
+                if (depth === 0)
+                    break;
+            }
+        }
+        else if (enclosure?.kind === "angle") {
+            if (ch === ">")
+                break;
+        }
+        else if (isBareDataUriTerminator(ch)) {
+            break;
+        }
+        if (comma < 0 && ch === ",")
+            comma = index;
+    }
+    return { end: index, comma, enclosure };
+}
+function headerEndsWithBase64(text, headerStart, comma) {
+    let end = comma;
+    while (end > headerStart && (text[end - 1] === " " || text[end - 1] === "\t"))
+        end--;
+    const token = "base64";
+    if (end - headerStart < token.length + 1)
+        return false;
+    const tokenStart = end - token.length;
+    for (let i = 0; i < token.length; i++) {
+        if (asciiLowerCode(text.charCodeAt(tokenStart + i)) !== token.charCodeAt(i))
+            return false;
+    }
+    return tokenStart > headerStart && text[tokenStart - 1] === ";";
+}
+function isMimeTokenChar(code) {
+    if (code < 33 || code > 126)
+        return false;
+    // RFC 2045 token = printable ASCII excluding tspecials. Slash is handled
+    // separately as the single type/subtype separator.
+    switch (code) {
+        case 34: // "
+        case 40: // (
+        case 41: // )
+        case 44: // ,
+        case 47: // /
+        case 58: // :
+        case 59: // ;
+        case 60: // <
+        case 61: // =
+        case 62: // >
+        case 63: // ?
+        case 64: // @
+        case 91: // [
+        case 92: // backslash
+        case 93: // ]
+            return false;
+        default:
+            return true;
+    }
+}
+function normalizeMimeType(text, headerStart, comma) {
+    let end = headerStart;
+    while (end < comma && text[end] !== ";")
+        end++;
+    if (end === headerStart)
+        return "text/plain";
+    if (end - headerStart > MAX_MIME_CHARS)
+        return "application/octet-stream";
+    let slashCount = 0;
+    for (let i = headerStart; i < end; i++) {
+        const code = text.charCodeAt(i);
+        if (code === 47) {
+            slashCount++;
+            if (i === headerStart || i === end - 1)
+                return "application/octet-stream";
+            continue;
+        }
+        if (!isMimeTokenChar(code))
+            return "application/octet-stream";
+    }
+    if (slashCount !== 1)
+        return "application/octet-stream";
+    return text.slice(headerStart, end).toLowerCase();
+}
+function hexValue(code) {
+    if (code >= 48 && code <= 57)
+        return code - 48;
+    if (code >= 65 && code <= 70)
+        return code - 55;
+    if (code >= 97 && code <= 102)
+        return code - 87;
+    return -1;
+}
+function decodePercentEncoded(payload) {
+    const output = Buffer.allocUnsafe(Buffer.byteLength(payload, "utf8"));
+    let outputOffset = 0;
+    let cursor = 0;
+    while (cursor < payload.length) {
+        const percent = payload.indexOf("%", cursor);
+        const runEnd = percent < 0 ? payload.length : percent;
+        if (runEnd > cursor) {
+            const run = Buffer.from(payload.slice(cursor, runEnd), "utf8");
+            run.copy(output, outputOffset);
+            outputOffset += run.length;
+        }
+        if (percent < 0)
+            break;
+        if (percent + 2 >= payload.length)
+            return { error: "invalid-percent-escape" };
+        const high = hexValue(payload.charCodeAt(percent + 1));
+        const low = hexValue(payload.charCodeAt(percent + 2));
+        if (high < 0 || low < 0)
+            return { error: "invalid-percent-escape" };
+        output[outputOffset++] = (high << 4) | low;
+        cursor = percent + 3;
+    }
+    return { bytes: output.subarray(0, outputOffset) };
+}
+function isBase64AlphabetByte(byte) {
+    return (byte >= 65 && byte <= 90)
+        || (byte >= 97 && byte <= 122)
+        || (byte >= 48 && byte <= 57)
+        || byte === 43
+        || byte === 47;
+}
+function decodeBase64Payload(payload) {
+    const percentDecoded = decodePercentEncoded(payload);
+    if ("error" in percentDecoded)
+        return percentDecoded;
+    const encoded = percentDecoded.bytes;
+    let paddingStart = encoded.length;
+    let paddingCount = 0;
+    for (let i = 0; i < encoded.length; i++) {
+        const byte = encoded[i];
+        if (byte > 127)
+            return { error: "non-ascii-base64" };
+        if (byte === 61) {
+            if (paddingStart === encoded.length)
+                paddingStart = i;
+            paddingCount++;
+            continue;
+        }
+        if (paddingStart !== encoded.length)
+            return { error: "invalid-base64-padding" };
+        if (!isBase64AlphabetByte(byte))
+            return { error: "invalid-base64-character" };
+    }
+    if (paddingCount > 2)
+        return { error: "invalid-base64-padding" };
+    if (paddingCount > 0 && encoded.length % 4 !== 0)
+        return { error: "invalid-base64-padding" };
+    if (paddingCount === 0 && encoded.length % 4 === 1)
+        return { error: "invalid-base64-length" };
+    if (paddingCount === 1 && paddingStart % 4 !== 3)
+        return { error: "invalid-base64-padding" };
+    if (paddingCount === 2 && paddingStart % 4 !== 2)
+        return { error: "invalid-base64-padding" };
+    return { bytes: Buffer.from(encoded.toString("ascii"), "base64") };
+}
+function safeMarkerSource(sourcePath) {
+    const source = sourcePath || "value";
+    let safe = "";
+    for (const ch of source) {
+        const code = ch.charCodeAt(0);
+        const allowed = (code >= 48 && code <= 57)
+            || (code >= 65 && code <= 90)
+            || (code >= 97 && code <= 122)
+            || ch === "."
+            || ch === "_"
+            || ch === "-"
+            || ch === "["
+            || ch === "]"
+            || ch === "*"
+            || ch === "$";
+        safe += allowed ? ch : "_";
+    }
+    if (safe.length <= MAX_MARKER_SOURCE_CHARS)
+        return safe;
+    const suffix = sha256Bytes(source).slice(0, 12);
+    return `${safe.slice(0, MAX_MARKER_SOURCE_CHARS - suffix.length - 1)}~${suffix}`;
+}
+function buildDataUriMarker(omission) {
+    const decoded = omission.decodedBytes === null ? "unknown" : String(omission.decodedBytes);
+    const decodeError = omission.decodeError ? `; decodeError=${omission.decodeError}` : "";
+    return `[pi-web-access inline data URI omitted; ordinal=${omission.ordinal}; source=${omission.sourcePath}; mime=${omission.mimeType}; encoding=${omission.encoding}; encodedBytes=${omission.encodedBytes}; decodedBytes=${decoded}${decodeError}; sha256=${omission.sha256}; digestBasis=${omission.digestBasis}; retrieval=not-retained]`;
+}
+function sanitizeTextInternal(text, sourcePath, state) {
+    let scanFrom = 0;
+    let retainedFrom = 0;
+    let outputLength = 0;
+    const pieces = [];
+    while (scanFrom < text.length) {
+        const start = findNextDataScheme(text, scanFrom);
+        if (start < 0)
+            break;
+        const candidate = scanDataUriCandidate(text, start);
+        const end = candidate.end;
+        const headerStart = start + 5;
+        const missingComma = candidate.comma < 0;
+        // A bare prose label such as "data: value" is not an inline URI. Enclosed
+        // or non-empty comma-less candidates are malformed URIs and are removed.
+        if (missingComma && end === headerStart && candidate.enclosure === null) {
+            scanFrom = end;
+            continue;
+        }
+        const headerEnd = missingComma ? end : candidate.comma;
+        const headerLength = headerEnd - headerStart;
+        const encoding = headerEndsWithBase64(text, headerStart, headerEnd)
+            ? "base64"
+            : "percent-encoded";
+        const mimeType = normalizeMimeType(text, headerStart, headerEnd);
+        // Without the required comma, the post-scheme bytes are ambiguous. Count
+        // and digest the whole omitted segment on the encoded basis rather than
+        // retaining any header/payload-looking suffix.
+        const payload = missingComma
+            ? text.slice(headerStart, end)
+            : text.slice(candidate.comma + 1, end);
+        const encodedBytes = Buffer.byteLength(payload, "utf8");
+        let decodedBytes = null;
+        let decodeError;
+        let digestBasis = "encoded";
+        let digest = "";
+        if (missingComma) {
+            decodeError = "missing-comma";
+            digest = sha256Bytes(payload);
+        }
+        else if (headerLength > MAX_DATA_URI_HEADER_CHARS) {
+            decodeError = "header-too-long";
+            digest = sha256Bytes(payload);
+        }
+        else {
+            const decoded = encoding === "base64"
+                ? decodeBase64Payload(payload)
+                : decodePercentEncoded(payload);
+            if ("error" in decoded) {
+                decodeError = decoded.error;
+                digest = sha256Bytes(payload);
+            }
+            else {
+                decodedBytes = decoded.bytes.length;
+                digestBasis = "decoded";
+                digest = sha256Bytes(decoded.bytes);
+            }
+        }
+        const omission = {
+            ordinal: state.nextOrdinal++,
+            sourcePath: safeMarkerSource(sourcePath),
+            mimeType,
+            encoding,
+            encodedBytes,
+            decodedBytes,
+            ...(decodeError ? { decodeError } : {}),
+            sha256: digest,
+            digestBasis,
+            retrieval: "not-retained",
+        };
+        const marker = buildDataUriMarker(omission);
+        const prefix = text.slice(retainedFrom, start);
+        pieces.push(prefix, marker);
+        outputLength += prefix.length;
+        const markerStart = outputLength;
+        outputLength += marker.length;
+        state.omissions.push({ ...omission, markerStart, markerEnd: outputLength });
+        retainedFrom = end;
+        scanFrom = end;
+    }
+    if (state.omissions.length === 0 && retainedFrom === 0)
+        return text;
+    pieces.push(text.slice(retainedFrom));
+    return pieces.join("");
+}
+/** Replace every recognized RFC 2397-style inline data URI in one string. */
+export function sanitizeInlineDataUris(text, sourcePath) {
+    const state = { nextOrdinal: 1, omissions: [] };
+    const sanitized = sanitizeTextInternal(text, sourcePath, state);
+    return {
+        text: sanitized,
+        omissions: state.omissions.map(({ markerStart: _start, markerEnd: _end, ...omission }) => omission),
+    };
+}
+/** Sanitize every string leaf in a JSON-compatible value with one ordinal sequence. */
+//# sourceMappingURL=data-uri-sanitize.js.map

package/dist/humain/web-access/vendor/data-uri-sanitize.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"data-uri-sanitize.js","sourceRoot":"","sources":["../../../../src/humain/web-access/vendor/data-uri-sanitize.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC,8EAA8E;AAC9E,8EAA8E;AAC9E,sEAAsE;AACtE,yEAAyE;AAEzE,MAAM,yBAAyB,GAAG,IAAI,CAAC;AACvC,MAAM,cAAc,GAAG,GAAG,CAAC;AAC3B,MAAM,uBAAuB,GAAG,GAAG,CAAC;AA8BpC,SAAS,WAAW,CAAC,KAA0B;IAC9C,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AACzD,CAAC;AAGD,SAAS,cAAc,CAAC,IAAY;IACnC,OAAO,IAAI,IAAI,EAAE,IAAI,IAAI,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;AACpD,CAAC;AAED,SAAS,iBAAiB,CAAC,IAAY,EAAE,KAAa;IACrD,OAAO,KAAK,GAAG,CAAC,IAAI,IAAI,CAAC,MAAM;WAC3B,cAAc,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,KAAK,GAAG;WAC9C,cAAc,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE;WACjD,cAAc,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,KAAK,GAAG;WAClD,cAAc,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE;WACjD,IAAI,CAAC,UAAU,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC;AACvC,CAAC;AAED,SAAS,gBAAgB,CAAC,IAAY,EAAE,KAAa;IACpD,IAAI,KAAK,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAC7B,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IACxC,MAAM,YAAY,GAAG,CAAC,IAAI,IAAI,EAAE,IAAI,IAAI,IAAI,EAAE,CAAC;WAC3C,CAAC,IAAI,IAAI,EAAE,IAAI,IAAI,IAAI,EAAE,CAAC;WAC1B,CAAC,IAAI,IAAI,EAAE,IAAI,IAAI,IAAI,GAAG,CAAC,CAAC;IAChC,OAAO,CAAC,YAAY,IAAI,IAAI,KAAK,EAAE,IAAI,IAAI,KAAK,EAAE,IAAI,IAAI,KAAK,EAAE,CAAC;AACnE,CAAC;AAED,SAAS,kBAAkB,CAAC,IAAY,EAAE,IAAY;IACrD,KAAK,IAAI,CAAC,GAAG,IAAI,EAAE,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC9C,IAAI,iBAAiB,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,gBAAgB,CAAC,IAAI,EAAE,CAAC,CAAC;YAAE,OAAO,CAAC,CAAC;IACvE,CAAC;IACD,OAAO,CAAC,CAAC,CAAC;AACX,CAAC;AAcD,SAAS,gBAAgB,CAAC,IAAY,EAAE,KAAa;IACpD,IAAI,KAAK,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAC7B,MAAM,cAAc,GAAG,KAAK,GAAG,CAAC,CAAC;IACjC,MAAM,SAAS,GAAG,IAAI,CAAC,cAAc,CAAC,CAAC;IACvC,IAAI,SAAS,KAAK,IAAI,IAAI,SAAS,KAAK,GAAG,IAAI,SAAS,KAAK,GAAG,EAAE,CAAC;QAClE,MAAM,WAAW,GAAG,cAAc,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,cAAc,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACvE,MAAM,kBAAkB,GAAG,cAAc,KAAK,CAAC;eAC3C,WAAW,KAAK,GAAG;eACnB,WAAW,KAAK,GAAG;eACnB,WAAW,KAAK,GAAG;eACnB,WAAW,KAAK,GAAG;eACnB,WAAW,KAAK,GAAG;eACnB,WAAW,KAAK,GAAG;eACnB,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAC3B,IAAI,kBAAkB;YAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;IACpE,CAAC;IAED,IAAI,WAAW,GAAG,cAAc,CAAC;IACjC,OAAO,WAAW,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,KAAK,GAAG,IAAI,IAAI,CAAC,WAAW,CAAC,KAAK,IAAI,CAAC;QAAE,WAAW,EAAE,CAAC;IACpG,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC;IACjC,IAAI,MAAM,KAAK,GAAG;QAAE,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,CAAC;IACnD,IAAI,MAAM,KAAK,GAAG;QAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;IAC7C,OAAO,IAAI,CAAC;AACb,CAAC;AAED,SAAS,uBAAuB,CAAC,EAAU;IAC1C,MAAM,IAAI,GAAG,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;IAC9B,OAAO,IAAI,IAAI,EAAE;WACb,IAAI,KAAK,GAAG;WACZ,EAAE,KAAK,IAAI;WACX,EAAE,KAAK,GAAG;WACV,EAAE,KAAK,GAAG;WACV,EAAE,KAAK,GAAG,CAAC;AAChB,CAAC;AAED;2EAC2E;AAC3E,SAAS,oBAAoB,CAAC,IAAY,EAAE,KAAa;IACxD,MAAM,SAAS,GAAG,gBAAgB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IAChD,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC;IACf,IAAI,KAAK,GAAG,SAAS,EAAE,IAAI,KAAK,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACtD,IAAI,KAAK,GAAG,KAAK,GAAG,CAAC,CAAC;IAEtB,OAAO,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;QACrC,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC;QACvB,IAAI,SAAS,EAAE,IAAI,KAAK,OAAO,EAAE,CAAC;YACjC,IAAI,EAAE,KAAK,SAAS,CAAC,KAAK;gBAAE,MAAM;QACnC,CAAC;aAAM,IAAI,SAAS,EAAE,IAAI,KAAK,aAAa,EAAE,CAAC;YAC9C,IAAI,EAAE,KAAK,GAAG,EAAE,CAAC;gBAChB,KAAK,EAAE,CAAC;YACT,CAAC;iBAAM,IAAI,EAAE,KAAK,GAAG,EAAE,CAAC;gBACvB,KAAK,EAAE,CAAC;gBACR,IAAI,KAAK,KAAK,CAAC;oBAAE,MAAM;YACxB,CAAC;QACF,CAAC;aAAM,IAAI,SAAS,EAAE,IAAI,KAAK,OAAO,EAAE,CAAC;YACxC,IAAI,EAAE,KAAK,GAAG;gBAAE,MAAM;QACvB,CAAC;aAAM,IAAI,uBAAuB,CAAC,EAAE,CAAC,EAAE,CAAC;YACxC,MAAM;QACP,CAAC;QACD,IAAI,KAAK,GAAG,CAAC,IAAI,EAAE,KAAK,GAAG;YAAE,KAAK,GAAG,KAAK,CAAC;IAC5C,CAAC;IAED,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;AACzC,CAAC;AAED,SAAS,oBAAoB,CAAC,IAAY,EAAE,WAAmB,EAAE,KAAa;IAC7E,IAAI,GAAG,GAAG,KAAK,CAAC;IAChB,OAAO,GAAG,GAAG,WAAW,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,GAAG,IAAI,IAAI,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC;QAAE,GAAG,EAAE,CAAC;IACrF,MAAM,KAAK,GAAG,QAAQ,CAAC;IACvB,IAAI,GAAG,GAAG,WAAW,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IACvD,MAAM,UAAU,GAAG,GAAG,GAAG,KAAK,CAAC,MAAM,CAAC;IACtC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,IAAI,cAAc,CAAC,IAAI,CAAC,UAAU,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC;YAAE,OAAO,KAAK,CAAC;IAC3F,CAAC;IACD,OAAO,UAAU,GAAG,WAAW,IAAI,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC,KAAK,GAAG,CAAC;AACjE,CAAC;AAED,SAAS,eAAe,CAAC,IAAY;IACpC,IAAI,IAAI,GAAG,EAAE,IAAI,IAAI,GAAG,GAAG;QAAE,OAAO,KAAK,CAAC;IAC1C,yEAAyE;IACzE,mDAAmD;IACnD,QAAQ,IAAI,EAAE,CAAC;QACd,KAAK,EAAE,CAAC,CAAC,IAAI;QACb,KAAK,EAAE,CAAC,CAAC,IAAI;QACb,KAAK,EAAE,CAAC,CAAC,IAAI;QACb,KAAK,EAAE,CAAC,CAAC,IAAI;QACb,KAAK,EAAE,CAAC,CAAC,IAAI;QACb,KAAK,EAAE,CAAC,CAAC,IAAI;QACb,KAAK,EAAE,CAAC,CAAC,IAAI;QACb,KAAK,EAAE,CAAC,CAAC,IAAI;QACb,KAAK,EAAE,CAAC,CAAC,IAAI;QACb,KAAK,EAAE,CAAC,CAAC,IAAI;QACb,KAAK,EAAE,CAAC,CAAC,IAAI;QACb,KAAK,EAAE,CAAC,CAAC,IAAI;QACb,KAAK,EAAE,CAAC,CAAC,IAAI;QACb,KAAK,EAAE,CAAC,CAAC,YAAY;QACrB,KAAK,EAAE,EAAE,IAAI;YACZ,OAAO,KAAK,CAAC;QACd;YACC,OAAO,IAAI,CAAC;IACd,CAAC;AACF,CAAC;AAED,SAAS,iBAAiB,CAAC,IAAY,EAAE,WAAmB,EAAE,KAAa;IAC1E,IAAI,GAAG,GAAG,WAAW,CAAC;IACtB,OAAO,GAAG,GAAG,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG;QAAE,GAAG,EAAE,CAAC;IAC/C,IAAI,GAAG,KAAK,WAAW;QAAE,OAAO,YAAY,CAAC;IAC7C,IAAI,GAAG,GAAG,WAAW,GAAG,cAAc;QAAE,OAAO,0BAA0B,CAAC;IAE1E,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,KAAK,IAAI,CAAC,GAAG,WAAW,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QAChC,IAAI,IAAI,KAAK,EAAE,EAAE,CAAC;YACjB,UAAU,EAAE,CAAC;YACb,IAAI,CAAC,KAAK,WAAW,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC;gBAAE,OAAO,0BAA0B,CAAC;YAC1E,SAAS;QACV,CAAC;QACD,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC;YAAE,OAAO,0BAA0B,CAAC;IAC/D,CAAC;IACD,IAAI,UAAU,KAAK,CAAC;QAAE,OAAO,0BAA0B,CAAC;IACxD,OAAO,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC;AACnD,CAAC;AAED,SAAS,QAAQ,CAAC,IAAY;IAC7B,IAAI,IAAI,IAAI,EAAE,IAAI,IAAI,IAAI,EAAE;QAAE,OAAO,IAAI,GAAG,EAAE,CAAC;IAC/C,IAAI,IAAI,IAAI,EAAE,IAAI,IAAI,IAAI,EAAE;QAAE,OAAO,IAAI,GAAG,EAAE,CAAC;IAC/C,IAAI,IAAI,IAAI,EAAE,IAAI,IAAI,IAAI,GAAG;QAAE,OAAO,IAAI,GAAG,EAAE,CAAC;IAChD,OAAO,CAAC,CAAC,CAAC;AACX,CAAC;AAED,SAAS,oBAAoB,CAAC,OAAe;IAC5C,MAAM,MAAM,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,UAAU,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC;IACtE,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,MAAM,GAAG,CAAC,CAAC;IAEf,OAAO,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;QAChC,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;QAC7C,MAAM,MAAM,GAAG,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC;QACtD,IAAI,MAAM,GAAG,MAAM,EAAE,CAAC;YACrB,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC;YAC/D,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;YAC/B,YAAY,IAAI,GAAG,CAAC,MAAM,CAAC;QAC5B,CAAC;QACD,IAAI,OAAO,GAAG,CAAC;YAAE,MAAM;QACvB,IAAI,OAAO,GAAG,CAAC,IAAI,OAAO,CAAC,MAAM;YAAE,OAAO,EAAE,KAAK,EAAE,wBAAwB,EAAE,CAAC;QAC9E,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC,UAAU,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC;QACvD,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,CAAC,UAAU,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC;QACtD,IAAI,IAAI,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC;YAAE,OAAO,EAAE,KAAK,EAAE,wBAAwB,EAAE,CAAC;QACpE,MAAM,CAAC,YAAY,EAAE,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC;QAC3C,MAAM,GAAG,OAAO,GAAG,CAAC,CAAC;IACtB,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,YAAY,CAAC,EAAE,CAAC;AACpD,CAAC;AAED,SAAS,oBAAoB,CAAC,IAAY;IACzC,OAAO,CAAC,IAAI,IAAI,EAAE,IAAI,IAAI,IAAI,EAAE,CAAC;WAC7B,CAAC,IAAI,IAAI,EAAE,IAAI,IAAI,IAAI,GAAG,CAAC;WAC3B,CAAC,IAAI,IAAI,EAAE,IAAI,IAAI,IAAI,EAAE,CAAC;WAC1B,IAAI,KAAK,EAAE;WACX,IAAI,KAAK,EAAE,CAAC;AACjB,CAAC;AAED,SAAS,mBAAmB,CAAC,OAAe;IAC3C,MAAM,cAAc,GAAG,oBAAoB,CAAC,OAAO,CAAC,CAAC;IACrD,IAAI,OAAO,IAAI,cAAc;QAAE,OAAO,cAAc,CAAC;IACrD,MAAM,OAAO,GAAG,cAAc,CAAC,KAAK,CAAC;IACrC,IAAI,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC;IAClC,IAAI,YAAY,GAAG,CAAC,CAAC;IAErB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACzC,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QACxB,IAAI,IAAI,GAAG,GAAG;YAAE,OAAO,EAAE,KAAK,EAAE,kBAAkB,EAAE,CAAC;QACrD,IAAI,IAAI,KAAK,EAAE,EAAE,CAAC;YACjB,IAAI,YAAY,KAAK,OAAO,CAAC,MAAM;gBAAE,YAAY,GAAG,CAAC,CAAC;YACtD,YAAY,EAAE,CAAC;YACf,SAAS;QACV,CAAC;QACD,IAAI,YAAY,KAAK,OAAO,CAAC,MAAM;YAAE,OAAO,EAAE,KAAK,EAAE,wBAAwB,EAAE,CAAC;QAChF,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC;YAAE,OAAO,EAAE,KAAK,EAAE,0BAA0B,EAAE,CAAC;IAC/E,CAAC;IAED,IAAI,YAAY,GAAG,CAAC;QAAE,OAAO,EAAE,KAAK,EAAE,wBAAwB,EAAE,CAAC;IACjE,IAAI,YAAY,GAAG,CAAC,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,KAAK,CAAC;QAAE,OAAO,EAAE,KAAK,EAAE,wBAAwB,EAAE,CAAC;IAC7F,IAAI,YAAY,KAAK,CAAC,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,KAAK,CAAC;QAAE,OAAO,EAAE,KAAK,EAAE,uBAAuB,EAAE,CAAC;IAC9F,IAAI,YAAY,KAAK,CAAC,IAAI,YAAY,GAAG,CAAC,KAAK,CAAC;QAAE,OAAO,EAAE,KAAK,EAAE,wBAAwB,EAAE,CAAC;IAC7F,IAAI,YAAY,KAAK,CAAC,IAAI,YAAY,GAAG,CAAC,KAAK,CAAC;QAAE,OAAO,EAAE,KAAK,EAAE,wBAAwB,EAAE,CAAC;IAE7F,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,QAAQ,CAAC,EAAE,CAAC;AACpE,CAAC;AAED,SAAS,gBAAgB,CAAC,UAAkB;IAC3C,MAAM,MAAM,GAAG,UAAU,IAAI,OAAO,CAAC;IACrC,IAAI,IAAI,GAAG,EAAE,CAAC;IACd,KAAK,MAAM,EAAE,IAAI,MAAM,EAAE,CAAC;QACzB,MAAM,IAAI,GAAG,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QAC9B,MAAM,OAAO,GAAG,CAAC,IAAI,IAAI,EAAE,IAAI,IAAI,IAAI,EAAE,CAAC;eACtC,CAAC,IAAI,IAAI,EAAE,IAAI,IAAI,IAAI,EAAE,CAAC;eAC1B,CAAC,IAAI,IAAI,EAAE,IAAI,IAAI,IAAI,GAAG,CAAC;eAC3B,EAAE,KAAK,GAAG;eACV,EAAE,KAAK,GAAG;eACV,EAAE,KAAK,GAAG;eACV,EAAE,KAAK,GAAG;eACV,EAAE,KAAK,GAAG;eACV,EAAE,KAAK,GAAG;eACV,EAAE,KAAK,GAAG,CAAC;QACf,IAAI,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;IAC5B,CAAC;IACD,IAAI,IAAI,CAAC,MAAM,IAAI,uBAAuB;QAAE,OAAO,IAAI,CAAC;IACxD,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAChD,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,uBAAuB,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,MAAM,EAAE,CAAC;AAClF,CAAC;AAED,SAAS,kBAAkB,CAAC,QAAyB;IACpD,MAAM,OAAO,GAAG,QAAQ,CAAC,YAAY,KAAK,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;IAC3F,MAAM,WAAW,GAAG,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,iBAAiB,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IACxF,OAAO,mDAAmD,QAAQ,CAAC,OAAO,YAAY,QAAQ,CAAC,UAAU,UAAU,QAAQ,CAAC,QAAQ,cAAc,QAAQ,CAAC,QAAQ,kBAAkB,QAAQ,CAAC,YAAY,kBAAkB,OAAO,GAAG,WAAW,YAAY,QAAQ,CAAC,MAAM,iBAAiB,QAAQ,CAAC,WAAW,2BAA2B,CAAC;AAC9U,CAAC;AAED,SAAS,oBAAoB,CAAC,IAAY,EAAE,UAAkB,EAAE,KAAwB;IACvF,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,MAAM,MAAM,GAAa,EAAE,CAAC;IAE5B,OAAO,QAAQ,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;QAC/B,MAAM,KAAK,GAAG,kBAAkB,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QACjD,IAAI,KAAK,GAAG,CAAC;YAAE,MAAM;QACrB,MAAM,SAAS,GAAG,oBAAoB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QACpD,MAAM,GAAG,GAAG,SAAS,CAAC,GAAG,CAAC;QAC1B,MAAM,WAAW,GAAG,KAAK,GAAG,CAAC,CAAC;QAC9B,MAAM,YAAY,GAAG,SAAS,CAAC,KAAK,GAAG,CAAC,CAAC;QACzC,0EAA0E;QAC1E,yEAAyE;QACzE,IAAI,YAAY,IAAI,GAAG,KAAK,WAAW,IAAI,SAAS,CAAC,SAAS,KAAK,IAAI,EAAE,CAAC;YACzE,QAAQ,GAAG,GAAG,CAAC;YACf,SAAS;QACV,CAAC;QACD,MAAM,SAAS,GAAG,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC;QACvD,MAAM,YAAY,GAAG,SAAS,GAAG,WAAW,CAAC;QAC7C,MAAM,QAAQ,GAAoB,oBAAoB,CAAC,IAAI,EAAE,WAAW,EAAE,SAAS,CAAC;YACnF,CAAC,CAAC,QAAQ;YACV,CAAC,CAAC,iBAAiB,CAAC;QACrB,MAAM,QAAQ,GAAG,iBAAiB,CAAC,IAAI,EAAE,WAAW,EAAE,SAAS,CAAC,CAAC;QACjE,yEAAyE;QACzE,wEAAwE;QACxE,+CAA+C;QAC/C,MAAM,OAAO,GAAG,YAAY;YAC3B,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,GAAG,CAAC;YAC9B,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC;QACxC,MAAM,YAAY,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAExD,IAAI,YAAY,GAAkB,IAAI,CAAC;QACvC,IAAI,WAA+B,CAAC;QACpC,IAAI,WAAW,GAAgB,SAAS,CAAC;QACzC,IAAI,MAAM,GAAG,EAAE,CAAC;QAEhB,IAAI,YAAY,EAAE,CAAC;YAClB,WAAW,GAAG,eAAe,CAAC;YAC9B,MAAM,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;QAC/B,CAAC;aAAM,IAAI,YAAY,GAAG,yBAAyB,EAAE,CAAC;YACrD,WAAW,GAAG,iBAAiB,CAAC;YAChC,MAAM,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;QAC/B,CAAC;aAAM,CAAC;YACP,MAAM,OAAO,GAAG,QAAQ,KAAK,QAAQ;gBACpC,CAAC,CAAC,mBAAmB,CAAC,OAAO,CAAC;gBAC9B,CAAC,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC;YACjC,IAAI,OAAO,IAAI,OAAO,EAAE,CAAC;gBACxB,WAAW,GAAG,OAAO,CAAC,KAAK,CAAC;gBAC5B,MAAM,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;YAC/B,CAAC;iBAAM,CAAC;gBACP,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC;gBACpC,WAAW,GAAG,SAAS,CAAC;gBACxB,MAAM,GAAG,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YACrC,CAAC;QACF,CAAC;QAED,MAAM,QAAQ,GAAoB;YACjC,OAAO,EAAE,KAAK,CAAC,WAAW,EAAE;YAC5B,UAAU,EAAE,gBAAgB,CAAC,UAAU,CAAC;YACxC,QAAQ;YACR,QAAQ;YACR,YAAY;YACZ,YAAY;YACZ,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACvC,MAAM,EAAE,MAAM;YACd,WAAW;YACX,SAAS,EAAE,cAAc;SACzB,CAAC;QACF,MAAM,MAAM,GAAG,kBAAkB,CAAC,QAAQ,CAAC,CAAC;QAC5C,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,EAAE,KAAK,CAAC,CAAC;QAC/C,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC5B,YAAY,IAAI,MAAM,CAAC,MAAM,CAAC;QAC9B,MAAM,WAAW,GAAG,YAAY,CAAC;QACjC,YAAY,IAAI,MAAM,CAAC,MAAM,CAAC;QAC9B,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,GAAG,QAAQ,EAAE,WAAW,EAAE,SAAS,EAAE,YAAY,EAAE,CAAC,CAAC;QAC5E,YAAY,GAAG,GAAG,CAAC;QACnB,QAAQ,GAAG,GAAG,CAAC;IAChB,CAAC;IAED,IAAI,KAAK,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC,IAAI,YAAY,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACpE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC;IACtC,OAAO,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;AACxB,CAAC;AAGD,6EAA6E;AAC7E,MAAM,UAAU,sBAAsB,CACrC,IAAY,EACZ,UAAkB;IAElB,MAAM,KAAK,GAAsB,EAAE,WAAW,EAAE,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC;IACnE,MAAM,SAAS,GAAG,oBAAoB,CAAC,IAAI,EAAE,UAAU,EAAE,KAAK,CAAC,CAAC;IAChE,OAAO;QACN,IAAI,EAAE,SAAS;QACf,SAAS,EAAE,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,GAAG,QAAQ,EAAE,EAAE,EAAE,CAAC,QAAQ,CAAC;KACnG,CAAC;AACH,CAAC;AAED,uFAAuF","sourcesContent":["import { Buffer } from \"node:buffer\";\nimport { createHash } from \"node:crypto\";\n\n// Replaces inline RFC 2397 data: URIs in extracted text with explicit bounded\n// omission markers so base64 payloads never reach model-visible tool results,\n// the fetch cache, or session persistence. Carved out of the original\n// persistence-safety layer; only the text-sanitization path is retained.\n\nconst MAX_DATA_URI_HEADER_CHARS = 1024;\nconst MAX_MIME_CHARS = 127;\nconst MAX_MARKER_SOURCE_CHARS = 180;\n\nexport type DataUriEncoding = \"base64\" | \"percent-encoded\";\nexport type DigestBasis = \"decoded\" | \"encoded\";\n\nexport interface DataUriOmission {\n\tordinal: number;\n\tsourcePath: string;\n\tmimeType: string;\n\tencoding: DataUriEncoding;\n\tencodedBytes: number;\n\tdecodedBytes: number | null;\n\tdecodeError?: string;\n\tsha256: string;\n\tdigestBasis: DigestBasis;\n\tretrieval: \"not-retained\";\n}\n\n\ninterface InternalDataUriOmission extends DataUriOmission {\n\tmarkerStart: number;\n\tmarkerEnd: number;\n}\n\ninterface SanitizationState {\n\tnextOrdinal: number;\n\tomissions: InternalDataUriOmission[];\n}\n\n\nfunction sha256Bytes(value: string | Uint8Array): string {\n\treturn createHash(\"sha256\").update(value).digest(\"hex\");\n}\n\n\nfunction asciiLowerCode(code: number): number {\n\treturn code >= 65 && code <= 90 ? code + 32 : code;\n}\n\nfunction matchesDataScheme(text: string, index: number): boolean {\n\treturn index + 5 <= text.length\n\t\t&& asciiLowerCode(text.charCodeAt(index)) === 100\n\t\t&& asciiLowerCode(text.charCodeAt(index + 1)) === 97\n\t\t&& asciiLowerCode(text.charCodeAt(index + 2)) === 116\n\t\t&& asciiLowerCode(text.charCodeAt(index + 3)) === 97\n\t\t&& text.charCodeAt(index + 4) === 58;\n}\n\nfunction isSchemeBoundary(text: string, index: number): boolean {\n\tif (index === 0) return true;\n\tconst code = text.charCodeAt(index - 1);\n\tconst alphaNumeric = (code >= 48 && code <= 57)\n\t\t|| (code >= 65 && code <= 90)\n\t\t|| (code >= 97 && code <= 122);\n\treturn !alphaNumeric && code !== 43 && code !== 45 && code !== 46;\n}\n\nfunction findNextDataScheme(text: string, from: number): number {\n\tfor (let i = from; i + 5 <= text.length; i++) {\n\t\tif (matchesDataScheme(text, i) && isSchemeBoundary(text, i)) return i;\n\t}\n\treturn -1;\n}\n\ntype DataUriEnclosure =\n\t| { kind: \"quote\"; close: \"\\\"\" | \"'\" | \"`\" }\n\t| { kind: \"parentheses\" }\n\t| { kind: \"angle\" }\n\t| null;\n\ninterface ScannedDataUriCandidate {\n\tend: number;\n\tcomma: number;\n\tenclosure: DataUriEnclosure;\n}\n\nfunction dataUriEnclosure(text: string, start: number): DataUriEnclosure {\n\tif (start === 0) return null;\n\tconst immediateIndex = start - 1;\n\tconst immediate = text[immediateIndex];\n\tif (immediate === \"\\\"\" || immediate === \"'\" || immediate === \"`\") {\n\t\tconst beforeQuote = immediateIndex > 0 ? text[immediateIndex - 1] : \"\";\n\t\tconst likelyOpeningQuote = immediateIndex === 0\n\t\t\t|| beforeQuote === \"=\"\n\t\t\t|| beforeQuote === \"(\"\n\t\t\t|| beforeQuote === \"[\"\n\t\t\t|| beforeQuote === \"{\"\n\t\t\t|| beforeQuote === \":\"\n\t\t\t|| beforeQuote === \",\"\n\t\t\t|| /\\s/.test(beforeQuote);\n\t\tif (likelyOpeningQuote) return { kind: \"quote\", close: immediate };\n\t}\n\n\tlet openerIndex = immediateIndex;\n\twhile (openerIndex >= 0 && (text[openerIndex] === \" \" || text[openerIndex] === \"\\t\")) openerIndex--;\n\tconst opener = text[openerIndex];\n\tif (opener === \"(\") return { kind: \"parentheses\" };\n\tif (opener === \"<\") return { kind: \"angle\" };\n\treturn null;\n}\n\nfunction isBareDataUriTerminator(ch: string): boolean {\n\tconst code = ch.charCodeAt(0);\n\treturn code <= 32\n\t\t|| code === 127\n\t\t|| ch === \"\\\"\"\n\t\t|| ch === \"`\"\n\t\t|| ch === \"<\"\n\t\t|| ch === \">\";\n}\n\n/** Scan one candidate exactly once. Enclosures consume malformed whitespace and\n * balanced inner parentheses so invalid payload suffixes cannot escape. */\nfunction scanDataUriCandidate(text: string, start: number): ScannedDataUriCandidate {\n\tconst enclosure = dataUriEnclosure(text, start);\n\tlet comma = -1;\n\tlet depth = enclosure?.kind === \"parentheses\" ? 1 : 0;\n\tlet index = start + 5;\n\n\tfor (; index < text.length; index++) {\n\t\tconst ch = text[index];\n\t\tif (enclosure?.kind === \"quote\") {\n\t\t\tif (ch === enclosure.close) break;\n\t\t} else if (enclosure?.kind === \"parentheses\") {\n\t\t\tif (ch === \"(\") {\n\t\t\t\tdepth++;\n\t\t\t} else if (ch === \")\") {\n\t\t\t\tdepth--;\n\t\t\t\tif (depth === 0) break;\n\t\t\t}\n\t\t} else if (enclosure?.kind === \"angle\") {\n\t\t\tif (ch === \">\") break;\n\t\t} else if (isBareDataUriTerminator(ch)) {\n\t\t\tbreak;\n\t\t}\n\t\tif (comma < 0 && ch === \",\") comma = index;\n\t}\n\n\treturn { end: index, comma, enclosure };\n}\n\nfunction headerEndsWithBase64(text: string, headerStart: number, comma: number): boolean {\n\tlet end = comma;\n\twhile (end > headerStart && (text[end - 1] === \" \" || text[end - 1] === \"\\t\")) end--;\n\tconst token = \"base64\";\n\tif (end - headerStart < token.length + 1) return false;\n\tconst tokenStart = end - token.length;\n\tfor (let i = 0; i < token.length; i++) {\n\t\tif (asciiLowerCode(text.charCodeAt(tokenStart + i)) !== token.charCodeAt(i)) return false;\n\t}\n\treturn tokenStart > headerStart && text[tokenStart - 1] === \";\";\n}\n\nfunction isMimeTokenChar(code: number): boolean {\n\tif (code < 33 || code > 126) return false;\n\t// RFC 2045 token = printable ASCII excluding tspecials. Slash is handled\n\t// separately as the single type/subtype separator.\n\tswitch (code) {\n\t\tcase 34: // \"\n\t\tcase 40: // (\n\t\tcase 41: // )\n\t\tcase 44: // ,\n\t\tcase 47: // /\n\t\tcase 58: // :\n\t\tcase 59: // ;\n\t\tcase 60: // <\n\t\tcase 61: // =\n\t\tcase 62: // >\n\t\tcase 63: // ?\n\t\tcase 64: // @\n\t\tcase 91: // [\n\t\tcase 92: // backslash\n\t\tcase 93: // ]\n\t\t\treturn false;\n\t\tdefault:\n\t\t\treturn true;\n\t}\n}\n\nfunction normalizeMimeType(text: string, headerStart: number, comma: number): string {\n\tlet end = headerStart;\n\twhile (end < comma && text[end] !== \";\") end++;\n\tif (end === headerStart) return \"text/plain\";\n\tif (end - headerStart > MAX_MIME_CHARS) return \"application/octet-stream\";\n\n\tlet slashCount = 0;\n\tfor (let i = headerStart; i < end; i++) {\n\t\tconst code = text.charCodeAt(i);\n\t\tif (code === 47) {\n\t\t\tslashCount++;\n\t\t\tif (i === headerStart || i === end - 1) return \"application/octet-stream\";\n\t\t\tcontinue;\n\t\t}\n\t\tif (!isMimeTokenChar(code)) return \"application/octet-stream\";\n\t}\n\tif (slashCount !== 1) return \"application/octet-stream\";\n\treturn text.slice(headerStart, end).toLowerCase();\n}\n\nfunction hexValue(code: number): number {\n\tif (code >= 48 && code <= 57) return code - 48;\n\tif (code >= 65 && code <= 70) return code - 55;\n\tif (code >= 97 && code <= 102) return code - 87;\n\treturn -1;\n}\n\nfunction decodePercentEncoded(payload: string): { bytes: Buffer } | { error: string } {\n\tconst output = Buffer.allocUnsafe(Buffer.byteLength(payload, \"utf8\"));\n\tlet outputOffset = 0;\n\tlet cursor = 0;\n\n\twhile (cursor < payload.length) {\n\t\tconst percent = payload.indexOf(\"%\", cursor);\n\t\tconst runEnd = percent < 0 ? payload.length : percent;\n\t\tif (runEnd > cursor) {\n\t\t\tconst run = Buffer.from(payload.slice(cursor, runEnd), \"utf8\");\n\t\t\trun.copy(output, outputOffset);\n\t\t\toutputOffset += run.length;\n\t\t}\n\t\tif (percent < 0) break;\n\t\tif (percent + 2 >= payload.length) return { error: \"invalid-percent-escape\" };\n\t\tconst high = hexValue(payload.charCodeAt(percent + 1));\n\t\tconst low = hexValue(payload.charCodeAt(percent + 2));\n\t\tif (high < 0 || low < 0) return { error: \"invalid-percent-escape\" };\n\t\toutput[outputOffset++] = (high << 4) | low;\n\t\tcursor = percent + 3;\n\t}\n\n\treturn { bytes: output.subarray(0, outputOffset) };\n}\n\nfunction isBase64AlphabetByte(byte: number): boolean {\n\treturn (byte >= 65 && byte <= 90)\n\t\t|| (byte >= 97 && byte <= 122)\n\t\t|| (byte >= 48 && byte <= 57)\n\t\t|| byte === 43\n\t\t|| byte === 47;\n}\n\nfunction decodeBase64Payload(payload: string): { bytes: Buffer } | { error: string } {\n\tconst percentDecoded = decodePercentEncoded(payload);\n\tif (\"error\" in percentDecoded) return percentDecoded;\n\tconst encoded = percentDecoded.bytes;\n\tlet paddingStart = encoded.length;\n\tlet paddingCount = 0;\n\n\tfor (let i = 0; i < encoded.length; i++) {\n\t\tconst byte = encoded[i];\n\t\tif (byte > 127) return { error: \"non-ascii-base64\" };\n\t\tif (byte === 61) {\n\t\t\tif (paddingStart === encoded.length) paddingStart = i;\n\t\t\tpaddingCount++;\n\t\t\tcontinue;\n\t\t}\n\t\tif (paddingStart !== encoded.length) return { error: \"invalid-base64-padding\" };\n\t\tif (!isBase64AlphabetByte(byte)) return { error: \"invalid-base64-character\" };\n\t}\n\n\tif (paddingCount > 2) return { error: \"invalid-base64-padding\" };\n\tif (paddingCount > 0 && encoded.length % 4 !== 0) return { error: \"invalid-base64-padding\" };\n\tif (paddingCount === 0 && encoded.length % 4 === 1) return { error: \"invalid-base64-length\" };\n\tif (paddingCount === 1 && paddingStart % 4 !== 3) return { error: \"invalid-base64-padding\" };\n\tif (paddingCount === 2 && paddingStart % 4 !== 2) return { error: \"invalid-base64-padding\" };\n\n\treturn { bytes: Buffer.from(encoded.toString(\"ascii\"), \"base64\") };\n}\n\nfunction safeMarkerSource(sourcePath: string): string {\n\tconst source = sourcePath || \"value\";\n\tlet safe = \"\";\n\tfor (const ch of source) {\n\t\tconst code = ch.charCodeAt(0);\n\t\tconst allowed = (code >= 48 && code <= 57)\n\t\t\t|| (code >= 65 && code <= 90)\n\t\t\t|| (code >= 97 && code <= 122)\n\t\t\t|| ch === \".\"\n\t\t\t|| ch === \"_\"\n\t\t\t|| ch === \"-\"\n\t\t\t|| ch === \"[\"\n\t\t\t|| ch === \"]\"\n\t\t\t|| ch === \"*\"\n\t\t\t|| ch === \"$\";\n\t\tsafe += allowed ? ch : \"_\";\n\t}\n\tif (safe.length <= MAX_MARKER_SOURCE_CHARS) return safe;\n\tconst suffix = sha256Bytes(source).slice(0, 12);\n\treturn `${safe.slice(0, MAX_MARKER_SOURCE_CHARS - suffix.length - 1)}~${suffix}`;\n}\n\nfunction buildDataUriMarker(omission: DataUriOmission): string {\n\tconst decoded = omission.decodedBytes === null ? \"unknown\" : String(omission.decodedBytes);\n\tconst decodeError = omission.decodeError ? `; decodeError=${omission.decodeError}` : \"\";\n\treturn `[pi-web-access inline data URI omitted; ordinal=${omission.ordinal}; source=${omission.sourcePath}; mime=${omission.mimeType}; encoding=${omission.encoding}; encodedBytes=${omission.encodedBytes}; decodedBytes=${decoded}${decodeError}; sha256=${omission.sha256}; digestBasis=${omission.digestBasis}; retrieval=not-retained]`;\n}\n\nfunction sanitizeTextInternal(text: string, sourcePath: string, state: SanitizationState): string {\n\tlet scanFrom = 0;\n\tlet retainedFrom = 0;\n\tlet outputLength = 0;\n\tconst pieces: string[] = [];\n\n\twhile (scanFrom < text.length) {\n\t\tconst start = findNextDataScheme(text, scanFrom);\n\t\tif (start < 0) break;\n\t\tconst candidate = scanDataUriCandidate(text, start);\n\t\tconst end = candidate.end;\n\t\tconst headerStart = start + 5;\n\t\tconst missingComma = candidate.comma < 0;\n\t\t// A bare prose label such as \"data: value\" is not an inline URI. Enclosed\n\t\t// or non-empty comma-less candidates are malformed URIs and are removed.\n\t\tif (missingComma && end === headerStart && candidate.enclosure === null) {\n\t\t\tscanFrom = end;\n\t\t\tcontinue;\n\t\t}\n\t\tconst headerEnd = missingComma ? end : candidate.comma;\n\t\tconst headerLength = headerEnd - headerStart;\n\t\tconst encoding: DataUriEncoding = headerEndsWithBase64(text, headerStart, headerEnd)\n\t\t\t? \"base64\"\n\t\t\t: \"percent-encoded\";\n\t\tconst mimeType = normalizeMimeType(text, headerStart, headerEnd);\n\t\t// Without the required comma, the post-scheme bytes are ambiguous. Count\n\t\t// and digest the whole omitted segment on the encoded basis rather than\n\t\t// retaining any header/payload-looking suffix.\n\t\tconst payload = missingComma\n\t\t\t? text.slice(headerStart, end)\n\t\t\t: text.slice(candidate.comma + 1, end);\n\t\tconst encodedBytes = Buffer.byteLength(payload, \"utf8\");\n\n\t\tlet decodedBytes: number | null = null;\n\t\tlet decodeError: string | undefined;\n\t\tlet digestBasis: DigestBasis = \"encoded\";\n\t\tlet digest = \"\";\n\n\t\tif (missingComma) {\n\t\t\tdecodeError = \"missing-comma\";\n\t\t\tdigest = sha256Bytes(payload);\n\t\t} else if (headerLength > MAX_DATA_URI_HEADER_CHARS) {\n\t\t\tdecodeError = \"header-too-long\";\n\t\t\tdigest = sha256Bytes(payload);\n\t\t} else {\n\t\t\tconst decoded = encoding === \"base64\"\n\t\t\t\t? decodeBase64Payload(payload)\n\t\t\t\t: decodePercentEncoded(payload);\n\t\t\tif (\"error\" in decoded) {\n\t\t\t\tdecodeError = decoded.error;\n\t\t\t\tdigest = sha256Bytes(payload);\n\t\t\t} else {\n\t\t\t\tdecodedBytes = decoded.bytes.length;\n\t\t\t\tdigestBasis = \"decoded\";\n\t\t\t\tdigest = sha256Bytes(decoded.bytes);\n\t\t\t}\n\t\t}\n\n\t\tconst omission: DataUriOmission = {\n\t\t\tordinal: state.nextOrdinal++,\n\t\t\tsourcePath: safeMarkerSource(sourcePath),\n\t\t\tmimeType,\n\t\t\tencoding,\n\t\t\tencodedBytes,\n\t\t\tdecodedBytes,\n\t\t\t...(decodeError ? { decodeError } : {}),\n\t\t\tsha256: digest,\n\t\t\tdigestBasis,\n\t\t\tretrieval: \"not-retained\",\n\t\t};\n\t\tconst marker = buildDataUriMarker(omission);\n\t\tconst prefix = text.slice(retainedFrom, start);\n\t\tpieces.push(prefix, marker);\n\t\toutputLength += prefix.length;\n\t\tconst markerStart = outputLength;\n\t\toutputLength += marker.length;\n\t\tstate.omissions.push({ ...omission, markerStart, markerEnd: outputLength });\n\t\tretainedFrom = end;\n\t\tscanFrom = end;\n\t}\n\n\tif (state.omissions.length === 0 && retainedFrom === 0) return text;\n\tpieces.push(text.slice(retainedFrom));\n\treturn pieces.join(\"\");\n}\n\n\n/** Replace every recognized RFC 2397-style inline data URI in one string. */\nexport function sanitizeInlineDataUris(\n\ttext: string,\n\tsourcePath: string,\n): { text: string; omissions: DataUriOmission[] } {\n\tconst state: SanitizationState = { nextOrdinal: 1, omissions: [] };\n\tconst sanitized = sanitizeTextInternal(text, sourcePath, state);\n\treturn {\n\t\ttext: sanitized,\n\t\tomissions: state.omissions.map(({ markerStart: _start, markerEnd: _end, ...omission }) => omission),\n\t};\n}\n\n/** Sanitize every string leaf in a JSON-compatible value with one ordinal sequence. */\n"]}

package/dist/humain/web-access/vendor/datalab-pdf-extract.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"datalab-pdf-extract.d.ts","sourceRoot":"","sources":["../../../../src/humain/web-access/vendor/datalab-pdf-extract.ts"],"names":[],"mappings":"AAiBA,MAAM,MAAM,WAAW,GAAG,MAAM,GAAG,UAAU,GAAG,UAAU,CAAC;AAC3D,MAAM,MAAM,yBAAyB,GAAG,IAAI,GAAG,IAAI,CAAC;AAEpD,eAAO,MAAM,mBAAmB,kBAI9B,CAAC;AACH,eAAO,MAAM,uBAAuB,gCAGlC,CAAC;AACH,eAAO,MAAM,oBAAoB,EAAE,WAAwB,CAAC;AAC5D,eAAO,MAAM,2BAA2B,EAAE,yBAAgC,CAAC;AAC3E,eAAO,MAAM,0BAA0B,SAAqB,CAAC;AAoC7D,wBAAgB,qBAAqB,IAAI,OAAO,CAM/C;AAED,wBAAsB,gBAAgB,CACrC,MAAM,CAAC,EAAE,WAAW,GAClB,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAOxB;AAED,wBAAgB,iBAAiB,IAAI,MAAM,CAM1C;AAED;;;;;;;GAOG;AACH,wBAAgB,4BAA4B,IAAI,yBAAyB,CAWxE;AAED,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,OAAO,GAAG,WAAW,CAQhE;AAED,MAAM,WAAW,wBAAwB;IACxC,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EAAE,WAAW,CAAC;IACnB,kBAAkB,CAAC,EAAE,yBAAyB,CAAC;CAC/C;AAED,MAAM,WAAW,uBAAuB;IACvC,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAuBD;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,wBAAsB,oBAAoB,CACzC,MAAM,EAAE,WAAW,EACnB,OAAO,EAAE,wBAAwB,GAC/B,OAAO,CAAC,uBAAuB,CAAC,CAsIlC","sourcesContent":["import { existsSync, readFileSync } from \"node:fs\";\nimport {\n\thasCredentialSource,\n\tresolveCredential,\n\tredactCredential,\n} from \"./credential-source.ts\";\nimport { getWebSearchConfigPath } from \"./utils.ts\";\n\nconst DEFAULT_API_HOST = \"https://www.datalab.to\";\nconst API_PREFIX = \"/api/v1\";\nconst DEFAULT_TIMEOUT_MS = 120_000;\nconst DEFAULT_POLL_INTERVAL_MS = 1_500;\nconst MAX_RESPONSE_BYTES = 4 * 1024 * 1024;\nconst MAX_ERROR_BODY_BYTES = 300;\nconst CLEANUP_TIMEOUT_MS = 5_000;\nconst CONFIG_PATH = getWebSearchConfigPath();\n\nexport type DatalabMode = \"fast\" | \"balanced\" | \"accurate\";\nexport type DatalabProcessingLocation = \"eu\" | \"us\";\n\nexport const DATALAB_MODE_VALUES = new Set<DatalabMode>([\n\t\"fast\",\n\t\"balanced\",\n\t\"accurate\",\n]);\nexport const DATALAB_LOCATION_VALUES = new Set<DatalabProcessingLocation>([\n\t\"eu\",\n\t\"us\",\n]);\nexport const DEFAULT_DATALAB_MODE: DatalabMode = \"balanced\";\nexport const DEFAULT_PROCESSING_LOCATION: DatalabProcessingLocation = \"us\";\nexport const DEFAULT_DATALAB_TIMEOUT_MS = DEFAULT_TIMEOUT_MS;\n\ninterface DatalabConfig {\n\tdatalabApiKey?: unknown;\n}\n\nlet cachedConfig: DatalabConfig | null = null;\n\nfunction loadConfig(): DatalabConfig {\n\tif (cachedConfig) return cachedConfig;\n\tif (!existsSync(CONFIG_PATH)) {\n\t\tcachedConfig = {};\n\t\treturn cachedConfig;\n\t}\n\n\tconst rawText = readFileSync(CONFIG_PATH, \"utf-8\");\n\ttry {\n\t\tcachedConfig = JSON.parse(rawText) as DatalabConfig;\n\t\treturn cachedConfig;\n\t} catch (err) {\n\t\tconst message = err instanceof Error ? err.message : String(err);\n\t\tthrow new Error(`Failed to parse ${CONFIG_PATH}: ${message}`);\n\t}\n}\n\nfunction normalizeString(value: unknown): string | null {\n\tif (typeof value !== \"string\") return null;\n\tconst normalized = value.trim();\n\treturn normalized.length > 0 ? normalized : null;\n}\n\nfunction normalizeFileId(value: unknown): string | null {\n\tif (typeof value === \"number\" && Number.isFinite(value)) return String(value);\n\treturn normalizeString(value);\n}\n\nexport function isDatalabApiAvailable(): boolean {\n\treturn hasCredentialSource({\n\t\tprovider: \"Datalab\",\n\t\tconfiguredValue: loadConfig().datalabApiKey,\n\t\tenvironmentValue: process.env.DATALAB_API_KEY,\n\t});\n}\n\nexport async function getDatalabApiKey(\n\tsignal?: AbortSignal,\n): Promise<string | null> {\n\treturn resolveCredential({\n\t\tprovider: \"Datalab\",\n\t\tconfiguredValue: loadConfig().datalabApiKey,\n\t\tenvironmentValue: process.env.DATALAB_API_KEY,\n\t\tsignal,\n\t});\n}\n\nexport function getDatalabApiBase(): string {\n\tconst normalized = normalizeString(process.env.DATALAB_API_BASE)?.replace(\n\t\t/\\/+$/,\n\t\t\"\",\n\t);\n\treturn normalized || `${DEFAULT_API_HOST}${API_PREFIX}`;\n}\n\n/**\n * Resolve the processing region. US storage and processing is the default;\n * set `DATALAB_PROCESSING_LOCATION=eu` to use EU data residency at Datalab's\n * documented 1.25× usage rate. Only `eu` and `us` are supported by the API.\n * Config errors are thrown with a `Failed to parse` prefix so the\n * extraction fallback chain treats them as fatal, mirroring how malformed\n * web-search config is handled elsewhere.\n */\nexport function getDatalabProcessingLocation(): DatalabProcessingLocation {\n\tconst value =\n\t\tnormalizeString(process.env.DATALAB_PROCESSING_LOCATION) ??\n\t\tDEFAULT_PROCESSING_LOCATION;\n\tconst normalized = value.toLowerCase() as DatalabProcessingLocation;\n\tif (!DATALAB_LOCATION_VALUES.has(normalized)) {\n\t\tthrow new Error(\n\t\t\t`Failed to parse DATALAB_PROCESSING_LOCATION: expected \"eu\" or \"us\", got \"${value}\"`,\n\t\t);\n\t}\n\treturn normalized;\n}\n\nexport function normalizeDatalabMode(value: unknown): DatalabMode {\n\tconst raw = normalizeString(value);\n\tif (!raw) return DEFAULT_DATALAB_MODE;\n\tconst normalized = raw.toLowerCase() as DatalabMode;\n\tif (DATALAB_MODE_VALUES.has(normalized)) return normalized;\n\tthrow new Error(\n\t\t`Failed to parse datalab mode: expected \"fast\", \"balanced\", or \"accurate\", got \"${value}\"`,\n\t);\n}\n\nexport interface DatalabPDFExtractOptions {\n\tmaxPages: number;\n\ttitle: string;\n\tsignal?: AbortSignal;\n\ttimeoutMs?: number;\n\tmode?: DatalabMode;\n\tprocessingLocation?: DatalabProcessingLocation;\n}\n\nexport interface DatalabPDFExtractResult {\n\tmarkdown: string;\n\tpages: number;\n\tparseQualityScore?: number;\n}\n\ninterface DatalabUploadRequest {\n\tfile_id?: unknown;\n\tupload_url?: unknown;\n\treference?: unknown;\n}\n\ninterface DatalabConfirmResponse {\n\tfile_id?: unknown;\n\treference?: unknown;\n}\n\ninterface DatalabConversionState {\n\tstatus?: unknown;\n\tsuccess?: unknown;\n\trequest_check_url?: unknown;\n\tmarkdown?: unknown;\n\tpage_count?: unknown;\n\tparse_quality_score?: unknown;\n\terror?: unknown;\n}\n\n/**\n * Convert a PDF buffer to Markdown through the Datalab hosted API.\n *\n * Datalab converts documents with a dedicated extraction engine (Marker),\n * which preserves tables, multi-column reading order, headings, and math\n * where the local `unpdf` fallback only produces flattened text. Conversion\n * is deterministic (no LLM transcription drift), and `accurate` mode handles\n * scanned pages. Completed responses may include a `parse_quality_score` (0–5)\n * for quality gating. Billing is per processed page; the free tier includes a\n * monthly credit (see README) so light use costs nothing.\n *\n * The flow is:\n *   1. request a presigned upload URL for the selected processing region,\n *   2. PUT the PDF bytes directly to storage,\n *   3. confirm the upload to obtain a `datalab://` reference,\n *   4. submit the conversion with that reference, then\n *   5. poll the returned `request_check_url` until `complete` or `failed`.\n *\n * The processing region is fixed at the upload step (US by default; set\n * `DATALAB_PROCESSING_LOCATION=eu` for EU data residency). The convert step\n * deliberately omits `processing_location` because the API rejects multipart form data that\n * carries it; the server processes the file from the storage region it was\n * uploaded to. The uploaded file is deleted best-effort after conversion.\n * The caller's signal and a shared deadline bound the whole exchange.\n */\nexport async function extractPDFViaDatalab(\n\tbuffer: ArrayBuffer,\n\toptions: DatalabPDFExtractOptions,\n): Promise<DatalabPDFExtractResult> {\n\toptions.signal?.throwIfAborted();\n\tconst apiKey = await getDatalabApiKey(options.signal);\n\toptions.signal?.throwIfAborted();\n\tif (!apiKey) {\n\t\tthrow new Error(\n\t\t\t\"Datalab PDF conversion requires a configured Datalab API key\",\n\t\t);\n\t}\n\n\tconst mode = options.mode ?? normalizeDatalabMode(process.env.DATALAB_MODE);\n\tconst processingLocation =\n\t\toptions.processingLocation ?? getDatalabProcessingLocation();\n\tconst timeoutMs =\n\t\ttypeof options.timeoutMs === \"number\" &&\n\t\tNumber.isFinite(options.timeoutMs) &&\n\t\toptions.timeoutMs > 0\n\t\t\t? Math.floor(options.timeoutMs)\n\t\t\t: DEFAULT_TIMEOUT_MS;\n\tconst deadline = Date.now() + timeoutMs;\n\n\tconst upload = await requestUploadUrl({\n\t\tapiKey,\n\t\ttitle: options.title,\n\t\tprocessingLocation,\n\t\tsignal: options.signal,\n\t\tdeadline,\n\t});\n\tif (typeof upload.upload_url !== \"string\" || !upload.upload_url) {\n\t\tthrow new Error(\"Datalab PDF conversion failed: missing upload_url\");\n\t}\n\tconst uploadFileId = normalizeFileId(upload.file_id);\n\tif (!uploadFileId) {\n\t\tthrow new Error(\"Datalab PDF conversion failed: missing file_id\");\n\t}\n\n\tlet fileId = uploadFileId;\n\tlet reference = normalizeString(upload.reference);\n\n\ttry {\n\t\tconst put = await fetchDatalab(\n\t\t\tupload.upload_url,\n\t\t\t{\n\t\t\t\tmethod: \"PUT\",\n\t\t\t\theaders: { \"content-type\": \"application/pdf\" },\n\t\t\t\tbody: new Blob([buffer], { type: \"application/pdf\" }),\n\t\t\t\tsignal: withTimeout(options.signal, remaining(deadline)),\n\t\t\t},\n\t\t\tapiKey,\n\t\t);\n\t\tif (!put.ok) {\n\t\t\tthrow new Error(\n\t\t\t\t`Datalab PDF upload failed: HTTP ${put.status} ${put.statusText}`,\n\t\t\t);\n\t\t}\n\n\t\tconst confirmed = await confirmUpload(\n\t\t\tapiKey,\n\t\t\tString(fileId),\n\t\t\toptions.signal,\n\t\t\tdeadline,\n\t\t);\n\t\tfileId = normalizeFileId(confirmed.file_id) ?? fileId;\n\t\treference = normalizeString(confirmed.reference) ?? reference;\n\t\tif (!reference) {\n\t\t\tthrow new Error(\"Datalab PDF conversion failed: missing file reference\");\n\t\t}\n\n\t\tconst form = new FormData();\n\t\tform.append(\"file_url\", reference);\n\t\tform.append(\"output_format\", \"markdown\");\n\t\tform.append(\"mode\", mode);\n\t\tform.append(\"max_pages\", String(options.maxPages));\n\t\tform.append(\"paginate\", \"true\");\n\n\t\t// The conversion itself must NOT repeat processing_location: the API\n\t\t// rejects multipart form data carrying processing_location (HTTP 403).\n\t\t// The region was already fixed when the presigned upload was issued for\n\t\t// storage in the selected region, so the server processes the file\n\t\t// wherever it was stored (US by default; use the env override for EU).\n\n\t\tconst submit = await fetchDatalab(\n\t\t\t`${getDatalabApiBase()}/convert`,\n\t\t\t{\n\t\t\t\tmethod: \"POST\",\n\t\t\t\theaders: { \"x-api-key\": apiKey },\n\t\t\t\tbody: form,\n\t\t\t\tsignal: withTimeout(options.signal, remaining(deadline)),\n\t\t\t},\n\t\t\tapiKey,\n\t\t);\n\t\tconst state = await readJsonResponse(submit, apiKey);\n\n\t\t// The submit response is an acceptance ack ({ success: true,\n\t\t// request_check_url }) with no status field and no markdown; only the\n\t\t// polled result carries a status. Key on status, never on success.\n\t\tif (state.status === \"complete\") return toResult(state);\n\t\tif (state.status === \"failed\") {\n\t\t\tthrow new Error(\n\t\t\t\t`Datalab PDF conversion failed: ${state.error ?? \"unknown error\"}`,\n\t\t\t);\n\t\t}\n\n\t\tconst checkUrl = normalizeCheckUrl(state.request_check_url);\n\t\twhile (Date.now() < deadline) {\n\t\t\tawait sleep(\n\t\t\t\tMath.min(DEFAULT_POLL_INTERVAL_MS, remaining(deadline)),\n\t\t\t\toptions.signal,\n\t\t\t);\n\t\t\tif (Date.now() >= deadline) break;\n\t\t\tconst poll = await fetchDatalab(\n\t\t\t\tcheckUrl,\n\t\t\t\t{\n\t\t\t\t\tmethod: \"GET\",\n\t\t\t\t\theaders: { \"x-api-key\": apiKey },\n\t\t\t\t\tsignal: withTimeout(options.signal, remaining(deadline)),\n\t\t\t\t},\n\t\t\t\tapiKey,\n\t\t\t);\n\t\t\tconst next = await readJsonResponse(poll, apiKey);\n\t\t\tif (next.status === \"complete\") return toResult(next);\n\t\t\tif (next.status === \"failed\") {\n\t\t\t\tthrow new Error(\n\t\t\t\t\t`Datalab PDF conversion failed: ${next.error ?? \"unknown error\"}`,\n\t\t\t\t);\n\t\t\t}\n\t\t}\n\n\t\tthrow new Error(\"Datalab PDF conversion timed out\");\n\t} finally {\n\t\t// File deletion is best-effort. Do not extend a caller's timeout or\n\t\t// cancellation while waiting for a remote cleanup request.\n\t\tvoid deleteDatalabFile(apiKey, fileId);\n\t}\n}\n\nasync function requestUploadUrl(options: {\n\tapiKey: string;\n\ttitle: string;\n\tprocessingLocation: DatalabProcessingLocation;\n\tsignal: AbortSignal | undefined;\n\tdeadline: number;\n}): Promise<DatalabUploadRequest> {\n\tconst { apiKey, title, processingLocation, signal, deadline } = options;\n\tconst response = await fetchDatalab(\n\t\t`${getDatalabApiBase()}/files/upload`,\n\t\t{\n\t\t\tmethod: \"POST\",\n\t\t\theaders: {\n\t\t\t\t\"x-api-key\": apiKey,\n\t\t\t\t\"content-type\": \"application/json\",\n\t\t\t},\n\t\t\tbody: JSON.stringify({\n\t\t\t\tfilename: `${datalabFilename(title)}.pdf`,\n\t\t\t\tcontent_type: \"application/pdf\",\n\t\t\t\tprocessing_location: processingLocation,\n\t\t\t}),\n\t\t\tsignal: withTimeout(signal, remaining(deadline)),\n\t\t},\n\t\tapiKey,\n\t);\n\treturn readJsonResponse(response, apiKey);\n}\n\nasync function confirmUpload(\n\tapiKey: string,\n\tfileId: string,\n\tsignal: AbortSignal | undefined,\n\tdeadline: number,\n): Promise<DatalabConfirmResponse> {\n\tconst response = await fetchDatalab(\n\t\t`${getDatalabApiBase()}/files/${encodeURIComponent(fileId)}/confirm`,\n\t\t{\n\t\t\tmethod: \"GET\",\n\t\t\theaders: { \"x-api-key\": apiKey },\n\t\t\tsignal: withTimeout(signal, remaining(deadline)),\n\t\t},\n\t\tapiKey,\n\t);\n\treturn readJsonResponse(response, apiKey);\n}\n\nasync function deleteDatalabFile(\n\tapiKey: string,\n\tfileId: string,\n): Promise<void> {\n\ttry {\n\t\tawait fetchDatalab(\n\t\t\t`${getDatalabApiBase()}/files/${encodeURIComponent(fileId)}`,\n\t\t\t{\n\t\t\t\tmethod: \"DELETE\",\n\t\t\t\theaders: { \"x-api-key\": apiKey },\n\t\t\t\tsignal: AbortSignal.timeout(CLEANUP_TIMEOUT_MS),\n\t\t\t},\n\t\t\tapiKey,\n\t\t);\n\t} catch {\n\t\t// Best-effort cleanup; a leftover file is not worth failing the conversion.\n\t}\n}\n\nfunction toResult(state: DatalabConversionState): DatalabPDFExtractResult {\n\tconst markdown = normalizeString(state.markdown);\n\tif (!markdown) {\n\t\tthrow new Error(\"Datalab PDF conversion returned empty markdown\");\n\t}\n\tconst quality = state.parse_quality_score;\n\treturn {\n\t\tmarkdown,\n\t\tpages: numericField(state.page_count) ?? countPageMarkers(markdown),\n\t\t...(typeof quality === \"number\" && Number.isFinite(quality)\n\t\t\t? { parseQualityScore: quality }\n\t\t\t: {}),\n\t};\n}\n\nfunction numericField(value: unknown): number | null {\n\treturn typeof value === \"number\" && Number.isFinite(value) && value > 0\n\t\t? Math.floor(value)\n\t\t: null;\n}\n\nfunction countPageMarkers(markdown: string): number {\n\treturn [...markdown.matchAll(/^<!-- Page (\\d+) -->$/gm)].length;\n}\n\nfunction normalizeCheckUrl(value: unknown): string {\n\tif (typeof value !== \"string\" || !value.trim()) {\n\t\tthrow new Error(\"Datalab PDF conversion failed: missing request_check_url\");\n\t}\n\tconst apiBase = getDatalabApiUrl();\n\tlet checkUrl: URL;\n\ttry {\n\t\tcheckUrl = new URL(value.trim(), apiBase);\n\t} catch {\n\t\tthrow new Error(\"Datalab PDF conversion failed: invalid request_check_url\");\n\t}\n\tif (checkUrl.origin !== apiBase.origin) {\n\t\tthrow new Error(\n\t\t\t\"Datalab PDF conversion failed: request_check_url has an unexpected origin\",\n\t\t);\n\t}\n\treturn checkUrl.toString();\n}\n\nfunction getDatalabApiUrl(): URL {\n\ttry {\n\t\treturn new URL(getDatalabApiBase());\n\t} catch {\n\t\tthrow new Error(\n\t\t\t\"Failed to parse DATALAB_API_BASE: expected an absolute URL\",\n\t\t);\n\t}\n}\n\nfunction remaining(deadline: number): number {\n\treturn Math.max(1, deadline - Date.now());\n}\n\nfunction withTimeout(\n\tsignal: AbortSignal | undefined,\n\ttimeoutMs: number,\n): AbortSignal {\n\tconst timeout = AbortSignal.timeout(timeoutMs);\n\treturn signal ? AbortSignal.any([signal, timeout]) : timeout;\n}\n\nfunction sleep(ms: number, signal?: AbortSignal): Promise<void> {\n\treturn new Promise((resolve, reject) => {\n\t\tlet timer: ReturnType<typeof setTimeout>;\n\t\tconst cleanup = () => signal?.removeEventListener(\"abort\", onAbort);\n\t\tconst onTimeout = () => {\n\t\t\tcleanup();\n\t\t\tresolve();\n\t\t};\n\t\tconst onAbort = () => {\n\t\t\tclearTimeout(timer);\n\t\t\tcleanup();\n\t\t\treject(signal?.reason);\n\t\t};\n\n\t\tif (signal?.aborted) {\n\t\t\tonAbort();\n\t\t\treturn;\n\t\t}\n\t\ttimer = setTimeout(onTimeout, ms);\n\t\tsignal?.addEventListener(\"abort\", onAbort, { once: true });\n\t});\n}\n\nasync function fetchDatalab(\n\turl: string,\n\tinit: RequestInit,\n\tapiKey: string,\n): Promise<Response> {\n\ttry {\n\t\treturn await fetch(url, { ...init, redirect: init.redirect ?? \"error\" });\n\t} catch (error) {\n\t\tconst message = error instanceof Error ? error.message : String(error);\n\t\tconst redacted = redactCredential(message, apiKey);\n\t\tif (redacted === message) throw error;\n\t\tconst redactedError = new Error(redacted);\n\t\tif (error instanceof Error) redactedError.name = error.name;\n\t\tthrow redactedError;\n\t}\n}\n\nasync function readJsonResponse(\n\tresponse: Response,\n\tapiKey: string,\n): Promise<Record<string, unknown>> {\n\tconst text = await readResponseText(response);\n\tif (!response.ok) {\n\t\tconst detail = text.trim().slice(0, MAX_ERROR_BODY_BYTES);\n\t\tconst message = detail\n\t\t\t? `Datalab PDF conversion failed: HTTP ${response.status} ${response.statusText}: ${detail}`\n\t\t\t: `Datalab PDF conversion failed: HTTP ${response.status} ${response.statusText}`;\n\t\tthrow new Error(redactCredential(message, apiKey));\n\t}\n\treturn parseJsonRecord(text);\n}\n\nfunction parseJsonRecord(text: string): Record<string, unknown> {\n\tlet parsed: unknown;\n\ttry {\n\t\tparsed = JSON.parse(text);\n\t} catch (error) {\n\t\tthrow new Error(\"Datalab PDF conversion returned invalid JSON\", { cause: error });\n\t}\n\tif (Object.prototype.toString.call(parsed) !== \"[object Object]\") {\n\t\tthrow new Error(\"Datalab PDF conversion returned invalid JSON object\");\n\t}\n\treturn parsed as Record<string, unknown>;\n}\n\nasync function readResponseText(response: Response): Promise<string> {\n\tconst contentLength = Number(response.headers.get(\"content-length\"));\n\tif (Number.isFinite(contentLength) && contentLength > MAX_RESPONSE_BYTES) {\n\t\tthrow new Error(\"Datalab PDF conversion response too large\");\n\t}\n\tif (!response.body) return \"\";\n\n\tconst reader = response.body.getReader();\n\tconst decoder = new TextDecoder();\n\tconst chunks: string[] = [];\n\tlet bytesRead = 0;\n\ttry {\n\t\twhile (true) {\n\t\t\tconst { done, value } = await reader.read();\n\t\t\tif (done) break;\n\t\t\tbytesRead += value.byteLength;\n\t\t\tif (bytesRead > MAX_RESPONSE_BYTES) {\n\t\t\t\ttry {\n\t\t\t\t\tawait reader.cancel();\n\t\t\t\t} catch {\n\t\t\t\t\t// The response is already too large; cancellation is best-effort.\n\t\t\t\t}\n\t\t\t\tthrow new Error(\"Datalab PDF conversion response too large\");\n\t\t\t}\n\t\t\tchunks.push(decoder.decode(value, { stream: true }));\n\t\t}\n\t\tchunks.push(decoder.decode());\n\t\treturn chunks.join(\"\");\n\t} finally {\n\t\treader.releaseLock();\n\t}\n}\n\nfunction datalabFilename(title: string): string {\n\treturn (\n\t\ttitle\n\t\t\t.toLowerCase()\n\t\t\t.replace(/[^a-z0-9\\s-]/g, \"\")\n\t\t\t.replace(/\\s+/g, \"-\")\n\t\t\t.replace(/-+/g, \"-\")\n\t\t\t.replace(/^-|-$/g, \"\")\n\t\t\t.slice(0, 60) || \"document\"\n\t);\n}\n"]}
+{"version":3,"file":"datalab-pdf-extract.d.ts","sourceRoot":"","sources":["../../../../src/humain/web-access/vendor/datalab-pdf-extract.ts"],"names":[],"mappings":"AAiBA,MAAM,MAAM,WAAW,GAAG,MAAM,GAAG,UAAU,GAAG,UAAU,CAAC;AAC3D,MAAM,MAAM,yBAAyB,GAAG,IAAI,GAAG,IAAI,CAAC;AAEpD,eAAO,MAAM,mBAAmB,kBAI9B,CAAC;AACH,eAAO,MAAM,uBAAuB,gCAGlC,CAAC;AACH,eAAO,MAAM,oBAAoB,EAAE,WAAwB,CAAC;AAC5D,eAAO,MAAM,2BAA2B,EAAE,yBAAgC,CAAC;AAC3E,eAAO,MAAM,0BAA0B,SAAqB,CAAC;AAoC7D,wBAAgB,qBAAqB,IAAI,OAAO,CAM/C;AAED,wBAAsB,gBAAgB,CACrC,MAAM,CAAC,EAAE,WAAW,GAClB,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAOxB;AAED,wBAAgB,iBAAiB,IAAI,MAAM,CAM1C;AAED;;;;;;;GAOG;AACH,wBAAgB,4BAA4B,IAAI,yBAAyB,CAWxE;AAED,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,OAAO,GAAG,WAAW,CAQhE;AAED,MAAM,WAAW,wBAAwB;IACxC,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EAAE,WAAW,CAAC;IACnB,kBAAkB,CAAC,EAAE,yBAAyB,CAAC;CAC/C;AAED,MAAM,WAAW,uBAAuB;IACvC,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAuBD;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,wBAAsB,oBAAoB,CACzC,MAAM,EAAE,WAAW,EACnB,OAAO,EAAE,wBAAwB,GAC/B,OAAO,CAAC,uBAAuB,CAAC,CAsIlC"}