@humain/terminal 0.0.2 → 0.0.3-beta.0

package/dist/humain/web-access/vendor/pdf-extract.js

@@ -0,0 +1,298 @@
+/**
+ * PDF Content Extractor
+ *
+ * Converts PDFs to Markdown. The `auto` provider chain runs Datalab
+ * (deterministic, layout-aware) first, then Gemini API, with unpdf as the
+ * deterministic local fallback; the chain can also be pinned with
+ * `pdf.provider`.
+ */
+import { existsSync, readFileSync } from "node:fs";
+import { writeFile, mkdir } from "node:fs/promises";
+import { join, basename } from "node:path";
+import { tmpdir } from "node:os";
+import { CredentialResolutionError } from "./credential-source.js";
+import { DATALAB_MODE_VALUES, DEFAULT_DATALAB_TIMEOUT_MS, isDatalabApiAvailable, normalizeDatalabMode, extractPDFViaDatalab, } from "./datalab-pdf-extract.js";
+import { isGeminiApiAvailable } from "./gemini-api.js";
+import { extractPDFViaGemini } from "./gemini-pdf-extract.js";
+import { getWebSearchConfigPath } from "./utils.js";
+export const PDF_PROVIDER_VALUES = new Set([
+    "auto",
+    "gemini",
+    "datalab",
+    "unpdf",
+]);
+export const DEFAULT_PDF_MAX_SIZE_MB = 20;
+export const MAX_PDF_MAX_SIZE_MB = 50;
+export const MAX_DATALAB_TIMEOUT_MS = 300_000;
+const DEFAULT_MAX_PAGES = 100;
+const DEFAULT_OUTPUT_DIR = join(tmpdir(), "pi-web-pdf");
+const CONFIG_PATH = getWebSearchConfigPath();
+const PAGE_MARKER_PATTERN = /^<!-- Page (\d+) -->$/gm;
+export function loadPDFConfig() {
+    if (!existsSync(CONFIG_PATH)) {
+        return {
+            enabled: true,
+            maxSizeMB: DEFAULT_PDF_MAX_SIZE_MB,
+            provider: "auto",
+            datalabMode: normalizeDatalabMode(process.env.DATALAB_MODE),
+            datalabTimeoutMs: DEFAULT_DATALAB_TIMEOUT_MS,
+        };
+    }
+    const rawText = readFileSync(CONFIG_PATH, "utf-8");
+    let raw;
+    try {
+        raw = JSON.parse(rawText);
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        throw new Error(`Failed to parse ${CONFIG_PATH}: ${message}`);
+    }
+    const root = raw && typeof raw === "object" ? raw : {};
+    const pdf = root.pdf && typeof root.pdf === "object"
+        ? root.pdf
+        : {};
+    const enabled = pdf.enabled !== false;
+    const configured = pdf.maxSizeMB;
+    const normalized = typeof configured === "number" &&
+        Number.isFinite(configured) &&
+        configured > 0
+        ? Math.min(configured, MAX_PDF_MAX_SIZE_MB)
+        : DEFAULT_PDF_MAX_SIZE_MB;
+    const provider = typeof pdf.provider === "string" &&
+        PDF_PROVIDER_VALUES.has(pdf.provider)
+        ? pdf.provider
+        : "auto";
+    const datalabMode = typeof pdf.datalabMode === "string" &&
+        DATALAB_MODE_VALUES.has(pdf.datalabMode)
+        ? pdf.datalabMode
+        : normalizeDatalabMode(process.env.DATALAB_MODE);
+    const configuredTimeout = pdf.datalabTimeoutMs;
+    const datalabTimeoutMs = typeof configuredTimeout === "number" &&
+        Number.isFinite(configuredTimeout) &&
+        configuredTimeout > 0
+        ? Math.min(configuredTimeout, MAX_DATALAB_TIMEOUT_MS)
+        : DEFAULT_DATALAB_TIMEOUT_MS;
+    return {
+        enabled,
+        maxSizeMB: normalized,
+        provider,
+        datalabMode,
+        datalabTimeoutMs,
+    };
+}
+async function getUnpdf() {
+    if (typeof Promise.try !==
+        "function") {
+        const { default: promiseTry } = await import("promise.try");
+        promiseTry.shim();
+    }
+    const [unpdf, pdfjs] = await Promise.all([
+        import("unpdf"),
+        import("unpdf/pdfjs"),
+    ]);
+    const { VerbosityLevel } = pdfjs;
+    return { getDocumentProxy: unpdf.getDocumentProxy, VerbosityLevel };
+}
+/**
+ * Extract text from a PDF buffer and save it to a Markdown file.
+ */
+export async function extractPDFToMarkdown(buffer, url, options = {}) {
+    const { maxPages = DEFAULT_MAX_PAGES, outputDir = DEFAULT_OUTPUT_DIR, filename, signal, geminiTimeoutMs, } = options;
+    const safeMaxPages = Number.isFinite(maxPages)
+        ? Math.max(1, Math.floor(maxPages))
+        : DEFAULT_MAX_PAGES;
+    const urlTitle = extractTitleFromURL(url);
+    const pdfConfig = loadPDFConfig();
+    const provider = pdfConfig.provider;
+    if (provider === "auto" || provider === "datalab") {
+        try {
+            if (isDatalabApiAvailable()) {
+                const result = await extractPDFViaDatalab(buffer, {
+                    maxPages: safeMaxPages,
+                    title: urlTitle,
+                    mode: pdfConfig.datalabMode,
+                    timeoutMs: pdfConfig.datalabTimeoutMs,
+                    ...(signal ? { signal } : {}),
+                });
+                return writeMarkdownResult({
+                    markdownBody: result.markdown,
+                    title: urlTitle,
+                    pages: result.pages,
+                    outputDir,
+                    filename,
+                    url,
+                });
+            }
+        }
+        catch (err) {
+            if (shouldRethrowExtractionError(err, signal))
+                throw err;
+        }
+    }
+    if (provider === "auto" || provider === "gemini") {
+        try {
+            if (isGeminiApiAvailable()) {
+                const markdownBody = await extractPDFViaGemini(buffer, {
+                    maxPages: safeMaxPages,
+                    title: urlTitle,
+                    ...(signal ? { signal } : {}),
+                    ...(geminiTimeoutMs !== undefined
+                        ? { timeoutMs: geminiTimeoutMs }
+                        : {}),
+                });
+                return writeMarkdownResult({
+                    markdownBody,
+                    title: urlTitle,
+                    pages: countPageMarkers(markdownBody),
+                    outputDir,
+                    filename,
+                    url,
+                });
+            }
+        }
+        catch (err) {
+            if (shouldRethrowExtractionError(err, signal))
+                throw err;
+        }
+    }
+    const { getDocumentProxy, VerbosityLevel } = await getUnpdf();
+    const pdf = await getDocumentProxy(new Uint8Array(buffer), {
+        verbosity: VerbosityLevel.ERRORS,
+    });
+    const metadata = await pdf.getMetadata();
+    const metadataInfo = metadata.info && typeof metadata.info === "object"
+        ? metadata.info
+        : null;
+    const metaTitle = typeof metadataInfo?.Title === "string" ? metadataInfo.Title : undefined;
+    const metaAuthor = typeof metadataInfo?.Author === "string" ? metadataInfo.Author : undefined;
+    const title = metaTitle?.trim() || urlTitle;
+    const pagesToExtract = Math.min(pdf.numPages, safeMaxPages);
+    const truncated = pdf.numPages > safeMaxPages;
+    const pages = [];
+    for (let i = 1; i <= pagesToExtract; i++) {
+        const page = await pdf.getPage(i);
+        const textContent = await page.getTextContent();
+        const pageText = textContent.items
+            .map((item) => {
+            const textItem = item;
+            return textItem.str || "";
+        })
+            .join(" ")
+            .replace(/\s+/g, " ")
+            .trim();
+        if (pageText) {
+            pages.push({ pageNum: i, text: pageText });
+        }
+    }
+    const bodyLines = [];
+    for (let i = 0; i < pages.length; i++) {
+        if (i > 0) {
+            bodyLines.push("");
+            bodyLines.push(`<!-- Page ${pages[i].pageNum} -->`);
+            bodyLines.push("");
+        }
+        bodyLines.push(pages[i].text);
+    }
+    return writeMarkdownResult({
+        markdownBody: bodyLines.join("\n"),
+        title,
+        pages: pdf.numPages,
+        outputDir,
+        filename,
+        url,
+        metaAuthor,
+        truncated,
+        pagesToExtract,
+    });
+}
+async function writeMarkdownResult(options) {
+    const lines = [];
+    lines.push(`# ${options.title}`);
+    lines.push("");
+    lines.push(`> Source: ${options.url}`);
+    lines.push(`> Pages: ${options.pages}${options.truncated ? ` (extracted first ${options.pagesToExtract})` : ""}`);
+    if (options.metaAuthor)
+        lines.push(`> Author: ${options.metaAuthor}`);
+    lines.push("");
+    lines.push("---");
+    lines.push("");
+    if (options.markdownBody)
+        lines.push(options.markdownBody);
+    if (options.truncated) {
+        lines.push("");
+        lines.push("---");
+        lines.push("");
+        lines.push(`*[Truncated: Only first ${options.pagesToExtract} of ${options.pages} pages extracted]*`);
+    }
+    const content = lines.join("\n");
+    const outputFilename = options.filename || sanitizeFilename(options.title) + ".md";
+    const outputPath = join(options.outputDir, outputFilename);
+    await mkdir(options.outputDir, { recursive: true });
+    await writeFile(outputPath, content, "utf-8");
+    return {
+        title: options.title,
+        pages: options.pages,
+        chars: content.length,
+        outputPath,
+    };
+}
+function countPageMarkers(markdown) {
+    return [...markdown.matchAll(PAGE_MARKER_PATTERN)].length;
+}
+function shouldRethrowExtractionError(err, signal) {
+    if (signal?.aborted)
+        return true;
+    if (err instanceof CredentialResolutionError)
+        return true;
+    const message = err instanceof Error ? err.message : String(err);
+    return message.startsWith("Failed to parse ");
+}
+/**
+ * Extract a reasonable title from URL
+ */
+function extractTitleFromURL(url) {
+    try {
+        const urlObj = new URL(url);
+        const pathname = urlObj.pathname;
+        let filename = basename(pathname, ".pdf");
+        if (urlObj.hostname.includes("arxiv.org")) {
+            const match = pathname.match(/\/(?:pdf|abs)\/(\d+\.\d+)/);
+            if (match) {
+                filename = `arxiv-${match[1]}`;
+            }
+        }
+        filename = filename.replace(/[_-]+/g, " ").replace(/\s+/g, " ").trim();
+        return filename || "document";
+    }
+    catch {
+        return "document";
+    }
+}
+/**
+ * Sanitize string for use as filename
+ */
+function sanitizeFilename(name) {
+    return (name
+        .toLowerCase()
+        .replace(/[^a-z0-9\s-]/g, "")
+        .replace(/\s+/g, "-")
+        .replace(/-+/g, "-")
+        .slice(0, 100)
+        .replace(/^-|-$/g, "") || "document");
+}
+/**
+ * Check if URL or content-type indicates a PDF
+ */
+export function isPDF(url, contentType) {
+    if (contentType?.includes("application/pdf")) {
+        return true;
+    }
+    try {
+        const urlObj = new URL(url);
+        return urlObj.pathname.toLowerCase().endsWith(".pdf");
+    }
+    catch {
+        return false;
+    }
+}
+//# sourceMappingURL=pdf-extract.js.map

package/dist/humain/web-access/vendor/pdf-extract.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pdf-extract.js","sourceRoot":"","sources":["../../../../src/humain/web-access/vendor/pdf-extract.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAC3C,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,yBAAyB,EAAE,MAAM,wBAAwB,CAAC;AACnE,OAAO,EACN,mBAAmB,EACnB,0BAA0B,EAC1B,qBAAqB,EACrB,oBAAoB,EACpB,oBAAoB,GAEpB,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AACvD,OAAO,EAAE,mBAAmB,EAAE,MAAM,yBAAyB,CAAC;AAC9D,OAAO,EAAE,sBAAsB,EAAE,MAAM,YAAY,CAAC;AAmBpD,MAAM,CAAC,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAc;IACvD,MAAM;IACN,QAAQ;IACR,SAAS;IACT,OAAO;CACP,CAAC,CAAC;AAUH,MAAM,CAAC,MAAM,uBAAuB,GAAG,EAAE,CAAC;AAC1C,MAAM,CAAC,MAAM,mBAAmB,GAAG,EAAE,CAAC;AACtC,MAAM,CAAC,MAAM,sBAAsB,GAAG,OAAO,CAAC;AAC9C,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAC9B,MAAM,kBAAkB,GAAG,IAAI,CAAC,MAAM,EAAE,EAAE,YAAY,CAAC,CAAC;AACxD,MAAM,WAAW,GAAG,sBAAsB,EAAE,CAAC;AAC7C,MAAM,mBAAmB,GAAG,yBAAyB,CAAC;AAEtD,MAAM,UAAU,aAAa,GAAc;IAC1C,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;QAC9B,OAAO;YACN,OAAO,EAAE,IAAI;YACb,SAAS,EAAE,uBAAuB;YAClC,QAAQ,EAAE,MAAM;YAChB,WAAW,EAAE,oBAAoB,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC;YAC3D,gBAAgB,EAAE,0BAA0B;SAC5C,CAAC;IACH,CAAC;IAED,MAAM,OAAO,GAAG,YAAY,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;IACnD,IAAI,GAAY,CAAC;IACjB,IAAI,CAAC;QACJ,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAY,CAAC;IACtC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACd,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACjE,MAAM,IAAI,KAAK,CAAC,mBAAmB,WAAW,KAAK,OAAO,EAAE,CAAC,CAAC;IAC/D,CAAC;IAED,MAAM,IAAI,GACT,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAE,GAA+B,CAAC,CAAC,CAAC,EAAE,CAAC;IACxE,MAAM,GAAG,GACR,IAAI,CAAC,GAAG,IAAI,OAAO,IAAI,CAAC,GAAG,KAAK,QAAQ;QACvC,CAAC,CAAE,IAAI,CAAC,GAA+B;QACvC,CAAC,CAAC,EAAE,CAAC;IACP,MAAM,OAAO,GAAG,GAAG,CAAC,OAAO,KAAK,KAAK,CAAC;IACtC,MAAM,UAAU,GAAG,GAAG,CAAC,SAAS,CAAC;IACjC,MAAM,UAAU,GACf,OAAO,UAAU,KAAK,QAAQ;QAC9B,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC;QAC3B,UAAU,GAAG,CAAC;QACb,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,mBAAmB,CAAC;QAC3C,CAAC,CAAC,uBAAuB,CAAC;IAE5B,MAAM,QAAQ,GACb,OAAO,GAAG,CAAC,QAAQ,KAAK,QAAQ;QAChC,mBAAmB,CAAC,GAAG,CAAC,GAAG,CAAC,QAAuB,CAAC;QACnD,CAAC,CAAE,GAAG,CAAC,QAAwB;QAC/B,CAAC,CAAC,MAAM,CAAC;IACX,MAAM,WAAW,GAChB,OAAO,GAAG,CAAC,WAAW,KAAK,QAAQ;QACnC,mBAAmB,CAAC,GAAG,CAAC,GAAG,CAAC,WAA0B,CAAC;QACtD,CAAC,CAAE,GAAG,CAAC,WAA2B;QAClC,CAAC,CAAC,oBAAoB,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,iBAAiB,GAAG,GAAG,CAAC,gBAAgB,CAAC;IAC/C,MAAM,gBAAgB,GACrB,OAAO,iBAAiB,KAAK,QAAQ;QACrC,MAAM,CAAC,QAAQ,CAAC,iBAAiB,CAAC;QAClC,iBAAiB,GAAG,CAAC;QACpB,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,iBAAiB,EAAE,sBAAsB,CAAC;QACrD,CAAC,CAAC,0BAA0B,CAAC;IAE/B,OAAO;QACN,OAAO;QACP,SAAS,EAAE,UAAU;QACrB,QAAQ;QACR,WAAW;QACX,gBAAgB;KAChB,CAAC;AAAA,CACF;AAED,KAAK,UAAU,QAAQ,GAAG;IACzB,IACC,OAAQ,OAAkD,CAAC,GAAG;QAC9D,UAAU,EACT,CAAC;QACF,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,CAAC;QAC5D,UAAU,CAAC,IAAI,EAAE,CAAC;IACnB,CAAC;IAED,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QACxC,MAAM,CAAC,OAAO,CAAC;QACf,MAAM,CAAC,aAAa,CAAC;KACrB,CAAC,CAAC;IACH,MAAM,EAAE,cAAc,EAAE,GAAG,KAE1B,CAAC;IAEF,OAAO,EAAE,gBAAgB,EAAE,KAAK,CAAC,gBAAgB,EAAE,cAAc,EAAE,CAAC;AAAA,CACpE;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACzC,MAAmB,EACnB,GAAW,EACX,OAAO,GAAsB,EAAE,EACH;IAC5B,MAAM,EACL,QAAQ,GAAG,iBAAiB,EAC5B,SAAS,GAAG,kBAAkB,EAC9B,QAAQ,EACR,MAAM,EACN,eAAe,GACf,GAAG,OAAO,CAAC;IAEZ,MAAM,YAAY,GAAG,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC;QAC7C,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QACnC,CAAC,CAAC,iBAAiB,CAAC;IACrB,MAAM,QAAQ,GAAG,mBAAmB,CAAC,GAAG,CAAC,CAAC;IAC1C,MAAM,SAAS,GAAG,aAAa,EAAE,CAAC;IAClC,MAAM,QAAQ,GAAG,SAAS,CAAC,QAAQ,CAAC;IAEpC,IAAI,QAAQ,KAAK,MAAM,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QACnD,IAAI,CAAC;YACJ,IAAI,qBAAqB,EAAE,EAAE,CAAC;gBAC7B,MAAM,MAAM,GAAG,MAAM,oBAAoB,CAAC,MAAM,EAAE;oBACjD,QAAQ,EAAE,YAAY;oBACtB,KAAK,EAAE,QAAQ;oBACf,IAAI,EAAE,SAAS,CAAC,WAAW;oBAC3B,SAAS,EAAE,SAAS,CAAC,gBAAgB;oBACrC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBAC7B,CAAC,CAAC;gBACH,OAAO,mBAAmB,CAAC;oBAC1B,YAAY,EAAE,MAAM,CAAC,QAAQ;oBAC7B,KAAK,EAAE,QAAQ;oBACf,KAAK,EAAE,MAAM,CAAC,KAAK;oBACnB,SAAS;oBACT,QAAQ;oBACR,GAAG;iBACH,CAAC,CAAC;YACJ,CAAC;QACF,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACd,IAAI,4BAA4B,CAAC,GAAG,EAAE,MAAM,CAAC;gBAAE,MAAM,GAAG,CAAC;QAC1D,CAAC;IACF,CAAC;IAED,IAAI,QAAQ,KAAK,MAAM,IAAI,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAClD,IAAI,CAAC;YACJ,IAAI,oBAAoB,EAAE,EAAE,CAAC;gBAC5B,MAAM,YAAY,GAAG,MAAM,mBAAmB,CAAC,MAAM,EAAE;oBACtD,QAAQ,EAAE,YAAY;oBACtB,KAAK,EAAE,QAAQ;oBACf,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBAC7B,GAAG,CAAC,eAAe,KAAK,SAAS;wBAChC,CAAC,CAAC,EAAE,SAAS,EAAE,eAAe,EAAE;wBAChC,CAAC,CAAC,EAAE,CAAC;iBACN,CAAC,CAAC;gBACH,OAAO,mBAAmB,CAAC;oBAC1B,YAAY;oBACZ,KAAK,EAAE,QAAQ;oBACf,KAAK,EAAE,gBAAgB,CAAC,YAAY,CAAC;oBACrC,SAAS;oBACT,QAAQ;oBACR,GAAG;iBACH,CAAC,CAAC;YACJ,CAAC;QACF,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACd,IAAI,4BAA4B,CAAC,GAAG,EAAE,MAAM,CAAC;gBAAE,MAAM,GAAG,CAAC;QAC1D,CAAC;IACF,CAAC;IAED,MAAM,EAAE,gBAAgB,EAAE,cAAc,EAAE,GAAG,MAAM,QAAQ,EAAE,CAAC;IAC9D,MAAM,GAAG,GAAG,MAAM,gBAAgB,CAAC,IAAI,UAAU,CAAC,MAAM,CAAC,EAAE;QAC1D,SAAS,EAAE,cAAc,CAAC,MAAM;KAChC,CAAC,CAAC;IACH,MAAM,QAAQ,GAAG,MAAM,GAAG,CAAC,WAAW,EAAE,CAAC;IACzC,MAAM,YAAY,GACjB,QAAQ,CAAC,IAAI,IAAI,OAAO,QAAQ,CAAC,IAAI,KAAK,QAAQ;QACjD,CAAC,CAAE,QAAQ,CAAC,IAAgC;QAC5C,CAAC,CAAC,IAAI,CAAC;IAET,MAAM,SAAS,GACd,OAAO,YAAY,EAAE,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;IAC1E,MAAM,UAAU,GACf,OAAO,YAAY,EAAE,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;IAC5E,MAAM,KAAK,GAAG,SAAS,EAAE,IAAI,EAAE,IAAI,QAAQ,CAAC;IAC5C,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;IAC5D,MAAM,SAAS,GAAG,GAAG,CAAC,QAAQ,GAAG,YAAY,CAAC;IAC9C,MAAM,KAAK,GAAwC,EAAE,CAAC;IAEtD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,cAAc,EAAE,CAAC,EAAE,EAAE,CAAC;QAC1C,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAClC,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAChD,MAAM,QAAQ,GAAG,WAAW,CAAC,KAAK;aAChC,GAAG,CAAC,CAAC,IAAa,EAAE,EAAE,CAAC;YACvB,MAAM,QAAQ,GAAG,IAAwB,CAAC;YAC1C,OAAO,QAAQ,CAAC,GAAG,IAAI,EAAE,CAAC;QAAA,CAC1B,CAAC;aACD,IAAI,CAAC,GAAG,CAAC;aACT,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;aACpB,IAAI,EAAE,CAAC;QAET,IAAI,QAAQ,EAAE,CAAC;YACd,KAAK,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC;QAC5C,CAAC;IACF,CAAC;IAED,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YACX,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACnB,SAAS,CAAC,IAAI,CAAC,aAAa,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,MAAM,CAAC,CAAC;YACpD,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACpB,CAAC;QACD,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAC/B,CAAC;IAED,OAAO,mBAAmB,CAAC;QAC1B,YAAY,EAAE,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC;QAClC,KAAK;QACL,KAAK,EAAE,GAAG,CAAC,QAAQ;QACnB,SAAS;QACT,QAAQ;QACR,GAAG;QACH,UAAU;QACV,SAAS;QACT,cAAc;KACd,CAAC,CAAC;AAAA,CACH;AAED,KAAK,UAAU,mBAAmB,CAAC,OAUlC,EAA6B;IAC7B,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,CAAC,IAAI,CAAC,KAAK,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;IACjC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,aAAa,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;IACvC,KAAK,CAAC,IAAI,CACT,YAAY,OAAO,CAAC,KAAK,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,qBAAqB,OAAO,CAAC,cAAc,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACrG,CAAC;IACF,IAAI,OAAO,CAAC,UAAU;QAAE,KAAK,CAAC,IAAI,CAAC,aAAa,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC;IACtE,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAClB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,IAAI,OAAO,CAAC,YAAY;QAAE,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;IAE3D,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;QACvB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAClB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CACT,2BAA2B,OAAO,CAAC,cAAc,OAAO,OAAO,CAAC,KAAK,oBAAoB,CACzF,CAAC;IACH,CAAC;IAED,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACjC,MAAM,cAAc,GACnB,OAAO,CAAC,QAAQ,IAAI,gBAAgB,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC;IAC7D,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC;IAE3D,MAAM,KAAK,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACpD,MAAM,SAAS,CAAC,UAAU,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IAE9C,OAAO;QACN,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,KAAK,EAAE,OAAO,CAAC,MAAM;QACrB,UAAU;KACV,CAAC;AAAA,CACF;AAED,SAAS,gBAAgB,CAAC,QAAgB,EAAU;IACnD,OAAO,CAAC,GAAG,QAAQ,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CAAC,CAAC,MAAM,CAAC;AAAA,CAC1D;AAED,SAAS,4BAA4B,CACpC,GAAY,EACZ,MAAoB,EACV;IACV,IAAI,MAAM,EAAE,OAAO;QAAE,OAAO,IAAI,CAAC;IACjC,IAAI,GAAG,YAAY,yBAAyB;QAAE,OAAO,IAAI,CAAC;IAC1D,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IACjE,OAAO,OAAO,CAAC,UAAU,CAAC,kBAAkB,CAAC,CAAC;AAAA,CAC9C;AAED;;GAEG;AACH,SAAS,mBAAmB,CAAC,GAAW,EAAU;IACjD,IAAI,CAAC;QACJ,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;QAEjC,IAAI,QAAQ,GAAG,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAE1C,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;YAC3C,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,2BAA2B,CAAC,CAAC;YAC1D,IAAI,KAAK,EAAE,CAAC;gBACX,QAAQ,GAAG,SAAS,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;YAChC,CAAC;QACF,CAAC;QAED,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QAEvE,OAAO,QAAQ,IAAI,UAAU,CAAC;IAC/B,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,UAAU,CAAC;IACnB,CAAC;AAAA,CACD;AAED;;GAEG;AACH,SAAS,gBAAgB,CAAC,IAAY,EAAU;IAC/C,OAAO,CACN,IAAI;SACF,WAAW,EAAE;SACb,OAAO,CAAC,eAAe,EAAE,EAAE,CAAC;SAC5B,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;SACpB,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC;SACnB,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;SACb,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,IAAI,UAAU,CACrC,CAAC;AAAA,CACF;AAED;;GAEG;AACH,MAAM,UAAU,KAAK,CAAC,GAAW,EAAE,WAAoB,EAAW;IACjE,IAAI,WAAW,EAAE,QAAQ,CAAC,iBAAiB,CAAC,EAAE,CAAC;QAC9C,OAAO,IAAI,CAAC;IACb,CAAC;IACD,IAAI,CAAC;QACJ,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,OAAO,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IACvD,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,KAAK,CAAC;IACd,CAAC;AAAA,CACD","sourcesContent":["/**\n * PDF Content Extractor\n *\n * Converts PDFs to Markdown. The `auto` provider chain runs Datalab\n * (deterministic, layout-aware) first, then Gemini API, with unpdf as the\n * deterministic local fallback; the chain can also be pinned with\n * `pdf.provider`.\n */\n\nimport { existsSync, readFileSync } from \"node:fs\";\nimport { writeFile, mkdir } from \"node:fs/promises\";\nimport { join, basename } from \"node:path\";\nimport { tmpdir } from \"node:os\";\nimport { CredentialResolutionError } from \"./credential-source.ts\";\nimport {\n\tDATALAB_MODE_VALUES,\n\tDEFAULT_DATALAB_TIMEOUT_MS,\n\tisDatalabApiAvailable,\n\tnormalizeDatalabMode,\n\textractPDFViaDatalab,\n\ttype DatalabMode,\n} from \"./datalab-pdf-extract.ts\";\nimport { isGeminiApiAvailable } from \"./gemini-api.ts\";\nimport { extractPDFViaGemini } from \"./gemini-pdf-extract.ts\";\nimport { getWebSearchConfigPath } from \"./utils.ts\";\n\nexport interface PDFExtractResult {\n\ttitle: string;\n\tpages: number;\n\tchars: number;\n\toutputPath: string;\n}\n\nexport interface PDFExtractOptions {\n\tmaxPages?: number;\n\toutputDir?: string;\n\tfilename?: string;\n\tsignal?: AbortSignal;\n\tgeminiTimeoutMs?: number;\n}\n\nexport type PDFProvider = \"auto\" | \"gemini\" | \"datalab\" | \"unpdf\";\n\nexport const PDF_PROVIDER_VALUES = new Set<PDFProvider>([\n\t\"auto\",\n\t\"gemini\",\n\t\"datalab\",\n\t\"unpdf\",\n]);\n\nexport interface PDFConfig {\n\tenabled: boolean;\n\tmaxSizeMB: number;\n\tprovider: PDFProvider;\n\tdatalabMode: DatalabMode;\n\tdatalabTimeoutMs: number;\n}\n\nexport const DEFAULT_PDF_MAX_SIZE_MB = 20;\nexport const MAX_PDF_MAX_SIZE_MB = 50;\nexport const MAX_DATALAB_TIMEOUT_MS = 300_000;\nconst DEFAULT_MAX_PAGES = 100;\nconst DEFAULT_OUTPUT_DIR = join(tmpdir(), \"pi-web-pdf\");\nconst CONFIG_PATH = getWebSearchConfigPath();\nconst PAGE_MARKER_PATTERN = /^<!-- Page (\\d+) -->$/gm;\n\nexport function loadPDFConfig(): PDFConfig {\n\tif (!existsSync(CONFIG_PATH)) {\n\t\treturn {\n\t\t\tenabled: true,\n\t\t\tmaxSizeMB: DEFAULT_PDF_MAX_SIZE_MB,\n\t\t\tprovider: \"auto\",\n\t\t\tdatalabMode: normalizeDatalabMode(process.env.DATALAB_MODE),\n\t\t\tdatalabTimeoutMs: DEFAULT_DATALAB_TIMEOUT_MS,\n\t\t};\n\t}\n\n\tconst rawText = readFileSync(CONFIG_PATH, \"utf-8\");\n\tlet raw: unknown;\n\ttry {\n\t\traw = JSON.parse(rawText) as unknown;\n\t} catch (err) {\n\t\tconst message = err instanceof Error ? err.message : String(err);\n\t\tthrow new Error(`Failed to parse ${CONFIG_PATH}: ${message}`);\n\t}\n\n\tconst root =\n\t\traw && typeof raw === \"object\" ? (raw as Record<string, unknown>) : {};\n\tconst pdf =\n\t\troot.pdf && typeof root.pdf === \"object\"\n\t\t\t? (root.pdf as Record<string, unknown>)\n\t\t\t: {};\n\tconst enabled = pdf.enabled !== false;\n\tconst configured = pdf.maxSizeMB;\n\tconst normalized =\n\t\ttypeof configured === \"number\" &&\n\t\tNumber.isFinite(configured) &&\n\t\tconfigured > 0\n\t\t\t? Math.min(configured, MAX_PDF_MAX_SIZE_MB)\n\t\t\t: DEFAULT_PDF_MAX_SIZE_MB;\n\n\tconst provider =\n\t\ttypeof pdf.provider === \"string\" &&\n\t\tPDF_PROVIDER_VALUES.has(pdf.provider as PDFProvider)\n\t\t\t? (pdf.provider as PDFProvider)\n\t\t\t: \"auto\";\n\tconst datalabMode =\n\t\ttypeof pdf.datalabMode === \"string\" &&\n\t\tDATALAB_MODE_VALUES.has(pdf.datalabMode as DatalabMode)\n\t\t\t? (pdf.datalabMode as DatalabMode)\n\t\t\t: normalizeDatalabMode(process.env.DATALAB_MODE);\n\tconst configuredTimeout = pdf.datalabTimeoutMs;\n\tconst datalabTimeoutMs =\n\t\ttypeof configuredTimeout === \"number\" &&\n\t\tNumber.isFinite(configuredTimeout) &&\n\t\tconfiguredTimeout > 0\n\t\t\t? Math.min(configuredTimeout, MAX_DATALAB_TIMEOUT_MS)\n\t\t\t: DEFAULT_DATALAB_TIMEOUT_MS;\n\n\treturn {\n\t\tenabled,\n\t\tmaxSizeMB: normalized,\n\t\tprovider,\n\t\tdatalabMode,\n\t\tdatalabTimeoutMs,\n\t};\n}\n\nasync function getUnpdf() {\n\tif (\n\t\ttypeof (Promise as PromiseConstructor & { try?: unknown }).try !==\n\t\t\"function\"\n\t) {\n\t\tconst { default: promiseTry } = await import(\"promise.try\");\n\t\tpromiseTry.shim();\n\t}\n\n\tconst [unpdf, pdfjs] = await Promise.all([\n\t\timport(\"unpdf\"),\n\t\timport(\"unpdf/pdfjs\"),\n\t]);\n\tconst { VerbosityLevel } = pdfjs as typeof pdfjs & {\n\t\tVerbosityLevel: { ERRORS: number };\n\t};\n\n\treturn { getDocumentProxy: unpdf.getDocumentProxy, VerbosityLevel };\n}\n\n/**\n * Extract text from a PDF buffer and save it to a Markdown file.\n */\nexport async function extractPDFToMarkdown(\n\tbuffer: ArrayBuffer,\n\turl: string,\n\toptions: PDFExtractOptions = {},\n): Promise<PDFExtractResult> {\n\tconst {\n\t\tmaxPages = DEFAULT_MAX_PAGES,\n\t\toutputDir = DEFAULT_OUTPUT_DIR,\n\t\tfilename,\n\t\tsignal,\n\t\tgeminiTimeoutMs,\n\t} = options;\n\n\tconst safeMaxPages = Number.isFinite(maxPages)\n\t\t? Math.max(1, Math.floor(maxPages))\n\t\t: DEFAULT_MAX_PAGES;\n\tconst urlTitle = extractTitleFromURL(url);\n\tconst pdfConfig = loadPDFConfig();\n\tconst provider = pdfConfig.provider;\n\n\tif (provider === \"auto\" || provider === \"datalab\") {\n\t\ttry {\n\t\t\tif (isDatalabApiAvailable()) {\n\t\t\t\tconst result = await extractPDFViaDatalab(buffer, {\n\t\t\t\t\tmaxPages: safeMaxPages,\n\t\t\t\t\ttitle: urlTitle,\n\t\t\t\t\tmode: pdfConfig.datalabMode,\n\t\t\t\t\ttimeoutMs: pdfConfig.datalabTimeoutMs,\n\t\t\t\t\t...(signal ? { signal } : {}),\n\t\t\t\t});\n\t\t\t\treturn writeMarkdownResult({\n\t\t\t\t\tmarkdownBody: result.markdown,\n\t\t\t\t\ttitle: urlTitle,\n\t\t\t\t\tpages: result.pages,\n\t\t\t\t\toutputDir,\n\t\t\t\t\tfilename,\n\t\t\t\t\turl,\n\t\t\t\t});\n\t\t\t}\n\t\t} catch (err) {\n\t\t\tif (shouldRethrowExtractionError(err, signal)) throw err;\n\t\t}\n\t}\n\n\tif (provider === \"auto\" || provider === \"gemini\") {\n\t\ttry {\n\t\t\tif (isGeminiApiAvailable()) {\n\t\t\t\tconst markdownBody = await extractPDFViaGemini(buffer, {\n\t\t\t\t\tmaxPages: safeMaxPages,\n\t\t\t\t\ttitle: urlTitle,\n\t\t\t\t\t...(signal ? { signal } : {}),\n\t\t\t\t\t...(geminiTimeoutMs !== undefined\n\t\t\t\t\t\t? { timeoutMs: geminiTimeoutMs }\n\t\t\t\t\t\t: {}),\n\t\t\t\t});\n\t\t\t\treturn writeMarkdownResult({\n\t\t\t\t\tmarkdownBody,\n\t\t\t\t\ttitle: urlTitle,\n\t\t\t\t\tpages: countPageMarkers(markdownBody),\n\t\t\t\t\toutputDir,\n\t\t\t\t\tfilename,\n\t\t\t\t\turl,\n\t\t\t\t});\n\t\t\t}\n\t\t} catch (err) {\n\t\t\tif (shouldRethrowExtractionError(err, signal)) throw err;\n\t\t}\n\t}\n\n\tconst { getDocumentProxy, VerbosityLevel } = await getUnpdf();\n\tconst pdf = await getDocumentProxy(new Uint8Array(buffer), {\n\t\tverbosity: VerbosityLevel.ERRORS,\n\t});\n\tconst metadata = await pdf.getMetadata();\n\tconst metadataInfo =\n\t\tmetadata.info && typeof metadata.info === \"object\"\n\t\t\t? (metadata.info as Record<string, unknown>)\n\t\t\t: null;\n\n\tconst metaTitle =\n\t\ttypeof metadataInfo?.Title === \"string\" ? metadataInfo.Title : undefined;\n\tconst metaAuthor =\n\t\ttypeof metadataInfo?.Author === \"string\" ? metadataInfo.Author : undefined;\n\tconst title = metaTitle?.trim() || urlTitle;\n\tconst pagesToExtract = Math.min(pdf.numPages, safeMaxPages);\n\tconst truncated = pdf.numPages > safeMaxPages;\n\tconst pages: { pageNum: number; text: string }[] = [];\n\n\tfor (let i = 1; i <= pagesToExtract; i++) {\n\t\tconst page = await pdf.getPage(i);\n\t\tconst textContent = await page.getTextContent();\n\t\tconst pageText = textContent.items\n\t\t\t.map((item: unknown) => {\n\t\t\t\tconst textItem = item as { str?: string };\n\t\t\t\treturn textItem.str || \"\";\n\t\t\t})\n\t\t\t.join(\" \")\n\t\t\t.replace(/\\s+/g, \" \")\n\t\t\t.trim();\n\n\t\tif (pageText) {\n\t\t\tpages.push({ pageNum: i, text: pageText });\n\t\t}\n\t}\n\n\tconst bodyLines: string[] = [];\n\tfor (let i = 0; i < pages.length; i++) {\n\t\tif (i > 0) {\n\t\t\tbodyLines.push(\"\");\n\t\t\tbodyLines.push(`<!-- Page ${pages[i].pageNum} -->`);\n\t\t\tbodyLines.push(\"\");\n\t\t}\n\t\tbodyLines.push(pages[i].text);\n\t}\n\n\treturn writeMarkdownResult({\n\t\tmarkdownBody: bodyLines.join(\"\\n\"),\n\t\ttitle,\n\t\tpages: pdf.numPages,\n\t\toutputDir,\n\t\tfilename,\n\t\turl,\n\t\tmetaAuthor,\n\t\ttruncated,\n\t\tpagesToExtract,\n\t});\n}\n\nasync function writeMarkdownResult(options: {\n\tmarkdownBody: string;\n\ttitle: string;\n\tpages: number;\n\toutputDir: string;\n\tfilename?: string;\n\turl: string;\n\tmetaAuthor?: string;\n\ttruncated?: boolean;\n\tpagesToExtract?: number;\n}): Promise<PDFExtractResult> {\n\tconst lines: string[] = [];\n\tlines.push(`# ${options.title}`);\n\tlines.push(\"\");\n\tlines.push(`> Source: ${options.url}`);\n\tlines.push(\n\t\t`> Pages: ${options.pages}${options.truncated ? ` (extracted first ${options.pagesToExtract})` : \"\"}`,\n\t);\n\tif (options.metaAuthor) lines.push(`> Author: ${options.metaAuthor}`);\n\tlines.push(\"\");\n\tlines.push(\"---\");\n\tlines.push(\"\");\n\tif (options.markdownBody) lines.push(options.markdownBody);\n\n\tif (options.truncated) {\n\t\tlines.push(\"\");\n\t\tlines.push(\"---\");\n\t\tlines.push(\"\");\n\t\tlines.push(\n\t\t\t`*[Truncated: Only first ${options.pagesToExtract} of ${options.pages} pages extracted]*`,\n\t\t);\n\t}\n\n\tconst content = lines.join(\"\\n\");\n\tconst outputFilename =\n\t\toptions.filename || sanitizeFilename(options.title) + \".md\";\n\tconst outputPath = join(options.outputDir, outputFilename);\n\n\tawait mkdir(options.outputDir, { recursive: true });\n\tawait writeFile(outputPath, content, \"utf-8\");\n\n\treturn {\n\t\ttitle: options.title,\n\t\tpages: options.pages,\n\t\tchars: content.length,\n\t\toutputPath,\n\t};\n}\n\nfunction countPageMarkers(markdown: string): number {\n\treturn [...markdown.matchAll(PAGE_MARKER_PATTERN)].length;\n}\n\nfunction shouldRethrowExtractionError(\n\terr: unknown,\n\tsignal?: AbortSignal,\n): boolean {\n\tif (signal?.aborted) return true;\n\tif (err instanceof CredentialResolutionError) return true;\n\tconst message = err instanceof Error ? err.message : String(err);\n\treturn message.startsWith(\"Failed to parse \");\n}\n\n/**\n * Extract a reasonable title from URL\n */\nfunction extractTitleFromURL(url: string): string {\n\ttry {\n\t\tconst urlObj = new URL(url);\n\t\tconst pathname = urlObj.pathname;\n\n\t\tlet filename = basename(pathname, \".pdf\");\n\n\t\tif (urlObj.hostname.includes(\"arxiv.org\")) {\n\t\t\tconst match = pathname.match(/\\/(?:pdf|abs)\\/(\\d+\\.\\d+)/);\n\t\t\tif (match) {\n\t\t\t\tfilename = `arxiv-${match[1]}`;\n\t\t\t}\n\t\t}\n\n\t\tfilename = filename.replace(/[_-]+/g, \" \").replace(/\\s+/g, \" \").trim();\n\n\t\treturn filename || \"document\";\n\t} catch {\n\t\treturn \"document\";\n\t}\n}\n\n/**\n * Sanitize string for use as filename\n */\nfunction sanitizeFilename(name: string): string {\n\treturn (\n\t\tname\n\t\t\t.toLowerCase()\n\t\t\t.replace(/[^a-z0-9\\s-]/g, \"\")\n\t\t\t.replace(/\\s+/g, \"-\")\n\t\t\t.replace(/-+/g, \"-\")\n\t\t\t.slice(0, 100)\n\t\t\t.replace(/^-|-$/g, \"\") || \"document\"\n\t);\n}\n\n/**\n * Check if URL or content-type indicates a PDF\n */\nexport function isPDF(url: string, contentType?: string): boolean {\n\tif (contentType?.includes(\"application/pdf\")) {\n\t\treturn true;\n\t}\n\ttry {\n\t\tconst urlObj = new URL(url);\n\t\treturn urlObj.pathname.toLowerCase().endsWith(\".pdf\");\n\t} catch {\n\t\treturn false;\n\t}\n}\n"]}

package/dist/humain/web-access/vendor/perplexity.d.ts

@@ -0,0 +1,20 @@
+import type { ExtractedContent } from "./extract.ts";
+export interface SearchResult {
+    title: string;
+    url: string;
+    snippet: string;
+}
+export interface SearchResponse {
+    answer: string;
+    results: SearchResult[];
+    inlineContent?: ExtractedContent[];
+}
+export interface SearchOptions {
+    numResults?: number;
+    recencyFilter?: "day" | "week" | "month" | "year";
+    domainFilter?: string[];
+    signal?: AbortSignal;
+}
+export declare function isPerplexityAvailable(): boolean;
+export declare function searchWithPerplexity(query: string, options?: SearchOptions): Promise<SearchResponse>;
+//# sourceMappingURL=perplexity.d.ts.map

package/dist/humain/web-access/vendor/perplexity.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"perplexity.d.ts","sourceRoot":"","sources":["../../../../src/humain/web-access/vendor/perplexity.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAcrD,MAAM,WAAW,YAAY;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,cAAc;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,YAAY,EAAE,CAAC;IACxB,aAAa,CAAC,EAAE,gBAAgB,EAAE,CAAC;CACnC;AAED,MAAM,WAAW,aAAa;IAC7B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,aAAa,CAAC,EAAE,KAAK,GAAG,MAAM,GAAG,OAAO,GAAG,MAAM,CAAC;IAClD,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,MAAM,CAAC,EAAE,WAAW,CAAC;CACrB;AAkED,wBAAgB,qBAAqB,IAAI,OAAO,CAM/C;AAED,wBAAsB,oBAAoB,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,cAAc,CAAC,CA8F9G","sourcesContent":["import { existsSync, readFileSync } from \"node:fs\";\nimport { activityMonitor } from \"./activity.ts\";\nimport type { ExtractedContent } from \"./extract.ts\";\nimport { hasCredentialSource, redactCredential, resolveCredential } from \"./credential-source.ts\";\nimport { getWebSearchConfigPath } from \"./utils.ts\";\n\nconst PERPLEXITY_API_URL = \"https://api.perplexity.ai/chat/completions\";\nconst CONFIG_PATH = getWebSearchConfigPath();\n\nconst RATE_LIMIT = {\n\tmaxRequests: 10,\n\twindowMs: 60 * 1000,\n};\n\nconst requestTimestamps: number[] = [];\n\nexport interface SearchResult {\n\ttitle: string;\n\turl: string;\n\tsnippet: string;\n}\n\nexport interface SearchResponse {\n\tanswer: string;\n\tresults: SearchResult[];\n\tinlineContent?: ExtractedContent[];\n}\n\nexport interface SearchOptions {\n\tnumResults?: number;\n\trecencyFilter?: \"day\" | \"week\" | \"month\" | \"year\";\n\tdomainFilter?: string[];\n\tsignal?: AbortSignal;\n}\n\ninterface WebSearchConfig {\n\tperplexityApiKey?: unknown;\n}\n\nlet cachedConfig: WebSearchConfig | null = null;\n\nfunction loadConfig(): WebSearchConfig {\n\tif (cachedConfig) return cachedConfig;\n\tif (!existsSync(CONFIG_PATH)) {\n\t\tcachedConfig = {};\n\t\treturn cachedConfig;\n\t}\n\n\tconst content = readFileSync(CONFIG_PATH, \"utf-8\");\n\ttry {\n\t\tcachedConfig = JSON.parse(content) as WebSearchConfig;\n\t\treturn cachedConfig;\n\t} catch (err) {\n\t\tconst message = err instanceof Error ? err.message : String(err);\n\t\tthrow new Error(`Failed to parse ${CONFIG_PATH}: ${message}`);\n\t}\n}\n\nasync function getApiKey(signal?: AbortSignal): Promise<string> {\n\tconst key = await resolveCredential({\n\t\tprovider: \"Perplexity\",\n\t\tconfiguredValue: loadConfig().perplexityApiKey,\n\t\tenvironmentValue: process.env.PERPLEXITY_API_KEY,\n\t\tsignal,\n\t});\n\tif (!key) {\n\t\tthrow new Error(\n\t\t\t\"Perplexity API key not found. Either:\\n\" +\n\t\t\t`  1. Create ${CONFIG_PATH} with { \"perplexityApiKey\": \"your-key\" }\\n` +\n\t\t\t\"  2. Set PERPLEXITY_API_KEY environment variable\\n\" +\n\t\t\t\"Get a key at https://perplexity.ai/settings/api\"\n\t\t);\n\t}\n\treturn key;\n}\n\nfunction checkRateLimit(): void {\n\tconst now = Date.now();\n\tconst windowStart = now - RATE_LIMIT.windowMs;\n\n\twhile (requestTimestamps.length > 0 && requestTimestamps[0] < windowStart) {\n\t\trequestTimestamps.shift();\n\t}\n\n\tif (requestTimestamps.length >= RATE_LIMIT.maxRequests) {\n\t\tconst waitMs = requestTimestamps[0] + RATE_LIMIT.windowMs - now;\n\t\tthrow new Error(`Rate limited. Try again in ${Math.ceil(waitMs / 1000)}s`);\n\t}\n\n\trequestTimestamps.push(now);\n}\n\nfunction validateDomainFilter(domains: string[]): string[] {\n\treturn domains.filter((d) => {\n\t\tconst domain = d.startsWith(\"-\") ? d.slice(1) : d;\n\t\treturn /^[a-zA-Z0-9][a-zA-Z0-9-_.]*\\.[a-zA-Z]{2,}$/.test(domain);\n\t});\n}\n\nexport function isPerplexityAvailable(): boolean {\n\treturn hasCredentialSource({\n\t\tprovider: \"Perplexity\",\n\t\tconfiguredValue: loadConfig().perplexityApiKey,\n\t\tenvironmentValue: process.env.PERPLEXITY_API_KEY,\n\t});\n}\n\nexport async function searchWithPerplexity(query: string, options: SearchOptions = {}): Promise<SearchResponse> {\n\tcheckRateLimit();\n\n\tconst activityId = activityMonitor.logStart({ type: \"api\", query });\n\n\tactivityMonitor.updateRateLimit({\n\t\tused: requestTimestamps.length,\n\t\tmax: RATE_LIMIT.maxRequests,\n\t\toldestTimestamp: requestTimestamps[0] ?? null,\n\t\twindowMs: RATE_LIMIT.windowMs,\n\t});\n\n\tconst apiKey = await getApiKey(options.signal);\n\tconst numResults = typeof options.numResults === \"number\" && Number.isFinite(options.numResults)\n\t\t? Math.max(1, Math.min(Math.floor(options.numResults), 20))\n\t\t: 5;\n\n\tconst requestBody: Record<string, unknown> = {\n\t\tmodel: \"sonar\",\n\t\tmessages: [{ role: \"user\", content: query }],\n\t\tmax_tokens: 1024,\n\t\treturn_related_questions: false,\n\t};\n\n\tif (options.recencyFilter) {\n\t\trequestBody.search_recency_filter = options.recencyFilter;\n\t}\n\n\tif (options.domainFilter && options.domainFilter.length > 0) {\n\t\tconst validated = validateDomainFilter(options.domainFilter);\n\t\tif (validated.length > 0) {\n\t\t\trequestBody.search_domain_filter = validated;\n\t\t}\n\t}\n\n\tlet response: Response;\n\ttry {\n\t\tresponse = await fetch(PERPLEXITY_API_URL, {\n\t\t\tmethod: \"POST\",\n\t\t\theaders: {\n\t\t\t\tAuthorization: `Bearer ${apiKey}`,\n\t\t\t\t\"Content-Type\": \"application/json\",\n\t\t\t},\n\t\t\tbody: JSON.stringify(requestBody),\n\t\t\t...(options.signal ? { signal: options.signal } : {}),\n\t\t});\n\t} catch (err) {\n\t\tconst message = err instanceof Error ? err.message : String(err);\n\t\tconst redactedMessage = redactCredential(message, apiKey);\n\t\tif (redactedMessage.toLowerCase().includes(\"abort\")) {\n\t\t\tactivityMonitor.logComplete(activityId, 0);\n\t\t} else {\n\t\t\tactivityMonitor.logError(activityId, redactedMessage);\n\t\t}\n\t\tif (redactedMessage === message) throw err;\n\t\tconst redactedError = new Error(redactedMessage);\n\t\tif (err instanceof Error) redactedError.name = err.name;\n\t\tthrow redactedError;\n\t}\n\n\tif (!response.ok) {\n\t\tactivityMonitor.logComplete(activityId, response.status);\n\t\tconst errorText = redactCredential(await response.text(), apiKey);\n\t\tthrow new Error(`Perplexity API error ${response.status}: ${errorText}`);\n\t}\n\n\tlet data: Record<string, unknown>;\n\ttry {\n\t\tdata = (await response.json()) as Record<string, unknown>;\n\t} catch (err) {\n\t\tactivityMonitor.logComplete(activityId, response.status);\n\t\tconst message = err instanceof Error ? err.message : String(err);\n\t\tthrow new Error(`Perplexity API returned invalid JSON: ${message}`);\n\t}\n\n\tconst answer = (data.choices as Array<{ message?: { content?: string } }>)?.[0]?.message?.content || \"\";\n\tconst citations = Array.isArray(data.citations) ? data.citations : [];\n\n\tconst results: SearchResult[] = [];\n\tfor (let i = 0; i < Math.min(citations.length, numResults); i++) {\n\t\tconst citation = citations[i];\n\t\tif (typeof citation === \"string\") {\n\t\t\tresults.push({ title: `Source ${i + 1}`, url: citation, snippet: \"\" });\n\t\t} else if (citation && typeof citation === \"object\" && typeof citation.url === \"string\") {\n\t\t\tresults.push({\n\t\t\t\ttitle: citation.title || `Source ${i + 1}`,\n\t\t\t\turl: citation.url,\n\t\t\t\tsnippet: \"\",\n\t\t\t});\n\t\t}\n\t}\n\n\tactivityMonitor.logComplete(activityId, response.status);\n\treturn { answer, results };\n}\n"]}

package/dist/humain/web-access/vendor/perplexity.js

@@ -0,0 +1,159 @@
+import { existsSync, readFileSync } from "node:fs";
+import { activityMonitor } from "./activity.js";
+import { hasCredentialSource, redactCredential, resolveCredential } from "./credential-source.js";
+import { getWebSearchConfigPath } from "./utils.js";
+const PERPLEXITY_API_URL = "https://api.perplexity.ai/chat/completions";
+const CONFIG_PATH = getWebSearchConfigPath();
+const RATE_LIMIT = {
+    maxRequests: 10,
+    windowMs: 60 * 1000,
+};
+const requestTimestamps = [];
+let cachedConfig = null;
+function loadConfig() {
+    if (cachedConfig)
+        return cachedConfig;
+    if (!existsSync(CONFIG_PATH)) {
+        cachedConfig = {};
+        return cachedConfig;
+    }
+    const content = readFileSync(CONFIG_PATH, "utf-8");
+    try {
+        cachedConfig = JSON.parse(content);
+        return cachedConfig;
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        throw new Error(`Failed to parse ${CONFIG_PATH}: ${message}`);
+    }
+}
+async function getApiKey(signal) {
+    const key = await resolveCredential({
+        provider: "Perplexity",
+        configuredValue: loadConfig().perplexityApiKey,
+        environmentValue: process.env.PERPLEXITY_API_KEY,
+        signal,
+    });
+    if (!key) {
+        throw new Error("Perplexity API key not found. Either:\n" +
+            `  1. Create ${CONFIG_PATH} with { "perplexityApiKey": "your-key" }\n` +
+            "  2. Set PERPLEXITY_API_KEY environment variable\n" +
+            "Get a key at https://perplexity.ai/settings/api");
+    }
+    return key;
+}
+function checkRateLimit() {
+    const now = Date.now();
+    const windowStart = now - RATE_LIMIT.windowMs;
+    while (requestTimestamps.length > 0 && requestTimestamps[0] < windowStart) {
+        requestTimestamps.shift();
+    }
+    if (requestTimestamps.length >= RATE_LIMIT.maxRequests) {
+        const waitMs = requestTimestamps[0] + RATE_LIMIT.windowMs - now;
+        throw new Error(`Rate limited. Try again in ${Math.ceil(waitMs / 1000)}s`);
+    }
+    requestTimestamps.push(now);
+}
+function validateDomainFilter(domains) {
+    return domains.filter((d) => {
+        const domain = d.startsWith("-") ? d.slice(1) : d;
+        return /^[a-zA-Z0-9][a-zA-Z0-9-_.]*\.[a-zA-Z]{2,}$/.test(domain);
+    });
+}
+export function isPerplexityAvailable() {
+    return hasCredentialSource({
+        provider: "Perplexity",
+        configuredValue: loadConfig().perplexityApiKey,
+        environmentValue: process.env.PERPLEXITY_API_KEY,
+    });
+}
+export async function searchWithPerplexity(query, options = {}) {
+    checkRateLimit();
+    const activityId = activityMonitor.logStart({ type: "api", query });
+    activityMonitor.updateRateLimit({
+        used: requestTimestamps.length,
+        max: RATE_LIMIT.maxRequests,
+        oldestTimestamp: requestTimestamps[0] ?? null,
+        windowMs: RATE_LIMIT.windowMs,
+    });
+    const apiKey = await getApiKey(options.signal);
+    const numResults = typeof options.numResults === "number" && Number.isFinite(options.numResults)
+        ? Math.max(1, Math.min(Math.floor(options.numResults), 20))
+        : 5;
+    const requestBody = {
+        model: "sonar",
+        messages: [{ role: "user", content: query }],
+        max_tokens: 1024,
+        return_related_questions: false,
+    };
+    if (options.recencyFilter) {
+        requestBody.search_recency_filter = options.recencyFilter;
+    }
+    if (options.domainFilter && options.domainFilter.length > 0) {
+        const validated = validateDomainFilter(options.domainFilter);
+        if (validated.length > 0) {
+            requestBody.search_domain_filter = validated;
+        }
+    }
+    let response;
+    try {
+        response = await fetch(PERPLEXITY_API_URL, {
+            method: "POST",
+            headers: {
+                Authorization: `Bearer ${apiKey}`,
+                "Content-Type": "application/json",
+            },
+            body: JSON.stringify(requestBody),
+            ...(options.signal ? { signal: options.signal } : {}),
+        });
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        const redactedMessage = redactCredential(message, apiKey);
+        if (redactedMessage.toLowerCase().includes("abort")) {
+            activityMonitor.logComplete(activityId, 0);
+        }
+        else {
+            activityMonitor.logError(activityId, redactedMessage);
+        }
+        if (redactedMessage === message)
+            throw err;
+        const redactedError = new Error(redactedMessage);
+        if (err instanceof Error)
+            redactedError.name = err.name;
+        throw redactedError;
+    }
+    if (!response.ok) {
+        activityMonitor.logComplete(activityId, response.status);
+        const errorText = redactCredential(await response.text(), apiKey);
+        throw new Error(`Perplexity API error ${response.status}: ${errorText}`);
+    }
+    let data;
+    try {
+        data = (await response.json());
+    }
+    catch (err) {
+        activityMonitor.logComplete(activityId, response.status);
+        const message = err instanceof Error ? err.message : String(err);
+        throw new Error(`Perplexity API returned invalid JSON: ${message}`);
+    }
+    const answer = data.choices?.[0]?.message?.content || "";
+    const citations = Array.isArray(data.citations) ? data.citations : [];
+    const results = [];
+    for (let i = 0; i < Math.min(citations.length, numResults); i++) {
+        const citation = citations[i];
+        if (typeof citation === "string") {
+            results.push({ title: `Source ${i + 1}`, url: citation, snippet: "" });
+        }
+        else if (citation && typeof citation === "object" && typeof citation.url === "string") {
+            results.push({
+                title: citation.title || `Source ${i + 1}`,
+                url: citation.url,
+                snippet: "",
+            });
+        }
+    }
+    activityMonitor.logComplete(activityId, response.status);
+    return { answer, results };
+}
+//# sourceMappingURL=perplexity.js.map

package/dist/humain/web-access/vendor/perplexity.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"perplexity.js","sourceRoot":"","sources":["../../../../src/humain/web-access/vendor/perplexity.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAEhD,OAAO,EAAE,mBAAmB,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAClG,OAAO,EAAE,sBAAsB,EAAE,MAAM,YAAY,CAAC;AAEpD,MAAM,kBAAkB,GAAG,4CAA4C,CAAC;AACxE,MAAM,WAAW,GAAG,sBAAsB,EAAE,CAAC;AAE7C,MAAM,UAAU,GAAG;IAClB,WAAW,EAAE,EAAE;IACf,QAAQ,EAAE,EAAE,GAAG,IAAI;CACnB,CAAC;AAEF,MAAM,iBAAiB,GAAa,EAAE,CAAC;AAyBvC,IAAI,YAAY,GAA2B,IAAI,CAAC;AAEhD,SAAS,UAAU,GAAoB;IACtC,IAAI,YAAY;QAAE,OAAO,YAAY,CAAC;IACtC,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;QAC9B,YAAY,GAAG,EAAE,CAAC;QAClB,OAAO,YAAY,CAAC;IACrB,CAAC;IAED,MAAM,OAAO,GAAG,YAAY,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;IACnD,IAAI,CAAC;QACJ,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAoB,CAAC;QACtD,OAAO,YAAY,CAAC;IACrB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACd,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACjE,MAAM,IAAI,KAAK,CAAC,mBAAmB,WAAW,KAAK,OAAO,EAAE,CAAC,CAAC;IAC/D,CAAC;AAAA,CACD;AAED,KAAK,UAAU,SAAS,CAAC,MAAoB,EAAmB;IAC/D,MAAM,GAAG,GAAG,MAAM,iBAAiB,CAAC;QACnC,QAAQ,EAAE,YAAY;QACtB,eAAe,EAAE,UAAU,EAAE,CAAC,gBAAgB;QAC9C,gBAAgB,EAAE,OAAO,CAAC,GAAG,CAAC,kBAAkB;QAChD,MAAM;KACN,CAAC,CAAC;IACH,IAAI,CAAC,GAAG,EAAE,CAAC;QACV,MAAM,IAAI,KAAK,CACd,yCAAyC;YACzC,eAAe,WAAW,4CAA4C;YACtE,oDAAoD;YACpD,iDAAiD,CACjD,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AAAA,CACX;AAED,SAAS,cAAc,GAAS;IAC/B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,MAAM,WAAW,GAAG,GAAG,GAAG,UAAU,CAAC,QAAQ,CAAC;IAE9C,OAAO,iBAAiB,CAAC,MAAM,GAAG,CAAC,IAAI,iBAAiB,CAAC,CAAC,CAAC,GAAG,WAAW,EAAE,CAAC;QAC3E,iBAAiB,CAAC,KAAK,EAAE,CAAC;IAC3B,CAAC;IAED,IAAI,iBAAiB,CAAC,MAAM,IAAI,UAAU,CAAC,WAAW,EAAE,CAAC;QACxD,MAAM,MAAM,GAAG,iBAAiB,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,QAAQ,GAAG,GAAG,CAAC;QAChE,MAAM,IAAI,KAAK,CAAC,8BAA8B,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;IAC5E,CAAC;IAED,iBAAiB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAAA,CAC5B;AAED,SAAS,oBAAoB,CAAC,OAAiB,EAAY;IAC1D,OAAO,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;QAC5B,MAAM,MAAM,GAAG,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAClD,OAAO,4CAA4C,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAAA,CACjE,CAAC,CAAC;AAAA,CACH;AAED,MAAM,UAAU,qBAAqB,GAAY;IAChD,OAAO,mBAAmB,CAAC;QAC1B,QAAQ,EAAE,YAAY;QACtB,eAAe,EAAE,UAAU,EAAE,CAAC,gBAAgB;QAC9C,gBAAgB,EAAE,OAAO,CAAC,GAAG,CAAC,kBAAkB;KAChD,CAAC,CAAC;AAAA,CACH;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,KAAa,EAAE,OAAO,GAAkB,EAAE,EAA2B;IAC/G,cAAc,EAAE,CAAC;IAEjB,MAAM,UAAU,GAAG,eAAe,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;IAEpE,eAAe,CAAC,eAAe,CAAC;QAC/B,IAAI,EAAE,iBAAiB,CAAC,MAAM;QAC9B,GAAG,EAAE,UAAU,CAAC,WAAW;QAC3B,eAAe,EAAE,iBAAiB,CAAC,CAAC,CAAC,IAAI,IAAI;QAC7C,QAAQ,EAAE,UAAU,CAAC,QAAQ;KAC7B,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAC/C,MAAM,UAAU,GAAG,OAAO,OAAO,CAAC,UAAU,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,UAAU,CAAC;QAC/F,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC;QAC3D,CAAC,CAAC,CAAC,CAAC;IAEL,MAAM,WAAW,GAA4B;QAC5C,KAAK,EAAE,OAAO;QACd,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;QAC5C,UAAU,EAAE,IAAI;QAChB,wBAAwB,EAAE,KAAK;KAC/B,CAAC;IAEF,IAAI,OAAO,CAAC,aAAa,EAAE,CAAC;QAC3B,WAAW,CAAC,qBAAqB,GAAG,OAAO,CAAC,aAAa,CAAC;IAC3D,CAAC;IAED,IAAI,OAAO,CAAC,YAAY,IAAI,OAAO,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7D,MAAM,SAAS,GAAG,oBAAoB,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;QAC7D,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1B,WAAW,CAAC,oBAAoB,GAAG,SAAS,CAAC;QAC9C,CAAC;IACF,CAAC;IAED,IAAI,QAAkB,CAAC;IACvB,IAAI,CAAC;QACJ,QAAQ,GAAG,MAAM,KAAK,CAAC,kBAAkB,EAAE;YAC1C,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACR,aAAa,EAAE,UAAU,MAAM,EAAE;gBACjC,cAAc,EAAE,kBAAkB;aAClC;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC;YACjC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACrD,CAAC,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACd,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACjE,MAAM,eAAe,GAAG,gBAAgB,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC1D,IAAI,eAAe,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;YACrD,eAAe,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;QAC5C,CAAC;aAAM,CAAC;YACP,eAAe,CAAC,QAAQ,CAAC,UAAU,EAAE,eAAe,CAAC,CAAC;QACvD,CAAC;QACD,IAAI,eAAe,KAAK,OAAO;YAAE,MAAM,GAAG,CAAC;QAC3C,MAAM,aAAa,GAAG,IAAI,KAAK,CAAC,eAAe,CAAC,CAAC;QACjD,IAAI,GAAG,YAAY,KAAK;YAAE,aAAa,CAAC,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;QACxD,MAAM,aAAa,CAAC;IACrB,CAAC;IAED,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QAClB,eAAe,CAAC,WAAW,CAAC,UAAU,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;QACzD,MAAM,SAAS,GAAG,gBAAgB,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,EAAE,MAAM,CAAC,CAAC;QAClE,MAAM,IAAI,KAAK,CAAC,wBAAwB,QAAQ,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC,CAAC;IAC1E,CAAC;IAED,IAAI,IAA6B,CAAC;IAClC,IAAI,CAAC;QACJ,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAA4B,CAAC;IAC3D,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACd,eAAe,CAAC,WAAW,CAAC,UAAU,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;QACzD,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACjE,MAAM,IAAI,KAAK,CAAC,yCAAyC,OAAO,EAAE,CAAC,CAAC;IACrE,CAAC;IAED,MAAM,MAAM,GAAI,IAAI,CAAC,OAAqD,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,IAAI,EAAE,CAAC;IACxG,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC;IAEtE,MAAM,OAAO,GAAmB,EAAE,CAAC;IACnC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,MAAM,EAAE,UAAU,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QACjE,MAAM,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;QAC9B,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE,CAAC;YAClC,OAAO,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,UAAU,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC;QACxE,CAAC;aAAM,IAAI,QAAQ,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,OAAO,QAAQ,CAAC,GAAG,KAAK,QAAQ,EAAE,CAAC;YACzF,OAAO,CAAC,IAAI,CAAC;gBACZ,KAAK,EAAE,QAAQ,CAAC,KAAK,IAAI,UAAU,CAAC,GAAG,CAAC,EAAE;gBAC1C,GAAG,EAAE,QAAQ,CAAC,GAAG;gBACjB,OAAO,EAAE,EAAE;aACX,CAAC,CAAC;QACJ,CAAC;IACF,CAAC;IAED,eAAe,CAAC,WAAW,CAAC,UAAU,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;IACzD,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;AAAA,CAC3B","sourcesContent":["import { existsSync, readFileSync } from \"node:fs\";\nimport { activityMonitor } from \"./activity.ts\";\nimport type { ExtractedContent } from \"./extract.ts\";\nimport { hasCredentialSource, redactCredential, resolveCredential } from \"./credential-source.ts\";\nimport { getWebSearchConfigPath } from \"./utils.ts\";\n\nconst PERPLEXITY_API_URL = \"https://api.perplexity.ai/chat/completions\";\nconst CONFIG_PATH = getWebSearchConfigPath();\n\nconst RATE_LIMIT = {\n\tmaxRequests: 10,\n\twindowMs: 60 * 1000,\n};\n\nconst requestTimestamps: number[] = [];\n\nexport interface SearchResult {\n\ttitle: string;\n\turl: string;\n\tsnippet: string;\n}\n\nexport interface SearchResponse {\n\tanswer: string;\n\tresults: SearchResult[];\n\tinlineContent?: ExtractedContent[];\n}\n\nexport interface SearchOptions {\n\tnumResults?: number;\n\trecencyFilter?: \"day\" | \"week\" | \"month\" | \"year\";\n\tdomainFilter?: string[];\n\tsignal?: AbortSignal;\n}\n\ninterface WebSearchConfig {\n\tperplexityApiKey?: unknown;\n}\n\nlet cachedConfig: WebSearchConfig | null = null;\n\nfunction loadConfig(): WebSearchConfig {\n\tif (cachedConfig) return cachedConfig;\n\tif (!existsSync(CONFIG_PATH)) {\n\t\tcachedConfig = {};\n\t\treturn cachedConfig;\n\t}\n\n\tconst content = readFileSync(CONFIG_PATH, \"utf-8\");\n\ttry {\n\t\tcachedConfig = JSON.parse(content) as WebSearchConfig;\n\t\treturn cachedConfig;\n\t} catch (err) {\n\t\tconst message = err instanceof Error ? err.message : String(err);\n\t\tthrow new Error(`Failed to parse ${CONFIG_PATH}: ${message}`);\n\t}\n}\n\nasync function getApiKey(signal?: AbortSignal): Promise<string> {\n\tconst key = await resolveCredential({\n\t\tprovider: \"Perplexity\",\n\t\tconfiguredValue: loadConfig().perplexityApiKey,\n\t\tenvironmentValue: process.env.PERPLEXITY_API_KEY,\n\t\tsignal,\n\t});\n\tif (!key) {\n\t\tthrow new Error(\n\t\t\t\"Perplexity API key not found. Either:\\n\" +\n\t\t\t`  1. Create ${CONFIG_PATH} with { \"perplexityApiKey\": \"your-key\" }\\n` +\n\t\t\t\"  2. Set PERPLEXITY_API_KEY environment variable\\n\" +\n\t\t\t\"Get a key at https://perplexity.ai/settings/api\"\n\t\t);\n\t}\n\treturn key;\n}\n\nfunction checkRateLimit(): void {\n\tconst now = Date.now();\n\tconst windowStart = now - RATE_LIMIT.windowMs;\n\n\twhile (requestTimestamps.length > 0 && requestTimestamps[0] < windowStart) {\n\t\trequestTimestamps.shift();\n\t}\n\n\tif (requestTimestamps.length >= RATE_LIMIT.maxRequests) {\n\t\tconst waitMs = requestTimestamps[0] + RATE_LIMIT.windowMs - now;\n\t\tthrow new Error(`Rate limited. Try again in ${Math.ceil(waitMs / 1000)}s`);\n\t}\n\n\trequestTimestamps.push(now);\n}\n\nfunction validateDomainFilter(domains: string[]): string[] {\n\treturn domains.filter((d) => {\n\t\tconst domain = d.startsWith(\"-\") ? d.slice(1) : d;\n\t\treturn /^[a-zA-Z0-9][a-zA-Z0-9-_.]*\\.[a-zA-Z]{2,}$/.test(domain);\n\t});\n}\n\nexport function isPerplexityAvailable(): boolean {\n\treturn hasCredentialSource({\n\t\tprovider: \"Perplexity\",\n\t\tconfiguredValue: loadConfig().perplexityApiKey,\n\t\tenvironmentValue: process.env.PERPLEXITY_API_KEY,\n\t});\n}\n\nexport async function searchWithPerplexity(query: string, options: SearchOptions = {}): Promise<SearchResponse> {\n\tcheckRateLimit();\n\n\tconst activityId = activityMonitor.logStart({ type: \"api\", query });\n\n\tactivityMonitor.updateRateLimit({\n\t\tused: requestTimestamps.length,\n\t\tmax: RATE_LIMIT.maxRequests,\n\t\toldestTimestamp: requestTimestamps[0] ?? null,\n\t\twindowMs: RATE_LIMIT.windowMs,\n\t});\n\n\tconst apiKey = await getApiKey(options.signal);\n\tconst numResults = typeof options.numResults === \"number\" && Number.isFinite(options.numResults)\n\t\t? Math.max(1, Math.min(Math.floor(options.numResults), 20))\n\t\t: 5;\n\n\tconst requestBody: Record<string, unknown> = {\n\t\tmodel: \"sonar\",\n\t\tmessages: [{ role: \"user\", content: query }],\n\t\tmax_tokens: 1024,\n\t\treturn_related_questions: false,\n\t};\n\n\tif (options.recencyFilter) {\n\t\trequestBody.search_recency_filter = options.recencyFilter;\n\t}\n\n\tif (options.domainFilter && options.domainFilter.length > 0) {\n\t\tconst validated = validateDomainFilter(options.domainFilter);\n\t\tif (validated.length > 0) {\n\t\t\trequestBody.search_domain_filter = validated;\n\t\t}\n\t}\n\n\tlet response: Response;\n\ttry {\n\t\tresponse = await fetch(PERPLEXITY_API_URL, {\n\t\t\tmethod: \"POST\",\n\t\t\theaders: {\n\t\t\t\tAuthorization: `Bearer ${apiKey}`,\n\t\t\t\t\"Content-Type\": \"application/json\",\n\t\t\t},\n\t\t\tbody: JSON.stringify(requestBody),\n\t\t\t...(options.signal ? { signal: options.signal } : {}),\n\t\t});\n\t} catch (err) {\n\t\tconst message = err instanceof Error ? err.message : String(err);\n\t\tconst redactedMessage = redactCredential(message, apiKey);\n\t\tif (redactedMessage.toLowerCase().includes(\"abort\")) {\n\t\t\tactivityMonitor.logComplete(activityId, 0);\n\t\t} else {\n\t\t\tactivityMonitor.logError(activityId, redactedMessage);\n\t\t}\n\t\tif (redactedMessage === message) throw err;\n\t\tconst redactedError = new Error(redactedMessage);\n\t\tif (err instanceof Error) redactedError.name = err.name;\n\t\tthrow redactedError;\n\t}\n\n\tif (!response.ok) {\n\t\tactivityMonitor.logComplete(activityId, response.status);\n\t\tconst errorText = redactCredential(await response.text(), apiKey);\n\t\tthrow new Error(`Perplexity API error ${response.status}: ${errorText}`);\n\t}\n\n\tlet data: Record<string, unknown>;\n\ttry {\n\t\tdata = (await response.json()) as Record<string, unknown>;\n\t} catch (err) {\n\t\tactivityMonitor.logComplete(activityId, response.status);\n\t\tconst message = err instanceof Error ? err.message : String(err);\n\t\tthrow new Error(`Perplexity API returned invalid JSON: ${message}`);\n\t}\n\n\tconst answer = (data.choices as Array<{ message?: { content?: string } }>)?.[0]?.message?.content || \"\";\n\tconst citations = Array.isArray(data.citations) ? data.citations : [];\n\n\tconst results: SearchResult[] = [];\n\tfor (let i = 0; i < Math.min(citations.length, numResults); i++) {\n\t\tconst citation = citations[i];\n\t\tif (typeof citation === \"string\") {\n\t\t\tresults.push({ title: `Source ${i + 1}`, url: citation, snippet: \"\" });\n\t\t} else if (citation && typeof citation === \"object\" && typeof citation.url === \"string\") {\n\t\t\tresults.push({\n\t\t\t\ttitle: citation.title || `Source ${i + 1}`,\n\t\t\t\turl: citation.url,\n\t\t\t\tsnippet: \"\",\n\t\t\t});\n\t\t}\n\t}\n\n\tactivityMonitor.logComplete(activityId, response.status);\n\treturn { answer, results };\n}\n"]}

package/dist/humain/web-access/vendor/querit.d.ts

@@ -0,0 +1,10 @@
+import type { ExtractedContent, ExtractOptions } from "./extract.ts";
+import type { SearchOptions, SearchResponse } from "./perplexity.ts";
+interface QueritSearchOptions extends SearchOptions {
+    includeContent?: boolean;
+}
+export declare function isQueritAvailable(): boolean;
+export declare function searchWithQuerit(query: string, options?: QueritSearchOptions): Promise<SearchResponse>;
+export declare function extractWithQuerit(url: string, signal?: AbortSignal, options?: ExtractOptions): Promise<ExtractedContent | null>;
+export {};
+//# sourceMappingURL=querit.d.ts.map

package/dist/humain/web-access/vendor/querit.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"querit.d.ts","sourceRoot":"","sources":["../../../../src/humain/web-access/vendor/querit.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,gBAAgB,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AACrE,OAAO,KAAK,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AA0DrE,UAAU,mBAAoB,SAAQ,aAAa;IAClD,cAAc,CAAC,EAAE,OAAO,CAAC;CACzB;AA4CD,wBAAgB,iBAAiB,IAAI,OAAO,CAM3C;AA4OD,wBAAsB,gBAAgB,CACrC,KAAK,EAAE,MAAM,EACb,OAAO,GAAE,mBAAwB,GAC/B,OAAO,CAAC,cAAc,CAAC,CA+BzB;AAED,wBAAsB,iBAAiB,CACtC,GAAG,EAAE,MAAM,EACX,MAAM,CAAC,EAAE,WAAW,EACpB,OAAO,GAAE,cAAmB,GAC1B,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC,CA0BlC","sourcesContent":["import { existsSync, readFileSync } from \"node:fs\";\nimport { activityMonitor } from \"./activity.ts\";\nimport { hasCredentialSource, redactCredential, resolveCredential } from \"./credential-source.ts\";\nimport type { ExtractedContent, ExtractOptions } from \"./extract.ts\";\nimport type { SearchOptions, SearchResponse } from \"./perplexity.ts\";\nimport { getWebSearchConfigPath } from \"./utils.ts\";\n\nconst QUERIT_SEARCH_URL = \"https://api.querit.ai/v1/search\";\nconst QUERIT_CONTENTS_URL = \"https://api.querit.ai/v1/contents\";\nconst CONFIG_PATH = getWebSearchConfigPath();\nconst SEARCH_TIMEOUT_MS = 60_000;\nconst CONTENTS_TIMEOUT_MS = 60_000;\nconst MAX_CONTENT_URLS = 10;\n\ninterface WebSearchConfig {\n\tqueritApiKey?: unknown;\n}\n\ninterface QueritSearchResult {\n\turl?: string | null;\n\ttitle?: string | null;\n\tsnippet?: string | null;\n\tpage_age?: string | null;\n\tsite_name?: string | null;\n}\n\ninterface QueritSearchResponse {\n\terror_code?: number | string;\n\terror_msg?: string;\n\tsearch_id?: number | string;\n\tresults?: { result?: QueritSearchResult[] };\n}\n\ninterface QueritContentMetadata {\n\ttitle?: string | null;\n\turl?: string | null;\n\tpublishTime?: string | null;\n\tsiteName?: string | null;\n\tsiteIcon?: string | null;\n}\n\ninterface QueritContentResult {\n\tid?: string | null;\n\turl?: string | null;\n\tcontent?: string | null;\n\textrasMeta?: QueritContentMetadata | null;\n}\n\ninterface QueritContentStatus {\n\tid?: string | null;\n\tstatus?: string | null;\n}\n\ninterface QueritContentsResponse {\n\terror_code?: number | string;\n\terror_msg?: string;\n\tsearch_id?: number | string;\n\tresults?: QueritContentResult[];\n\tstatuses?: QueritContentStatus[];\n\tsearchTime?: number;\n}\n\ninterface QueritSearchOptions extends SearchOptions {\n\tincludeContent?: boolean;\n}\n\nlet cachedConfig: WebSearchConfig | null = null;\n\nfunction loadConfig(): WebSearchConfig {\n\tif (cachedConfig) return cachedConfig;\n\tif (!existsSync(CONFIG_PATH)) {\n\t\tcachedConfig = {};\n\t\treturn cachedConfig;\n\t}\n\n\tconst raw = readFileSync(CONFIG_PATH, \"utf-8\");\n\tlet parsed: unknown;\n\ttry {\n\t\tparsed = JSON.parse(raw);\n\t} catch (err) {\n\t\tconst message = err instanceof Error ? err.message : String(err);\n\t\tthrow new Error(`Failed to parse ${CONFIG_PATH}: ${message}`);\n\t}\n\tif (!parsed || typeof parsed !== \"object\" || Array.isArray(parsed)) {\n\t\tthrow new Error(`Invalid config in ${CONFIG_PATH}: expected a JSON object`);\n\t}\n\tcachedConfig = parsed as WebSearchConfig;\n\treturn cachedConfig;\n}\n\nasync function getApiKey(signal?: AbortSignal): Promise<string> {\n\tconst key = await resolveCredential({\n\t\tprovider: \"Querit\",\n\t\tconfiguredValue: loadConfig().queritApiKey,\n\t\tenvironmentValue: process.env.QUERIT_API_KEY,\n\t\tsignal,\n\t});\n\tif (!key) {\n\t\tthrow new Error(\n\t\t\t\"Querit API key not found. Either:\\n\" +\n\t\t\t`  1. Create ${CONFIG_PATH} with { \"queritApiKey\": \"your-key\" }\\n` +\n\t\t\t\"  2. Set QUERIT_API_KEY environment variable\\n\" +\n\t\t\t\"Create a key at https://www.querit.ai/en/dashboard/api-keys\",\n\t\t);\n\t}\n\treturn key;\n}\n\nexport function isQueritAvailable(): boolean {\n\treturn hasCredentialSource({\n\t\tprovider: \"Querit\",\n\t\tconfiguredValue: loadConfig().queritApiKey,\n\t\tenvironmentValue: process.env.QUERIT_API_KEY,\n\t});\n}\n\nfunction errorMessage(err: unknown): string {\n\treturn err instanceof Error ? err.message : String(err);\n}\n\nfunction requestSignal(signal: AbortSignal | undefined, timeoutMs: number): AbortSignal {\n\tconst timeout = AbortSignal.timeout(timeoutMs);\n\treturn signal ? AbortSignal.any([signal, timeout]) : timeout;\n}\n\nfunction normalizeNumResults(value: number | undefined): number {\n\tif (typeof value !== \"number\" || !Number.isFinite(value)) return 5;\n\treturn Math.max(1, Math.min(Math.floor(value), 20));\n}\n\nfunction normalizeDomain(value: string): string | null {\n\tlet input = value.trim().toLowerCase();\n\tif (!input) return null;\n\tif (input.startsWith(\"-\")) input = input.slice(1).trim();\n\tif (!input) return null;\n\ttry {\n\t\tconst parsed = input.includes(\"://\") ? new URL(input) : new URL(`https://${input}`);\n\t\tinput = parsed.hostname;\n\t} catch {\n\t\tinput = input.split(\"/\")[0]?.split(\":\")[0] ?? \"\";\n\t}\n\tinput = input.replace(/^\\.+|\\.+$/g, \"\");\n\treturn /^[a-z0-9][a-z0-9.-]*\\.[a-z]{2,}$/i.test(input) ? input : null;\n}\n\nfunction mapDomainFilter(domainFilter: string[] | undefined): { include: string[]; exclude: string[] } {\n\tconst include: string[] = [];\n\tconst exclude: string[] = [];\n\tfor (const raw of domainFilter ?? []) {\n\t\tconst domain = normalizeDomain(raw);\n\t\tif (!domain) continue;\n\t\tconst target = raw.trim().startsWith(\"-\") ? exclude : include;\n\t\tif (!target.includes(domain)) target.push(domain);\n\t}\n\treturn { include, exclude };\n}\n\nfunction mapRecencyFilter(value: SearchOptions[\"recencyFilter\"]): string | undefined {\n\tif (value === \"day\") return \"d1\";\n\tif (value === \"week\") return \"w1\";\n\tif (value === \"month\") return \"m1\";\n\tif (value === \"year\") return \"y1\";\n\treturn undefined;\n}\n\nfunction buildSearchBody(query: string, options: QueritSearchOptions): Record<string, unknown> {\n\tconst { include, exclude } = mapDomainFilter(options.domainFilter);\n\tconst date = mapRecencyFilter(options.recencyFilter);\n\tconst filters: Record<string, unknown> = {};\n\tif (include.length > 0 || exclude.length > 0) {\n\t\tfilters.sites = {\n\t\t\t...(include.length > 0 ? { include } : {}),\n\t\t\t...(exclude.length > 0 ? { exclude } : {}),\n\t\t};\n\t}\n\tif (date) filters.timeRange = { date };\n\treturn {\n\t\tquery,\n\t\tcount: normalizeNumResults(options.numResults),\n\t\t...(Object.keys(filters).length > 0 ? { filters } : {}),\n\t};\n}\n\nfunction normalizeTimeoutMs(value: number | undefined): number {\n\tif (typeof value !== \"number\" || !Number.isFinite(value)) return CONTENTS_TIMEOUT_MS;\n\treturn Math.max(1, Math.floor(value));\n}\n\nfunction crawlTimeoutSeconds(value: number | undefined): number {\n\tif (typeof value !== \"number\" || !Number.isFinite(value)) return 10;\n\treturn Math.max(1, Math.min(Math.ceil(value / 1_000), 60));\n}\n\nfunction buildContentsBody(urls: string[], options: ExtractOptions = {}): Record<string, unknown> {\n\treturn {\n\t\turls,\n\t\tformat: \"markdown\",\n\t\tcrawlTimeout: crawlTimeoutSeconds(options.timeoutMs),\n\t\textrasMeta: true,\n\t};\n}\n\nasync function queritJsonRequest<T>(\n\tlabel: \"Search\" | \"Contents\",\n\turl: string,\n\tapiKey: string,\n\tbody: Record<string, unknown>,\n\ttimeoutMs: number,\n\tsignal?: AbortSignal,\n): Promise<T> {\n\tlet response: Response;\n\ttry {\n\t\tresponse = await fetch(url, {\n\t\t\tmethod: \"POST\",\n\t\t\theaders: {\n\t\t\t\tAuthorization: `Bearer ${apiKey}`,\n\t\t\t\tAccept: \"application/json\",\n\t\t\t\t\"Content-Type\": \"application/json\",\n\t\t\t},\n\t\t\tbody: JSON.stringify(body),\n\t\t\tsignal: requestSignal(signal, timeoutMs),\n\t\t});\n\t} catch (err) {\n\t\tconst message = errorMessage(err);\n\t\tconst isRequestAbort = err instanceof Error\n\t\t\t? err.name === \"AbortError\" || err.name === \"TimeoutError\" || /abort|timeout/i.test(message)\n\t\t\t: /abort|timeout/i.test(message);\n\t\tif (!signal?.aborted && isRequestAbort) {\n\t\t\tconst timeoutError = new Error(`Querit ${label} API request timed out after ${Math.ceil(timeoutMs / 1_000)} seconds`);\n\t\t\ttimeoutError.name = \"TimeoutError\";\n\t\t\tthrow timeoutError;\n\t\t}\n\t\tconst redactedMessage = redactCredential(message, apiKey);\n\t\tif (redactedMessage === message) throw err;\n\t\tconst redactedError = new Error(redactedMessage);\n\t\tif (err instanceof Error) redactedError.name = err.name;\n\t\tthrow redactedError;\n\t}\n\n\tconst raw = await response.text();\n\tif (!response.ok) {\n\t\tthrow new Error(`Querit ${label} API error ${response.status}: ${redactCredential(raw, apiKey).slice(0, 300)}`);\n\t}\n\ttry {\n\t\treturn JSON.parse(raw) as T;\n\t} catch (err) {\n\t\tthrow new Error(`Querit ${label} API returned invalid JSON: ${errorMessage(err)}`);\n\t}\n}\n\nfunction assertApiSuccess(label: \"Search\" | \"Contents\", data: QueritSearchResponse | QueritContentsResponse): void {\n\tconst code = Number(data.error_code);\n\tif (!Number.isFinite(code) || code !== 200) {\n\t\tconst renderedCode = data.error_code === undefined ? \"unknown\" : String(data.error_code);\n\t\tconst message = typeof data.error_msg === \"string\" && data.error_msg.trim() ? `: ${data.error_msg.trim()}` : \"\";\n\t\tthrow new Error(`Querit ${label} API returned error ${renderedCode}${message}`);\n\t}\n}\n\nfunction mapSearchResults(data: QueritSearchResponse): SearchResponse[\"results\"] {\n\tconst items = data.results?.result;\n\tif (!Array.isArray(items)) {\n\t\tthrow new Error(\"Querit Search API returned an unexpected response shape\");\n\t}\n\treturn items.flatMap((item) => {\n\t\tif (!item || typeof item.url !== \"string\" || item.url.trim().length === 0) return [];\n\t\tconst url = item.url.trim();\n\t\treturn [{\n\t\t\ttitle: typeof item.title === \"string\" && item.title.trim() ? item.title.trim() : url,\n\t\t\turl,\n\t\t\tsnippet: typeof item.snippet === \"string\" ? item.snippet.replace(/\\s+/g, \" \").trim() : \"\",\n\t\t}];\n\t});\n}\n\nfunction buildAnswer(results: SearchResponse[\"results\"]): string {\n\treturn results.map((result) => {\n\t\tif (result.snippet) return `${result.snippet}\\nSource: ${result.title} (${result.url})`;\n\t\treturn `Source: ${result.title} (${result.url})`;\n\t}).join(\"\\n\\n\");\n}\n\nfunction mapContentResult(result: QueritContentResult | undefined, requestedUrl: string): ExtractedContent | null {\n\tif (!result || typeof result.content !== \"string\" || result.content.trim().length === 0) return null;\n\tconst metadata = result.extrasMeta;\n\treturn {\n\t\turl: requestedUrl,\n\t\ttitle: metadata && typeof metadata.title === \"string\" ? metadata.title.trim() : \"\",\n\t\tcontent: result.content.trim(),\n\t\terror: null,\n\t};\n}\n\nfunction findContentResult(\n\tdata: QueritContentsResponse,\n\trequestedUrl: string,\n\tindex: number,\n\trequestedCount: number,\n): QueritContentResult | undefined {\n\tif (!Array.isArray(data.results)) return undefined;\n\tconst exact = data.results.find((item) => item?.url === requestedUrl || item?.extrasMeta?.url === requestedUrl);\n\tif (exact) return exact;\n\tif (requestedCount === 1) return data.results[0];\n\treturn data.results.length === requestedCount ? data.results[index] : undefined;\n}\n\nfunction failedContentStatus(data: QueritContentsResponse, result: QueritContentResult | undefined, index: number): boolean {\n\tif (!Array.isArray(data.statuses)) return false;\n\tconst status = result?.id\n\t\t? data.statuses.find((item) => item?.id === result.id)\n\t\t: data.statuses[index];\n\treturn status?.status === \"failed\";\n}\n\nasync function fetchContentsBatch(\n\turls: string[],\n\tapiKey: string,\n\tsignal?: AbortSignal,\n\toptions: ExtractOptions = {},\n): Promise<QueritContentsResponse> {\n\tconst data = await queritJsonRequest<QueritContentsResponse>(\n\t\t\"Contents\",\n\t\tQUERIT_CONTENTS_URL,\n\t\tapiKey,\n\t\tbuildContentsBody(urls, options),\n\t\tnormalizeTimeoutMs(options.timeoutMs),\n\t\tsignal,\n\t);\n\tassertApiSuccess(\"Contents\", data);\n\tif (!Array.isArray(data.results) || !Array.isArray(data.statuses)) {\n\t\tthrow new Error(\"Querit Contents API returned an unexpected response shape\");\n\t}\n\treturn data;\n}\n\nasync function fetchInlineContent(urls: string[], apiKey: string, signal?: AbortSignal): Promise<ExtractedContent[]> {\n\tconst content: ExtractedContent[] = [];\n\tfor (let offset = 0; offset < urls.length; offset += MAX_CONTENT_URLS) {\n\t\tconst batch = urls.slice(offset, offset + MAX_CONTENT_URLS);\n\t\tconst data = await fetchContentsBatch(batch, apiKey, signal);\n\t\tfor (let index = 0; index < batch.length; index++) {\n\t\t\tconst requestedUrl = batch[index];\n\t\t\tconst result = findContentResult(data, requestedUrl, index, batch.length);\n\t\t\tconst mapped = mapContentResult(result, requestedUrl);\n\t\t\tif (mapped) content.push(mapped);\n\t\t}\n\t}\n\treturn content;\n}\n\nexport async function searchWithQuerit(\n\tquery: string,\n\toptions: QueritSearchOptions = {},\n): Promise<SearchResponse> {\n\tconst apiKey = await getApiKey(options.signal);\n\tconst activityId = activityMonitor.logStart({ type: \"api\", query });\n\ttry {\n\t\tconst data = await queritJsonRequest<QueritSearchResponse>(\n\t\t\t\"Search\",\n\t\t\tQUERIT_SEARCH_URL,\n\t\t\tapiKey,\n\t\t\tbuildSearchBody(query, options),\n\t\t\tSEARCH_TIMEOUT_MS,\n\t\t\toptions.signal,\n\t\t);\n\t\tassertApiSuccess(\"Search\", data);\n\t\tconst results = mapSearchResults(data);\n\t\tconst response: SearchResponse = { answer: buildAnswer(results), results };\n\t\tif (options.includeContent && results.length > 0) {\n\t\t\tconst inlineContent = await fetchInlineContent(results.map((result) => result.url), apiKey, options.signal);\n\t\t\tif (inlineContent.length > 0) response.inlineContent = inlineContent;\n\t\t}\n\t\tactivityMonitor.logComplete(activityId, 200);\n\t\treturn response;\n\t} catch (err) {\n\t\tconst message = errorMessage(err);\n\t\tconst redactedMessage = redactCredential(message, apiKey);\n\t\tif (options.signal?.aborted) activityMonitor.logComplete(activityId, 0);\n\t\telse activityMonitor.logError(activityId, redactedMessage);\n\t\tif (redactedMessage === message) throw err;\n\t\tconst redactedError = new Error(redactedMessage);\n\t\tif (err instanceof Error) redactedError.name = err.name;\n\t\tthrow redactedError;\n\t}\n}\n\nexport async function extractWithQuerit(\n\turl: string,\n\tsignal?: AbortSignal,\n\toptions: ExtractOptions = {},\n): Promise<ExtractedContent | null> {\n\tconst apiKey = await getApiKey(signal);\n\tconst activityId = activityMonitor.logStart({ type: \"fetch\", url });\n\ttry {\n\t\tconst data = await fetchContentsBatch([url], apiKey, signal, options);\n\t\tconst result = findContentResult(data, url, 0, 1);\n\t\tconst mapped = mapContentResult(result, url);\n\t\tif (mapped) {\n\t\t\tactivityMonitor.logComplete(activityId, 200);\n\t\t\treturn mapped;\n\t\t}\n\t\tif (failedContentStatus(data, result, 0)) {\n\t\t\tthrow new Error(`Querit Contents API failed to crawl ${url}`);\n\t\t}\n\t\tactivityMonitor.logComplete(activityId, 200);\n\t\treturn null;\n\t} catch (err) {\n\t\tconst message = errorMessage(err);\n\t\tconst redactedMessage = redactCredential(message, apiKey);\n\t\tif (signal?.aborted) activityMonitor.logComplete(activityId, 0);\n\t\telse activityMonitor.logError(activityId, redactedMessage);\n\t\tif (redactedMessage === message) throw err;\n\t\tconst redactedError = new Error(redactedMessage);\n\t\tif (err instanceof Error) redactedError.name = err.name;\n\t\tthrow redactedError;\n\t}\n}\n"]}