@humain/terminal 0.0.6 → 0.0.8

package/dist/humain/web-access/vendor/extract.js

@@ -8,6 +8,7 @@ import { activityMonitor } from "./activity.js";
 import { extractRSCContent } from "./rsc-extract.js";
 import { extractPDFToMarkdown, isPDF, loadPDFConfig } from "./pdf-extract.js";
 import { extractGitHub } from "./github-extract.js";
+import { extractGitHubIssuePr } from "./github-issue-pr.js";
 import { isYouTubeURL, isYouTubeEnabled, extractYouTube, extractYouTubeFrame, extractYouTubeFrames, getYouTubeStreamInfo } from "./youtube-extract.js";
 import { CredentialResolutionError } from "./credential-source.js";
 import { extractWithUrlContext, extractWithGeminiWeb } from "./gemini-url-context.js";
@@ -29,14 +30,85 @@ import { assertAuthFetchUrl, authFetchRedirectGuard } from "./auth-fetch.js";
 import { getBrowserCookiesForHosts, getLastBrowserCookieDiagnostic } from "./chrome-cookies.js";
 import { sanitizeInlineDataUris } from "./data-uri-sanitize.js";
 const DEFAULT_TIMEOUT_MS = 30000;
+const MAX_CONFIGURED_TIMEOUT_MS = 2_147_483_647;
 const CONCURRENT_LIMIT = 3;
+const WEB_SEARCH_CONFIG_PATH = getWebSearchConfigPath();
+function loadFetchTimeoutMs() {
+    if (!existsSync(WEB_SEARCH_CONFIG_PATH))
+        return DEFAULT_TIMEOUT_MS;
+    let raw;
+    try {
+        raw = JSON.parse(readFileSync(WEB_SEARCH_CONFIG_PATH, "utf-8"));
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        throw new Error(`Failed to parse ${WEB_SEARCH_CONFIG_PATH}: ${message}`);
+    }
+    if (!raw || typeof raw !== "object" || Array.isArray(raw)) {
+        throw new Error(`Invalid config in ${WEB_SEARCH_CONFIG_PATH}: expected a JSON object`);
+    }
+    const fetchConfig = raw.fetch;
+    if (fetchConfig === undefined)
+        return DEFAULT_TIMEOUT_MS;
+    if (!fetchConfig || typeof fetchConfig !== "object" || Array.isArray(fetchConfig)) {
+        throw new Error(`fetch in ${WEB_SEARCH_CONFIG_PATH} must be an object`);
+    }
+    const value = fetchConfig.timeout;
+    if (value === undefined)
+        return DEFAULT_TIMEOUT_MS;
+    if (typeof value !== "number" || !Number.isFinite(value) || value <= 0) {
+        throw new Error(`Invalid fetch.timeout in ${WEB_SEARCH_CONFIG_PATH}: expected a positive finite number of seconds, got ${JSON.stringify(value)}`);
+    }
+    const timeoutMs = Math.ceil(value * 1000);
+    if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1 || timeoutMs > MAX_CONFIGURED_TIMEOUT_MS) {
+        throw new Error(`Invalid fetch.timeout in ${WEB_SEARCH_CONFIG_PATH}: converted timeout must be a finite safe integer from 1 through ${MAX_CONFIGURED_TIMEOUT_MS} milliseconds`);
+    }
+    return Math.max(1, timeoutMs);
+}
 const NON_RECOVERABLE_ERRORS = ["Unsupported content type", "Response too large", "PDF extraction is disabled", "Image fetching is disabled"];
 const MIN_USEFUL_CONTENT = 500;
 const SUPPORTED_IMAGE_TYPES = new Set(["image/png", "image/jpeg", "image/webp", "image/gif"]);
-const WEB_SEARCH_CONFIG_PATH = getWebSearchConfigPath();
 const FETCH_PROVIDERS = ["http", "firecrawl", "jina", "tinyfish", "search1api", "querit", "kagi", "ollama", "parallel", "parallel-mcp", "brightdata", "gemini"];
 const DEFAULT_FETCH_PROVIDER_ORDER = ["http", "firecrawl", "jina", "tinyfish", "search1api", "querit", "kagi", "ollama", "parallel", "brightdata", "gemini"];
 const REMOTE_HOSTED_FETCH_PROVIDERS = new Set(["jina", "tinyfish", "search1api", "querit", "kagi", "ollama", "parallel", "parallel-mcp", "brightdata", "gemini"]);
+function isDefuddleConsoleError(args) {
+    const prefix = args[0];
+    return prefix === "Defuddle" || (typeof prefix === "string" && /^Defuddle(?:\s|:)/.test(prefix));
+}
+async function extractWithDefuddle(text, url) {
+    const { Defuddle } = await import("defuddle/node");
+    const { document } = parseHTML(text);
+    Object.defineProperty(document, "location", {
+        value: new URL(url),
+        configurable: true,
+    });
+    let processingError;
+    const originalConsoleError = console.error;
+    console.error = (...args) => {
+        if (isDefuddleConsoleError(args)) {
+            if (args[0] === "Defuddle" && args[1] === "Error processing document:") {
+                processingError = args[2];
+            }
+            return;
+        }
+        originalConsoleError(...args);
+    };
+    let resultPromise;
+    try {
+        // With useAsync:false, Defuddle parses synchronously before returning its promise.
+        // Keep the console interception limited to that call so unrelated Pi output is
+        // never routed through this fallback's handler.
+        resultPromise = Defuddle(document, url, { markdown: true, useAsync: false });
+    }
+    finally {
+        console.error = originalConsoleError;
+    }
+    const result = await resultPromise;
+    if (processingError !== undefined) {
+        throw new Error(`Defuddle failed to process document: ${errorMessage(processingError)}`);
+    }
+    return typeof result.content === "string" ? { title: result.title, content: result.content } : null;
+}
 export { loadSsrfConfig } from "./ssrf-protection.js";
 export function loadSsrfAllowRanges() {
     return loadSsrfConfig().allowRanges;
@@ -162,6 +234,26 @@ function loadFetchRouting() {
     }
     return { providers, allowRemoteHostedProviders: allowRemoteHostedProvidersValue === true };
 }
+/** Guidance for definitive origin 404/410 responses: no extraction provider
+ * can retrieve a page the origin says is gone, so point at the registered
+ * search/fetch tools (when the caller knows them) instead of provider config. */
+function notFoundGuidance(result, toolNames) {
+    const lines = [
+        result.error ?? `HTTP ${result.status}`,
+        "",
+        `The origin server says this page does not exist (HTTP ${result.status}), so extraction providers cannot retrieve it.`,
+    ];
+    if (toolNames?.webSearch && toolNames.fetchContent) {
+        lines.push(`The page may have moved or been renamed. Use ${toolNames.webSearch} to find the current URL, then retry ${toolNames.fetchContent} with it.`);
+    }
+    else if (toolNames?.webSearch) {
+        lines.push(`The page may have moved or been renamed. Use ${toolNames.webSearch} to find the current URL.`);
+    }
+    else {
+        lines.push("The page may have moved or been renamed. Find the current URL, then retry the fetch with it.");
+    }
+    return lines.join("\n");
+}
 function abortedResult(url) {
     return { url, title: "", content: "", error: "Aborted" };
 }
@@ -170,9 +262,12 @@ const turndown = new TurndownService({
     codeBlockStyle: "fenced",
 });
 const fetchLimit = pLimit(CONCURRENT_LIMIT);
+/** Resolve the direct HTTP/Jina fetch budget, with a per-call override taking precedence. */
+export function resolveFetchTimeoutMs(options) {
+    return options?.timeoutMs ?? loadFetchTimeoutMs();
+}
 const JINA_READER_BASE = "https://r.jina.ai/";
-const JINA_TIMEOUT_MS = 30000;
-async function extractWithJinaReader(url, signal, lookup) {
+async function extractWithJinaReader(url, timeoutMs, signal, lookup) {
     const jinaUrl = JINA_READER_BASE + url;
     const activityId = activityMonitor.logStart({ type: "api", query: `jina: ${url}` });
     try {
@@ -190,7 +285,7 @@ async function extractWithJinaReader(url, signal, lookup) {
                 "X-No-Cache": "true",
             },
             signal: AbortSignal.any([
-                AbortSignal.timeout(JINA_TIMEOUT_MS),
+                AbortSignal.timeout(timeoutMs),
                 ...(signal ? [signal] : []),
             ]),
         });
@@ -326,17 +421,14 @@ export async function extractContent(url, signal, options) {
             return { url, title: "", content: "", error: errorMessage(err) };
         }
     }
-    if (options?.authFetchProfile) {
+    if (options?.authFetchProfile || options?.mode === "raw") {
         try {
-            return await extractViaHttp(url, signal, options);
+            return await extractViaHttp(url, resolveFetchTimeoutMs(options), signal, options);
         }
         catch (err) {
             return { url, title: "", content: "", error: errorMessage(err) };
         }
     }
-    if (options?.mode === "raw") {
-        return extractViaHttp(url, signal, options);
-    }
     if (options?.frames || options?.timestamp) {
         const disabled = imageGateError();
         if (disabled)
@@ -488,6 +580,21 @@ export async function extractContent(url, signal, options) {
     catch (err) {
         return { url, title: "", content: "", error: errorMessage(err) };
     }
+    try {
+        const ghIssuePrResult = await extractGitHubIssuePr(url, signal, options);
+        if (ghIssuePrResult)
+            return ghIssuePrResult;
+        if (signal?.aborted)
+            return abortedResult(url);
+    }
+    catch (err) {
+        const message = errorMessage(err);
+        if (isAbortError(err))
+            return abortedResult(url);
+        if (isConfigParseError(err)) {
+            return { url, title: "", content: "", error: message };
+        }
+    }
     try {
         const ghResult = await extractGitHub(url, signal, options?.forceClone);
         if (ghResult)
@@ -534,6 +641,13 @@ export async function extractContent(url, signal, options) {
     }
     if (signal?.aborted)
         return abortedResult(url);
+    let fetchTimeoutMs;
+    try {
+        fetchTimeoutMs = resolveFetchTimeoutMs(options);
+    }
+    catch (err) {
+        return { url, title: "", content: "", error: errorMessage(err) };
+    }
     let fetchRouting;
     try {
         fetchRouting = loadFetchRouting();
@@ -562,7 +676,7 @@ export async function extractContent(url, signal, options) {
         ? { ...httpResult, error: message }
         : { url, title: "", content: "", error: message };
     const runHttpProvider = async () => {
-        const { declaredLinks: discoveredLinks = [], ...result } = await extractViaHttp(url, signal, options);
+        const { declaredLinks: discoveredLinks = [], ...result } = await extractViaHttp(url, fetchTimeoutMs, signal, options);
         httpResult = result;
         declaredLinks = discoveredLinks;
         if (signal?.aborted)
@@ -620,7 +734,7 @@ export async function extractContent(url, signal, options) {
             continue;
         }
         if (provider === "jina") {
-            const jinaResult = await extractWithJinaReader(url, signal, options?.lookup);
+            const jinaResult = await extractWithJinaReader(url, fetchTimeoutMs, signal, options?.lookup);
             if (jinaResult)
                 return withDeclaredLinks(jinaResult);
             continue;
@@ -796,6 +910,13 @@ export async function extractContent(url, signal, options) {
     const finalHttpResult = httpResult;
     if (finalHttpResult && declaredLinks.length > 0)
         return { ...finalHttpResult, error: null };
+    // A definitive 404/410 from the origin means no extraction provider can
+    // retrieve the page, so the provider-configuration checklist below would
+    // send users down the wrong path. Point at search instead.
+    if (finalHttpResult?.status === 404 || finalHttpResult?.status === 410) {
+        return { ...finalHttpResult, error: notFoundGuidance(finalHttpResult, options?.toolNames) };
+    }
+    const searchToolName = options?.toolNames?.webSearch;
     const guidance = [
         finalHttpResult?.error ?? "No fetch_content provider returned content",
         ...(firecrawlError ? [`Firecrawl fallback failed: ${firecrawlError}`] : []),
@@ -819,26 +940,22 @@ export async function extractContent(url, signal, options) {
         `  • Set brightdataApiKey and brightdataUnlockerZone in ${WEB_SEARCH_CONFIG_PATH} or BRIGHTDATA_API_KEY and BRIGHTDATA_UNLOCKER_ZONE`,
         `  • Set GEMINI_API_KEY in ${WEB_SEARCH_CONFIG_PATH}`,
         "  • Sign into gemini.google.com in Chrome",
-        "  • Use web_search to find content about this topic",
+        ...(searchToolName ? [`  • Use ${searchToolName} to find content about this topic`] : []),
     ].join("\n");
     return { ...(finalHttpResult ?? { url, title: "", content: "", error: null }), error: guidance };
 }
 function isLikelyJSRendered(html) {
-    // Extract body content
     const bodyMatch = html.match(/<body[^>]*>([\s\S]*?)<\/body>/i);
     if (!bodyMatch)
         return false;
     const bodyHtml = bodyMatch[1];
-    // Strip tags to get text content
     const textContent = bodyHtml
         .replace(/<script[\s\S]*?<\/script>/gi, "")
         .replace(/<style[\s\S]*?<\/style>/gi, "")
         .replace(/<[^>]+>/g, "")
         .replace(/\s+/g, " ")
         .trim();
-    // Count scripts
     const scriptCount = (html.match(/<script/gi) || []).length;
-    // Heuristic: little text content but many scripts suggests JS rendering
     return textContent.length < 500 && scriptCount > 3;
 }
 export async function readPDFResponseBuffer(response, maxSizeMB) {
@@ -909,8 +1026,7 @@ function pdfSizeLimitError(maxSizeMB) {
 function responseSizeLimitError(maxBytes) {
     return new Error(`Response too large (${Math.round(maxBytes / 1024 / 1024)}MB)`);
 }
-async function extractViaHttp(url, signal, options) {
-    const timeoutMs = options?.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+async function extractViaHttp(url, timeoutMs, signal, options) {
     const activityId = activityMonitor.logStart({ type: "fetch", url });
     const controller = new AbortController();
     const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
@@ -920,8 +1036,10 @@ async function extractViaHttp(url, signal, options) {
         const ssrf = loadSsrfConfig();
         const domainPolicy = loadFetchContentDomainPolicy();
         const authProfile = options?.authFetchProfile;
+        const trustEnvProxy = options?.proxy === undefined && ssrf.trustEnvProxy;
         const requestInit = {
             signal: controller.signal,
+            __proxy: options?.proxy,
             headers: {
                 "User-Agent": "OpenAI File Downloader, XaiImageApiFetch/1.0",
                 "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
@@ -935,10 +1053,10 @@ async function extractViaHttp(url, signal, options) {
             },
         };
         const response = authProfile
-            ? await fetchAuthenticatedRemoteUrl(url, requestInit, { ssrf, domainPolicy, ...(options?.lookup ? { lookup: options.lookup } : {}) }, authProfile)
+            ? await fetchAuthenticatedRemoteUrl(url, requestInit, { ssrf: { ...ssrf, trustEnvProxy }, domainPolicy, ...(options?.lookup ? { lookup: options.lookup } : {}) }, authProfile)
             : await fetchRemoteUrl(url, requestInit, {
                 allowRanges: ssrf.allowRanges,
-                trustEnvProxy: ssrf.trustEnvProxy,
+                trustEnvProxy,
                 domainPolicy,
                 ...(options?.lookup ? { lookup: options.lookup } : {}),
             });
@@ -1077,6 +1195,19 @@ async function extractViaHttp(url, signal, options) {
                     declaredLinks,
                 };
             }
+            controller.signal.throwIfAborted();
+            const defuddleResult = await extractWithDefuddle(text, response.url || url);
+            controller.signal.throwIfAborted();
+            if (defuddleResult && defuddleResult.content.length >= MIN_USEFUL_CONTENT) {
+                activityMonitor.logComplete(activityId, response.status);
+                return {
+                    url,
+                    title: documentTitle || defuddleResult.title,
+                    content: appendDeclaredWebLinks(defuddleResult.content, declaredLinks),
+                    error: null,
+                    declaredLinks,
+                };
+            }
             activityMonitor.logComplete(activityId, response.status);
             const jsRendered = isLikelyJSRendered(text);
             const errorMsg = jsRendered
@@ -1106,6 +1237,18 @@ async function extractViaHttp(url, signal, options) {
                     declaredLinks,
                 };
             }
+            controller.signal.throwIfAborted();
+            const defuddleResult = await extractWithDefuddle(text, response.url || url);
+            controller.signal.throwIfAborted();
+            if (defuddleResult && defuddleResult.content.length >= MIN_USEFUL_CONTENT) {
+                return {
+                    url,
+                    title: article.title || documentTitle || defuddleResult.title,
+                    content: appendDeclaredWebLinks(defuddleResult.content, declaredLinks),
+                    error: null,
+                    declaredLinks,
+                };
+            }
             return {
                 url,
                 title: article.title || documentTitle,