@humain/terminal 0.1.0 → 0.2.1

package/dist/humain/web-access/vendor/extract.js

@@ -1,8 +1,4 @@
 import { existsSync, readFileSync } from "node:fs";
-import { Readability } from "@mozilla/readability";
-import { resizeImage } from "../host.js";
-import { parseHTML } from "linkedom";
-import TurndownService from "turndown";
 import pLimit from "p-limit";
 import { activityMonitor } from "./activity.js";
 import { extractRSCContent } from "./rsc-extract.js";
@@ -20,6 +16,7 @@ import { extractWithQuerit, isQueritAvailable } from "./querit.js";
 import { extractWithKagi, isKagiExtractAvailable } from "./kagi.js";
 import { extractWithOllama, isOllamaFetchAvailable } from "./ollama.js";
 import { extractWithFirecrawl, isFirecrawlAvailable } from "./firecrawl.js";
+import { extractWithCrawl4ai, isCrawl4aiAvailable } from "./crawl4ai.js";
 import { extractWithBrightDataUnlocker, isBrightDataUnlockerAvailable } from "./brightdata-unlocker.js";
 import { isVideoFile, extractVideo, extractVideoFrame, getLocalVideoDuration } from "./video-extract.js";
 import { appendDeclaredWebLinks, discoverDeclaredWebLinks } from "./declared-web-links.js";
@@ -68,8 +65,8 @@ function loadFetchTimeoutMs() {
 const NON_RECOVERABLE_ERRORS = ["Unsupported content type", "Response too large", "PDF extraction is disabled", "Image fetching is disabled"];
 const MIN_USEFUL_CONTENT = 500;
 const SUPPORTED_IMAGE_TYPES = new Set(["image/png", "image/jpeg", "image/webp", "image/gif"]);
-const FETCH_PROVIDERS = ["http", "firecrawl", "jina", "tinyfish", "search1api", "querit", "kagi", "ollama", "parallel", "parallel-mcp", "brightdata", "gemini"];
-const DEFAULT_FETCH_PROVIDER_ORDER = ["http", "firecrawl", "jina", "tinyfish", "search1api", "querit", "kagi", "ollama", "parallel", "brightdata", "gemini"];
+const FETCH_PROVIDERS = ["http", "firecrawl", "crawl4ai", "jina", "tinyfish", "search1api", "querit", "kagi", "ollama", "parallel", "parallel-mcp", "brightdata", "gemini"];
+const DEFAULT_FETCH_PROVIDER_ORDER = ["http", "firecrawl", "crawl4ai", "jina", "tinyfish", "search1api", "querit", "kagi", "ollama", "parallel", "brightdata", "gemini"];
 const REMOTE_HOSTED_FETCH_PROVIDERS = new Set(["jina", "tinyfish", "search1api", "querit", "kagi", "ollama", "parallel", "parallel-mcp", "brightdata", "gemini"]);
 function isDefuddleConsoleError(args) {
     const prefix = args[0];
@@ -77,6 +74,7 @@ function isDefuddleConsoleError(args) {
 }
 async function extractWithDefuddle(text, url) {
     const { Defuddle } = await import("defuddle/node");
+    const { parseHTML } = await import("linkedom");
     const { document } = parseHTML(text);
     Object.defineProperty(document, "location", {
         value: new URL(url),
@@ -122,6 +120,9 @@ function isConfigParseError(err) {
 function isAbortError(err) {
     return errorMessage(err).toLowerCase().includes("abort");
 }
+function isAbortException(err) {
+    return err instanceof DOMException && (err.name === "AbortError" || err.name === "TimeoutError");
+}
 function isRedirectPolicyError(message) {
     return message.startsWith("Authenticated fetch refused cross-origin redirect") ||
         message.startsWith("Blocked internal ") ||
@@ -257,10 +258,19 @@ function notFoundGuidance(result, toolNames) {
 function abortedResult(url) {
     return { url, title: "", content: "", error: "Aborted" };
 }
-const turndown = new TurndownService({
-    headingStyle: "atx",
-    codeBlockStyle: "fenced",
-});
+// Share first-use initialization without loading Turndown at extension startup.
+let turndownInstance;
+async function loadTurndown() {
+    const { default: TurndownService } = await import("turndown");
+    return new TurndownService({
+        headingStyle: "atx",
+        codeBlockStyle: "fenced",
+    });
+}
+function getTurndown() {
+    turndownInstance ??= loadTurndown();
+    return turndownInstance;
+}
 const fetchLimit = pLimit(CONCURRENT_LIMIT);
 /** Resolve the direct HTTP/Jina fetch budget, with a per-call override taking precedence. */
 export function resolveFetchTimeoutMs(options) {
@@ -689,6 +699,7 @@ export async function extractContent(url, signal, options) {
         return null;
     };
     let firecrawlError = null;
+    let crawl4aiError = null;
     let tinyfishError = null;
     let search1apiError = null;
     let queritError = null;
@@ -733,6 +744,28 @@ export async function extractContent(url, signal, options) {
             }
             continue;
         }
+        if (provider === "crawl4ai") {
+            try {
+                if (isCrawl4aiAvailable()) {
+                    const ssrf = loadSsrfConfig();
+                    const crawl4aiResult = await extractWithCrawl4ai(url, signal, {
+                        timeoutMs: options?.timeoutMs,
+                        ...(options?.lookup ? { lookup: options.lookup } : {}),
+                        ssrf,
+                    });
+                    if (crawl4aiResult)
+                        return withDeclaredLinks(crawl4aiResult);
+                }
+            }
+            catch (err) {
+                if (signal?.aborted || isAbortException(err))
+                    return abortedResult(url);
+                crawl4aiError = errorMessage(err);
+                if (isConfigParseError(err))
+                    return parseErrorResult(crawl4aiError);
+            }
+            continue;
+        }
         if (provider === "jina") {
             const jinaResult = await extractWithJinaReader(url, fetchTimeoutMs, signal, options?.lookup);
             if (jinaResult)
@@ -920,6 +953,7 @@ export async function extractContent(url, signal, options) {
     const guidance = [
         finalHttpResult?.error ?? "No fetch_content provider returned content",
         ...(firecrawlError ? [`Firecrawl fallback failed: ${firecrawlError}`] : []),
+        ...(crawl4aiError ? [`Crawl4AI fallback failed: ${crawl4aiError}`] : []),
         ...(tinyfishError ? [`TinyFish fallback failed: ${tinyfishError}`] : []),
         ...(search1apiError ? [`Search1API fallback failed: ${search1apiError}`] : []),
         ...(queritError ? [`Querit fallback failed: ${queritError}`] : []),
@@ -931,6 +965,7 @@ export async function extractContent(url, signal, options) {
         "",
         "Fallback options:",
         `  • Set firecrawlBaseUrl in ${WEB_SEARCH_CONFIG_PATH} to a self-hosted Firecrawl instance`,
+        `  • Set crawl4aiBaseUrl in ${WEB_SEARCH_CONFIG_PATH} to a self-hosted Crawl4AI instance`,
         `  • Set tinyfishApiKey in ${WEB_SEARCH_CONFIG_PATH} or TINYFISH_API_KEY`,
         `  • Set search1apiApiKey in ${WEB_SEARCH_CONFIG_PATH} or SEARCH1API_KEY`,
         `  • Set queritApiKey in ${WEB_SEARCH_CONFIG_PATH} or QUERIT_API_KEY`,
@@ -1029,6 +1064,7 @@ function responseSizeLimitError(maxBytes) {
 async function extractViaHttp(url, timeoutMs, signal, options) {
     const activityId = activityMonitor.logStart({ type: "fetch", url });
     const controller = new AbortController();
+    const startedAt = Date.now();
     const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
     const onAbort = () => controller.abort();
     signal?.addEventListener("abort", onAbort);
@@ -1111,6 +1147,7 @@ async function extractViaHttp(url, timeoutMs, signal, options) {
             }
             try {
                 const buffer = await readResponseBufferWithLimit(response, maxResponseSize, () => responseSizeLimitError(maxResponseSize));
+                const { resizeImage } = await import("../host.js");
                 const resized = await resizeImage(new Uint8Array(buffer), mimeType, { maxWidth: 2000, maxHeight: 2000 });
                 activityMonitor.logComplete(activityId, response.status);
                 if (!resized)
@@ -1178,9 +1215,11 @@ async function extractViaHttp(url, timeoutMs, signal, options) {
             const title = extractTextTitle(text, url);
             return { url, title, content: text, error: null };
         }
+        const { parseHTML } = await import("linkedom");
         const { document } = parseHTML(text);
         const documentTitle = document.title?.trim() ?? "";
         const declaredLinks = discoverDeclaredWebLinks(document, response.headers.get("link"), response.url || url);
+        const { Readability } = await import("@mozilla/readability");
         const reader = new Readability(document);
         const article = reader.parse();
         if (!article) {
@@ -1224,7 +1263,7 @@ async function extractViaHttp(url, timeoutMs, signal, options) {
         if (typeof article.content !== "string") {
             throw new Error("Readability returned invalid article content");
         }
-        const markdown = turndown.turndown(article.content);
+        const markdown = (await getTurndown()).turndown(article.content);
         activityMonitor.logComplete(activityId, response.status);
         if (markdown.length < MIN_USEFUL_CONTENT) {
             const rscResult = extractRSCContent(text);
@@ -1280,6 +1319,14 @@ async function extractViaHttp(url, timeoutMs, signal, options) {
     finally {
         clearTimeout(timeoutId);
         signal?.removeEventListener("abort", onAbort);
+        // Imports and CPU-bound processing need not observe the fetch signal, and
+        // can finish before an expired timer gets a turn. Guard every exit, with
+        // caller cancellation taking precedence over the internal deadline.
+        if (signal?.aborted)
+            return abortedResult(url);
+        if (controller.signal.aborted || Date.now() - startedAt >= timeoutMs) {
+            return { url, title: "", content: "", error: "The operation was aborted." };
+        }
     }
 }
 export function extractHeadingTitle(text) {