@humain/terminal 0.0.2 → 0.0.3-beta.0

package/dist/humain/web-access/vendor/extract.js

@@ -0,0 +1,1064 @@
+import { existsSync, readFileSync } from "node:fs";
+import { Readability } from "@mozilla/readability";
+import { resizeImage } from "../host.js";
+import { parseHTML } from "linkedom";
+import TurndownService from "turndown";
+import pLimit from "p-limit";
+import { activityMonitor } from "./activity.js";
+import { extractRSCContent } from "./rsc-extract.js";
+import { extractPDFToMarkdown, isPDF, loadPDFConfig } from "./pdf-extract.js";
+import { extractGitHub } from "./github-extract.js";
+import { isYouTubeURL, isYouTubeEnabled, extractYouTube, extractYouTubeFrame, extractYouTubeFrames, getYouTubeStreamInfo } from "./youtube-extract.js";
+import { CredentialResolutionError } from "./credential-source.js";
+import { extractWithUrlContext, extractWithGeminiWeb } from "./gemini-url-context.js";
+import { extractWithParallel, isParallelAvailable } from "./parallel.js";
+import { extractWithTinyFish, isTinyFishAvailable } from "./tinyfish.js";
+import { extractWithSearch1API, isSearch1APIAvailable } from "./search1api.js";
+import { extractWithQuerit, isQueritAvailable } from "./querit.js";
+import { extractWithKagi, isKagiExtractAvailable } from "./kagi.js";
+import { extractWithOllama, isOllamaFetchAvailable } from "./ollama.js";
+import { extractWithFirecrawl, isFirecrawlAvailable } from "./firecrawl.js";
+import { extractWithBrightDataUnlocker, isBrightDataUnlockerAvailable } from "./brightdata-unlocker.js";
+import { isVideoFile, extractVideo, extractVideoFrame, getLocalVideoDuration } from "./video-extract.js";
+import { appendDeclaredWebLinks, discoverDeclaredWebLinks } from "./declared-web-links.js";
+import { fetchRemoteUrl, loadFetchContentDomainPolicy, loadSsrfConfig, validateRemoteUrl } from "./ssrf-protection.js";
+import { formatSeconds, getWebSearchConfigPath } from "./utils.js";
+import { isImageEnabled } from "./feature-config.js";
+const DEFAULT_TIMEOUT_MS = 30000;
+const CONCURRENT_LIMIT = 3;
+const NON_RECOVERABLE_ERRORS = ["Unsupported content type", "Response too large", "PDF extraction is disabled", "Image fetching is disabled"];
+const MIN_USEFUL_CONTENT = 500;
+const SUPPORTED_IMAGE_TYPES = new Set(["image/png", "image/jpeg", "image/webp", "image/gif"]);
+const WEB_SEARCH_CONFIG_PATH = getWebSearchConfigPath();
+const FETCH_PROVIDERS = ["http", "firecrawl", "jina", "tinyfish", "search1api", "querit", "kagi", "ollama", "parallel", "brightdata", "gemini"];
+const DEFAULT_FETCH_PROVIDER_ORDER = ["http", "firecrawl", "jina", "tinyfish", "search1api", "querit", "kagi", "ollama", "parallel", "brightdata", "gemini"];
+const REMOTE_HOSTED_FETCH_PROVIDERS = new Set(["jina", "tinyfish", "search1api", "querit", "kagi", "ollama", "parallel", "brightdata", "gemini"]);
+export { loadSsrfConfig } from "./ssrf-protection.js";
+export function loadSsrfAllowRanges() {
+    return loadSsrfConfig().allowRanges;
+}
+function errorMessage(err) {
+    return err instanceof Error ? err.message : String(err);
+}
+function isConfigParseError(err) {
+    return errorMessage(err).startsWith("Failed to parse ");
+}
+function isAbortError(err) {
+    return errorMessage(err).toLowerCase().includes("abort");
+}
+function isRedirectPolicyError(message) {
+    return message.startsWith("Blocked internal ") ||
+        message.startsWith("Blocked hostname by fetch_content domain policy") ||
+        message.startsWith("Hostname not allowed by fetch_content domain policy") ||
+        message.startsWith("Too many redirects fetching ") ||
+        message === "Only HTTP and HTTPS URLs can be fetched remotely" ||
+        message === "URL must include a hostname" ||
+        message.startsWith("Failed to resolve ");
+}
+function imageGateError() {
+    try {
+        return isImageEnabled() ? null : "Image fetching is disabled by image.enabled";
+    }
+    catch (err) {
+        return errorMessage(err);
+    }
+}
+function loadFetchRouting() {
+    if (!existsSync(WEB_SEARCH_CONFIG_PATH)) {
+        return { providers: DEFAULT_FETCH_PROVIDER_ORDER, allowRemoteHostedProviders: false };
+    }
+    let raw;
+    try {
+        const parsed = JSON.parse(readFileSync(WEB_SEARCH_CONFIG_PATH, "utf-8"));
+        if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
+            throw new Error("expected a JSON object");
+        }
+        raw = parsed;
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        throw new Error(`Failed to parse ${WEB_SEARCH_CONFIG_PATH}: ${message}`);
+    }
+    if (!Object.hasOwn(raw, "fetchRouting")) {
+        return { providers: DEFAULT_FETCH_PROVIDER_ORDER, allowRemoteHostedProviders: false };
+    }
+    const routing = raw.fetchRouting;
+    if (!routing || typeof routing !== "object" || Array.isArray(routing)) {
+        throw new Error(`fetchRouting in ${WEB_SEARCH_CONFIG_PATH} must be an object`);
+    }
+    const routingConfig = routing;
+    const providersValue = routingConfig.providers;
+    let providers = DEFAULT_FETCH_PROVIDER_ORDER;
+    if (providersValue !== undefined) {
+        if (!Array.isArray(providersValue) || providersValue.length === 0) {
+            throw new Error(`fetchRouting.providers in ${WEB_SEARCH_CONFIG_PATH} must be a non-empty array`);
+        }
+        providers = [];
+        for (const provider of providersValue) {
+            const normalized = typeof provider === "string" ? provider.trim().toLowerCase() : "";
+            if (!FETCH_PROVIDERS.includes(normalized)) {
+                throw new Error(`fetchRouting.providers in ${WEB_SEARCH_CONFIG_PATH} contains an invalid provider: ${String(provider)}`);
+            }
+            if (providers.includes(normalized)) {
+                throw new Error(`fetchRouting.providers in ${WEB_SEARCH_CONFIG_PATH} must not contain duplicates: ${normalized}`);
+            }
+            providers.push(normalized);
+        }
+    }
+    const allowRemoteHostedProvidersValue = routingConfig.allowRemoteHostedProviders;
+    if (allowRemoteHostedProvidersValue !== undefined && typeof allowRemoteHostedProvidersValue !== "boolean") {
+        throw new Error(`fetchRouting.allowRemoteHostedProviders in ${WEB_SEARCH_CONFIG_PATH} must be a boolean`);
+    }
+    return { providers, allowRemoteHostedProviders: allowRemoteHostedProvidersValue === true };
+}
+function abortedResult(url) {
+    return { url, title: "", content: "", error: "Aborted" };
+}
+const turndown = new TurndownService({
+    headingStyle: "atx",
+    codeBlockStyle: "fenced",
+});
+const fetchLimit = pLimit(CONCURRENT_LIMIT);
+const JINA_READER_BASE = "https://r.jina.ai/";
+const JINA_TIMEOUT_MS = 30000;
+async function extractWithJinaReader(url, signal, lookup) {
+    const jinaUrl = JINA_READER_BASE + url;
+    const activityId = activityMonitor.logStart({ type: "api", query: `jina: ${url}` });
+    try {
+        const ssrf = loadSsrfConfig();
+        const domainPolicy = loadFetchContentDomainPolicy();
+        await validateRemoteUrl(url, {
+            allowRanges: ssrf.allowRanges,
+            trustEnvProxy: ssrf.trustEnvProxy,
+            domainPolicy,
+            ...(lookup ? { lookup } : {}),
+        });
+        const res = await fetch(jinaUrl, {
+            headers: {
+                "Accept": "text/markdown",
+                "X-No-Cache": "true",
+            },
+            signal: AbortSignal.any([
+                AbortSignal.timeout(JINA_TIMEOUT_MS),
+                ...(signal ? [signal] : []),
+            ]),
+        });
+        if (!res.ok) {
+            activityMonitor.logComplete(activityId, res.status);
+            return null;
+        }
+        const content = await res.text();
+        activityMonitor.logComplete(activityId, res.status);
+        const contentStart = content.indexOf("Markdown Content:");
+        if (contentStart < 0) {
+            return null;
+        }
+        const markdownPart = content.slice(contentStart + 17).trim(); // 17 = "Markdown Content:".length
+        // Check for failed JS rendering or minimal content
+        if (markdownPart.length < 100 ||
+            markdownPart.startsWith("Loading...") ||
+            markdownPart.startsWith("Please enable JavaScript")) {
+            return null;
+        }
+        const title = extractHeadingTitle(markdownPart) ?? (new URL(url).pathname.split("/").pop() || url);
+        return { url, title, content: markdownPart, error: null };
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        if (message.toLowerCase().includes("abort")) {
+            activityMonitor.logComplete(activityId, 0);
+        }
+        else {
+            activityMonitor.logError(activityId, message);
+        }
+        return null;
+    }
+}
+function parseTimestamp(ts) {
+    const num = Number(ts);
+    if (!isNaN(num) && num >= 0)
+        return Math.floor(num);
+    const parts = ts.split(":").map(Number);
+    if (parts.some(p => isNaN(p) || p < 0))
+        return null;
+    if (parts.length === 3)
+        return Math.floor(parts[0] * 3600 + parts[1] * 60 + parts[2]);
+    if (parts.length === 2)
+        return Math.floor(parts[0] * 60 + parts[1]);
+    return null;
+}
+function parseTimestampSpec(ts) {
+    const dashIdx = ts.indexOf("-", 1);
+    if (dashIdx > 0) {
+        const start = parseTimestamp(ts.slice(0, dashIdx));
+        const end = parseTimestamp(ts.slice(dashIdx + 1));
+        if (start !== null && end !== null && end > start)
+            return { type: "range", start, end };
+    }
+    const seconds = parseTimestamp(ts);
+    return seconds !== null ? { type: "single", seconds } : null;
+}
+const DEFAULT_RANGE_FRAMES = 6;
+const MIN_FRAME_INTERVAL = 5;
+function computeRangeTimestamps(start, end, maxFrames = DEFAULT_RANGE_FRAMES) {
+    if (maxFrames <= 1)
+        return [start];
+    const duration = end - start;
+    const idealInterval = duration / (maxFrames - 1);
+    if (idealInterval < MIN_FRAME_INTERVAL) {
+        const timestamps = [];
+        for (let t = start; t <= end && timestamps.length < maxFrames; t += MIN_FRAME_INTERVAL) {
+            timestamps.push(t);
+        }
+        return timestamps;
+    }
+    return Array.from({ length: maxFrames }, (_, i) => Math.round(start + i * idealInterval));
+}
+function buildFrameResult(url, label, requestedCount, frames, error, duration) {
+    if (frames.length === 0) {
+        const msg = error ?? "Frame extraction failed";
+        return { url, title: `Frames ${label} (0/${requestedCount})`, content: msg, error: msg };
+    }
+    return {
+        url,
+        title: `Frames ${label} (${frames.length}/${requestedCount})`,
+        content: `${frames.length} frames extracted from ${label}`,
+        error: null,
+        frames,
+        ...(duration !== undefined ? { duration } : {}),
+    };
+}
+async function extractLocalFrames(filePath, timestamps) {
+    const results = await Promise.all(timestamps.map(async (t) => {
+        const frame = await extractVideoFrame(filePath, t);
+        if ("error" in frame)
+            return { error: frame.error };
+        return { ...frame, timestamp: formatSeconds(t) };
+    }));
+    const frames = results.filter((f) => "data" in f);
+    const firstError = results.find((f) => "error" in f);
+    return { frames, error: frames.length === 0 && firstError ? firstError.error : null };
+}
+function safeVideoInfo(url) {
+    try {
+        return { info: isVideoFile(url) };
+    }
+    catch (err) {
+        return { info: null, error: errorMessage(err) };
+    }
+}
+export async function extractContent(url, signal, options) {
+    if (signal?.aborted) {
+        return { url, title: "", content: "", error: "Aborted" };
+    }
+    let remoteUrl = null;
+    try {
+        const parsed = new URL(url);
+        if (parsed.protocol === "http:" || parsed.protocol === "https:")
+            remoteUrl = parsed;
+    }
+    catch {
+    }
+    if (remoteUrl) {
+        try {
+            const ssrf = loadSsrfConfig();
+            const domainPolicy = loadFetchContentDomainPolicy();
+            await validateRemoteUrl(remoteUrl, {
+                allowRanges: ssrf.allowRanges,
+                trustEnvProxy: ssrf.trustEnvProxy,
+                domainPolicy,
+                ...(options?.lookup ? { lookup: options.lookup } : {}),
+            });
+        }
+        catch (err) {
+            return { url, title: "", content: "", error: errorMessage(err) };
+        }
+    }
+    if (options?.mode === "raw") {
+        return extractViaHttp(url, signal, options);
+    }
+    if (options?.frames || options?.timestamp) {
+        const disabled = imageGateError();
+        if (disabled)
+            return { url, title: "", content: "", error: disabled };
+    }
+    if (options?.frames && !options.timestamp) {
+        const frameCount = options.frames;
+        const ytInfo = isYouTubeURL(url);
+        if (ytInfo.isYouTube && ytInfo.videoId) {
+            const streamInfo = await getYouTubeStreamInfo(ytInfo.videoId);
+            if ("error" in streamInfo) {
+                return { url, title: "Frames", content: streamInfo.error, error: streamInfo.error };
+            }
+            if (streamInfo.duration === null) {
+                const error = "Cannot determine video duration. Use a timestamp range instead.";
+                return { url, title: "Frames", content: error, error };
+            }
+            const dur = Math.floor(streamInfo.duration);
+            const timestamps = computeRangeTimestamps(0, dur, frameCount);
+            const result = await extractYouTubeFrames(ytInfo.videoId, timestamps, streamInfo);
+            const label = `${formatSeconds(0)}-${formatSeconds(dur)}`;
+            return buildFrameResult(url, label, timestamps.length, result.frames, result.error, streamInfo.duration);
+        }
+        const localVideo = safeVideoInfo(url);
+        if (localVideo.error) {
+            return { url, title: "", content: "", error: localVideo.error };
+        }
+        if (localVideo.info) {
+            const durationResult = await getLocalVideoDuration(localVideo.info.absolutePath);
+            if (typeof durationResult !== "number") {
+                return { url, title: "Frames", content: durationResult.error, error: durationResult.error };
+            }
+            const dur = Math.floor(durationResult);
+            const timestamps = computeRangeTimestamps(0, dur, frameCount);
+            const result = await extractLocalFrames(localVideo.info.absolutePath, timestamps);
+            const label = `${formatSeconds(0)}-${formatSeconds(dur)}`;
+            return buildFrameResult(url, label, timestamps.length, result.frames, result.error, durationResult);
+        }
+        return { url, title: "", content: "", error: "Frame extraction only works with YouTube and local video files" };
+    }
+    if (options?.timestamp) {
+        const spec = parseTimestampSpec(options.timestamp);
+        if (!spec) {
+            return {
+                url,
+                title: "",
+                content: "",
+                error: `Invalid timestamp format: "${options.timestamp}". Use "H:MM:SS", "MM:SS", "85", or "start-end".`,
+            };
+        }
+        const frameCount = options.frames;
+        const ytInfo = isYouTubeURL(url);
+        if (ytInfo.isYouTube && ytInfo.videoId) {
+            const streamInfo = await getYouTubeStreamInfo(ytInfo.videoId);
+            if ("error" in streamInfo) {
+                if (spec.type === "range") {
+                    const label = `${formatSeconds(spec.start)}-${formatSeconds(spec.end)}`;
+                    return { url, title: `Frames ${label}`, content: streamInfo.error, error: streamInfo.error };
+                }
+                if (frameCount) {
+                    const end = spec.seconds + (frameCount - 1) * MIN_FRAME_INTERVAL;
+                    const label = `${formatSeconds(spec.seconds)}-${formatSeconds(end)}`;
+                    return { url, title: `Frames ${label}`, content: streamInfo.error, error: streamInfo.error };
+                }
+                return { url, title: `Frame at ${options.timestamp}`, content: streamInfo.error, error: streamInfo.error };
+            }
+            if (spec.type === "range") {
+                const label = `${formatSeconds(spec.start)}-${formatSeconds(spec.end)}`;
+                if (streamInfo.duration !== null && spec.end > streamInfo.duration) {
+                    const error = `Timestamp ${formatSeconds(spec.end)} exceeds video duration (${formatSeconds(Math.floor(streamInfo.duration))})`;
+                    return { url, title: `Frames ${label}`, content: error, error };
+                }
+                const timestamps = frameCount
+                    ? computeRangeTimestamps(spec.start, spec.end, frameCount)
+                    : computeRangeTimestamps(spec.start, spec.end);
+                const result = await extractYouTubeFrames(ytInfo.videoId, timestamps, streamInfo);
+                return buildFrameResult(url, label, timestamps.length, result.frames, result.error, result.duration ?? undefined);
+            }
+            if (frameCount) {
+                const end = spec.seconds + (frameCount - 1) * MIN_FRAME_INTERVAL;
+                const label = `${formatSeconds(spec.seconds)}-${formatSeconds(end)}`;
+                if (streamInfo.duration !== null && end > streamInfo.duration) {
+                    const error = `Timestamp ${formatSeconds(end)} exceeds video duration (${formatSeconds(Math.floor(streamInfo.duration))})`;
+                    return { url, title: `Frames ${label}`, content: error, error };
+                }
+                const timestamps = computeRangeTimestamps(spec.seconds, end, frameCount);
+                const result = await extractYouTubeFrames(ytInfo.videoId, timestamps, streamInfo);
+                return buildFrameResult(url, label, timestamps.length, result.frames, result.error, result.duration ?? undefined);
+            }
+            if (streamInfo.duration !== null && spec.seconds > streamInfo.duration) {
+                const error = `Timestamp ${formatSeconds(spec.seconds)} exceeds video duration (${formatSeconds(Math.floor(streamInfo.duration))})`;
+                return { url, title: `Frame at ${options.timestamp}`, content: error, error };
+            }
+            const frame = await extractYouTubeFrame(ytInfo.videoId, spec.seconds, streamInfo);
+            if ("error" in frame) {
+                return { url, title: `Frame at ${options.timestamp}`, content: frame.error, error: frame.error };
+            }
+            return { url, title: `Frame at ${options.timestamp}`, content: `Video frame at ${options.timestamp}`, error: null, thumbnail: frame };
+        }
+        const localVideo = safeVideoInfo(url);
+        if (localVideo.error) {
+            return { url, title: "", content: "", error: localVideo.error };
+        }
+        if (localVideo.info) {
+            if (spec.type === "range") {
+                const timestamps = frameCount
+                    ? computeRangeTimestamps(spec.start, spec.end, frameCount)
+                    : computeRangeTimestamps(spec.start, spec.end);
+                const result = await extractLocalFrames(localVideo.info.absolutePath, timestamps);
+                const label = `${formatSeconds(spec.start)}-${formatSeconds(spec.end)}`;
+                return buildFrameResult(url, label, timestamps.length, result.frames, result.error);
+            }
+            if (frameCount) {
+                const end = spec.seconds + (frameCount - 1) * MIN_FRAME_INTERVAL;
+                const timestamps = computeRangeTimestamps(spec.seconds, end, frameCount);
+                const result = await extractLocalFrames(localVideo.info.absolutePath, timestamps);
+                const label = `${formatSeconds(spec.seconds)}-${formatSeconds(end)}`;
+                return buildFrameResult(url, label, timestamps.length, result.frames, result.error);
+            }
+            const frame = await extractVideoFrame(localVideo.info.absolutePath, spec.seconds);
+            if ("error" in frame) {
+                return { url, title: `Frame at ${options.timestamp}`, content: frame.error, error: frame.error };
+            }
+            return { url, title: `Frame at ${options.timestamp}`, content: `Video frame at ${options.timestamp}`, error: null, thumbnail: frame };
+        }
+        return { url, title: "", content: "", error: "Timestamp extraction only works with YouTube and local video files" };
+    }
+    const localVideo = safeVideoInfo(url);
+    if (localVideo.error) {
+        return { url, title: "", content: "", error: localVideo.error };
+    }
+    if (localVideo.info) {
+        try {
+            const result = await extractVideo(localVideo.info, signal, options);
+            if (signal?.aborted)
+                return abortedResult(url);
+            return result ?? { url, title: "", content: "", error: `Video analysis requires Gemini access. Either:\n  1. Sign into gemini.google.com in Chrome (free, uses cookies)\n  2. Set GEMINI_API_KEY in ${WEB_SEARCH_CONFIG_PATH}` };
+        }
+        catch (err) {
+            if (isAbortError(err))
+                return abortedResult(url);
+            return { url, title: "", content: "", error: errorMessage(err) };
+        }
+    }
+    try {
+        if (!remoteUrl)
+            new URL(url);
+    }
+    catch (err) {
+        return { url, title: "", content: "", error: errorMessage(err) };
+    }
+    try {
+        const ghResult = await extractGitHub(url, signal, options?.forceClone);
+        if (ghResult)
+            return ghResult;
+        if (signal?.aborted)
+            return abortedResult(url);
+    }
+    catch (err) {
+        const message = errorMessage(err);
+        if (isAbortError(err))
+            return abortedResult(url);
+        if (isConfigParseError(err)) {
+            return { url, title: "", content: "", error: message };
+        }
+    }
+    const ytInfo = isYouTubeURL(url);
+    let youtubeEnabled = false;
+    try {
+        youtubeEnabled = isYouTubeEnabled();
+    }
+    catch (err) {
+        return { url, title: "", content: "", error: errorMessage(err) };
+    }
+    if (ytInfo.isYouTube && youtubeEnabled) {
+        try {
+            const ytResult = await extractYouTube(url, signal, options?.prompt, options?.model);
+            if (ytResult)
+                return ytResult;
+            if (signal?.aborted)
+                return abortedResult(url);
+        }
+        catch (err) {
+            const message = errorMessage(err);
+            if (isAbortError(err))
+                return abortedResult(url);
+            return { url, title: "", content: "", error: message };
+        }
+        return {
+            url,
+            title: "",
+            content: "",
+            error: "Could not extract YouTube video content. Sign into Google in Chrome for automatic access, or set GEMINI_API_KEY.",
+        };
+    }
+    if (signal?.aborted)
+        return abortedResult(url);
+    let fetchRouting;
+    try {
+        fetchRouting = loadFetchRouting();
+    }
+    catch (err) {
+        return { url, title: "", content: "", error: errorMessage(err) };
+    }
+    const providerOrder = remoteUrl && !fetchRouting.allowRemoteHostedProviders
+        ? fetchRouting.providers.filter(provider => !REMOTE_HOSTED_FETCH_PROVIDERS.has(provider))
+        : fetchRouting.providers;
+    if (providerOrder.length === 0) {
+        return {
+            url,
+            title: "",
+            content: "",
+            error: "Remote hosted fetch providers are disabled unless fetchRouting.allowRemoteHostedProviders is true",
+        };
+    }
+    let httpResult = null;
+    let declaredLinks = [];
+    const withDeclaredLinks = (result) => ({
+        ...result,
+        content: appendDeclaredWebLinks(result.content, declaredLinks),
+    });
+    const parseErrorResult = (message) => httpResult
+        ? { ...httpResult, error: message }
+        : { url, title: "", content: "", error: message };
+    const runHttpProvider = async () => {
+        const { declaredLinks: discoveredLinks = [], ...result } = await extractViaHttp(url, signal, options);
+        httpResult = result;
+        declaredLinks = discoveredLinks;
+        if (signal?.aborted)
+            return abortedResult(url);
+        if (!httpResult.error)
+            return httpResult;
+        if (NON_RECOVERABLE_ERRORS.some(prefix => httpResult.error.startsWith(prefix)) || isRedirectPolicyError(httpResult.error) || isConfigParseError(httpResult.error)) {
+            return httpResult;
+        }
+        return null;
+    };
+    let firecrawlError = null;
+    let tinyfishError = null;
+    let search1apiError = null;
+    let queritError = null;
+    let kagiError = null;
+    let ollamaError = null;
+    let parallelError = null;
+    let brightdataError = null;
+    if (remoteUrl && providerOrder[0] !== "http") {
+        const httpGateResult = await runHttpProvider();
+        if (httpGateResult)
+            return httpGateResult;
+    }
+    for (const provider of providerOrder) {
+        if (signal?.aborted)
+            return abortedResult(url);
+        if (provider === "http") {
+            const result = await runHttpProvider();
+            if (result)
+                return result;
+            continue;
+        }
+        if (provider === "firecrawl") {
+            try {
+                if (isFirecrawlAvailable()) {
+                    const ssrf = loadSsrfConfig();
+                    const firecrawlResult = await extractWithFirecrawl(url, signal, {
+                        timeoutMs: options?.timeoutMs,
+                        ...(options?.lookup ? { lookup: options.lookup } : {}),
+                        ssrf,
+                    });
+                    if (firecrawlResult)
+                        return withDeclaredLinks(firecrawlResult);
+                }
+            }
+            catch (err) {
+                if (isAbortError(err))
+                    return abortedResult(url);
+                firecrawlError = errorMessage(err);
+                if (isConfigParseError(err))
+                    return parseErrorResult(firecrawlError);
+            }
+            continue;
+        }
+        if (provider === "jina") {
+            const jinaResult = await extractWithJinaReader(url, signal, options?.lookup);
+            if (jinaResult)
+                return withDeclaredLinks(jinaResult);
+            continue;
+        }
+        if (provider === "tinyfish") {
+            try {
+                if (isTinyFishAvailable()) {
+                    const tinyfishResult = await extractWithTinyFish(url, signal, options);
+                    if (tinyfishResult)
+                        return withDeclaredLinks(tinyfishResult);
+                }
+            }
+            catch (err) {
+                if (isAbortError(err))
+                    return abortedResult(url);
+                tinyfishError = errorMessage(err);
+                if (isConfigParseError(err))
+                    return parseErrorResult(tinyfishError);
+            }
+            continue;
+        }
+        if (provider === "search1api") {
+            try {
+                if (isSearch1APIAvailable()) {
+                    const search1apiResult = await extractWithSearch1API(url, signal, options);
+                    if (search1apiResult)
+                        return withDeclaredLinks(search1apiResult);
+                }
+            }
+            catch (err) {
+                if (isAbortError(err))
+                    return abortedResult(url);
+                search1apiError = errorMessage(err);
+                if (isConfigParseError(err))
+                    return parseErrorResult(search1apiError);
+            }
+            continue;
+        }
+        if (provider === "querit") {
+            try {
+                if (isQueritAvailable()) {
+                    const queritResult = await extractWithQuerit(url, signal, options);
+                    if (queritResult)
+                        return withDeclaredLinks(queritResult);
+                }
+            }
+            catch (err) {
+                if (isAbortError(err))
+                    return abortedResult(url);
+                queritError = errorMessage(err);
+                if (isConfigParseError(err))
+                    return parseErrorResult(queritError);
+            }
+            continue;
+        }
+        if (provider === "kagi") {
+            try {
+                if (isKagiExtractAvailable()) {
+                    const ssrf = loadSsrfConfig();
+                    const kagiResult = await extractWithKagi(url, signal, {
+                        timeoutMs: options?.timeoutMs,
+                        ...(options?.lookup ? { lookup: options.lookup } : {}),
+                        ssrf,
+                    });
+                    if (kagiResult)
+                        return withDeclaredLinks(kagiResult);
+                }
+            }
+            catch (err) {
+                if (isAbortError(err))
+                    return abortedResult(url);
+                kagiError = errorMessage(err);
+                if (isConfigParseError(err))
+                    return parseErrorResult(kagiError);
+            }
+            continue;
+        }
+        if (provider === "ollama") {
+            try {
+                if (isOllamaFetchAvailable()) {
+                    const ssrf = loadSsrfConfig();
+                    const ollamaResult = await extractWithOllama(url, signal, {
+                        timeoutMs: options?.timeoutMs,
+                        ...(options?.lookup ? { lookup: options.lookup } : {}),
+                        ssrf,
+                    });
+                    if (ollamaResult)
+                        return withDeclaredLinks(ollamaResult);
+                }
+            }
+            catch (err) {
+                if (isAbortError(err))
+                    return abortedResult(url);
+                ollamaError = errorMessage(err);
+                if (isConfigParseError(err))
+                    return parseErrorResult(ollamaError);
+            }
+            continue;
+        }
+        if (provider === "parallel") {
+            try {
+                if (isParallelAvailable()) {
+                    const parallelResult = await extractWithParallel(url, signal, options);
+                    if (parallelResult)
+                        return withDeclaredLinks(parallelResult);
+                }
+            }
+            catch (err) {
+                if (isAbortError(err))
+                    return abortedResult(url);
+                parallelError = errorMessage(err);
+                if (isConfigParseError(err))
+                    return parseErrorResult(parallelError);
+            }
+            continue;
+        }
+        if (provider === "brightdata") {
+            try {
+                if (isBrightDataUnlockerAvailable()) {
+                    const ssrf = loadSsrfConfig();
+                    const brightdataResult = await extractWithBrightDataUnlocker(url, signal, {
+                        timeoutMs: options?.timeoutMs,
+                        ...(options?.lookup ? { lookup: options.lookup } : {}),
+                        ssrf,
+                    });
+                    if (brightdataResult)
+                        return withDeclaredLinks(brightdataResult);
+                }
+            }
+            catch (err) {
+                if (isAbortError(err))
+                    return abortedResult(url);
+                brightdataError = errorMessage(err);
+                if (isConfigParseError(err))
+                    return parseErrorResult(brightdataError);
+            }
+            continue;
+        }
+        if (provider === "gemini") {
+            let geminiResult = null;
+            try {
+                geminiResult = await extractWithUrlContext(url, signal)
+                    ?? await extractWithGeminiWeb(url, signal);
+            }
+            catch (err) {
+                if (isAbortError(err))
+                    return abortedResult(url);
+                if (err instanceof CredentialResolutionError || isConfigParseError(err)) {
+                    return parseErrorResult(errorMessage(err));
+                }
+            }
+            if (geminiResult)
+                return withDeclaredLinks(geminiResult);
+        }
+    }
+    if (signal?.aborted)
+        return abortedResult(url);
+    const finalHttpResult = httpResult;
+    if (finalHttpResult && declaredLinks.length > 0)
+        return { ...finalHttpResult, error: null };
+    const guidance = [
+        finalHttpResult?.error ?? "No fetch_content provider returned content",
+        ...(firecrawlError ? [`Firecrawl fallback failed: ${firecrawlError}`] : []),
+        ...(tinyfishError ? [`TinyFish fallback failed: ${tinyfishError}`] : []),
+        ...(search1apiError ? [`Search1API fallback failed: ${search1apiError}`] : []),
+        ...(queritError ? [`Querit fallback failed: ${queritError}`] : []),
+        ...(kagiError ? [`Kagi fallback failed: ${kagiError}`] : []),
+        ...(ollamaError ? [`Ollama fallback failed: ${ollamaError}`] : []),
+        ...(parallelError ? [`Parallel fallback failed: ${parallelError}`] : []),
+        ...(brightdataError ? [`Bright Data fallback failed: ${brightdataError}`] : []),
+        "",
+        "Fallback options:",
+        `  • Set firecrawlBaseUrl in ${WEB_SEARCH_CONFIG_PATH} to a self-hosted Firecrawl instance`,
+        `  • Set tinyfishApiKey in ${WEB_SEARCH_CONFIG_PATH} or TINYFISH_API_KEY`,
+        `  • Set search1apiApiKey in ${WEB_SEARCH_CONFIG_PATH} or SEARCH1API_KEY`,
+        `  • Set queritApiKey in ${WEB_SEARCH_CONFIG_PATH} or QUERIT_API_KEY`,
+        `  • Set kagiApiKey in ${WEB_SEARCH_CONFIG_PATH} or KAGI_API_KEY`,
+        `  • Set ollamaApiKey in ${WEB_SEARCH_CONFIG_PATH} or OLLAMA_API_KEY`,
+        `  • Set parallelApiKey in ${WEB_SEARCH_CONFIG_PATH} or PARALLEL_API_KEY`,
+        `  • Set brightdataApiKey and brightdataUnlockerZone in ${WEB_SEARCH_CONFIG_PATH} or BRIGHTDATA_API_KEY and BRIGHTDATA_UNLOCKER_ZONE`,
+        `  • Set GEMINI_API_KEY in ${WEB_SEARCH_CONFIG_PATH}`,
+        "  • Sign into gemini.google.com in Chrome",
+        "  • Use web_search to find content about this topic",
+    ].join("\n");
+    return { ...(finalHttpResult ?? { url, title: "", content: "", error: null }), error: guidance };
+}
+function isLikelyJSRendered(html) {
+    // Extract body content
+    const bodyMatch = html.match(/<body[^>]*>([\s\S]*?)<\/body>/i);
+    if (!bodyMatch)
+        return false;
+    const bodyHtml = bodyMatch[1];
+    // Strip tags to get text content
+    const textContent = bodyHtml
+        .replace(/<script[\s\S]*?<\/script>/gi, "")
+        .replace(/<style[\s\S]*?<\/style>/gi, "")
+        .replace(/<[^>]+>/g, "")
+        .replace(/\s+/g, " ")
+        .trim();
+    // Count scripts
+    const scriptCount = (html.match(/<script/gi) || []).length;
+    // Heuristic: little text content but many scripts suggests JS rendering
+    return textContent.length < 500 && scriptCount > 3;
+}
+export async function readPDFResponseBuffer(response, maxSizeMB) {
+    const maxBytes = maxSizeMB * 1024 * 1024;
+    return readResponseBufferWithLimit(response, maxBytes, () => pdfSizeLimitError(maxSizeMB));
+}
+async function readTextResponseWithLimit(response, maxBytes) {
+    const buffer = await readResponseBufferWithLimit(response, maxBytes, () => responseSizeLimitError(maxBytes));
+    const charset = response.headers.get("content-type")?.match(/charset\s*=\s*["']?([^;"'\s]+)/i)?.[1];
+    try {
+        return new TextDecoder(charset || "utf-8").decode(buffer);
+    }
+    catch {
+        return new TextDecoder("utf-8").decode(buffer);
+    }
+}
+function isTextContentType(contentType) {
+    const mimeType = contentType.split(";", 1)[0]?.trim().toLowerCase() ?? "";
+    return mimeType.startsWith("text/") ||
+        mimeType === "application/json" ||
+        mimeType === "application/ld+json" ||
+        mimeType === "application/xml" ||
+        mimeType === "application/xhtml+xml" ||
+        mimeType === "application/javascript" ||
+        mimeType === "application/x-javascript" ||
+        mimeType.endsWith("+json") ||
+        mimeType.endsWith("+xml");
+}
+async function readResponseBufferWithLimit(response, maxBytes, buildError) {
+    const reader = response.body?.getReader();
+    if (!reader) {
+        const buffer = await response.arrayBuffer();
+        if (buffer.byteLength > maxBytes)
+            throw buildError();
+        return buffer;
+    }
+    const chunks = [];
+    let totalBytes = 0;
+    try {
+        while (true) {
+            const { done, value } = await reader.read();
+            if (done)
+                break;
+            if (!value)
+                continue;
+            totalBytes += value.byteLength;
+            if (totalBytes > maxBytes) {
+                await reader.cancel();
+                throw buildError();
+            }
+            chunks.push(value);
+        }
+    }
+    finally {
+        reader.releaseLock();
+    }
+    const combined = new Uint8Array(totalBytes);
+    let offset = 0;
+    for (const chunk of chunks) {
+        combined.set(chunk, offset);
+        offset += chunk.byteLength;
+    }
+    return combined.buffer;
+}
+function pdfSizeLimitError(maxSizeMB) {
+    return new Error(`PDF exceeds configured pdf.maxSizeMB limit (${maxSizeMB} MB)`);
+}
+function responseSizeLimitError(maxBytes) {
+    return new Error(`Response too large (${Math.round(maxBytes / 1024 / 1024)}MB)`);
+}
+async function extractViaHttp(url, signal, options) {
+    const timeoutMs = options?.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+    const activityId = activityMonitor.logStart({ type: "fetch", url });
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
+    const onAbort = () => controller.abort();
+    signal?.addEventListener("abort", onAbort);
+    try {
+        const ssrf = loadSsrfConfig();
+        const domainPolicy = loadFetchContentDomainPolicy();
+        const response = await fetchRemoteUrl(url, {
+            signal: controller.signal,
+            headers: {
+                "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36",
+                "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
+                "Accept-Language": "en-US,en;q=0.9",
+                "Cache-Control": "no-cache",
+                "Sec-Fetch-Dest": "document",
+                "Sec-Fetch-Mode": "navigate",
+                "Sec-Fetch-Site": "none",
+                "Sec-Fetch-User": "?1",
+                "Upgrade-Insecure-Requests": "1",
+            },
+        }, {
+            allowRanges: ssrf.allowRanges,
+            trustEnvProxy: ssrf.trustEnvProxy,
+            domainPolicy,
+            ...(options?.lookup ? { lookup: options.lookup } : {}),
+        });
+        if (!response.ok && options?.mode !== "raw") {
+            activityMonitor.logComplete(activityId, response.status);
+            return {
+                url,
+                title: "",
+                content: "",
+                error: `HTTP ${response.status}: ${response.statusText}`,
+                status: response.status,
+            };
+        }
+        const contentLengthHeader = response.headers.get("content-length");
+        const contentType = response.headers.get("content-type") || "";
+        const mimeType = contentType.split(";", 1)[0]?.trim().toLowerCase() ?? "";
+        const isPDFContent = isPDF(url, contentType);
+        const pdfConfig = isPDFContent ? loadPDFConfig() : null;
+        if (isPDFContent && pdfConfig && !pdfConfig.enabled) {
+            activityMonitor.logComplete(activityId, response.status);
+            return { url, title: "", content: "", error: "PDF extraction is disabled by pdf.enabled", mimeType, status: response.status };
+        }
+        const maxResponseSize = (pdfConfig?.maxSizeMB ?? 5) * 1024 * 1024;
+        if (contentLengthHeader) {
+            const contentLength = Number.parseInt(contentLengthHeader, 10);
+            if (Number.isFinite(contentLength) && contentLength > maxResponseSize) {
+                activityMonitor.logComplete(activityId, response.status);
+                return {
+                    url,
+                    title: "",
+                    content: "",
+                    error: pdfConfig
+                        ? pdfSizeLimitError(pdfConfig.maxSizeMB).message
+                        : `Response too large (${Math.round(contentLength / 1024 / 1024)}MB)`,
+                };
+            }
+        }
+        if (options?.mode === "raw") {
+            if (!isTextContentType(contentType)) {
+                activityMonitor.logComplete(activityId, response.status);
+                return { url, title: "", content: "", error: `Unsupported content type in raw mode: ${mimeType || "missing"}`, mimeType, status: response.status };
+            }
+            const text = await readTextResponseWithLimit(response, maxResponseSize);
+            activityMonitor.logComplete(activityId, response.status);
+            return { url, title: extractTextTitle(text, url), content: text, error: null, mimeType, status: response.status };
+        }
+        if (SUPPORTED_IMAGE_TYPES.has(mimeType)) {
+            const disabled = imageGateError();
+            if (disabled) {
+                activityMonitor.logComplete(activityId, response.status);
+                return { url, title: "", content: "", error: disabled, mimeType, status: response.status };
+            }
+            try {
+                const buffer = await readResponseBufferWithLimit(response, maxResponseSize, () => responseSizeLimitError(maxResponseSize));
+                const resized = await resizeImage(new Uint8Array(buffer), mimeType, { maxWidth: 2000, maxHeight: 2000 });
+                activityMonitor.logComplete(activityId, response.status);
+                if (!resized)
+                    return { url, title: "", content: "", error: `Could not decode image: ${mimeType}`, mimeType, status: response.status };
+                const title = new URL(response.url || url).pathname.split("/").pop() || url;
+                return {
+                    url,
+                    title,
+                    content: `Image fetched (${resized.width}×${resized.height}, ${resized.mimeType})`,
+                    error: null,
+                    thumbnail: { data: resized.data, mimeType: resized.mimeType },
+                    mimeType: resized.mimeType,
+                    status: response.status,
+                };
+            }
+            catch (err) {
+                const message = errorMessage(err);
+                activityMonitor.logError(activityId, message);
+                return { url, title: "", content: "", error: message, mimeType, status: response.status };
+            }
+        }
+        if (isPDFContent && pdfConfig) {
+            try {
+                const buffer = await readPDFResponseBuffer(response, pdfConfig.maxSizeMB);
+                if (signal?.aborted)
+                    return abortedResult(url);
+                const result = await extractPDFToMarkdown(buffer, url, { signal });
+                activityMonitor.logComplete(activityId, response.status);
+                return {
+                    url,
+                    title: result.title,
+                    content: `PDF extracted and saved to: ${result.outputPath}\n\nPages: ${result.pages}\nCharacters: ${result.chars}`,
+                    error: null,
+                };
+            }
+            catch (err) {
+                const message = err instanceof Error ? err.message : String(err);
+                activityMonitor.logError(activityId, message);
+                if (message.startsWith("PDF exceeds configured pdf.maxSizeMB limit")) {
+                    return { url, title: "", content: "", error: message };
+                }
+                if (err instanceof CredentialResolutionError || isConfigParseError(err)) {
+                    return { url, title: "", content: "", error: message };
+                }
+                return { url, title: "", content: "", error: `PDF extraction failed: ${message}` };
+            }
+        }
+        if (contentType.includes("application/octet-stream") ||
+            contentType.includes("image/") ||
+            contentType.includes("audio/") ||
+            contentType.includes("video/") ||
+            contentType.includes("application/zip")) {
+            activityMonitor.logComplete(activityId, response.status);
+            return {
+                url,
+                title: "",
+                content: "",
+                error: `Unsupported content type: ${contentType.split(";")[0]}`,
+            };
+        }
+        const text = await readTextResponseWithLimit(response, maxResponseSize);
+        const isHTML = contentType.includes("text/html") || contentType.includes("application/xhtml+xml");
+        if (!isHTML) {
+            activityMonitor.logComplete(activityId, response.status);
+            const title = extractTextTitle(text, url);
+            return { url, title, content: text, error: null };
+        }
+        const { document } = parseHTML(text);
+        const documentTitle = document.title?.trim() ?? "";
+        const declaredLinks = discoverDeclaredWebLinks(document, response.headers.get("link"), response.url || url);
+        const reader = new Readability(document);
+        const article = reader.parse();
+        if (!article) {
+            const rscResult = extractRSCContent(text);
+            if (rscResult) {
+                activityMonitor.logComplete(activityId, response.status);
+                return {
+                    url,
+                    title: rscResult.title,
+                    content: appendDeclaredWebLinks(rscResult.content, declaredLinks),
+                    error: null,
+                    declaredLinks,
+                };
+            }
+            activityMonitor.logComplete(activityId, response.status);
+            const jsRendered = isLikelyJSRendered(text);
+            const errorMsg = jsRendered
+                ? "Page appears to be JavaScript-rendered (content loads dynamically)"
+                : "Could not extract readable content from HTML structure";
+            return {
+                url,
+                title: documentTitle,
+                content: appendDeclaredWebLinks("", declaredLinks),
+                error: errorMsg,
+                declaredLinks,
+            };
+        }
+        if (typeof article.content !== "string") {
+            throw new Error("Readability returned invalid article content");
+        }
+        const markdown = turndown.turndown(article.content);
+        activityMonitor.logComplete(activityId, response.status);
+        if (markdown.length < MIN_USEFUL_CONTENT) {
+            return {
+                url,
+                title: article.title || documentTitle,
+                content: appendDeclaredWebLinks(markdown, declaredLinks),
+                error: isLikelyJSRendered(text)
+                    ? "Page appears to be JavaScript-rendered (content loads dynamically)"
+                    : "Extracted content appears incomplete",
+                declaredLinks,
+            };
+        }
+        return {
+            url,
+            title: article.title || documentTitle,
+            content: appendDeclaredWebLinks(markdown, declaredLinks),
+            error: null,
+            declaredLinks,
+        };
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        if (message.toLowerCase().includes("abort")) {
+            activityMonitor.logComplete(activityId, 0);
+        }
+        else {
+            activityMonitor.logError(activityId, message);
+        }
+        return { url, title: "", content: "", error: message };
+    }
+    finally {
+        clearTimeout(timeoutId);
+        signal?.removeEventListener("abort", onAbort);
+    }
+}
+export function extractHeadingTitle(text) {
+    const match = text.match(/^#{1,2}\s+(.+)/m);
+    if (!match)
+        return null;
+    const cleaned = match[1].replace(/\*+/g, "").trim();
+    return cleaned || null;
+}
+function extractTextTitle(text, url) {
+    return extractHeadingTitle(text) ?? (new URL(url).pathname.split("/").pop() || url);
+}
+export async function fetchAllContent(urls, signal, options) {
+    return Promise.all(urls.map((url) => fetchLimit(() => extractContent(url, signal, options))));
+}
+//# sourceMappingURL=extract.js.map