@humain/terminal 0.0.2 → 0.0.3-beta.0

package/dist/humain/web-access/vendor/extract.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"extract.d.ts","sourceRoot":"","sources":["../../../../src/humain/web-access/vendor/extract.ts"],"names":[],"mappings":"AAuBA,OAAO,EAAmF,KAAK,MAAM,EAAE,MAAM,sBAAsB,CAAC;AAiBpI,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAEtD,wBAAgB,mBAAmB,IAAI,MAAM,EAAE,CAE9C;AAiGD,MAAM,WAAW,UAAU;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,MAAM,SAAS,GAAG;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,CAAC;AAC3D,MAAM,MAAM,WAAW,GAAG,SAAS,GAAG;IAAE,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC;AAExD,MAAM,WAAW,gBAAgB;IAChC,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,SAAS,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC;IAC/C,MAAM,CAAC,EAAE,UAAU,EAAE,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;CAChB;AAID,MAAM,WAAW,cAAc;IAC9B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,UAAU,GAAG,KAAK,GAAG,QAAQ,CAAC;IACrC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,2EAA2E;IAC3E,MAAM,CAAC,EAAE,MAAM,CAAC;CAChB;AAoJD,wBAAsB,cAAc,CACnC,GAAG,EAAE,MAAM,EACX,MAAM,CAAC,EAAE,WAAW,EACpB,OAAO,CAAC,EAAE,cAAc,GACtB,OAAO,CAAC,gBAAgB,CAAC,CAyd3B;AAwBD,wBAAsB,qBAAqB,CAAC,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CAGvG;AAwTD,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAK/D;AAMD,wBAAsB,eAAe,CACpC,IAAI,EAAE,MAAM,EAAE,EACd,MAAM,CAAC,EAAE,WAAW,EACpB,OAAO,CAAC,EAAE,cAAc,GACtB,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAE7B","sourcesContent":["import { existsSync, readFileSync } from \"node:fs\";\nimport { Readability } from \"@mozilla/readability\";\nimport { resizeImage } from \"../host.ts\";\nimport { parseHTML } from \"linkedom\";\nimport TurndownService from \"turndown\";\nimport pLimit from \"p-limit\";\nimport { activityMonitor } from \"./activity.ts\";\nimport { extractRSCContent } from \"./rsc-extract.ts\";\nimport { extractPDFToMarkdown, isPDF, loadPDFConfig } from \"./pdf-extract.ts\";\nimport { extractGitHub } from \"./github-extract.ts\";\nimport { isYouTubeURL, isYouTubeEnabled, extractYouTube, extractYouTubeFrame, extractYouTubeFrames, getYouTubeStreamInfo } from \"./youtube-extract.ts\";\nimport { CredentialResolutionError } from \"./credential-source.ts\";\nimport { extractWithUrlContext, extractWithGeminiWeb } from \"./gemini-url-context.ts\";\nimport { extractWithParallel, isParallelAvailable } from \"./parallel.ts\";\nimport { extractWithTinyFish, isTinyFishAvailable } from \"./tinyfish.ts\";\nimport { extractWithSearch1API, isSearch1APIAvailable } from \"./search1api.ts\";\nimport { extractWithQuerit, isQueritAvailable } from \"./querit.ts\";\nimport { extractWithKagi, isKagiExtractAvailable } from \"./kagi.ts\";\nimport { extractWithOllama, isOllamaFetchAvailable } from \"./ollama.ts\";\nimport { extractWithFirecrawl, isFirecrawlAvailable } from \"./firecrawl.ts\";\nimport { extractWithBrightDataUnlocker, isBrightDataUnlockerAvailable } from \"./brightdata-unlocker.ts\";\nimport { isVideoFile, extractVideo, extractVideoFrame, getLocalVideoDuration } from \"./video-extract.ts\";\nimport { appendDeclaredWebLinks, discoverDeclaredWebLinks, type DeclaredWebLink } from \"./declared-web-links.ts\";\nimport { fetchRemoteUrl, loadFetchContentDomainPolicy, loadSsrfConfig, validateRemoteUrl, type Lookup } from \"./ssrf-protection.ts\";\nimport { formatSeconds, getWebSearchConfigPath } from \"./utils.ts\";\nimport { isImageEnabled } from \"./feature-config.ts\";\n\nconst DEFAULT_TIMEOUT_MS = 30000;\nconst CONCURRENT_LIMIT = 3;\n\nconst NON_RECOVERABLE_ERRORS = [\"Unsupported content type\", \"Response too large\", \"PDF extraction is disabled\", \"Image fetching is disabled\"];\nconst MIN_USEFUL_CONTENT = 500;\nconst SUPPORTED_IMAGE_TYPES = new Set([\"image/png\", \"image/jpeg\", \"image/webp\", \"image/gif\"]);\nconst WEB_SEARCH_CONFIG_PATH = getWebSearchConfigPath();\nconst FETCH_PROVIDERS = [\"http\", \"firecrawl\", \"jina\", \"tinyfish\", \"search1api\", \"querit\", \"kagi\", \"ollama\", \"parallel\", \"brightdata\", \"gemini\"] as const;\ntype FetchProvider = typeof FETCH_PROVIDERS[number];\ntype FetchRouting = { providers: FetchProvider[]; allowRemoteHostedProviders: boolean };\nconst DEFAULT_FETCH_PROVIDER_ORDER: FetchProvider[] = [\"http\", \"firecrawl\", \"jina\", \"tinyfish\", \"search1api\", \"querit\", \"kagi\", \"ollama\", \"parallel\", \"brightdata\", \"gemini\"];\nconst REMOTE_HOSTED_FETCH_PROVIDERS = new Set<FetchProvider>([\"jina\", \"tinyfish\", \"search1api\", \"querit\", \"kagi\", \"ollama\", \"parallel\", \"brightdata\", \"gemini\"]);\n\nexport { loadSsrfConfig } from \"./ssrf-protection.ts\";\n\nexport function loadSsrfAllowRanges(): string[] {\n\treturn loadSsrfConfig().allowRanges;\n}\n\nfunction errorMessage(err: unknown): string {\n\treturn err instanceof Error ? err.message : String(err);\n}\n\nfunction isConfigParseError(err: unknown): boolean {\n\treturn errorMessage(err).startsWith(\"Failed to parse \");\n}\n\nfunction isAbortError(err: unknown): boolean {\n\treturn errorMessage(err).toLowerCase().includes(\"abort\");\n}\n\nfunction isRedirectPolicyError(message: string): boolean {\n\treturn message.startsWith(\"Blocked internal \") ||\n\t\tmessage.startsWith(\"Blocked hostname by fetch_content domain policy\") ||\n\t\tmessage.startsWith(\"Hostname not allowed by fetch_content domain policy\") ||\n\t\tmessage.startsWith(\"Too many redirects fetching \") ||\n\t\tmessage === \"Only HTTP and HTTPS URLs can be fetched remotely\" ||\n\t\tmessage === \"URL must include a hostname\" ||\n\t\tmessage.startsWith(\"Failed to resolve \");\n}\n\nfunction imageGateError(): string | null {\n\ttry {\n\t\treturn isImageEnabled() ? null : \"Image fetching is disabled by image.enabled\";\n\t} catch (err) {\n\t\treturn errorMessage(err);\n\t}\n}\n\nfunction loadFetchRouting(): FetchRouting {\n\tif (!existsSync(WEB_SEARCH_CONFIG_PATH)) {\n\t\treturn { providers: DEFAULT_FETCH_PROVIDER_ORDER, allowRemoteHostedProviders: false };\n\t}\n\n\tlet raw: Record<string, unknown>;\n\ttry {\n\t\tconst parsed: unknown = JSON.parse(readFileSync(WEB_SEARCH_CONFIG_PATH, \"utf-8\"));\n\t\tif (!parsed || typeof parsed !== \"object\" || Array.isArray(parsed)) {\n\t\t\tthrow new Error(\"expected a JSON object\");\n\t\t}\n\t\traw = parsed as Record<string, unknown>;\n\t} catch (err) {\n\t\tconst message = err instanceof Error ? err.message : String(err);\n\t\tthrow new Error(`Failed to parse ${WEB_SEARCH_CONFIG_PATH}: ${message}`);\n\t}\n\n\tif (!Object.hasOwn(raw, \"fetchRouting\")) {\n\t\treturn { providers: DEFAULT_FETCH_PROVIDER_ORDER, allowRemoteHostedProviders: false };\n\t}\n\tconst routing = raw.fetchRouting;\n\tif (!routing || typeof routing !== \"object\" || Array.isArray(routing)) {\n\t\tthrow new Error(`fetchRouting in ${WEB_SEARCH_CONFIG_PATH} must be an object`);\n\t}\n\n\tconst routingConfig = routing as Record<string, unknown>;\n\tconst providersValue = routingConfig.providers;\n\tlet providers = DEFAULT_FETCH_PROVIDER_ORDER;\n\tif (providersValue !== undefined) {\n\t\tif (!Array.isArray(providersValue) || providersValue.length === 0) {\n\t\t\tthrow new Error(`fetchRouting.providers in ${WEB_SEARCH_CONFIG_PATH} must be a non-empty array`);\n\t\t}\n\n\t\tproviders = [];\n\t\tfor (const provider of providersValue) {\n\t\t\tconst normalized = typeof provider === \"string\" ? provider.trim().toLowerCase() : \"\";\n\t\t\tif (!FETCH_PROVIDERS.includes(normalized as FetchProvider)) {\n\t\t\t\tthrow new Error(`fetchRouting.providers in ${WEB_SEARCH_CONFIG_PATH} contains an invalid provider: ${String(provider)}`);\n\t\t\t}\n\t\t\tif (providers.includes(normalized as FetchProvider)) {\n\t\t\t\tthrow new Error(`fetchRouting.providers in ${WEB_SEARCH_CONFIG_PATH} must not contain duplicates: ${normalized}`);\n\t\t\t}\n\t\t\tproviders.push(normalized as FetchProvider);\n\t\t}\n\t}\n\n\tconst allowRemoteHostedProvidersValue = routingConfig.allowRemoteHostedProviders;\n\tif (allowRemoteHostedProvidersValue !== undefined && typeof allowRemoteHostedProvidersValue !== \"boolean\") {\n\t\tthrow new Error(`fetchRouting.allowRemoteHostedProviders in ${WEB_SEARCH_CONFIG_PATH} must be a boolean`);\n\t}\n\n\treturn { providers, allowRemoteHostedProviders: allowRemoteHostedProvidersValue === true };\n}\n\nfunction abortedResult(url: string): ExtractedContent {\n\treturn { url, title: \"\", content: \"\", error: \"Aborted\" };\n}\n\nconst turndown = new TurndownService({\n\theadingStyle: \"atx\",\n\tcodeBlockStyle: \"fenced\",\n});\n\nconst fetchLimit = pLimit(CONCURRENT_LIMIT);\n\nexport interface VideoFrame {\n\tdata: string;\n\tmimeType: string;\n\ttimestamp: string;\n}\n\nexport type FrameData = { data: string; mimeType: string };\nexport type FrameResult = FrameData | { error: string };\n\nexport interface ExtractedContent {\n\turl: string;\n\ttitle: string;\n\tcontent: string;\n\terror: string | null;\n\tthumbnail?: { data: string; mimeType: string };\n\tframes?: VideoFrame[];\n\tduration?: number;\n\tmimeType?: string;\n\tstatus?: number;\n}\n\ntype HttpExtractedContent = ExtractedContent & { declaredLinks?: DeclaredWebLink[] };\n\nexport interface ExtractOptions {\n\ttimeoutMs?: number;\n\tforceClone?: boolean;\n\tprompt?: string;\n\ttimestamp?: string;\n\tframes?: number;\n\tmodel?: string;\n\tmode?: \"readable\" | \"raw\" | \"answer\";\n\tanswerModel?: string;\n\t/** Custom DNS resolver used for SSRF validation. Primarily a test seam. */\n\tlookup?: Lookup;\n}\n\nconst JINA_READER_BASE = \"https://r.jina.ai/\";\nconst JINA_TIMEOUT_MS = 30000;\n\nasync function extractWithJinaReader(\n\turl: string,\n\tsignal?: AbortSignal,\n\tlookup?: Lookup,\n): Promise<ExtractedContent | null> {\n\tconst jinaUrl = JINA_READER_BASE + url;\n\n\tconst activityId = activityMonitor.logStart({ type: \"api\", query: `jina: ${url}` });\n\n\ttry {\n\t\tconst ssrf = loadSsrfConfig();\n\t\tconst domainPolicy = loadFetchContentDomainPolicy();\n\t\tawait validateRemoteUrl(url, {\n\t\t\tallowRanges: ssrf.allowRanges,\n\t\t\ttrustEnvProxy: ssrf.trustEnvProxy,\n\t\t\tdomainPolicy,\n\t\t\t...(lookup ? { lookup } : {}),\n\t\t});\n\t\tconst res = await fetch(jinaUrl, {\n\t\t\theaders: {\n\t\t\t\t\"Accept\": \"text/markdown\",\n\t\t\t\t\"X-No-Cache\": \"true\",\n\t\t\t},\n\t\t\tsignal: AbortSignal.any([\n\t\t\t\tAbortSignal.timeout(JINA_TIMEOUT_MS),\n\t\t\t\t...(signal ? [signal] : []),\n\t\t\t]),\n\t\t});\n\n\t\tif (!res.ok) {\n\t\t\tactivityMonitor.logComplete(activityId, res.status);\n\t\t\treturn null;\n\t\t}\n\n\t\tconst content = await res.text();\n\t\tactivityMonitor.logComplete(activityId, res.status);\n\n\t\tconst contentStart = content.indexOf(\"Markdown Content:\");\n\t\tif (contentStart < 0) {\n\t\t\treturn null;\n\t\t}\n\n\t\tconst markdownPart = content.slice(contentStart + 17).trim(); // 17 = \"Markdown Content:\".length\n\n\t\t// Check for failed JS rendering or minimal content\n\t\tif (markdownPart.length < 100 ||\n\t\t\tmarkdownPart.startsWith(\"Loading...\") ||\n\t\t\tmarkdownPart.startsWith(\"Please enable JavaScript\")) {\n\t\t\treturn null;\n\t\t}\n\n\t\tconst title = extractHeadingTitle(markdownPart) ?? (new URL(url).pathname.split(\"/\").pop() || url);\n\t\treturn { url, title, content: markdownPart, error: null };\n\t} catch (err) {\n\t\tconst message = err instanceof Error ? err.message : String(err);\n\t\tif (message.toLowerCase().includes(\"abort\")) {\n\t\t\tactivityMonitor.logComplete(activityId, 0);\n\t\t} else {\n\t\t\tactivityMonitor.logError(activityId, message);\n\t\t}\n\t\treturn null;\n\t}\n}\n\nfunction parseTimestamp(ts: string): number | null {\n\tconst num = Number(ts);\n\tif (!isNaN(num) && num >= 0) return Math.floor(num);\n\tconst parts = ts.split(\":\").map(Number);\n\tif (parts.some(p => isNaN(p) || p < 0)) return null;\n\tif (parts.length === 3) return Math.floor(parts[0] * 3600 + parts[1] * 60 + parts[2]);\n\tif (parts.length === 2) return Math.floor(parts[0] * 60 + parts[1]);\n\treturn null;\n}\n\ntype TimestampSpec = { type: \"single\"; seconds: number } | { type: \"range\"; start: number; end: number };\n\nfunction parseTimestampSpec(ts: string): TimestampSpec | null {\n\tconst dashIdx = ts.indexOf(\"-\", 1);\n\tif (dashIdx > 0) {\n\t\tconst start = parseTimestamp(ts.slice(0, dashIdx));\n\t\tconst end = parseTimestamp(ts.slice(dashIdx + 1));\n\t\tif (start !== null && end !== null && end > start) return { type: \"range\", start, end };\n\t}\n\tconst seconds = parseTimestamp(ts);\n\treturn seconds !== null ? { type: \"single\", seconds } : null;\n}\n\nconst DEFAULT_RANGE_FRAMES = 6;\nconst MIN_FRAME_INTERVAL = 5;\n\nfunction computeRangeTimestamps(start: number, end: number, maxFrames: number = DEFAULT_RANGE_FRAMES): number[] {\n\tif (maxFrames <= 1) return [start];\n\tconst duration = end - start;\n\tconst idealInterval = duration / (maxFrames - 1);\n\tif (idealInterval < MIN_FRAME_INTERVAL) {\n\t\tconst timestamps: number[] = [];\n\t\tfor (let t = start; t <= end && timestamps.length < maxFrames; t += MIN_FRAME_INTERVAL) {\n\t\t\ttimestamps.push(t);\n\t\t}\n\t\treturn timestamps;\n\t}\n\treturn Array.from({ length: maxFrames }, (_, i) => Math.round(start + i * idealInterval));\n}\n\nfunction buildFrameResult(\n\turl: string, label: string, requestedCount: number,\n\tframes: VideoFrame[], error: string | null, duration?: number,\n): ExtractedContent {\n\tif (frames.length === 0) {\n\t\tconst msg = error ?? \"Frame extraction failed\";\n\t\treturn { url, title: `Frames ${label} (0/${requestedCount})`, content: msg, error: msg };\n\t}\n\treturn {\n\t\turl,\n\t\ttitle: `Frames ${label} (${frames.length}/${requestedCount})`,\n\t\tcontent: `${frames.length} frames extracted from ${label}`,\n\t\terror: null,\n\t\tframes,\n\t\t...(duration !== undefined ? { duration } : {}),\n\t};\n}\n\nasync function extractLocalFrames(\n\tfilePath: string, timestamps: number[],\n): Promise<{ frames: VideoFrame[]; error: string | null }> {\n\tconst results = await Promise.all(timestamps.map(async (t) => {\n\t\tconst frame = await extractVideoFrame(filePath, t);\n\t\tif (\"error\" in frame) return { error: frame.error };\n\t\treturn { ...frame, timestamp: formatSeconds(t) };\n\t}));\n\tconst frames = results.filter((f): f is VideoFrame => \"data\" in f);\n\tconst firstError = results.find((f): f is { error: string } => \"error\" in f);\n\treturn { frames, error: frames.length === 0 && firstError ? firstError.error : null };\n}\n\nfunction safeVideoInfo(url: string): { info: ReturnType<typeof isVideoFile>; error?: string } {\n\ttry {\n\t\treturn { info: isVideoFile(url) };\n\t} catch (err) {\n\t\treturn { info: null, error: errorMessage(err) };\n\t}\n}\n\nexport async function extractContent(\n\turl: string,\n\tsignal?: AbortSignal,\n\toptions?: ExtractOptions,\n): Promise<ExtractedContent> {\n\tif (signal?.aborted) {\n\t\treturn { url, title: \"\", content: \"\", error: \"Aborted\" };\n\t}\n\n\tlet remoteUrl: URL | null = null;\n\ttry {\n\t\tconst parsed = new URL(url);\n\t\tif (parsed.protocol === \"http:\" || parsed.protocol === \"https:\") remoteUrl = parsed;\n\t} catch {\n\t}\n\tif (remoteUrl) {\n\t\ttry {\n\t\t\tconst ssrf = loadSsrfConfig();\n\t\t\tconst domainPolicy = loadFetchContentDomainPolicy();\n\t\t\tawait validateRemoteUrl(remoteUrl, {\n\t\t\t\tallowRanges: ssrf.allowRanges,\n\t\t\t\ttrustEnvProxy: ssrf.trustEnvProxy,\n\t\t\t\tdomainPolicy,\n\t\t\t\t...(options?.lookup ? { lookup: options.lookup } : {}),\n\t\t\t});\n\t\t} catch (err) {\n\t\t\treturn { url, title: \"\", content: \"\", error: errorMessage(err) };\n\t\t}\n\t}\n\n\tif (options?.mode === \"raw\") {\n\t\treturn extractViaHttp(url, signal, options);\n\t}\n\n\tif (options?.frames || options?.timestamp) {\n\t\tconst disabled = imageGateError();\n\t\tif (disabled) return { url, title: \"\", content: \"\", error: disabled };\n\t}\n\n\tif (options?.frames && !options.timestamp) {\n\t\tconst frameCount = options.frames;\n\t\tconst ytInfo = isYouTubeURL(url);\n\t\tif (ytInfo.isYouTube && ytInfo.videoId) {\n\t\t\tconst streamInfo = await getYouTubeStreamInfo(ytInfo.videoId);\n\t\t\tif (\"error\" in streamInfo) {\n\t\t\t\treturn { url, title: \"Frames\", content: streamInfo.error, error: streamInfo.error };\n\t\t\t}\n\t\t\tif (streamInfo.duration === null) {\n\t\t\t\tconst error = \"Cannot determine video duration. Use a timestamp range instead.\";\n\t\t\t\treturn { url, title: \"Frames\", content: error, error };\n\t\t\t}\n\t\t\tconst dur = Math.floor(streamInfo.duration);\n\t\t\tconst timestamps = computeRangeTimestamps(0, dur, frameCount);\n\t\t\tconst result = await extractYouTubeFrames(ytInfo.videoId, timestamps, streamInfo);\n\t\t\tconst label = `${formatSeconds(0)}-${formatSeconds(dur)}`;\n\t\t\treturn buildFrameResult(url, label, timestamps.length, result.frames, result.error, streamInfo.duration);\n\t\t}\n\n\t\tconst localVideo = safeVideoInfo(url);\n\t\tif (localVideo.error) {\n\t\t\treturn { url, title: \"\", content: \"\", error: localVideo.error };\n\t\t}\n\t\tif (localVideo.info) {\n\t\t\tconst durationResult = await getLocalVideoDuration(localVideo.info.absolutePath);\n\t\t\tif (typeof durationResult !== \"number\") {\n\t\t\t\treturn { url, title: \"Frames\", content: durationResult.error, error: durationResult.error };\n\t\t\t}\n\t\t\tconst dur = Math.floor(durationResult);\n\t\t\tconst timestamps = computeRangeTimestamps(0, dur, frameCount);\n\t\t\tconst result = await extractLocalFrames(localVideo.info.absolutePath, timestamps);\n\t\t\tconst label = `${formatSeconds(0)}-${formatSeconds(dur)}`;\n\t\t\treturn buildFrameResult(url, label, timestamps.length, result.frames, result.error, durationResult);\n\t\t}\n\n\t\treturn { url, title: \"\", content: \"\", error: \"Frame extraction only works with YouTube and local video files\" };\n\t}\n\n\tif (options?.timestamp) {\n\t\tconst spec = parseTimestampSpec(options.timestamp);\n\t\tif (!spec) {\n\t\t\treturn {\n\t\t\t\turl,\n\t\t\t\ttitle: \"\",\n\t\t\t\tcontent: \"\",\n\t\t\t\terror: `Invalid timestamp format: \"${options.timestamp}\". Use \"H:MM:SS\", \"MM:SS\", \"85\", or \"start-end\".`,\n\t\t\t};\n\t\t}\n\n\t\tconst frameCount = options.frames;\n\t\tconst ytInfo = isYouTubeURL(url);\n\t\tif (ytInfo.isYouTube && ytInfo.videoId) {\n\t\t\tconst streamInfo = await getYouTubeStreamInfo(ytInfo.videoId);\n\t\t\tif (\"error\" in streamInfo) {\n\t\t\t\tif (spec.type === \"range\") {\n\t\t\t\t\tconst label = `${formatSeconds(spec.start)}-${formatSeconds(spec.end)}`;\n\t\t\t\t\treturn { url, title: `Frames ${label}`, content: streamInfo.error, error: streamInfo.error };\n\t\t\t\t}\n\t\t\t\tif (frameCount) {\n\t\t\t\t\tconst end = spec.seconds + (frameCount - 1) * MIN_FRAME_INTERVAL;\n\t\t\t\t\tconst label = `${formatSeconds(spec.seconds)}-${formatSeconds(end)}`;\n\t\t\t\t\treturn { url, title: `Frames ${label}`, content: streamInfo.error, error: streamInfo.error };\n\t\t\t\t}\n\t\t\t\treturn { url, title: `Frame at ${options.timestamp}`, content: streamInfo.error, error: streamInfo.error };\n\t\t\t}\n\n\t\t\tif (spec.type === \"range\") {\n\t\t\t\tconst label = `${formatSeconds(spec.start)}-${formatSeconds(spec.end)}`;\n\t\t\t\tif (streamInfo.duration !== null && spec.end > streamInfo.duration) {\n\t\t\t\t\tconst error = `Timestamp ${formatSeconds(spec.end)} exceeds video duration (${formatSeconds(Math.floor(streamInfo.duration))})`;\n\t\t\t\t\treturn { url, title: `Frames ${label}`, content: error, error };\n\t\t\t\t}\n\t\t\t\tconst timestamps = frameCount\n\t\t\t\t\t? computeRangeTimestamps(spec.start, spec.end, frameCount)\n\t\t\t\t\t: computeRangeTimestamps(spec.start, spec.end);\n\t\t\t\tconst result = await extractYouTubeFrames(ytInfo.videoId, timestamps, streamInfo);\n\t\t\t\treturn buildFrameResult(url, label, timestamps.length, result.frames, result.error, result.duration ?? undefined);\n\t\t\t}\n\n\t\t\tif (frameCount) {\n\t\t\t\tconst end = spec.seconds + (frameCount - 1) * MIN_FRAME_INTERVAL;\n\t\t\t\tconst label = `${formatSeconds(spec.seconds)}-${formatSeconds(end)}`;\n\t\t\t\tif (streamInfo.duration !== null && end > streamInfo.duration) {\n\t\t\t\t\tconst error = `Timestamp ${formatSeconds(end)} exceeds video duration (${formatSeconds(Math.floor(streamInfo.duration))})`;\n\t\t\t\t\treturn { url, title: `Frames ${label}`, content: error, error };\n\t\t\t\t}\n\t\t\t\tconst timestamps = computeRangeTimestamps(spec.seconds, end, frameCount);\n\t\t\t\tconst result = await extractYouTubeFrames(ytInfo.videoId, timestamps, streamInfo);\n\t\t\t\treturn buildFrameResult(url, label, timestamps.length, result.frames, result.error, result.duration ?? undefined);\n\t\t\t}\n\n\t\t\tif (streamInfo.duration !== null && spec.seconds > streamInfo.duration) {\n\t\t\t\tconst error = `Timestamp ${formatSeconds(spec.seconds)} exceeds video duration (${formatSeconds(Math.floor(streamInfo.duration))})`;\n\t\t\t\treturn { url, title: `Frame at ${options.timestamp}`, content: error, error };\n\t\t\t}\n\t\t\tconst frame = await extractYouTubeFrame(ytInfo.videoId, spec.seconds, streamInfo);\n\t\t\tif (\"error\" in frame) {\n\t\t\t\treturn { url, title: `Frame at ${options.timestamp}`, content: frame.error, error: frame.error };\n\t\t\t}\n\t\t\treturn { url, title: `Frame at ${options.timestamp}`, content: `Video frame at ${options.timestamp}`, error: null, thumbnail: frame };\n\t\t}\n\n\t\tconst localVideo = safeVideoInfo(url);\n\t\tif (localVideo.error) {\n\t\t\treturn { url, title: \"\", content: \"\", error: localVideo.error };\n\t\t}\n\t\tif (localVideo.info) {\n\t\t\tif (spec.type === \"range\") {\n\t\t\t\tconst timestamps = frameCount\n\t\t\t\t\t? computeRangeTimestamps(spec.start, spec.end, frameCount)\n\t\t\t\t\t: computeRangeTimestamps(spec.start, spec.end);\n\t\t\t\tconst result = await extractLocalFrames(localVideo.info.absolutePath, timestamps);\n\t\t\t\tconst label = `${formatSeconds(spec.start)}-${formatSeconds(spec.end)}`;\n\t\t\t\treturn buildFrameResult(url, label, timestamps.length, result.frames, result.error);\n\t\t\t}\n\n\t\t\tif (frameCount) {\n\t\t\t\tconst end = spec.seconds + (frameCount - 1) * MIN_FRAME_INTERVAL;\n\t\t\t\tconst timestamps = computeRangeTimestamps(spec.seconds, end, frameCount);\n\t\t\t\tconst result = await extractLocalFrames(localVideo.info.absolutePath, timestamps);\n\t\t\t\tconst label = `${formatSeconds(spec.seconds)}-${formatSeconds(end)}`;\n\t\t\t\treturn buildFrameResult(url, label, timestamps.length, result.frames, result.error);\n\t\t\t}\n\n\t\t\tconst frame = await extractVideoFrame(localVideo.info.absolutePath, spec.seconds);\n\t\t\tif (\"error\" in frame) {\n\t\t\t\treturn { url, title: `Frame at ${options.timestamp}`, content: frame.error, error: frame.error };\n\t\t\t}\n\t\t\treturn { url, title: `Frame at ${options.timestamp}`, content: `Video frame at ${options.timestamp}`, error: null, thumbnail: frame };\n\t\t}\n\n\t\treturn { url, title: \"\", content: \"\", error: \"Timestamp extraction only works with YouTube and local video files\" };\n\t}\n\n\tconst localVideo = safeVideoInfo(url);\n\tif (localVideo.error) {\n\t\treturn { url, title: \"\", content: \"\", error: localVideo.error };\n\t}\n\tif (localVideo.info) {\n\t\ttry {\n\t\t\tconst result = await extractVideo(localVideo.info, signal, options);\n\t\t\tif (signal?.aborted) return abortedResult(url);\n\t\t\treturn result ?? { url, title: \"\", content: \"\", error: `Video analysis requires Gemini access. Either:\\n  1. Sign into gemini.google.com in Chrome (free, uses cookies)\\n  2. Set GEMINI_API_KEY in ${WEB_SEARCH_CONFIG_PATH}` };\n\t\t} catch (err) {\n\t\t\tif (isAbortError(err)) return abortedResult(url);\n\t\t\treturn { url, title: \"\", content: \"\", error: errorMessage(err) };\n\t\t}\n\t}\n\n\ttry {\n\t\tif (!remoteUrl) new URL(url);\n\t} catch (err) {\n\t\treturn { url, title: \"\", content: \"\", error: errorMessage(err) };\n\t}\n\n\ttry {\n\t\tconst ghResult = await extractGitHub(url, signal, options?.forceClone);\n\t\tif (ghResult) return ghResult;\n\t\tif (signal?.aborted) return abortedResult(url);\n\t} catch (err) {\n\t\tconst message = errorMessage(err);\n\t\tif (isAbortError(err)) return abortedResult(url);\n\t\tif (isConfigParseError(err)) {\n\t\t\treturn { url, title: \"\", content: \"\", error: message };\n\t\t}\n\t}\n\n\tconst ytInfo = isYouTubeURL(url);\n\tlet youtubeEnabled = false;\n\ttry {\n\t\tyoutubeEnabled = isYouTubeEnabled();\n\t} catch (err) {\n\t\treturn { url, title: \"\", content: \"\", error: errorMessage(err) };\n\t}\n\tif (ytInfo.isYouTube && youtubeEnabled) {\n\t\ttry {\n\t\t\tconst ytResult = await extractYouTube(url, signal, options?.prompt, options?.model);\n\t\t\tif (ytResult) return ytResult;\n\t\t\tif (signal?.aborted) return abortedResult(url);\n\t\t} catch (err) {\n\t\t\tconst message = errorMessage(err);\n\t\t\tif (isAbortError(err)) return abortedResult(url);\n\t\t\treturn { url, title: \"\", content: \"\", error: message };\n\t\t}\n\t\treturn {\n\t\t\turl,\n\t\t\ttitle: \"\",\n\t\t\tcontent: \"\",\n\t\t\terror: \"Could not extract YouTube video content. Sign into Google in Chrome for automatic access, or set GEMINI_API_KEY.\",\n\t\t};\n\t}\n\n\tif (signal?.aborted) return abortedResult(url);\n\n\tlet fetchRouting: FetchRouting;\n\ttry {\n\t\tfetchRouting = loadFetchRouting();\n\t} catch (err) {\n\t\treturn { url, title: \"\", content: \"\", error: errorMessage(err) };\n\t}\n\tconst providerOrder = remoteUrl && !fetchRouting.allowRemoteHostedProviders\n\t\t? fetchRouting.providers.filter(provider => !REMOTE_HOSTED_FETCH_PROVIDERS.has(provider))\n\t\t: fetchRouting.providers;\n\tif (providerOrder.length === 0) {\n\t\treturn {\n\t\t\turl,\n\t\t\ttitle: \"\",\n\t\t\tcontent: \"\",\n\t\t\terror: \"Remote hosted fetch providers are disabled unless fetchRouting.allowRemoteHostedProviders is true\",\n\t\t};\n\t}\n\n\tlet httpResult: ExtractedContent | null = null;\n\tlet declaredLinks: DeclaredWebLink[] = [];\n\tconst withDeclaredLinks = (result: ExtractedContent): ExtractedContent => ({\n\t\t...result,\n\t\tcontent: appendDeclaredWebLinks(result.content, declaredLinks),\n\t});\n\tconst parseErrorResult = (message: string): ExtractedContent => httpResult\n\t\t? { ...httpResult, error: message }\n\t\t: { url, title: \"\", content: \"\", error: message };\n\tconst runHttpProvider = async (): Promise<ExtractedContent | null> => {\n\t\tconst { declaredLinks: discoveredLinks = [], ...result } = await extractViaHttp(url, signal, options);\n\t\thttpResult = result;\n\t\tdeclaredLinks = discoveredLinks;\n\t\tif (signal?.aborted) return abortedResult(url);\n\t\tif (!httpResult.error) return httpResult;\n\t\tif (NON_RECOVERABLE_ERRORS.some(prefix => httpResult!.error!.startsWith(prefix)) || isRedirectPolicyError(httpResult.error) || isConfigParseError(httpResult.error)) {\n\t\t\treturn httpResult;\n\t\t}\n\t\treturn null;\n\t};\n\n\tlet firecrawlError: string | null = null;\n\tlet tinyfishError: string | null = null;\n\tlet search1apiError: string | null = null;\n\tlet queritError: string | null = null;\n\tlet kagiError: string | null = null;\n\tlet ollamaError: string | null = null;\n\tlet parallelError: string | null = null;\n\tlet brightdataError: string | null = null;\n\n\tif (remoteUrl && providerOrder[0] !== \"http\") {\n\t\tconst httpGateResult = await runHttpProvider();\n\t\tif (httpGateResult) return httpGateResult;\n\t}\n\n\tfor (const provider of providerOrder) {\n\t\tif (signal?.aborted) return abortedResult(url);\n\n\t\tif (provider === \"http\") {\n\t\t\tconst result = await runHttpProvider();\n\t\t\tif (result) return result;\n\t\t\tcontinue;\n\t\t}\n\n\t\tif (provider === \"firecrawl\") {\n\t\t\ttry {\n\t\t\t\tif (isFirecrawlAvailable()) {\n\t\t\t\t\tconst ssrf = loadSsrfConfig();\n\t\t\t\t\tconst firecrawlResult = await extractWithFirecrawl(url, signal, {\n\t\t\t\t\t\ttimeoutMs: options?.timeoutMs,\n\t\t\t\t\t\t...(options?.lookup ? { lookup: options.lookup } : {}),\n\t\t\t\t\t\tssrf,\n\t\t\t\t\t});\n\t\t\t\t\tif (firecrawlResult) return withDeclaredLinks(firecrawlResult);\n\t\t\t\t}\n\t\t\t} catch (err) {\n\t\t\t\tif (isAbortError(err)) return abortedResult(url);\n\t\t\t\tfirecrawlError = errorMessage(err);\n\t\t\t\tif (isConfigParseError(err)) return parseErrorResult(firecrawlError);\n\t\t\t}\n\t\t\tcontinue;\n\t\t}\n\n\t\tif (provider === \"jina\") {\n\t\t\tconst jinaResult = await extractWithJinaReader(url, signal, options?.lookup);\n\t\t\tif (jinaResult) return withDeclaredLinks(jinaResult);\n\t\t\tcontinue;\n\t\t}\n\n\t\tif (provider === \"tinyfish\") {\n\t\t\ttry {\n\t\t\t\tif (isTinyFishAvailable()) {\n\t\t\t\t\tconst tinyfishResult = await extractWithTinyFish(url, signal, options);\n\t\t\t\t\tif (tinyfishResult) return withDeclaredLinks(tinyfishResult);\n\t\t\t\t}\n\t\t\t} catch (err) {\n\t\t\t\tif (isAbortError(err)) return abortedResult(url);\n\t\t\t\ttinyfishError = errorMessage(err);\n\t\t\t\tif (isConfigParseError(err)) return parseErrorResult(tinyfishError);\n\t\t\t}\n\t\t\tcontinue;\n\t\t}\n\n\t\tif (provider === \"search1api\") {\n\t\t\ttry {\n\t\t\t\tif (isSearch1APIAvailable()) {\n\t\t\t\t\tconst search1apiResult = await extractWithSearch1API(url, signal, options);\n\t\t\t\t\tif (search1apiResult) return withDeclaredLinks(search1apiResult);\n\t\t\t\t}\n\t\t\t} catch (err) {\n\t\t\t\tif (isAbortError(err)) return abortedResult(url);\n\t\t\t\tsearch1apiError = errorMessage(err);\n\t\t\t\tif (isConfigParseError(err)) return parseErrorResult(search1apiError);\n\t\t\t}\n\t\t\tcontinue;\n\t\t}\n\n\t\tif (provider === \"querit\") {\n\t\t\ttry {\n\t\t\t\tif (isQueritAvailable()) {\n\t\t\t\t\tconst queritResult = await extractWithQuerit(url, signal, options);\n\t\t\t\t\tif (queritResult) return withDeclaredLinks(queritResult);\n\t\t\t\t}\n\t\t\t} catch (err) {\n\t\t\t\tif (isAbortError(err)) return abortedResult(url);\n\t\t\t\tqueritError = errorMessage(err);\n\t\t\t\tif (isConfigParseError(err)) return parseErrorResult(queritError);\n\t\t\t}\n\t\t\tcontinue;\n\t\t}\n\n\t\tif (provider === \"kagi\") {\n\t\t\ttry {\n\t\t\t\tif (isKagiExtractAvailable()) {\n\t\t\t\t\tconst ssrf = loadSsrfConfig();\n\t\t\t\t\tconst kagiResult = await extractWithKagi(url, signal, {\n\t\t\t\t\t\ttimeoutMs: options?.timeoutMs,\n\t\t\t\t\t\t...(options?.lookup ? { lookup: options.lookup } : {}),\n\t\t\t\t\t\tssrf,\n\t\t\t\t\t});\n\t\t\t\t\tif (kagiResult) return withDeclaredLinks(kagiResult);\n\t\t\t\t}\n\t\t\t} catch (err) {\n\t\t\t\tif (isAbortError(err)) return abortedResult(url);\n\t\t\t\tkagiError = errorMessage(err);\n\t\t\t\tif (isConfigParseError(err)) return parseErrorResult(kagiError);\n\t\t\t}\n\t\t\tcontinue;\n\t\t}\n\n\t\tif (provider === \"ollama\") {\n\t\t\ttry {\n\t\t\t\tif (isOllamaFetchAvailable()) {\n\t\t\t\t\tconst ssrf = loadSsrfConfig();\n\t\t\t\t\tconst ollamaResult = await extractWithOllama(url, signal, {\n\t\t\t\t\t\ttimeoutMs: options?.timeoutMs,\n\t\t\t\t\t\t...(options?.lookup ? { lookup: options.lookup } : {}),\n\t\t\t\t\t\tssrf,\n\t\t\t\t\t});\n\t\t\t\t\tif (ollamaResult) return withDeclaredLinks(ollamaResult);\n\t\t\t\t}\n\t\t\t} catch (err) {\n\t\t\t\tif (isAbortError(err)) return abortedResult(url);\n\t\t\t\tollamaError = errorMessage(err);\n\t\t\t\tif (isConfigParseError(err)) return parseErrorResult(ollamaError);\n\t\t\t}\n\t\t\tcontinue;\n\t\t}\n\n\t\tif (provider === \"parallel\") {\n\t\t\ttry {\n\t\t\t\tif (isParallelAvailable()) {\n\t\t\t\t\tconst parallelResult = await extractWithParallel(url, signal, options);\n\t\t\t\t\tif (parallelResult) return withDeclaredLinks(parallelResult);\n\t\t\t\t}\n\t\t\t} catch (err) {\n\t\t\t\tif (isAbortError(err)) return abortedResult(url);\n\t\t\t\tparallelError = errorMessage(err);\n\t\t\t\tif (isConfigParseError(err)) return parseErrorResult(parallelError);\n\t\t\t}\n\t\t\tcontinue;\n\t\t}\n\n\t\tif (provider === \"brightdata\") {\n\t\t\ttry {\n\t\t\t\tif (isBrightDataUnlockerAvailable()) {\n\t\t\t\t\tconst ssrf = loadSsrfConfig();\n\t\t\t\t\tconst brightdataResult = await extractWithBrightDataUnlocker(url, signal, {\n\t\t\t\t\t\ttimeoutMs: options?.timeoutMs,\n\t\t\t\t\t\t...(options?.lookup ? { lookup: options.lookup } : {}),\n\t\t\t\t\t\tssrf,\n\t\t\t\t\t});\n\t\t\t\t\tif (brightdataResult) return withDeclaredLinks(brightdataResult);\n\t\t\t\t}\n\t\t\t} catch (err) {\n\t\t\t\tif (isAbortError(err)) return abortedResult(url);\n\t\t\t\tbrightdataError = errorMessage(err);\n\t\t\t\tif (isConfigParseError(err)) return parseErrorResult(brightdataError);\n\t\t\t}\n\t\t\tcontinue;\n\t\t}\n\n\t\tif (provider === \"gemini\") {\n\t\t\tlet geminiResult: ExtractedContent | null = null;\n\t\t\ttry {\n\t\t\t\tgeminiResult = await extractWithUrlContext(url, signal)\n\t\t\t\t\t?? await extractWithGeminiWeb(url, signal);\n\t\t\t} catch (err) {\n\t\t\t\tif (isAbortError(err)) return abortedResult(url);\n\t\t\t\tif (err instanceof CredentialResolutionError || isConfigParseError(err)) {\n\t\t\t\t\treturn parseErrorResult(errorMessage(err));\n\t\t\t\t}\n\t\t\t}\n\t\t\tif (geminiResult) return withDeclaredLinks(geminiResult);\n\t\t}\n\t}\n\n\tif (signal?.aborted) return abortedResult(url);\n\tconst finalHttpResult = httpResult as ExtractedContent | null;\n\tif (finalHttpResult && declaredLinks.length > 0) return { ...finalHttpResult, error: null };\n\n\tconst guidance = [\n\t\tfinalHttpResult?.error ?? \"No fetch_content provider returned content\",\n\t\t...(firecrawlError ? [`Firecrawl fallback failed: ${firecrawlError}`] : []),\n\t\t...(tinyfishError ? [`TinyFish fallback failed: ${tinyfishError}`] : []),\n\t\t...(search1apiError ? [`Search1API fallback failed: ${search1apiError}`] : []),\n\t\t...(queritError ? [`Querit fallback failed: ${queritError}`] : []),\n\t\t...(kagiError ? [`Kagi fallback failed: ${kagiError}`] : []),\n\t\t...(ollamaError ? [`Ollama fallback failed: ${ollamaError}`] : []),\n\t\t...(parallelError ? [`Parallel fallback failed: ${parallelError}`] : []),\n\t\t...(brightdataError ? [`Bright Data fallback failed: ${brightdataError}`] : []),\n\t\t\"\",\n\t\t\"Fallback options:\",\n\t\t`  • Set firecrawlBaseUrl in ${WEB_SEARCH_CONFIG_PATH} to a self-hosted Firecrawl instance`,\n\t\t`  • Set tinyfishApiKey in ${WEB_SEARCH_CONFIG_PATH} or TINYFISH_API_KEY`,\n\t\t`  • Set search1apiApiKey in ${WEB_SEARCH_CONFIG_PATH} or SEARCH1API_KEY`,\n\t\t`  • Set queritApiKey in ${WEB_SEARCH_CONFIG_PATH} or QUERIT_API_KEY`,\n\t\t`  • Set kagiApiKey in ${WEB_SEARCH_CONFIG_PATH} or KAGI_API_KEY`,\n\t\t`  • Set ollamaApiKey in ${WEB_SEARCH_CONFIG_PATH} or OLLAMA_API_KEY`,\n\t\t`  • Set parallelApiKey in ${WEB_SEARCH_CONFIG_PATH} or PARALLEL_API_KEY`,\n\t\t`  • Set brightdataApiKey and brightdataUnlockerZone in ${WEB_SEARCH_CONFIG_PATH} or BRIGHTDATA_API_KEY and BRIGHTDATA_UNLOCKER_ZONE`,\n\t\t`  • Set GEMINI_API_KEY in ${WEB_SEARCH_CONFIG_PATH}`,\n\t\t\"  • Sign into gemini.google.com in Chrome\",\n\t\t\"  • Use web_search to find content about this topic\",\n\t].join(\"\\n\");\n\treturn { ...(finalHttpResult ?? { url, title: \"\", content: \"\", error: null }), error: guidance };\n}\n\nfunction isLikelyJSRendered(html: string): boolean {\n\t// Extract body content\n\tconst bodyMatch = html.match(/<body[^>]*>([\\s\\S]*?)<\\/body>/i);\n\tif (!bodyMatch) return false;\n\n\tconst bodyHtml = bodyMatch[1];\n\n\t// Strip tags to get text content\n\tconst textContent = bodyHtml\n\t\t.replace(/<script[\\s\\S]*?<\\/script>/gi, \"\")\n\t\t.replace(/<style[\\s\\S]*?<\\/style>/gi, \"\")\n\t\t.replace(/<[^>]+>/g, \"\")\n\t\t.replace(/\\s+/g, \" \")\n\t\t.trim();\n\n\t// Count scripts\n\tconst scriptCount = (html.match(/<script/gi) || []).length;\n\n\t// Heuristic: little text content but many scripts suggests JS rendering\n\treturn textContent.length < 500 && scriptCount > 3;\n}\n\nexport async function readPDFResponseBuffer(response: Response, maxSizeMB: number): Promise<ArrayBuffer> {\n\tconst maxBytes = maxSizeMB * 1024 * 1024;\n\treturn readResponseBufferWithLimit(response, maxBytes, () => pdfSizeLimitError(maxSizeMB));\n}\n\nasync function readTextResponseWithLimit(response: Response, maxBytes: number): Promise<string> {\n\tconst buffer = await readResponseBufferWithLimit(response, maxBytes, () => responseSizeLimitError(maxBytes));\n\tconst charset = response.headers.get(\"content-type\")?.match(/charset\\s*=\\s*[\"']?([^;\"'\\s]+)/i)?.[1];\n\ttry {\n\t\treturn new TextDecoder(charset || \"utf-8\").decode(buffer);\n\t} catch {\n\t\treturn new TextDecoder(\"utf-8\").decode(buffer);\n\t}\n}\n\nfunction isTextContentType(contentType: string): boolean {\n\tconst mimeType = contentType.split(\";\", 1)[0]?.trim().toLowerCase() ?? \"\";\n\treturn mimeType.startsWith(\"text/\") ||\n\t\tmimeType === \"application/json\" ||\n\t\tmimeType === \"application/ld+json\" ||\n\t\tmimeType === \"application/xml\" ||\n\t\tmimeType === \"application/xhtml+xml\" ||\n\t\tmimeType === \"application/javascript\" ||\n\t\tmimeType === \"application/x-javascript\" ||\n\t\tmimeType.endsWith(\"+json\") ||\n\t\tmimeType.endsWith(\"+xml\");\n}\n\nasync function readResponseBufferWithLimit(\n\tresponse: Response,\n\tmaxBytes: number,\n\tbuildError: () => Error,\n): Promise<ArrayBuffer> {\n\tconst reader = response.body?.getReader();\n\tif (!reader) {\n\t\tconst buffer = await response.arrayBuffer();\n\t\tif (buffer.byteLength > maxBytes) throw buildError();\n\t\treturn buffer;\n\t}\n\n\tconst chunks: Uint8Array[] = [];\n\tlet totalBytes = 0;\n\ttry {\n\t\twhile (true) {\n\t\t\tconst { done, value } = await reader.read();\n\t\t\tif (done) break;\n\t\t\tif (!value) continue;\n\t\t\ttotalBytes += value.byteLength;\n\t\t\tif (totalBytes > maxBytes) {\n\t\t\t\tawait reader.cancel();\n\t\t\t\tthrow buildError();\n\t\t\t}\n\t\t\tchunks.push(value);\n\t\t}\n\t} finally {\n\t\treader.releaseLock();\n\t}\n\n\tconst combined = new Uint8Array(totalBytes);\n\tlet offset = 0;\n\tfor (const chunk of chunks) {\n\t\tcombined.set(chunk, offset);\n\t\toffset += chunk.byteLength;\n\t}\n\treturn combined.buffer;\n}\n\nfunction pdfSizeLimitError(maxSizeMB: number): Error {\n\treturn new Error(`PDF exceeds configured pdf.maxSizeMB limit (${maxSizeMB} MB)`);\n}\n\nfunction responseSizeLimitError(maxBytes: number): Error {\n\treturn new Error(`Response too large (${Math.round(maxBytes / 1024 / 1024)}MB)`);\n}\n\nasync function extractViaHttp(\n\turl: string,\n\tsignal?: AbortSignal,\n\toptions?: ExtractOptions,\n): Promise<HttpExtractedContent> {\n\tconst timeoutMs = options?.timeoutMs ?? DEFAULT_TIMEOUT_MS;\n\tconst activityId = activityMonitor.logStart({ type: \"fetch\", url });\n\n\tconst controller = new AbortController();\n\tconst timeoutId = setTimeout(() => controller.abort(), timeoutMs);\n\n\tconst onAbort = () => controller.abort();\n\tsignal?.addEventListener(\"abort\", onAbort);\n\n\ttry {\n\t\tconst ssrf = loadSsrfConfig();\n\t\tconst domainPolicy = loadFetchContentDomainPolicy();\n\t\tconst response = await fetchRemoteUrl(\n\t\t\turl,\n\t\t\t{\n\t\t\t\tsignal: controller.signal,\n\t\t\t\theaders: {\n\t\t\t\t\t\"User-Agent\": \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36\",\n\t\t\t\t\t\"Accept\": \"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8\",\n\t\t\t\t\t\"Accept-Language\": \"en-US,en;q=0.9\",\n\t\t\t\t\t\"Cache-Control\": \"no-cache\",\n\t\t\t\t\t\"Sec-Fetch-Dest\": \"document\",\n\t\t\t\t\t\"Sec-Fetch-Mode\": \"navigate\",\n\t\t\t\t\t\"Sec-Fetch-Site\": \"none\",\n\t\t\t\t\t\"Sec-Fetch-User\": \"?1\",\n\t\t\t\t\t\"Upgrade-Insecure-Requests\": \"1\",\n\t\t\t\t},\n\t\t\t},\n\t\t\t{\n\t\t\t\tallowRanges: ssrf.allowRanges,\n\t\t\t\ttrustEnvProxy: ssrf.trustEnvProxy,\n\t\t\t\tdomainPolicy,\n\t\t\t\t...(options?.lookup ? { lookup: options.lookup } : {}),\n\t\t\t},\n\t\t);\n\n\t\tif (!response.ok && options?.mode !== \"raw\") {\n\t\t\tactivityMonitor.logComplete(activityId, response.status);\n\t\t\treturn {\n\t\t\t\turl,\n\t\t\t\ttitle: \"\",\n\t\t\t\tcontent: \"\",\n\t\t\t\terror: `HTTP ${response.status}: ${response.statusText}`,\n\t\t\t\tstatus: response.status,\n\t\t\t};\n\t\t}\n\n\t\tconst contentLengthHeader = response.headers.get(\"content-length\");\n\t\tconst contentType = response.headers.get(\"content-type\") || \"\";\n\t\tconst mimeType = contentType.split(\";\", 1)[0]?.trim().toLowerCase() ?? \"\";\n\t\tconst isPDFContent = isPDF(url, contentType);\n\t\tconst pdfConfig = isPDFContent ? loadPDFConfig() : null;\n\t\tif (isPDFContent && pdfConfig && !pdfConfig.enabled) {\n\t\t\tactivityMonitor.logComplete(activityId, response.status);\n\t\t\treturn { url, title: \"\", content: \"\", error: \"PDF extraction is disabled by pdf.enabled\", mimeType, status: response.status };\n\t\t}\n\t\tconst maxResponseSize = (pdfConfig?.maxSizeMB ?? 5) * 1024 * 1024;\n\t\tif (contentLengthHeader) {\n\t\t\tconst contentLength = Number.parseInt(contentLengthHeader, 10);\n\t\t\tif (Number.isFinite(contentLength) && contentLength > maxResponseSize) {\n\t\t\t\tactivityMonitor.logComplete(activityId, response.status);\n\t\t\t\treturn {\n\t\t\t\t\turl,\n\t\t\t\t\ttitle: \"\",\n\t\t\t\t\tcontent: \"\",\n\t\t\t\t\terror: pdfConfig\n\t\t\t\t\t\t? pdfSizeLimitError(pdfConfig.maxSizeMB).message\n\t\t\t\t\t\t: `Response too large (${Math.round(contentLength / 1024 / 1024)}MB)`,\n\t\t\t\t};\n\t\t\t}\n\t\t}\n\n\t\tif (options?.mode === \"raw\") {\n\t\t\tif (!isTextContentType(contentType)) {\n\t\t\t\tactivityMonitor.logComplete(activityId, response.status);\n\t\t\t\treturn { url, title: \"\", content: \"\", error: `Unsupported content type in raw mode: ${mimeType || \"missing\"}`, mimeType, status: response.status };\n\t\t\t}\n\t\t\tconst text = await readTextResponseWithLimit(response, maxResponseSize);\n\t\t\tactivityMonitor.logComplete(activityId, response.status);\n\t\t\treturn { url, title: extractTextTitle(text, url), content: text, error: null, mimeType, status: response.status };\n\t\t}\n\n\t\tif (SUPPORTED_IMAGE_TYPES.has(mimeType)) {\n\t\t\tconst disabled = imageGateError();\n\t\t\tif (disabled) {\n\t\t\t\tactivityMonitor.logComplete(activityId, response.status);\n\t\t\t\treturn { url, title: \"\", content: \"\", error: disabled, mimeType, status: response.status };\n\t\t\t}\n\t\t\ttry {\n\t\t\t\tconst buffer = await readResponseBufferWithLimit(response, maxResponseSize, () => responseSizeLimitError(maxResponseSize));\n\t\t\t\tconst resized = await resizeImage(new Uint8Array(buffer), mimeType, { maxWidth: 2000, maxHeight: 2000 });\n\t\t\t\tactivityMonitor.logComplete(activityId, response.status);\n\t\t\t\tif (!resized) return { url, title: \"\", content: \"\", error: `Could not decode image: ${mimeType}`, mimeType, status: response.status };\n\t\t\t\tconst title = new URL(response.url || url).pathname.split(\"/\").pop() || url;\n\t\t\t\treturn {\n\t\t\t\t\turl,\n\t\t\t\t\ttitle,\n\t\t\t\t\tcontent: `Image fetched (${resized.width}×${resized.height}, ${resized.mimeType})`,\n\t\t\t\t\terror: null,\n\t\t\t\t\tthumbnail: { data: resized.data, mimeType: resized.mimeType },\n\t\t\t\t\tmimeType: resized.mimeType,\n\t\t\t\t\tstatus: response.status,\n\t\t\t\t};\n\t\t\t} catch (err) {\n\t\t\t\tconst message = errorMessage(err);\n\t\t\t\tactivityMonitor.logError(activityId, message);\n\t\t\t\treturn { url, title: \"\", content: \"\", error: message, mimeType, status: response.status };\n\t\t\t}\n\t\t}\n\n\t\tif (isPDFContent && pdfConfig) {\n\t\t\ttry {\n\t\t\t\tconst buffer = await readPDFResponseBuffer(response, pdfConfig.maxSizeMB);\n\t\t\t\tif (signal?.aborted) return abortedResult(url);\n\t\t\t\tconst result = await extractPDFToMarkdown(buffer, url, { signal });\n\t\t\t\tactivityMonitor.logComplete(activityId, response.status);\n\t\t\t\treturn {\n\t\t\t\t\turl,\n\t\t\t\t\ttitle: result.title,\n\t\t\t\t\tcontent: `PDF extracted and saved to: ${result.outputPath}\\n\\nPages: ${result.pages}\\nCharacters: ${result.chars}`,\n\t\t\t\t\terror: null,\n\t\t\t\t};\n\t\t\t} catch (err) {\n\t\t\t\tconst message = err instanceof Error ? err.message : String(err);\n\t\t\t\tactivityMonitor.logError(activityId, message);\n\t\t\t\tif (message.startsWith(\"PDF exceeds configured pdf.maxSizeMB limit\")) {\n\t\t\t\t\treturn { url, title: \"\", content: \"\", error: message };\n\t\t\t\t}\n\t\t\t\tif (err instanceof CredentialResolutionError || isConfigParseError(err)) {\n\t\t\t\t\treturn { url, title: \"\", content: \"\", error: message };\n\t\t\t\t}\n\t\t\t\treturn { url, title: \"\", content: \"\", error: `PDF extraction failed: ${message}` };\n\t\t\t}\n\t\t}\n\n\t\tif (contentType.includes(\"application/octet-stream\") ||\n\t\t\tcontentType.includes(\"image/\") ||\n\t\t\tcontentType.includes(\"audio/\") ||\n\t\t\tcontentType.includes(\"video/\") ||\n\t\t\tcontentType.includes(\"application/zip\")) {\n\t\t\tactivityMonitor.logComplete(activityId, response.status);\n\t\t\treturn {\n\t\t\t\turl,\n\t\t\t\ttitle: \"\",\n\t\t\t\tcontent: \"\",\n\t\t\t\terror: `Unsupported content type: ${contentType.split(\";\")[0]}`,\n\t\t\t};\n\t\t}\n\n\t\tconst text = await readTextResponseWithLimit(response, maxResponseSize);\n\t\tconst isHTML = contentType.includes(\"text/html\") || contentType.includes(\"application/xhtml+xml\");\n\n\t\tif (!isHTML) {\n\t\t\tactivityMonitor.logComplete(activityId, response.status);\n\t\t\tconst title = extractTextTitle(text, url);\n\t\t\treturn { url, title, content: text, error: null };\n\t\t}\n\n\t\tconst { document } = parseHTML(text);\n\t\tconst documentTitle = document.title?.trim() ?? \"\";\n\t\tconst declaredLinks = discoverDeclaredWebLinks(\n\t\t\tdocument,\n\t\t\tresponse.headers.get(\"link\"),\n\t\t\tresponse.url || url,\n\t\t);\n\t\tconst reader = new Readability(document as unknown as ConstructorParameters<typeof Readability>[0]);\n\t\tconst article = reader.parse();\n\n\t\tif (!article) {\n\t\t\tconst rscResult = extractRSCContent(text);\n\t\t\tif (rscResult) {\n\t\t\t\tactivityMonitor.logComplete(activityId, response.status);\n\t\t\t\treturn {\n\t\t\t\t\turl,\n\t\t\t\t\ttitle: rscResult.title,\n\t\t\t\t\tcontent: appendDeclaredWebLinks(rscResult.content, declaredLinks),\n\t\t\t\t\terror: null,\n\t\t\t\t\tdeclaredLinks,\n\t\t\t\t};\n\t\t\t}\n\n\t\t\tactivityMonitor.logComplete(activityId, response.status);\n\t\t\tconst jsRendered = isLikelyJSRendered(text);\n\t\t\tconst errorMsg = jsRendered\n\t\t\t\t? \"Page appears to be JavaScript-rendered (content loads dynamically)\"\n\t\t\t\t: \"Could not extract readable content from HTML structure\";\n\n\t\t\treturn {\n\t\t\t\turl,\n\t\t\t\ttitle: documentTitle,\n\t\t\t\tcontent: appendDeclaredWebLinks(\"\", declaredLinks),\n\t\t\t\terror: errorMsg,\n\t\t\t\tdeclaredLinks,\n\t\t\t};\n\t\t}\n\n\t\tif (typeof article.content !== \"string\") {\n\t\t\tthrow new Error(\"Readability returned invalid article content\");\n\t\t}\n\t\tconst markdown = turndown.turndown(article.content);\n\t\tactivityMonitor.logComplete(activityId, response.status);\n\n\t\tif (markdown.length < MIN_USEFUL_CONTENT) {\n\t\t\treturn {\n\t\t\t\turl,\n\t\t\t\ttitle: article.title || documentTitle,\n\t\t\t\tcontent: appendDeclaredWebLinks(markdown, declaredLinks),\n\t\t\t\terror: isLikelyJSRendered(text)\n\t\t\t\t\t? \"Page appears to be JavaScript-rendered (content loads dynamically)\"\n\t\t\t\t\t: \"Extracted content appears incomplete\",\n\t\t\t\tdeclaredLinks,\n\t\t\t};\n\t\t}\n\n\t\treturn {\n\t\t\turl,\n\t\t\ttitle: article.title || documentTitle,\n\t\t\tcontent: appendDeclaredWebLinks(markdown, declaredLinks),\n\t\t\terror: null,\n\t\t\tdeclaredLinks,\n\t\t};\n\t} catch (err) {\n\t\tconst message = err instanceof Error ? err.message : String(err);\n\t\tif (message.toLowerCase().includes(\"abort\")) {\n\t\t\tactivityMonitor.logComplete(activityId, 0);\n\t\t} else {\n\t\t\tactivityMonitor.logError(activityId, message);\n\t\t}\n\t\treturn { url, title: \"\", content: \"\", error: message };\n\t} finally {\n\t\tclearTimeout(timeoutId);\n\t\tsignal?.removeEventListener(\"abort\", onAbort);\n\t}\n}\n\nexport function extractHeadingTitle(text: string): string | null {\n\tconst match = text.match(/^#{1,2}\\s+(.+)/m);\n\tif (!match) return null;\n\tconst cleaned = match[1].replace(/\\*+/g, \"\").trim();\n\treturn cleaned || null;\n}\n\nfunction extractTextTitle(text: string, url: string): string {\n\treturn extractHeadingTitle(text) ?? (new URL(url).pathname.split(\"/\").pop() || url);\n}\n\nexport async function fetchAllContent(\n\turls: string[],\n\tsignal?: AbortSignal,\n\toptions?: ExtractOptions,\n): Promise<ExtractedContent[]> {\n\treturn Promise.all(urls.map((url) => fetchLimit(() => extractContent(url, signal, options))));\n}\n"]}