@ashx-j/lunr 0.1.0

package/dist/builtin-extensions/pi-web-access/extract.d.ts

@@ -0,0 +1,48 @@
+import { type Lookup } from "./ssrf-protection.ts";
+/**
+ * Read `ssrf.allowRanges` (CIDR strings) from web-search.json. Returns [] when
+ * the file is missing, unreadable, or the key is unset so SSRF protection stays
+ * fully on by default. Throws when `ssrf.allowRanges` is present but not an array
+ * so a mistyped value (e.g. a bare string instead of a JSON array) fails loudly
+ * instead of being silently ignored. Exempts synthetic ranges used by TUN/fake-IP
+ * proxies (e.g. 198.18.0.0/15).
+ */
+export declare function loadSsrfAllowRanges(): string[];
+export interface VideoFrame {
+    data: string;
+    mimeType: string;
+    timestamp: string;
+}
+export type FrameData = {
+    data: string;
+    mimeType: string;
+};
+export type FrameResult = FrameData | {
+    error: string;
+};
+export interface ExtractedContent {
+    url: string;
+    title: string;
+    content: string;
+    error: string | null;
+    thumbnail?: {
+        data: string;
+        mimeType: string;
+    };
+    frames?: VideoFrame[];
+    duration?: number;
+}
+export interface ExtractOptions {
+    timeoutMs?: number;
+    forceClone?: boolean;
+    prompt?: string;
+    timestamp?: string;
+    frames?: number;
+    model?: string;
+    /** Custom DNS resolver used for SSRF validation. Primarily a test seam. */
+    lookup?: Lookup;
+}
+export declare function extractContent(url: string, signal?: AbortSignal, options?: ExtractOptions): Promise<ExtractedContent>;
+export declare function extractHeadingTitle(text: string): string | null;
+export declare function fetchAllContent(urls: string[], signal?: AbortSignal, options?: ExtractOptions): Promise<ExtractedContent[]>;
+//# sourceMappingURL=extract.d.ts.map

package/dist/builtin-extensions/pi-web-access/extract.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"extract.d.ts","sourceRoot":"","sources":["../../../src/builtin-extensions/pi-web-access/extract.ts"],"names":[],"mappings":"AAcA,OAAO,EAAqC,KAAK,MAAM,EAAE,MAAM,sBAAsB,CAAC;AAUtF;;;;;;;GAOG;AACH,wBAAgB,mBAAmB,IAAI,MAAM,EAAE,CAuB9C;AAyBD,MAAM,WAAW,UAAU;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,MAAM,SAAS,GAAG;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,CAAC;AAC3D,MAAM,MAAM,WAAW,GAAG,SAAS,GAAG;IAAE,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC;AAExD,MAAM,WAAW,gBAAgB;IAChC,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,SAAS,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC;IAC/C,MAAM,CAAC,EAAE,UAAU,EAAE,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,cAAc;IAC9B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,2EAA2E;IAC3E,MAAM,CAAC,EAAE,MAAM,CAAC;CAChB;AA6ID,wBAAsB,cAAc,CACnC,GAAG,EAAE,MAAM,EACX,MAAM,CAAC,EAAE,WAAW,EACpB,OAAO,CAAC,EAAE,cAAc,GACtB,OAAO,CAAC,gBAAgB,CAAC,CA4P3B;AAsLD,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAK/D;AAMD,wBAAsB,eAAe,CACpC,IAAI,EAAE,MAAM,EAAE,EACd,MAAM,CAAC,EAAE,WAAW,EACpB,OAAO,CAAC,EAAE,cAAc,GACtB,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAE7B","sourcesContent":["// @ts-nocheck\nimport { Readability } from \"@mozilla/readability\";\nimport { parseHTML } from \"linkedom\";\nimport TurndownService from \"turndown\";\nimport pLimit from \"p-limit\";\nimport { activityMonitor } from \"./activity.ts\";\nimport { extractRSCContent } from \"./rsc-extract.ts\";\nimport { extractPDFToMarkdown, isPDF } from \"./pdf-extract.ts\";\nimport { extractGitHub } from \"./github-extract.ts\";\nimport { isYouTubeURL, isYouTubeEnabled, extractYouTube, extractYouTubeFrame, extractYouTubeFrames, getYouTubeStreamInfo } from \"./youtube-extract.ts\";\nimport { extractWithUrlContext, extractWithGeminiWeb } from \"./gemini-url-context.ts\";\nimport { extractWithParallel, isParallelAvailable } from \"./parallel.ts\";\nimport { isVideoFile, extractVideo, extractVideoFrame, getLocalVideoDuration } from \"./video-extract.ts\";\nimport { existsSync, readFileSync } from \"node:fs\";\nimport { fetchRemoteUrl, validateRemoteUrl, type Lookup } from \"./ssrf-protection.ts\";\nimport { formatSeconds, getWebSearchConfigPath } from \"./utils.ts\";\n\nconst DEFAULT_TIMEOUT_MS = 30000;\nconst CONCURRENT_LIMIT = 3;\n\nconst NON_RECOVERABLE_ERRORS = [\"Unsupported content type\", \"Response too large\"];\nconst MIN_USEFUL_CONTENT = 500;\nconst WEB_SEARCH_CONFIG_PATH = getWebSearchConfigPath();\n\n/**\n * Read `ssrf.allowRanges` (CIDR strings) from web-search.json. Returns [] when\n * the file is missing, unreadable, or the key is unset so SSRF protection stays\n * fully on by default. Throws when `ssrf.allowRanges` is present but not an array\n * so a mistyped value (e.g. a bare string instead of a JSON array) fails loudly\n * instead of being silently ignored. Exempts synthetic ranges used by TUN/fake-IP\n * proxies (e.g. 198.18.0.0/15).\n */\nexport function loadSsrfAllowRanges(): string[] {\n\tlet value: unknown;\n\ttry {\n\t\tif (!existsSync(WEB_SEARCH_CONFIG_PATH)) return [];\n\t\tconst raw = readFileSync(WEB_SEARCH_CONFIG_PATH, \"utf-8\");\n\t\tvalue = (JSON.parse(raw) as { ssrf?: { allowRanges?: unknown } })?.ssrf?.allowRanges;\n\t} catch {\n\t\t// Missing/unreadable file or invalid JSON: fail safe with SSRF fully on.\n\t\treturn [];\n\t}\n\tif (value === undefined || value === null) return [];\n\tif (!Array.isArray(value)) {\n\t\tthrow new Error(`ssrf.allowRanges in ${WEB_SEARCH_CONFIG_PATH} must be an array of CIDR strings`);\n\t}\n\tconst ranges: string[] = [];\n\tfor (const [index, entry] of value.entries()) {\n\t\tif (typeof entry !== \"string\") {\n\t\t\tthrow new Error(`ssrf.allowRanges in ${WEB_SEARCH_CONFIG_PATH} must contain only CIDR strings; entry ${index + 1} is ${typeof entry}`);\n\t\t}\n\t\tconst trimmed = entry.trim();\n\t\tif (trimmed) ranges.push(trimmed);\n\t}\n\treturn ranges;\n}\n\nfunction errorMessage(err: unknown): string {\n\treturn err instanceof Error ? err.message : String(err);\n}\n\nfunction isConfigParseError(err: unknown): boolean {\n\treturn errorMessage(err).startsWith(\"Failed to parse \");\n}\n\nfunction isAbortError(err: unknown): boolean {\n\treturn errorMessage(err).toLowerCase().includes(\"abort\");\n}\n\nfunction abortedResult(url: string): ExtractedContent {\n\treturn { url, title: \"\", content: \"\", error: \"Aborted\" };\n}\n\nconst turndown = new TurndownService({\n\theadingStyle: \"atx\",\n\tcodeBlockStyle: \"fenced\",\n});\n\nconst fetchLimit = pLimit(CONCURRENT_LIMIT);\n\nexport interface VideoFrame {\n\tdata: string;\n\tmimeType: string;\n\ttimestamp: string;\n}\n\nexport type FrameData = { data: string; mimeType: string };\nexport type FrameResult = FrameData | { error: string };\n\nexport interface ExtractedContent {\n\turl: string;\n\ttitle: string;\n\tcontent: string;\n\terror: string | null;\n\tthumbnail?: { data: string; mimeType: string };\n\tframes?: VideoFrame[];\n\tduration?: number;\n}\n\nexport interface ExtractOptions {\n\ttimeoutMs?: number;\n\tforceClone?: boolean;\n\tprompt?: string;\n\ttimestamp?: string;\n\tframes?: number;\n\tmodel?: string;\n\t/** Custom DNS resolver used for SSRF validation. Primarily a test seam. */\n\tlookup?: Lookup;\n}\n\nconst JINA_READER_BASE = \"https://r.jina.ai/\";\nconst JINA_TIMEOUT_MS = 30000;\n\nasync function extractWithJinaReader(\n\turl: string,\n\tsignal?: AbortSignal,\n\tlookup?: Lookup,\n): Promise<ExtractedContent | null> {\n\tconst jinaUrl = JINA_READER_BASE + url;\n\n\tconst activityId = activityMonitor.logStart({ type: \"api\", query: `jina: ${url}` });\n\n\ttry {\n\t\tawait validateRemoteUrl(url, { allowRanges: loadSsrfAllowRanges(), lookup });\n\t\tconst res = await fetch(jinaUrl, {\n\t\t\theaders: {\n\t\t\t\t\"Accept\": \"text/markdown\",\n\t\t\t\t\"X-No-Cache\": \"true\",\n\t\t\t},\n\t\t\tsignal: AbortSignal.any([\n\t\t\t\tAbortSignal.timeout(JINA_TIMEOUT_MS),\n\t\t\t\t...(signal ? [signal] : []),\n\t\t\t]),\n\t\t});\n\n\t\tif (!res.ok) {\n\t\t\tactivityMonitor.logComplete(activityId, res.status);\n\t\t\treturn null;\n\t\t}\n\n\t\tconst content = await res.text();\n\t\tactivityMonitor.logComplete(activityId, res.status);\n\n\t\tconst contentStart = content.indexOf(\"Markdown Content:\");\n\t\tif (contentStart < 0) {\n\t\t\treturn null;\n\t\t}\n\n\t\tconst markdownPart = content.slice(contentStart + 17).trim(); // 17 = \"Markdown Content:\".length\n\n\t\t// Check for failed JS rendering or minimal content\n\t\tif (markdownPart.length < 100 ||\n\t\t\tmarkdownPart.startsWith(\"Loading...\") ||\n\t\t\tmarkdownPart.startsWith(\"Please enable JavaScript\")) {\n\t\t\treturn null;\n\t\t}\n\n\t\tconst title = extractHeadingTitle(markdownPart) ?? (new URL(url).pathname.split(\"/\").pop() || url);\n\t\treturn { url, title, content: markdownPart, error: null };\n\t} catch (err) {\n\t\tconst message = err instanceof Error ? err.message : String(err);\n\t\tif (message.toLowerCase().includes(\"abort\")) {\n\t\t\tactivityMonitor.logComplete(activityId, 0);\n\t\t} else {\n\t\t\tactivityMonitor.logError(activityId, message);\n\t\t}\n\t\treturn null;\n\t}\n}\n\nfunction parseTimestamp(ts: string): number | null {\n\tconst num = Number(ts);\n\tif (!isNaN(num) && num >= 0) return Math.floor(num);\n\tconst parts = ts.split(\":\").map(Number);\n\tif (parts.some(p => isNaN(p) || p < 0)) return null;\n\tif (parts.length === 3) return Math.floor(parts[0] * 3600 + parts[1] * 60 + parts[2]);\n\tif (parts.length === 2) return Math.floor(parts[0] * 60 + parts[1]);\n\treturn null;\n}\n\ntype TimestampSpec = { type: \"single\"; seconds: number } | { type: \"range\"; start: number; end: number };\n\nfunction parseTimestampSpec(ts: string): TimestampSpec | null {\n\tconst dashIdx = ts.indexOf(\"-\", 1);\n\tif (dashIdx > 0) {\n\t\tconst start = parseTimestamp(ts.slice(0, dashIdx));\n\t\tconst end = parseTimestamp(ts.slice(dashIdx + 1));\n\t\tif (start !== null && end !== null && end > start) return { type: \"range\", start, end };\n\t}\n\tconst seconds = parseTimestamp(ts);\n\treturn seconds !== null ? { type: \"single\", seconds } : null;\n}\n\nconst DEFAULT_RANGE_FRAMES = 6;\nconst MIN_FRAME_INTERVAL = 5;\n\nfunction computeRangeTimestamps(start: number, end: number, maxFrames: number = DEFAULT_RANGE_FRAMES): number[] {\n\tif (maxFrames <= 1) return [start];\n\tconst duration = end - start;\n\tconst idealInterval = duration / (maxFrames - 1);\n\tif (idealInterval < MIN_FRAME_INTERVAL) {\n\t\tconst timestamps: number[] = [];\n\t\tfor (let t = start; t <= end && timestamps.length < maxFrames; t += MIN_FRAME_INTERVAL) {\n\t\t\ttimestamps.push(t);\n\t\t}\n\t\treturn timestamps;\n\t}\n\treturn Array.from({ length: maxFrames }, (_, i) => Math.round(start + i * idealInterval));\n}\n\nfunction buildFrameResult(\n\turl: string, label: string, requestedCount: number,\n\tframes: VideoFrame[], error: string | null, duration?: number,\n): ExtractedContent {\n\tif (frames.length === 0) {\n\t\tconst msg = error ?? \"Frame extraction failed\";\n\t\treturn { url, title: `Frames ${label} (0/${requestedCount})`, content: msg, error: msg };\n\t}\n\treturn {\n\t\turl,\n\t\ttitle: `Frames ${label} (${frames.length}/${requestedCount})`,\n\t\tcontent: `${frames.length} frames extracted from ${label}`,\n\t\terror: null,\n\t\tframes,\n\t\tduration,\n\t};\n}\n\nasync function extractLocalFrames(\n\tfilePath: string, timestamps: number[],\n): Promise<{ frames: VideoFrame[]; error: string | null }> {\n\tconst results = await Promise.all(timestamps.map(async (t) => {\n\t\tconst frame = await extractVideoFrame(filePath, t);\n\t\tif (\"error\" in frame) return { error: frame.error };\n\t\treturn { ...frame, timestamp: formatSeconds(t) };\n\t}));\n\tconst frames = results.filter((f): f is VideoFrame => \"data\" in f);\n\tconst firstError = results.find((f): f is { error: string } => \"error\" in f);\n\treturn { frames, error: frames.length === 0 && firstError ? firstError.error : null };\n}\n\nfunction safeVideoInfo(url: string): { info: ReturnType<typeof isVideoFile>; error?: string } {\n\ttry {\n\t\treturn { info: isVideoFile(url) };\n\t} catch (err) {\n\t\treturn { info: null, error: errorMessage(err) };\n\t}\n}\n\nexport async function extractContent(\n\turl: string,\n\tsignal?: AbortSignal,\n\toptions?: ExtractOptions,\n): Promise<ExtractedContent> {\n\tif (signal?.aborted) {\n\t\treturn { url, title: \"\", content: \"\", error: \"Aborted\" };\n\t}\n\n\tif (options?.frames && !options.timestamp) {\n\t\tconst frameCount = options.frames;\n\t\tconst ytInfo = isYouTubeURL(url);\n\t\tif (ytInfo.isYouTube && ytInfo.videoId) {\n\t\t\tconst streamInfo = await getYouTubeStreamInfo(ytInfo.videoId);\n\t\t\tif (\"error\" in streamInfo) {\n\t\t\t\treturn { url, title: \"Frames\", content: streamInfo.error, error: streamInfo.error };\n\t\t\t}\n\t\t\tif (streamInfo.duration === null) {\n\t\t\t\tconst error = \"Cannot determine video duration. Use a timestamp range instead.\";\n\t\t\t\treturn { url, title: \"Frames\", content: error, error };\n\t\t\t}\n\t\t\tconst dur = Math.floor(streamInfo.duration);\n\t\t\tconst timestamps = computeRangeTimestamps(0, dur, frameCount);\n\t\t\tconst result = await extractYouTubeFrames(ytInfo.videoId, timestamps, streamInfo);\n\t\t\tconst label = `${formatSeconds(0)}-${formatSeconds(dur)}`;\n\t\t\treturn buildFrameResult(url, label, timestamps.length, result.frames, result.error, streamInfo.duration);\n\t\t}\n\n\t\tconst localVideo = safeVideoInfo(url);\n\t\tif (localVideo.error) {\n\t\t\treturn { url, title: \"\", content: \"\", error: localVideo.error };\n\t\t}\n\t\tif (localVideo.info) {\n\t\t\tconst durationResult = await getLocalVideoDuration(localVideo.info.absolutePath);\n\t\t\tif (typeof durationResult !== \"number\") {\n\t\t\t\treturn { url, title: \"Frames\", content: durationResult.error, error: durationResult.error };\n\t\t\t}\n\t\t\tconst dur = Math.floor(durationResult);\n\t\t\tconst timestamps = computeRangeTimestamps(0, dur, frameCount);\n\t\t\tconst result = await extractLocalFrames(localVideo.info.absolutePath, timestamps);\n\t\t\tconst label = `${formatSeconds(0)}-${formatSeconds(dur)}`;\n\t\t\treturn buildFrameResult(url, label, timestamps.length, result.frames, result.error, durationResult);\n\t\t}\n\n\t\treturn { url, title: \"\", content: \"\", error: \"Frame extraction only works with YouTube and local video files\" };\n\t}\n\n\tif (options?.timestamp) {\n\t\tconst spec = parseTimestampSpec(options.timestamp);\n\t\tif (!spec) {\n\t\t\treturn {\n\t\t\t\turl,\n\t\t\t\ttitle: \"\",\n\t\t\t\tcontent: \"\",\n\t\t\t\terror: `Invalid timestamp format: \"${options.timestamp}\". Use \"H:MM:SS\", \"MM:SS\", \"85\", or \"start-end\".`,\n\t\t\t};\n\t\t}\n\n\t\tconst frameCount = options.frames;\n\t\tconst ytInfo = isYouTubeURL(url);\n\t\tif (ytInfo.isYouTube && ytInfo.videoId) {\n\t\t\tconst streamInfo = await getYouTubeStreamInfo(ytInfo.videoId);\n\t\t\tif (\"error\" in streamInfo) {\n\t\t\t\tif (spec.type === \"range\") {\n\t\t\t\t\tconst label = `${formatSeconds(spec.start)}-${formatSeconds(spec.end)}`;\n\t\t\t\t\treturn { url, title: `Frames ${label}`, content: streamInfo.error, error: streamInfo.error };\n\t\t\t\t}\n\t\t\t\tif (frameCount) {\n\t\t\t\t\tconst end = spec.seconds + (frameCount - 1) * MIN_FRAME_INTERVAL;\n\t\t\t\t\tconst label = `${formatSeconds(spec.seconds)}-${formatSeconds(end)}`;\n\t\t\t\t\treturn { url, title: `Frames ${label}`, content: streamInfo.error, error: streamInfo.error };\n\t\t\t\t}\n\t\t\t\treturn { url, title: `Frame at ${options.timestamp}`, content: streamInfo.error, error: streamInfo.error };\n\t\t\t}\n\n\t\t\tif (spec.type === \"range\") {\n\t\t\t\tconst label = `${formatSeconds(spec.start)}-${formatSeconds(spec.end)}`;\n\t\t\t\tif (streamInfo.duration !== null && spec.end > streamInfo.duration) {\n\t\t\t\t\tconst error = `Timestamp ${formatSeconds(spec.end)} exceeds video duration (${formatSeconds(Math.floor(streamInfo.duration))})`;\n\t\t\t\t\treturn { url, title: `Frames ${label}`, content: error, error };\n\t\t\t\t}\n\t\t\t\tconst timestamps = frameCount\n\t\t\t\t\t? computeRangeTimestamps(spec.start, spec.end, frameCount)\n\t\t\t\t\t: computeRangeTimestamps(spec.start, spec.end);\n\t\t\t\tconst result = await extractYouTubeFrames(ytInfo.videoId, timestamps, streamInfo);\n\t\t\t\treturn buildFrameResult(url, label, timestamps.length, result.frames, result.error, result.duration ?? undefined);\n\t\t\t}\n\n\t\t\tif (frameCount) {\n\t\t\t\tconst end = spec.seconds + (frameCount - 1) * MIN_FRAME_INTERVAL;\n\t\t\t\tconst label = `${formatSeconds(spec.seconds)}-${formatSeconds(end)}`;\n\t\t\t\tif (streamInfo.duration !== null && end > streamInfo.duration) {\n\t\t\t\t\tconst error = `Timestamp ${formatSeconds(end)} exceeds video duration (${formatSeconds(Math.floor(streamInfo.duration))})`;\n\t\t\t\t\treturn { url, title: `Frames ${label}`, content: error, error };\n\t\t\t\t}\n\t\t\t\tconst timestamps = computeRangeTimestamps(spec.seconds, end, frameCount);\n\t\t\t\tconst result = await extractYouTubeFrames(ytInfo.videoId, timestamps, streamInfo);\n\t\t\t\treturn buildFrameResult(url, label, timestamps.length, result.frames, result.error, result.duration ?? undefined);\n\t\t\t}\n\n\t\t\tif (streamInfo.duration !== null && spec.seconds > streamInfo.duration) {\n\t\t\t\tconst error = `Timestamp ${formatSeconds(spec.seconds)} exceeds video duration (${formatSeconds(Math.floor(streamInfo.duration))})`;\n\t\t\t\treturn { url, title: `Frame at ${options.timestamp}`, content: error, error };\n\t\t\t}\n\t\t\tconst frame = await extractYouTubeFrame(ytInfo.videoId, spec.seconds, streamInfo);\n\t\t\tif (\"error\" in frame) {\n\t\t\t\treturn { url, title: `Frame at ${options.timestamp}`, content: frame.error, error: frame.error };\n\t\t\t}\n\t\t\treturn { url, title: `Frame at ${options.timestamp}`, content: `Video frame at ${options.timestamp}`, error: null, thumbnail: frame };\n\t\t}\n\n\t\tconst localVideo = safeVideoInfo(url);\n\t\tif (localVideo.error) {\n\t\t\treturn { url, title: \"\", content: \"\", error: localVideo.error };\n\t\t}\n\t\tif (localVideo.info) {\n\t\t\tif (spec.type === \"range\") {\n\t\t\t\tconst timestamps = frameCount\n\t\t\t\t\t? computeRangeTimestamps(spec.start, spec.end, frameCount)\n\t\t\t\t\t: computeRangeTimestamps(spec.start, spec.end);\n\t\t\t\tconst result = await extractLocalFrames(localVideo.info.absolutePath, timestamps);\n\t\t\t\tconst label = `${formatSeconds(spec.start)}-${formatSeconds(spec.end)}`;\n\t\t\t\treturn buildFrameResult(url, label, timestamps.length, result.frames, result.error);\n\t\t\t}\n\n\t\t\tif (frameCount) {\n\t\t\t\tconst end = spec.seconds + (frameCount - 1) * MIN_FRAME_INTERVAL;\n\t\t\t\tconst timestamps = computeRangeTimestamps(spec.seconds, end, frameCount);\n\t\t\t\tconst result = await extractLocalFrames(localVideo.info.absolutePath, timestamps);\n\t\t\t\tconst label = `${formatSeconds(spec.seconds)}-${formatSeconds(end)}`;\n\t\t\t\treturn buildFrameResult(url, label, timestamps.length, result.frames, result.error);\n\t\t\t}\n\n\t\t\tconst frame = await extractVideoFrame(localVideo.info.absolutePath, spec.seconds);\n\t\t\tif (\"error\" in frame) {\n\t\t\t\treturn { url, title: `Frame at ${options.timestamp}`, content: frame.error, error: frame.error };\n\t\t\t}\n\t\t\treturn { url, title: `Frame at ${options.timestamp}`, content: `Video frame at ${options.timestamp}`, error: null, thumbnail: frame };\n\t\t}\n\n\t\treturn { url, title: \"\", content: \"\", error: \"Timestamp extraction only works with YouTube and local video files\" };\n\t}\n\n\tconst localVideo = safeVideoInfo(url);\n\tif (localVideo.error) {\n\t\treturn { url, title: \"\", content: \"\", error: localVideo.error };\n\t}\n\tif (localVideo.info) {\n\t\ttry {\n\t\t\tconst result = await extractVideo(localVideo.info, signal, options);\n\t\t\tif (signal?.aborted) return abortedResult(url);\n\t\t\treturn result ?? { url, title: \"\", content: \"\", error: `Video analysis requires Gemini access. Either:\\n  1. Sign into gemini.google.com in Chrome (free, uses cookies)\\n  2. Set GEMINI_API_KEY in ${WEB_SEARCH_CONFIG_PATH}` };\n\t\t} catch (err) {\n\t\t\tif (isAbortError(err)) return abortedResult(url);\n\t\t\treturn { url, title: \"\", content: \"\", error: errorMessage(err) };\n\t\t}\n\t}\n\n\ttry {\n\t\tconst parsed = new URL(url);\n\t\tif (parsed.protocol === \"http:\" || parsed.protocol === \"https:\") {\n\t\t\tawait validateRemoteUrl(parsed, { allowRanges: loadSsrfAllowRanges(), lookup: options?.lookup });\n\t\t}\n\t} catch (err) {\n\t\treturn { url, title: \"\", content: \"\", error: errorMessage(err) };\n\t}\n\n\ttry {\n\t\tconst ghResult = await extractGitHub(url, signal, options?.forceClone);\n\t\tif (ghResult) return ghResult;\n\t\tif (signal?.aborted) return abortedResult(url);\n\t} catch (err) {\n\t\tconst message = errorMessage(err);\n\t\tif (isAbortError(err)) return abortedResult(url);\n\t\tif (isConfigParseError(err)) {\n\t\t\treturn { url, title: \"\", content: \"\", error: message };\n\t\t}\n\t}\n\n\tconst ytInfo = isYouTubeURL(url);\n\tlet youtubeEnabled = false;\n\ttry {\n\t\tyoutubeEnabled = isYouTubeEnabled();\n\t} catch (err) {\n\t\treturn { url, title: \"\", content: \"\", error: errorMessage(err) };\n\t}\n\tif (ytInfo.isYouTube && youtubeEnabled) {\n\t\ttry {\n\t\t\tconst ytResult = await extractYouTube(url, signal, options?.prompt, options?.model);\n\t\t\tif (ytResult) return ytResult;\n\t\t\tif (signal?.aborted) return abortedResult(url);\n\t\t} catch (err) {\n\t\t\tconst message = errorMessage(err);\n\t\t\tif (isAbortError(err)) return abortedResult(url);\n\t\t\treturn { url, title: \"\", content: \"\", error: message };\n\t\t}\n\t\treturn {\n\t\t\turl,\n\t\t\ttitle: \"\",\n\t\t\tcontent: \"\",\n\t\t\terror: \"Could not extract YouTube video content. Sign into Google in Chrome for automatic access, or set GEMINI_API_KEY.\",\n\t\t};\n\t}\n\n\tif (signal?.aborted) return abortedResult(url);\n\n\tconst httpResult = await extractViaHttp(url, signal, options);\n\n\tif (signal?.aborted) return abortedResult(url);\n\tif (!httpResult.error) return httpResult;\n\tif (NON_RECOVERABLE_ERRORS.some(prefix => httpResult.error!.startsWith(prefix))) return httpResult;\n\n\tconst jinaResult = await extractWithJinaReader(url, signal, options?.lookup);\n\tif (jinaResult) return jinaResult;\n\tif (signal?.aborted) return abortedResult(url);\n\n\tlet parallelError: string | null = null;\n\ttry {\n\t\tif (isParallelAvailable()) {\n\t\t\tconst parallelResult = await extractWithParallel(url, signal, options);\n\t\t\tif (parallelResult) return parallelResult;\n\t\t}\n\t} catch (err) {\n\t\tif (isAbortError(err)) return abortedResult(url);\n\t\tparallelError = errorMessage(err);\n\t\tif (isConfigParseError(err)) {\n\t\t\treturn { ...httpResult, error: parallelError };\n\t\t}\n\t}\n\tif (signal?.aborted) return abortedResult(url);\n\n\tlet geminiResult: ExtractedContent | null = null;\n\ttry {\n\t\tgeminiResult = await extractWithUrlContext(url, signal)\n\t\t\t?? await extractWithGeminiWeb(url, signal);\n\t} catch (err) {\n\t\tif (isAbortError(err)) return abortedResult(url);\n\t\tif (isConfigParseError(err)) {\n\t\t\treturn { ...httpResult, error: errorMessage(err) };\n\t\t}\n\t}\n\n\tif (geminiResult) return geminiResult;\n\tif (signal?.aborted) return abortedResult(url);\n\n\tconst guidance = [\n\t\thttpResult.error,\n\t\t...(parallelError ? [`Parallel fallback failed: ${parallelError}`] : []),\n\t\t\"\",\n\t\t\"Fallback options:\",\n\t\t`  \\u2022 Set PARALLEL_API_KEY in ${WEB_SEARCH_CONFIG_PATH}`,\n\t\t`  \\u2022 Set GEMINI_API_KEY in ${WEB_SEARCH_CONFIG_PATH}`,\n\t\t\"  \\u2022 Sign into gemini.google.com in Chrome\",\n\t\t\"  \\u2022 Use web_search to find content about this topic\",\n\t].join(\"\\n\");\n\treturn { ...httpResult, error: guidance };\n}\n\nfunction isLikelyJSRendered(html: string): boolean {\n\t// Extract body content\n\tconst bodyMatch = html.match(/<body[^>]*>([\\s\\S]*?)<\\/body>/i);\n\tif (!bodyMatch) return false;\n\n\tconst bodyHtml = bodyMatch[1];\n\n\t// Strip tags to get text content\n\tconst textContent = bodyHtml\n\t\t.replace(/<script[\\s\\S]*?<\\/script>/gi, \"\")\n\t\t.replace(/<style[\\s\\S]*?<\\/style>/gi, \"\")\n\t\t.replace(/<[^>]+>/g, \"\")\n\t\t.replace(/\\s+/g, \" \")\n\t\t.trim();\n\n\t// Count scripts\n\tconst scriptCount = (html.match(/<script/gi) || []).length;\n\n\t// Heuristic: little text content but many scripts suggests JS rendering\n\treturn textContent.length < 500 && scriptCount > 3;\n}\n\nasync function extractViaHttp(\n\turl: string,\n\tsignal?: AbortSignal,\n\toptions?: ExtractOptions,\n): Promise<ExtractedContent> {\n\tconst timeoutMs = options?.timeoutMs ?? DEFAULT_TIMEOUT_MS;\n\tconst activityId = activityMonitor.logStart({ type: \"fetch\", url });\n\n\tconst controller = new AbortController();\n\tconst timeoutId = setTimeout(() => controller.abort(), timeoutMs);\n\n\tconst onAbort = () => controller.abort();\n\tsignal?.addEventListener(\"abort\", onAbort);\n\n\ttry {\n\t\tconst response = await fetchRemoteUrl(\n\t\t\turl,\n\t\t\t{\n\t\t\t\tsignal: controller.signal,\n\t\t\t\theaders: {\n\t\t\t\t\t\"User-Agent\": \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36\",\n\t\t\t\t\t\"Accept\": \"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8\",\n\t\t\t\t\t\"Accept-Language\": \"en-US,en;q=0.9\",\n\t\t\t\t\t\"Cache-Control\": \"no-cache\",\n\t\t\t\t\t\"Sec-Fetch-Dest\": \"document\",\n\t\t\t\t\t\"Sec-Fetch-Mode\": \"navigate\",\n\t\t\t\t\t\"Sec-Fetch-Site\": \"none\",\n\t\t\t\t\t\"Sec-Fetch-User\": \"?1\",\n\t\t\t\t\t\"Upgrade-Insecure-Requests\": \"1\",\n\t\t\t\t},\n\t\t\t},\n\t\t\t{ allowRanges: loadSsrfAllowRanges(), lookup: options?.lookup },\n\t\t);\n\n\t\tif (!response.ok) {\n\t\t\tactivityMonitor.logComplete(activityId, response.status);\n\t\t\treturn {\n\t\t\t\turl,\n\t\t\t\ttitle: \"\",\n\t\t\t\tcontent: \"\",\n\t\t\t\terror: `HTTP ${response.status}: ${response.statusText}`,\n\t\t\t};\n\t\t}\n\n\t\tconst contentLengthHeader = response.headers.get(\"content-length\");\n\t\tconst contentType = response.headers.get(\"content-type\") || \"\";\n\t\tconst isPDFContent = isPDF(url, contentType);\n\t\tconst maxResponseSize = isPDFContent ? 20 * 1024 * 1024 : 5 * 1024 * 1024;\n\t\tif (contentLengthHeader) {\n\t\t\tconst contentLength = parseInt(contentLengthHeader, 10);\n\t\t\tif (contentLength > maxResponseSize) {\n\t\t\t\tactivityMonitor.logComplete(activityId, response.status);\n\t\t\t\treturn {\n\t\t\t\t\turl,\n\t\t\t\t\ttitle: \"\",\n\t\t\t\t\tcontent: \"\",\n\t\t\t\t\terror: `Response too large (${Math.round(contentLength / 1024 / 1024)}MB)`,\n\t\t\t\t};\n\t\t\t}\n\t\t}\n\n\t\tif (isPDFContent) {\n\t\t\ttry {\n\t\t\t\tconst buffer = await response.arrayBuffer();\n\t\t\t\tconst result = await extractPDFToMarkdown(buffer, url);\n\t\t\t\tactivityMonitor.logComplete(activityId, response.status);\n\t\t\t\treturn {\n\t\t\t\t\turl,\n\t\t\t\t\ttitle: result.title,\n\t\t\t\t\tcontent: `PDF extracted and saved to: ${result.outputPath}\\n\\nPages: ${result.pages}\\nCharacters: ${result.chars}`,\n\t\t\t\t\terror: null,\n\t\t\t\t};\n\t\t\t} catch (err) {\n\t\t\t\tconst message = err instanceof Error ? err.message : String(err);\n\t\t\t\tactivityMonitor.logError(activityId, message);\n\t\t\t\treturn { url, title: \"\", content: \"\", error: `PDF extraction failed: ${message}` };\n\t\t\t}\n\t\t}\n\n\t\tif (contentType.includes(\"application/octet-stream\") ||\n\t\t\tcontentType.includes(\"image/\") ||\n\t\t\tcontentType.includes(\"audio/\") ||\n\t\t\tcontentType.includes(\"video/\") ||\n\t\t\tcontentType.includes(\"application/zip\")) {\n\t\t\tactivityMonitor.logComplete(activityId, response.status);\n\t\t\treturn {\n\t\t\t\turl,\n\t\t\t\ttitle: \"\",\n\t\t\t\tcontent: \"\",\n\t\t\t\terror: `Unsupported content type: ${contentType.split(\";\")[0]}`,\n\t\t\t};\n\t\t}\n\n\t\tconst text = await response.text();\n\t\tconst isHTML = contentType.includes(\"text/html\") || contentType.includes(\"application/xhtml+xml\");\n\n\t\tif (!isHTML) {\n\t\t\tactivityMonitor.logComplete(activityId, response.status);\n\t\t\tconst title = extractTextTitle(text, url);\n\t\t\treturn { url, title, content: text, error: null };\n\t\t}\n\n\t\tconst { document } = parseHTML(text);\n\t\tconst reader = new Readability(document as unknown as Document);\n\t\tconst article = reader.parse();\n\n\t\tif (!article) {\n\t\t\tconst rscResult = extractRSCContent(text);\n\t\t\tif (rscResult) {\n\t\t\t\tactivityMonitor.logComplete(activityId, response.status);\n\t\t\t\treturn { url, title: rscResult.title, content: rscResult.content, error: null };\n\t\t\t}\n\n\t\t\tactivityMonitor.logComplete(activityId, response.status);\n\n\t\t\t// Provide more specific error message\n\t\t\tconst jsRendered = isLikelyJSRendered(text);\n\t\t\tconst errorMsg = jsRendered\n\t\t\t\t? \"Page appears to be JavaScript-rendered (content loads dynamically)\"\n\t\t\t\t: \"Could not extract readable content from HTML structure\";\n\n\t\t\treturn {\n\t\t\t\turl,\n\t\t\t\ttitle: \"\",\n\t\t\t\tcontent: \"\",\n\t\t\t\terror: errorMsg,\n\t\t\t};\n\t\t}\n\n\t\tconst markdown = turndown.turndown(article.content);\n\t\tactivityMonitor.logComplete(activityId, response.status);\n\n\t\tif (markdown.length < MIN_USEFUL_CONTENT) {\n\t\t\treturn {\n\t\t\t\turl,\n\t\t\t\ttitle: article.title || \"\",\n\t\t\t\tcontent: markdown,\n\t\t\t\terror: isLikelyJSRendered(text)\n\t\t\t\t\t? \"Page appears to be JavaScript-rendered (content loads dynamically)\"\n\t\t\t\t\t: \"Extracted content appears incomplete\",\n\t\t\t};\n\t\t}\n\n\t\treturn { url, title: article.title || \"\", content: markdown, error: null };\n\t} catch (err) {\n\t\tconst message = err instanceof Error ? err.message : String(err);\n\t\tif (message.toLowerCase().includes(\"abort\")) {\n\t\t\tactivityMonitor.logComplete(activityId, 0);\n\t\t} else {\n\t\t\tactivityMonitor.logError(activityId, message);\n\t\t}\n\t\treturn { url, title: \"\", content: \"\", error: message };\n\t} finally {\n\t\tclearTimeout(timeoutId);\n\t\tsignal?.removeEventListener(\"abort\", onAbort);\n\t}\n}\n\nexport function extractHeadingTitle(text: string): string | null {\n\tconst match = text.match(/^#{1,2}\\s+(.+)/m);\n\tif (!match) return null;\n\tconst cleaned = match[1].replace(/\\*+/g, \"\").trim();\n\treturn cleaned || null;\n}\n\nfunction extractTextTitle(text: string, url: string): string {\n\treturn extractHeadingTitle(text) ?? (new URL(url).pathname.split(\"/\").pop() || url);\n}\n\nexport async function fetchAllContent(\n\turls: string[],\n\tsignal?: AbortSignal,\n\toptions?: ExtractOptions,\n): Promise<ExtractedContent[]> {\n\treturn Promise.all(urls.map((url) => fetchLimit(() => extractContent(url, signal, options))));\n}\n"]}