@humain/terminal 0.0.3-beta.0 → 0.0.3-beta.2

package/dist/humain/web-access/vendor/firecrawl.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"firecrawl.d.ts","sourceRoot":"","sources":["../../../../src/humain/web-access/vendor/firecrawl.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,gBAAgB,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAYrE,MAAM,WAAW,oBAAoB;IACpC,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,aAAa,EAAE,OAAO,CAAC;CACvB;AAED,MAAM,WAAW,uBAAwB,SAAQ,IAAI,CAAC,cAAc,EAAE,WAAW,GAAG,QAAQ,CAAC;IAC5F,IAAI,CAAC,EAAE,oBAAoB,CAAC;CAC5B;AAsCD,wBAAgB,yBAAyB,IAAI,IAAI,CAEhD;AA6LD,wBAAgB,oBAAoB,IAAI,OAAO,CAE9C;AAED,wBAAsB,oBAAoB,CACzC,GAAG,EAAE,MAAM,EACX,MAAM,CAAC,EAAE,WAAW,EACpB,OAAO,CAAC,EAAE,uBAAuB,GAC/B,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC,CAmBlC","sourcesContent":["import { existsSync, readFileSync } from \"node:fs\";\nimport { activityMonitor } from \"./activity.ts\";\nimport { redactCredential, resolveCredential } from \"./credential-source.ts\";\nimport type { ExtractedContent, ExtractOptions } from \"./extract.ts\";\nimport { validateRemoteUrl, type Lookup } from \"./ssrf-protection.ts\";\nimport { getWebSearchConfigPath } from \"./utils.ts\";\n\nconst CONFIG_PATH = getWebSearchConfigPath();\nconst DEFAULT_API_VERSION = \"v2\";\nconst EXTRACT_TIMEOUT_MS = 60_000;\nconst DEFAULT_MAX_REDIRECTS = 5;\nconst REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);\nconst SUPPORTED_API_VERSIONS = [\"v1\", \"v2\"] as const;\ntype FirecrawlApiVersion = (typeof SUPPORTED_API_VERSIONS)[number];\n\nexport interface FirecrawlSsrfOptions {\n\tallowRanges: string[];\n\ttrustEnvProxy: boolean;\n}\n\nexport interface FirecrawlExtractOptions extends Pick<ExtractOptions, \"timeoutMs\" | \"lookup\"> {\n\tssrf?: FirecrawlSsrfOptions;\n}\n\ninterface FirecrawlConfig {\n\tfirecrawlBaseUrl?: unknown;\n\tfirecrawlApiKey?: unknown;\n\tfirecrawlApiVersion?: unknown;\n\tfirecrawlFreshScrape?: unknown;\n}\n\ninterface FirecrawlScrapeData {\n\ttitle?: unknown;\n\tmarkdown?: unknown;\n\tmetadata?: { title?: unknown };\n}\n\nlet cachedConfig: FirecrawlConfig | null = null;\n\nfunction loadConfig(): FirecrawlConfig {\n\tif (cachedConfig) return cachedConfig;\n\tif (!existsSync(CONFIG_PATH)) {\n\t\tcachedConfig = {};\n\t\treturn cachedConfig;\n\t}\n\tconst raw = readFileSync(CONFIG_PATH, \"utf8\");\n\tlet parsed: unknown;\n\ttry {\n\t\tparsed = JSON.parse(raw);\n\t} catch (err) {\n\t\tconst message = err instanceof Error ? err.message : String(err);\n\t\tthrow new Error(`Failed to parse ${CONFIG_PATH}: ${message}`);\n\t}\n\tif (!parsed || typeof parsed !== \"object\" || Array.isArray(parsed)) {\n\t\tthrow new Error(`Invalid config in ${CONFIG_PATH}: expected a JSON object`);\n\t}\n\tcachedConfig = parsed as FirecrawlConfig;\n\treturn cachedConfig;\n}\n\nexport function clearFirecrawlConfigCache(): void {\n\tcachedConfig = null;\n}\n\nfunction normalizeBaseUrl(value: unknown): string | null {\n\tif (typeof value !== \"string\") return null;\n\tconst trimmed = value.trim();\n\tif (!trimmed) return null;\n\tlet parsed: URL;\n\ttry {\n\t\tparsed = new URL(trimmed);\n\t} catch {\n\t\tthrow new Error(`Invalid Firecrawl base URL in ${CONFIG_PATH}: expected an HTTP or HTTPS URL`);\n\t}\n\tif (parsed.protocol !== \"http:\" && parsed.protocol !== \"https:\") {\n\t\tthrow new Error(`Invalid Firecrawl base URL in ${CONFIG_PATH}: expected an HTTP or HTTPS URL`);\n\t}\n\tif (parsed.username || parsed.password) {\n\t\tthrow new Error(`Invalid Firecrawl base URL in ${CONFIG_PATH}: URL credentials are not allowed`);\n\t}\n\tparsed.pathname = parsed.pathname.replace(/\\/+$/, \"\");\n\tparsed.search = \"\";\n\tparsed.hash = \"\";\n\treturn parsed.toString().replace(/\\/+$/, \"\");\n}\n\nfunction getBaseUrl(): string | null {\n\treturn normalizeBaseUrl(process.env.FIRECRAWL_BASE_URL) ?? normalizeBaseUrl(loadConfig().firecrawlBaseUrl);\n}\n\nfunction requireBaseUrl(): string {\n\tconst baseUrl = getBaseUrl();\n\tif (!baseUrl) {\n\t\tthrow new Error(\n\t\t\t\"Firecrawl base URL not configured. Either:\\n\" +\n\t\t\t`  1. Set firecrawlBaseUrl in ${CONFIG_PATH}\\n` +\n\t\t\t\"  2. Set FIRECRAWL_BASE_URL environment variable\",\n\t\t);\n\t}\n\treturn baseUrl;\n}\n\nfunction getApiVersion(): FirecrawlApiVersion {\n\tconst environmentValue = typeof process.env.FIRECRAWL_API_VERSION === \"string\"\n\t\t? process.env.FIRECRAWL_API_VERSION.trim()\n\t\t: \"\";\n\tconst raw = environmentValue || loadConfig().firecrawlApiVersion;\n\tif (raw === undefined || raw === null) return DEFAULT_API_VERSION;\n\tif (typeof raw !== \"string\") {\n\t\tthrow new Error(`firecrawlApiVersion in ${CONFIG_PATH} must be a string (\"v1\" or \"v2\")`);\n\t}\n\tconst normalized = raw.trim().toLowerCase();\n\tif (!normalized) return DEFAULT_API_VERSION;\n\tif (!SUPPORTED_API_VERSIONS.includes(normalized as FirecrawlApiVersion)) {\n\t\tthrow new Error(`Unsupported Firecrawl API version \"${raw}\". Supported versions: ${SUPPORTED_API_VERSIONS.join(\", \")}`);\n\t}\n\treturn normalized as FirecrawlApiVersion;\n}\n\nfunction allowFreshScrape(): boolean {\n\tconst environmentValue = process.env.FIRECRAWL_FRESH_SCRAPE;\n\tif (environmentValue !== undefined) return environmentValue === \"1\" || environmentValue.toLowerCase() === \"true\";\n\tconst configured = loadConfig().firecrawlFreshScrape;\n\tif (configured === undefined || configured === null) return false;\n\tif (typeof configured !== \"boolean\") throw new Error(`firecrawlFreshScrape in ${CONFIG_PATH} must be a boolean`);\n\treturn configured;\n}\n\nasync function getApiKey(signal?: AbortSignal): Promise<string | null> {\n\tconst configKey = loadConfig().firecrawlApiKey;\n\treturn resolveCredential({\n\t\tprovider: \"Firecrawl\",\n\t\tconfiguredValue: configKey,\n\t\tenvironmentValue: process.env.FIRECRAWL_API_KEY,\n\t\tsignal,\n\t});\n}\n\nfunction requestSignal(timeoutMs: number, signal?: AbortSignal): AbortSignal {\n\tconst timeout = AbortSignal.timeout(timeoutMs);\n\treturn signal ? AbortSignal.any([timeout, signal]) : timeout;\n}\n\nfunction errorMessage(err: unknown): string {\n\treturn err instanceof Error ? err.message : String(err);\n}\n\nfunction isAbortError(err: unknown): boolean {\n\treturn errorMessage(err).toLowerCase().includes(\"abort\");\n}\n\nfunction ssrfOptions(options?: FirecrawlExtractOptions): { lookup?: Lookup; allowRanges: string[]; trustEnvProxy: boolean } {\n\treturn {\n\t\tallowRanges: options?.ssrf?.allowRanges ?? [],\n\t\ttrustEnvProxy: options?.ssrf?.trustEnvProxy ?? false,\n\t\t...(options?.lookup ? { lookup: options.lookup } : {}),\n\t};\n}\n\nfunction withoutSensitiveHeaders(headers: Record<string, string>): Record<string, string> {\n\tconst next = { ...headers };\n\tdelete next.Authorization;\n\tdelete next.authorization;\n\tdelete next.Cookie;\n\tdelete next.cookie;\n\tdelete next[\"X-API-Key\"];\n\tdelete next[\"x-api-key\"];\n\treturn next;\n}\n\nasync function fetchFirecrawlApi(\n\turl: string,\n\tinit: { method: string; headers: Record<string, string>; body: string; signal: AbortSignal },\n\toptions: FirecrawlExtractOptions | undefined,\n): Promise<Response> {\n\tlet current = await validateRemoteUrl(url, ssrfOptions(options));\n\tlet headers = init.headers;\n\tfor (let redirects = 0; redirects <= DEFAULT_MAX_REDIRECTS; redirects++) {\n\t\tconst response = await fetch(current, { ...init, headers, redirect: \"manual\" });\n\t\tif (!REDIRECT_STATUSES.has(response.status)) return response;\n\n\t\tconst location = response.headers.get(\"location\");\n\t\tif (!location) return response;\n\t\tif (redirects === DEFAULT_MAX_REDIRECTS) throw new Error(`Too many redirects fetching ${current.toString()}`);\n\n\t\tconst next = await validateRemoteUrl(new URL(location, current), ssrfOptions(options));\n\t\tif (next.origin !== current.origin) headers = withoutSensitiveHeaders(headers);\n\t\tcurrent = next;\n\t}\n\tthrow new Error(`Too many redirects fetching ${current.toString()}`);\n}\n\nfunction scrapeBody(url: string): Record<string, unknown> {\n\treturn {\n\t\turl,\n\t\tformats: [\"markdown\"],\n\t\tonlyMainContent: true,\n\t\t...(allowFreshScrape() ? {} : { lockdown: true }),\n\t};\n}\n\nasync function firecrawlFetch(\n\tendpoint: string,\n\tbody: Record<string, unknown>,\n\tsignal: AbortSignal | undefined,\n\toptions: FirecrawlExtractOptions | undefined,\n): Promise<Record<string, unknown>> {\n\tconst baseUrl = requireBaseUrl();\n\tconst version = getApiVersion();\n\tconst apiKey = await getApiKey(signal);\n\tconst headers: Record<string, string> = { \"Content-Type\": \"application/json\" };\n\tif (apiKey) headers.Authorization = `Bearer ${apiKey}`;\n\tconst requestUrl = `${baseUrl}/${version}/${endpoint}`;\n\tconst activityId = activityMonitor.logStart({ type: \"fetch\", url: requestUrl });\n\ttry {\n\t\tconst response = await fetchFirecrawlApi(requestUrl, {\n\t\t\tmethod: \"POST\",\n\t\t\theaders,\n\t\t\tbody: JSON.stringify(body),\n\t\t\tsignal: requestSignal(options?.timeoutMs ?? EXTRACT_TIMEOUT_MS, signal),\n\t\t}, options);\n\t\tif (!response.ok) {\n\t\t\tconst text = await response.text().catch(() => \"\");\n\t\t\tthrow new Error(`Firecrawl scrape error ${response.status}: ${redactCredential(text.slice(0, 300), apiKey)}`);\n\t\t}\n\t\tlet data: unknown;\n\t\ttry {\n\t\t\tdata = await response.json();\n\t\t} catch (err) {\n\t\t\tthrow new Error(`Firecrawl scrape returned invalid JSON: ${errorMessage(err)}`);\n\t\t}\n\t\tif (!data || typeof data !== \"object\" || Array.isArray(data)) {\n\t\t\tthrow new Error(\"Firecrawl scrape returned an unexpected response shape\");\n\t\t}\n\t\tconst envelope = data as Record<string, unknown>;\n\t\tif (envelope.success === false) {\n\t\t\tconst reason = typeof envelope.error === \"string\" && envelope.error.trim() ? envelope.error : \"unknown error\";\n\t\t\tthrow new Error(`Firecrawl scrape unsuccessful: ${redactCredential(reason, apiKey)}`);\n\t\t}\n\t\tif (envelope.success !== true) {\n\t\t\tthrow new Error(\"Firecrawl scrape returned an unexpected response shape\");\n\t\t}\n\t\tactivityMonitor.logComplete(activityId, response.status);\n\t\treturn envelope;\n\t} catch (err) {\n\t\tif (isAbortError(err)) activityMonitor.logComplete(activityId, 0);\n\t\telse activityMonitor.logError(activityId, errorMessage(err));\n\t\tthrow err;\n\t}\n}\n\nexport function isFirecrawlAvailable(): boolean {\n\treturn getBaseUrl() !== null;\n}\n\nexport async function extractWithFirecrawl(\n\turl: string,\n\tsignal?: AbortSignal,\n\toptions?: FirecrawlExtractOptions,\n): Promise<ExtractedContent | null> {\n\trequireBaseUrl();\n\tawait validateRemoteUrl(url, ssrfOptions(options));\n\tconst envelope = await firecrawlFetch(\"scrape\", scrapeBody(url), signal, options);\n\tconst data = envelope.data;\n\tif (!data || typeof data !== \"object\" || Array.isArray(data)) {\n\t\tthrow new Error(\"Firecrawl scrape returned an unexpected data shape\");\n\t}\n\tconst scrape = data as FirecrawlScrapeData;\n\tif (typeof scrape.markdown !== \"string\") {\n\t\tthrow new Error(\"Firecrawl scrape returned markdown in an unexpected shape\");\n\t}\n\tconst content = scrape.markdown.trim();\n\tif (!content) return null;\n\tconst metadataTitle = scrape.metadata?.title;\n\tconst title = typeof metadataTitle === \"string\" && metadataTitle.trim()\n\t\t? metadataTitle.trim()\n\t\t: typeof scrape.title === \"string\" ? scrape.title.trim() : \"\";\n\treturn { url, title, content, error: null };\n}\n"]}
+{"version":3,"file":"firecrawl.d.ts","sourceRoot":"","sources":["../../../../src/humain/web-access/vendor/firecrawl.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,gBAAgB,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AACrE,OAAO,KAAK,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAarE,MAAM,WAAW,oBAAoB;IACpC,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,aAAa,EAAE,OAAO,CAAC;CACvB;AAED,MAAM,WAAW,uBAAwB,SAAQ,IAAI,CAAC,cAAc,EAAE,WAAW,GAAG,QAAQ,CAAC;IAC5F,IAAI,CAAC,EAAE,oBAAoB,CAAC;CAC5B;AAED,UAAU,sBAAuB,SAAQ,aAAa,EAAE,IAAI,CAAC,cAAc,EAAE,WAAW,GAAG,QAAQ,CAAC;IACnG,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,IAAI,CAAC,EAAE,oBAAoB,CAAC;CAC5B;AAyDD,wBAAgB,yBAAyB,IAAI,IAAI,CAEhD;AAiTD,wBAAgB,oBAAoB,IAAI,OAAO,CAE9C;AAED,wBAAsB,mBAAmB,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE,sBAA2B,GAAG,OAAO,CAAC,cAAc,CAAC,CAgBtH;AAED,wBAAsB,oBAAoB,CACzC,GAAG,EAAE,MAAM,EACX,MAAM,CAAC,EAAE,WAAW,EACpB,OAAO,CAAC,EAAE,uBAAuB,GAC/B,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC,CAmBlC","sourcesContent":["import { existsSync, readFileSync } from \"node:fs\";\nimport { activityMonitor } from \"./activity.ts\";\nimport { redactCredential, resolveCredential } from \"./credential-source.ts\";\nimport type { ExtractedContent, ExtractOptions } from \"./extract.ts\";\nimport type { SearchOptions, SearchResponse } from \"./perplexity.ts\";\nimport { loadSsrfConfig, validateRemoteUrl, type Lookup } from \"./ssrf-protection.ts\";\nimport { getWebSearchConfigPath } from \"./utils.ts\";\n\nconst CONFIG_PATH = getWebSearchConfigPath();\nconst DEFAULT_API_VERSION = \"v2\";\nconst EXTRACT_TIMEOUT_MS = 60_000;\nconst SEARCH_TIMEOUT_MS = 60_000;\nconst DEFAULT_MAX_REDIRECTS = 5;\nconst REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);\nconst SUPPORTED_API_VERSIONS = [\"v1\", \"v2\"] as const;\ntype FirecrawlApiVersion = (typeof SUPPORTED_API_VERSIONS)[number];\n\nexport interface FirecrawlSsrfOptions {\n\tallowRanges: string[];\n\ttrustEnvProxy: boolean;\n}\n\nexport interface FirecrawlExtractOptions extends Pick<ExtractOptions, \"timeoutMs\" | \"lookup\"> {\n\tssrf?: FirecrawlSsrfOptions;\n}\n\ninterface FirecrawlSearchOptions extends SearchOptions, Pick<ExtractOptions, \"timeoutMs\" | \"lookup\"> {\n\tincludeContent?: boolean;\n\tssrf?: FirecrawlSsrfOptions;\n}\n\ninterface FirecrawlConfig {\n\tfirecrawlBaseUrl?: unknown;\n\tfirecrawlApiKey?: unknown;\n\tfirecrawlApiVersion?: unknown;\n\tfirecrawlFreshScrape?: unknown;\n}\n\ninterface FirecrawlScrapeData {\n\ttitle?: unknown;\n\tmarkdown?: unknown;\n\tmetadata?: { title?: unknown };\n}\n\ninterface FirecrawlSearchItem {\n\ttitle?: unknown;\n\tdescription?: unknown;\n\tsnippet?: unknown;\n\turl?: unknown;\n\tmarkdown?: unknown;\n\tmetadata?: {\n\t\ttitle?: unknown;\n\t\tdescription?: unknown;\n\t\tsourceURL?: unknown;\n\t\turl?: unknown;\n\t};\n}\n\ninterface DomainFilters {\n\tinclude: string[];\n\texclude: string[];\n}\n\nlet cachedConfig: FirecrawlConfig | null = null;\n\nfunction loadConfig(): FirecrawlConfig {\n\tif (cachedConfig) return cachedConfig;\n\tif (!existsSync(CONFIG_PATH)) {\n\t\tcachedConfig = {};\n\t\treturn cachedConfig;\n\t}\n\tconst raw = readFileSync(CONFIG_PATH, \"utf8\");\n\tlet parsed: unknown;\n\ttry {\n\t\tparsed = JSON.parse(raw);\n\t} catch (err) {\n\t\tconst message = err instanceof Error ? err.message : String(err);\n\t\tthrow new Error(`Failed to parse ${CONFIG_PATH}: ${message}`);\n\t}\n\tif (!parsed || typeof parsed !== \"object\" || Array.isArray(parsed)) {\n\t\tthrow new Error(`Invalid config in ${CONFIG_PATH}: expected a JSON object`);\n\t}\n\tcachedConfig = parsed as FirecrawlConfig;\n\treturn cachedConfig;\n}\n\nexport function clearFirecrawlConfigCache(): void {\n\tcachedConfig = null;\n}\n\nfunction normalizeBaseUrl(value: unknown): string | null {\n\tif (typeof value !== \"string\") return null;\n\tconst trimmed = value.trim();\n\tif (!trimmed) return null;\n\tlet parsed: URL;\n\ttry {\n\t\tparsed = new URL(trimmed);\n\t} catch {\n\t\tthrow new Error(`Invalid Firecrawl base URL in ${CONFIG_PATH}: expected an HTTP or HTTPS URL`);\n\t}\n\tif (parsed.protocol !== \"http:\" && parsed.protocol !== \"https:\") {\n\t\tthrow new Error(`Invalid Firecrawl base URL in ${CONFIG_PATH}: expected an HTTP or HTTPS URL`);\n\t}\n\tif (parsed.username || parsed.password) {\n\t\tthrow new Error(`Invalid Firecrawl base URL in ${CONFIG_PATH}: URL credentials are not allowed`);\n\t}\n\tparsed.pathname = parsed.pathname.replace(/\\/+$/, \"\");\n\tparsed.search = \"\";\n\tparsed.hash = \"\";\n\treturn parsed.toString().replace(/\\/+$/, \"\");\n}\n\nfunction getBaseUrl(): string | null {\n\treturn normalizeBaseUrl(process.env.FIRECRAWL_BASE_URL) ?? normalizeBaseUrl(loadConfig().firecrawlBaseUrl);\n}\n\nfunction requireBaseUrl(): string {\n\tconst baseUrl = getBaseUrl();\n\tif (!baseUrl) {\n\t\tthrow new Error(\n\t\t\t\"Firecrawl base URL not configured. Either:\\n\" +\n\t\t\t`  1. Set firecrawlBaseUrl in ${CONFIG_PATH}\\n` +\n\t\t\t\"  2. Set FIRECRAWL_BASE_URL environment variable\",\n\t\t);\n\t}\n\treturn baseUrl;\n}\n\nfunction getApiVersion(): FirecrawlApiVersion {\n\tconst environmentValue = typeof process.env.FIRECRAWL_API_VERSION === \"string\"\n\t\t? process.env.FIRECRAWL_API_VERSION.trim()\n\t\t: \"\";\n\tconst raw = environmentValue || loadConfig().firecrawlApiVersion;\n\tif (raw === undefined || raw === null) return DEFAULT_API_VERSION;\n\tif (typeof raw !== \"string\") {\n\t\tthrow new Error(`firecrawlApiVersion in ${CONFIG_PATH} must be a string (\"v1\" or \"v2\")`);\n\t}\n\tconst normalized = raw.trim().toLowerCase();\n\tif (!normalized) return DEFAULT_API_VERSION;\n\tif (!SUPPORTED_API_VERSIONS.includes(normalized as FirecrawlApiVersion)) {\n\t\tthrow new Error(`Unsupported Firecrawl API version \"${raw}\". Supported versions: ${SUPPORTED_API_VERSIONS.join(\", \")}`);\n\t}\n\treturn normalized as FirecrawlApiVersion;\n}\n\nfunction allowFreshScrape(): boolean {\n\tconst environmentValue = process.env.FIRECRAWL_FRESH_SCRAPE;\n\tif (environmentValue !== undefined) return environmentValue === \"1\" || environmentValue.toLowerCase() === \"true\";\n\tconst configured = loadConfig().firecrawlFreshScrape;\n\tif (configured === undefined || configured === null) return false;\n\tif (typeof configured !== \"boolean\") throw new Error(`firecrawlFreshScrape in ${CONFIG_PATH} must be a boolean`);\n\treturn configured;\n}\n\nasync function getApiKey(signal?: AbortSignal): Promise<string | null> {\n\tconst configKey = loadConfig().firecrawlApiKey;\n\treturn resolveCredential({\n\t\tprovider: \"Firecrawl\",\n\t\tconfiguredValue: configKey,\n\t\tenvironmentValue: process.env.FIRECRAWL_API_KEY,\n\t\tsignal,\n\t});\n}\n\nfunction requestSignal(timeoutMs: number, signal?: AbortSignal): AbortSignal {\n\tconst timeout = AbortSignal.timeout(timeoutMs);\n\treturn signal ? AbortSignal.any([timeout, signal]) : timeout;\n}\n\nfunction errorMessage(err: unknown): string {\n\treturn err instanceof Error ? err.message : String(err);\n}\n\nfunction isAbortError(err: unknown): boolean {\n\treturn errorMessage(err).toLowerCase().includes(\"abort\");\n}\n\nfunction ssrfOptions(options?: FirecrawlExtractOptions | FirecrawlSearchOptions): { lookup?: Lookup; allowRanges: string[]; trustEnvProxy: boolean } {\n\tconst config = loadSsrfConfig();\n\treturn {\n\t\tallowRanges: options?.ssrf?.allowRanges ?? config.allowRanges,\n\t\ttrustEnvProxy: options?.ssrf?.trustEnvProxy ?? config.trustEnvProxy,\n\t\t...(options?.lookup ? { lookup: options.lookup } : {}),\n\t};\n}\n\nfunction withoutSensitiveHeaders(headers: Record<string, string>): Record<string, string> {\n\tconst next = { ...headers };\n\tdelete next.Authorization;\n\tdelete next.authorization;\n\tdelete next.Cookie;\n\tdelete next.cookie;\n\tdelete next[\"X-API-Key\"];\n\tdelete next[\"x-api-key\"];\n\treturn next;\n}\n\nasync function fetchFirecrawlApi(\n\turl: string,\n\tinit: { method: string; headers: Record<string, string>; body: string; signal: AbortSignal },\n\toptions: FirecrawlExtractOptions | FirecrawlSearchOptions | undefined,\n): Promise<Response> {\n\tlet current = await validateRemoteUrl(url, ssrfOptions(options));\n\tlet headers = init.headers;\n\tfor (let redirects = 0; redirects <= DEFAULT_MAX_REDIRECTS; redirects++) {\n\t\tconst response = await fetch(current, { ...init, headers, redirect: \"manual\" });\n\t\tif (!REDIRECT_STATUSES.has(response.status)) return response;\n\n\t\tconst location = response.headers.get(\"location\");\n\t\tif (!location) return response;\n\t\tif (redirects === DEFAULT_MAX_REDIRECTS) throw new Error(`Too many redirects fetching ${current.toString()}`);\n\n\t\tconst next = await validateRemoteUrl(new URL(location, current), ssrfOptions(options));\n\t\tif (next.origin !== current.origin) headers = withoutSensitiveHeaders(headers);\n\t\tcurrent = next;\n\t}\n\tthrow new Error(`Too many redirects fetching ${current.toString()}`);\n}\n\nfunction scrapeBody(url: string): Record<string, unknown> {\n\treturn {\n\t\turl,\n\t\tformats: [\"markdown\"],\n\t\tonlyMainContent: true,\n\t\t...(allowFreshScrape() ? {} : { lockdown: true }),\n\t};\n}\n\nfunction normalizeCount(value: number | undefined): number {\n\tif (typeof value !== \"number\" || !Number.isFinite(value)) return 5;\n\treturn Math.max(1, Math.min(Math.floor(value), 20));\n}\n\nfunction normalizeDomain(value: string): string | null {\n\tlet input = value.trim().toLowerCase();\n\tif (!input) return null;\n\tif (input.startsWith(\"-\")) input = input.slice(1).trim();\n\tif (!input) return null;\n\ttry {\n\t\tconst parsed = input.includes(\"://\") ? new URL(input) : new URL(`https://${input}`);\n\t\tinput = parsed.hostname;\n\t} catch {\n\t\tinput = input.split(\"/\")[0]?.split(\":\")[0] ?? \"\";\n\t}\n\tinput = input.replace(/^\\.+|\\.+$/g, \"\");\n\treturn /^[a-z0-9][a-z0-9.-]*\\.[a-z]{2,}$/i.test(input) ? input : null;\n}\n\nfunction parseDomainFilter(domainFilter: string[] | undefined): DomainFilters {\n\tconst filters: DomainFilters = { include: [], exclude: [] };\n\tfor (const raw of domainFilter ?? []) {\n\t\tconst domain = normalizeDomain(raw);\n\t\tif (!domain) continue;\n\t\tconst target = raw.trim().startsWith(\"-\") ? filters.exclude : filters.include;\n\t\tif (!target.includes(domain)) target.push(domain);\n\t}\n\treturn filters;\n}\n\nfunction passesDomainFilters(url: string, filters: DomainFilters): boolean {\n\tif (filters.include.length === 0 && filters.exclude.length === 0) return true;\n\tlet hostname: string;\n\ttry {\n\t\thostname = new URL(url).hostname.toLowerCase();\n\t} catch {\n\t\treturn false;\n\t}\n\tconst matches = (domain: string) => hostname === domain || hostname.endsWith(`.${domain}`);\n\tif (filters.exclude.some(matches)) return false;\n\treturn filters.include.length === 0 || filters.include.some(matches);\n}\n\nfunction mapRecencyFilter(value: SearchOptions[\"recencyFilter\"]): string | undefined {\n\tswitch (value) {\n\t\tcase \"day\": return \"qdr:d\";\n\t\tcase \"week\": return \"qdr:w\";\n\t\tcase \"month\": return \"qdr:m\";\n\t\tcase \"year\": return \"qdr:y\";\n\t\tdefault: return undefined;\n\t}\n}\n\nfunction searchBody(query: string, options: FirecrawlSearchOptions, numResults: number, filters: DomainFilters): Record<string, unknown> {\n\treturn {\n\t\tquery,\n\t\tlimit: numResults,\n\t\tsources: [\"web\"],\n\t\t...(filters.include.length > 0 ? { includeDomains: filters.include } : {}),\n\t\t...(filters.include.length === 0 && filters.exclude.length > 0 ? { excludeDomains: filters.exclude } : {}),\n\t\t...(mapRecencyFilter(options.recencyFilter) ? { tbs: mapRecencyFilter(options.recencyFilter) } : {}),\n\t\t...(options.includeContent ? {\n\t\t\tscrapeOptions: {\n\t\t\t\tformats: [\"markdown\"],\n\t\t\t\tonlyMainContent: true,\n\t\t\t\t...(allowFreshScrape() ? {} : { lockdown: true }),\n\t\t\t},\n\t\t} : {}),\n\t};\n}\n\nfunction firstString(...values: unknown[]): string | null {\n\tfor (const value of values) {\n\t\tif (typeof value === \"string\" && value.trim()) return value.trim();\n\t}\n\treturn null;\n}\n\nfunction buildAnswer(results: SearchResponse[\"results\"]): string {\n\treturn results.map((result) => {\n\t\tif (result.snippet) return `${result.snippet}\\nSource: ${result.title} (${result.url})`;\n\t\treturn `Source: ${result.title} (${result.url})`;\n\t}).join(\"\\n\\n\");\n}\n\nfunction mapSearchResults(data: unknown, numResults: number, filters: DomainFilters): {\n\tresults: SearchResponse[\"results\"];\n\tinlineContent: ExtractedContent[];\n} {\n\tconst web = Array.isArray(data)\n\t\t? data\n\t\t: data && typeof data === \"object\" ? (data as Record<string, unknown>).web : undefined;\n\tif (!Array.isArray(web)) throw new Error(\"Firecrawl search returned an unexpected web result shape\");\n\tconst results: SearchResponse[\"results\"] = [];\n\tconst inlineContent: ExtractedContent[] = [];\n\tconst seen = new Set<string>();\n\tfor (const rawItem of web) {\n\t\tif (!rawItem || typeof rawItem !== \"object\" || Array.isArray(rawItem)) continue;\n\t\tconst item = rawItem as FirecrawlSearchItem;\n\t\tconst url = firstString(item.url, item.metadata?.sourceURL, item.metadata?.url);\n\t\tif (!url || seen.has(url) || !passesDomainFilters(url, filters)) continue;\n\t\tseen.add(url);\n\t\tconst title = firstString(item.title, item.metadata?.title) ?? url;\n\t\tconst snippet = firstString(item.description, item.snippet, item.metadata?.description) ?? \"\";\n\t\tresults.push({ title, url, snippet });\n\t\tconst markdown = firstString(item.markdown);\n\t\tif (markdown) inlineContent.push({ url, title, content: markdown, error: null });\n\t\tif (results.length >= numResults) break;\n\t}\n\treturn { results, inlineContent };\n}\n\nasync function firecrawlFetch(\n\tendpoint: string,\n\tbody: Record<string, unknown>,\n\tsignal: AbortSignal | undefined,\n\toptions: FirecrawlExtractOptions | FirecrawlSearchOptions | undefined,\n\tlabel = endpoint,\n\tactivity: { type: \"api\"; query: string } | { type: \"fetch\"; url: string } | undefined = undefined,\n): Promise<Record<string, unknown>> {\n\tconst baseUrl = requireBaseUrl();\n\tconst version = getApiVersion();\n\tconst apiKey = await getApiKey(signal);\n\tconst headers: Record<string, string> = { \"Content-Type\": \"application/json\" };\n\tif (apiKey) headers.Authorization = `Bearer ${apiKey}`;\n\tconst requestUrl = `${baseUrl}/${version}/${endpoint}`;\n\tconst activityId = activityMonitor.logStart(activity ?? { type: \"fetch\", url: requestUrl });\n\ttry {\n\t\tconst response = await fetchFirecrawlApi(requestUrl, {\n\t\t\tmethod: \"POST\",\n\t\t\theaders,\n\t\t\tbody: JSON.stringify(body),\n\t\t\tsignal: requestSignal(options?.timeoutMs ?? EXTRACT_TIMEOUT_MS, signal),\n\t\t}, options);\n\t\tif (!response.ok) {\n\t\t\tconst text = await response.text().catch(() => \"\");\n\t\t\tthrow new Error(`Firecrawl ${label} error ${response.status}: ${redactCredential(text.slice(0, 300), apiKey)}`);\n\t\t}\n\t\tlet data: unknown;\n\t\ttry {\n\t\t\tdata = await response.json();\n\t\t} catch (err) {\n\t\t\tthrow new Error(`Firecrawl ${label} returned invalid JSON: ${errorMessage(err)}`);\n\t\t}\n\t\tif (!data || typeof data !== \"object\" || Array.isArray(data)) {\n\t\t\tthrow new Error(`Firecrawl ${label} returned an unexpected response shape`);\n\t\t}\n\t\tconst envelope = data as Record<string, unknown>;\n\t\tif (envelope.success === false) {\n\t\t\tconst reason = typeof envelope.error === \"string\" && envelope.error.trim() ? envelope.error : \"unknown error\";\n\t\t\tthrow new Error(`Firecrawl ${label} unsuccessful: ${redactCredential(reason, apiKey)}`);\n\t\t}\n\t\tif (envelope.success !== true) {\n\t\t\tthrow new Error(`Firecrawl ${label} returned an unexpected response shape`);\n\t\t}\n\t\tactivityMonitor.logComplete(activityId, response.status);\n\t\treturn envelope;\n\t} catch (err) {\n\t\tif (isAbortError(err)) activityMonitor.logComplete(activityId, 0);\n\t\telse activityMonitor.logError(activityId, errorMessage(err));\n\t\tthrow err;\n\t}\n}\n\nexport function isFirecrawlAvailable(): boolean {\n\treturn getBaseUrl() !== null;\n}\n\nexport async function searchWithFirecrawl(query: string, options: FirecrawlSearchOptions = {}): Promise<SearchResponse> {\n\trequireBaseUrl();\n\tconst numResults = normalizeCount(options.numResults);\n\tconst filters = parseDomainFilter(options.domainFilter);\n\tconst envelope = await firecrawlFetch(\n\t\t\"search\",\n\t\tsearchBody(query, options, numResults, filters),\n\t\toptions.signal,\n\t\t{ ...options, timeoutMs: options.timeoutMs ?? SEARCH_TIMEOUT_MS },\n\t\t\"search\",\n\t\t{ type: \"api\", query },\n\t);\n\tconst mapped = mapSearchResults(envelope.data, numResults, filters);\n\tconst response: SearchResponse = { answer: buildAnswer(mapped.results), results: mapped.results };\n\tif (options.includeContent && mapped.inlineContent.length > 0) response.inlineContent = mapped.inlineContent;\n\treturn response;\n}\n\nexport async function extractWithFirecrawl(\n\turl: string,\n\tsignal?: AbortSignal,\n\toptions?: FirecrawlExtractOptions,\n): Promise<ExtractedContent | null> {\n\trequireBaseUrl();\n\tawait validateRemoteUrl(url, ssrfOptions(options));\n\tconst envelope = await firecrawlFetch(\"scrape\", scrapeBody(url), signal, options);\n\tconst data = envelope.data;\n\tif (!data || typeof data !== \"object\" || Array.isArray(data)) {\n\t\tthrow new Error(\"Firecrawl scrape returned an unexpected data shape\");\n\t}\n\tconst scrape = data as FirecrawlScrapeData;\n\tif (typeof scrape.markdown !== \"string\") {\n\t\tthrow new Error(\"Firecrawl scrape returned markdown in an unexpected shape\");\n\t}\n\tconst content = scrape.markdown.trim();\n\tif (!content) return null;\n\tconst metadataTitle = scrape.metadata?.title;\n\tconst title = typeof metadataTitle === \"string\" && metadataTitle.trim()\n\t\t? metadataTitle.trim()\n\t\t: typeof scrape.title === \"string\" ? scrape.title.trim() : \"\";\n\treturn { url, title, content, error: null };\n}\n"]}

package/dist/humain/web-access/vendor/firecrawl.js

@@ -1,11 +1,12 @@
 import { existsSync, readFileSync } from "node:fs";
 import { activityMonitor } from "./activity.js";
 import { redactCredential, resolveCredential } from "./credential-source.js";
-import { validateRemoteUrl } from "./ssrf-protection.js";
+import { loadSsrfConfig, validateRemoteUrl } from "./ssrf-protection.js";
 import { getWebSearchConfigPath } from "./utils.js";
 const CONFIG_PATH = getWebSearchConfigPath();
 const DEFAULT_API_VERSION = "v2";
 const EXTRACT_TIMEOUT_MS = 60_000;
+const SEARCH_TIMEOUT_MS = 60_000;
 const DEFAULT_MAX_REDIRECTS = 5;
 const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
 const SUPPORTED_API_VERSIONS = ["v1", "v2"];
@@ -120,9 +121,10 @@ function isAbortError(err) {
     return errorMessage(err).toLowerCase().includes("abort");
 }
 function ssrfOptions(options) {
+    const config = loadSsrfConfig();
     return {
-        allowRanges: options?.ssrf?.allowRanges ?? [],
-        trustEnvProxy: options?.ssrf?.trustEnvProxy ?? false,
+        allowRanges: options?.ssrf?.allowRanges ?? config.allowRanges,
+        trustEnvProxy: options?.ssrf?.trustEnvProxy ?? config.trustEnvProxy,
         ...(options?.lookup ? { lookup: options.lookup } : {}),
     };
 }
@@ -163,7 +165,125 @@ function scrapeBody(url) {
         ...(allowFreshScrape() ? {} : { lockdown: true }),
     };
 }
-async function firecrawlFetch(endpoint, body, signal, options) {
+function normalizeCount(value) {
+    if (typeof value !== "number" || !Number.isFinite(value))
+        return 5;
+    return Math.max(1, Math.min(Math.floor(value), 20));
+}
+function normalizeDomain(value) {
+    let input = value.trim().toLowerCase();
+    if (!input)
+        return null;
+    if (input.startsWith("-"))
+        input = input.slice(1).trim();
+    if (!input)
+        return null;
+    try {
+        const parsed = input.includes("://") ? new URL(input) : new URL(`https://${input}`);
+        input = parsed.hostname;
+    }
+    catch {
+        input = input.split("/")[0]?.split(":")[0] ?? "";
+    }
+    input = input.replace(/^\.+|\.+$/g, "");
+    return /^[a-z0-9][a-z0-9.-]*\.[a-z]{2,}$/i.test(input) ? input : null;
+}
+function parseDomainFilter(domainFilter) {
+    const filters = { include: [], exclude: [] };
+    for (const raw of domainFilter ?? []) {
+        const domain = normalizeDomain(raw);
+        if (!domain)
+            continue;
+        const target = raw.trim().startsWith("-") ? filters.exclude : filters.include;
+        if (!target.includes(domain))
+            target.push(domain);
+    }
+    return filters;
+}
+function passesDomainFilters(url, filters) {
+    if (filters.include.length === 0 && filters.exclude.length === 0)
+        return true;
+    let hostname;
+    try {
+        hostname = new URL(url).hostname.toLowerCase();
+    }
+    catch {
+        return false;
+    }
+    const matches = (domain) => hostname === domain || hostname.endsWith(`.${domain}`);
+    if (filters.exclude.some(matches))
+        return false;
+    return filters.include.length === 0 || filters.include.some(matches);
+}
+function mapRecencyFilter(value) {
+    switch (value) {
+        case "day": return "qdr:d";
+        case "week": return "qdr:w";
+        case "month": return "qdr:m";
+        case "year": return "qdr:y";
+        default: return undefined;
+    }
+}
+function searchBody(query, options, numResults, filters) {
+    return {
+        query,
+        limit: numResults,
+        sources: ["web"],
+        ...(filters.include.length > 0 ? { includeDomains: filters.include } : {}),
+        ...(filters.include.length === 0 && filters.exclude.length > 0 ? { excludeDomains: filters.exclude } : {}),
+        ...(mapRecencyFilter(options.recencyFilter) ? { tbs: mapRecencyFilter(options.recencyFilter) } : {}),
+        ...(options.includeContent ? {
+            scrapeOptions: {
+                formats: ["markdown"],
+                onlyMainContent: true,
+                ...(allowFreshScrape() ? {} : { lockdown: true }),
+            },
+        } : {}),
+    };
+}
+function firstString(...values) {
+    for (const value of values) {
+        if (typeof value === "string" && value.trim())
+            return value.trim();
+    }
+    return null;
+}
+function buildAnswer(results) {
+    return results.map((result) => {
+        if (result.snippet)
+            return `${result.snippet}\nSource: ${result.title} (${result.url})`;
+        return `Source: ${result.title} (${result.url})`;
+    }).join("\n\n");
+}
+function mapSearchResults(data, numResults, filters) {
+    const web = Array.isArray(data)
+        ? data
+        : data && typeof data === "object" ? data.web : undefined;
+    if (!Array.isArray(web))
+        throw new Error("Firecrawl search returned an unexpected web result shape");
+    const results = [];
+    const inlineContent = [];
+    const seen = new Set();
+    for (const rawItem of web) {
+        if (!rawItem || typeof rawItem !== "object" || Array.isArray(rawItem))
+            continue;
+        const item = rawItem;
+        const url = firstString(item.url, item.metadata?.sourceURL, item.metadata?.url);
+        if (!url || seen.has(url) || !passesDomainFilters(url, filters))
+            continue;
+        seen.add(url);
+        const title = firstString(item.title, item.metadata?.title) ?? url;
+        const snippet = firstString(item.description, item.snippet, item.metadata?.description) ?? "";
+        results.push({ title, url, snippet });
+        const markdown = firstString(item.markdown);
+        if (markdown)
+            inlineContent.push({ url, title, content: markdown, error: null });
+        if (results.length >= numResults)
+            break;
+    }
+    return { results, inlineContent };
+}
+async function firecrawlFetch(endpoint, body, signal, options, label = endpoint, activity = undefined) {
     const baseUrl = requireBaseUrl();
     const version = getApiVersion();
     const apiKey = await getApiKey(signal);
@@ -171,7 +291,7 @@ async function firecrawlFetch(endpoint, body, signal, options) {
     if (apiKey)
         headers.Authorization = `Bearer ${apiKey}`;
     const requestUrl = `${baseUrl}/${version}/${endpoint}`;
-    const activityId = activityMonitor.logStart({ type: "fetch", url: requestUrl });
+    const activityId = activityMonitor.logStart(activity ?? { type: "fetch", url: requestUrl });
     try {
         const response = await fetchFirecrawlApi(requestUrl, {
             method: "POST",
@@ -181,25 +301,25 @@ async function firecrawlFetch(endpoint, body, signal, options) {
         }, options);
         if (!response.ok) {
             const text = await response.text().catch(() => "");
-            throw new Error(`Firecrawl scrape error ${response.status}: ${redactCredential(text.slice(0, 300), apiKey)}`);
+            throw new Error(`Firecrawl ${label} error ${response.status}: ${redactCredential(text.slice(0, 300), apiKey)}`);
         }
         let data;
         try {
             data = await response.json();
         }
         catch (err) {
-            throw new Error(`Firecrawl scrape returned invalid JSON: ${errorMessage(err)}`);
+            throw new Error(`Firecrawl ${label} returned invalid JSON: ${errorMessage(err)}`);
         }
         if (!data || typeof data !== "object" || Array.isArray(data)) {
-            throw new Error("Firecrawl scrape returned an unexpected response shape");
+            throw new Error(`Firecrawl ${label} returned an unexpected response shape`);
         }
         const envelope = data;
         if (envelope.success === false) {
             const reason = typeof envelope.error === "string" && envelope.error.trim() ? envelope.error : "unknown error";
-            throw new Error(`Firecrawl scrape unsuccessful: ${redactCredential(reason, apiKey)}`);
+            throw new Error(`Firecrawl ${label} unsuccessful: ${redactCredential(reason, apiKey)}`);
         }
         if (envelope.success !== true) {
-            throw new Error("Firecrawl scrape returned an unexpected response shape");
+            throw new Error(`Firecrawl ${label} returned an unexpected response shape`);
         }
         activityMonitor.logComplete(activityId, response.status);
         return envelope;
@@ -215,6 +335,17 @@ async function firecrawlFetch(endpoint, body, signal, options) {
 export function isFirecrawlAvailable() {
     return getBaseUrl() !== null;
 }
+export async function searchWithFirecrawl(query, options = {}) {
+    requireBaseUrl();
+    const numResults = normalizeCount(options.numResults);
+    const filters = parseDomainFilter(options.domainFilter);
+    const envelope = await firecrawlFetch("search", searchBody(query, options, numResults, filters), options.signal, { ...options, timeoutMs: options.timeoutMs ?? SEARCH_TIMEOUT_MS }, "search", { type: "api", query });
+    const mapped = mapSearchResults(envelope.data, numResults, filters);
+    const response = { answer: buildAnswer(mapped.results), results: mapped.results };
+    if (options.includeContent && mapped.inlineContent.length > 0)
+        response.inlineContent = mapped.inlineContent;
+    return response;
+}
 export async function extractWithFirecrawl(url, signal, options) {
     requireBaseUrl();
     await validateRemoteUrl(url, ssrfOptions(options));

package/dist/humain/web-access/vendor/firecrawl.js.map

@@ -1 +1 @@
-{"version":3,"file":"firecrawl.js","sourceRoot":"","sources":["../../../../src/humain/web-access/vendor/firecrawl.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAChD,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAE7E,OAAO,EAAE,iBAAiB,EAAe,MAAM,sBAAsB,CAAC;AACtE,OAAO,EAAE,sBAAsB,EAAE,MAAM,YAAY,CAAC;AAEpD,MAAM,WAAW,GAAG,sBAAsB,EAAE,CAAC;AAC7C,MAAM,mBAAmB,GAAG,IAAI,CAAC;AACjC,MAAM,kBAAkB,GAAG,MAAM,CAAC;AAClC,MAAM,qBAAqB,GAAG,CAAC,CAAC;AAChC,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAC7D,MAAM,sBAAsB,GAAG,CAAC,IAAI,EAAE,IAAI,CAAU,CAAC;AAyBrD,IAAI,YAAY,GAA2B,IAAI,CAAC;AAEhD,SAAS,UAAU,GAAoB;IACtC,IAAI,YAAY;QAAE,OAAO,YAAY,CAAC;IACtC,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;QAC9B,YAAY,GAAG,EAAE,CAAC;QAClB,OAAO,YAAY,CAAC;IACrB,CAAC;IACD,MAAM,GAAG,GAAG,YAAY,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;IAC9C,IAAI,MAAe,CAAC;IACpB,IAAI,CAAC;QACJ,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC1B,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACd,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACjE,MAAM,IAAI,KAAK,CAAC,mBAAmB,WAAW,KAAK,OAAO,EAAE,CAAC,CAAC;IAC/D,CAAC;IACD,IAAI,CAAC,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QACpE,MAAM,IAAI,KAAK,CAAC,qBAAqB,WAAW,0BAA0B,CAAC,CAAC;IAC7E,CAAC;IACD,YAAY,GAAG,MAAyB,CAAC;IACzC,OAAO,YAAY,CAAC;AAAA,CACpB;AAED,MAAM,UAAU,yBAAyB,GAAS;IACjD,YAAY,GAAG,IAAI,CAAC;AAAA,CACpB;AAED,SAAS,gBAAgB,CAAC,KAAc,EAAiB;IACxD,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IAC3C,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;IAC7B,IAAI,CAAC,OAAO;QAAE,OAAO,IAAI,CAAC;IAC1B,IAAI,MAAW,CAAC;IAChB,IAAI,CAAC;QACJ,MAAM,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC;IAC3B,CAAC;IAAC,MAAM,CAAC;QACR,MAAM,IAAI,KAAK,CAAC,iCAAiC,WAAW,iCAAiC,CAAC,CAAC;IAChG,CAAC;IACD,IAAI,MAAM,CAAC,QAAQ,KAAK,OAAO,IAAI,MAAM,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QACjE,MAAM,IAAI,KAAK,CAAC,iCAAiC,WAAW,iCAAiC,CAAC,CAAC;IAChG,CAAC;IACD,IAAI,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;QACxC,MAAM,IAAI,KAAK,CAAC,iCAAiC,WAAW,mCAAmC,CAAC,CAAC;IAClG,CAAC;IACD,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IACtD,MAAM,CAAC,MAAM,GAAG,EAAE,CAAC;IACnB,MAAM,CAAC,IAAI,GAAG,EAAE,CAAC;IACjB,OAAO,MAAM,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;AAAA,CAC7C;AAED,SAAS,UAAU,GAAkB;IACpC,OAAO,gBAAgB,CAAC,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,IAAI,gBAAgB,CAAC,UAAU,EAAE,CAAC,gBAAgB,CAAC,CAAC;AAAA,CAC3G;AAED,SAAS,cAAc,GAAW;IACjC,MAAM,OAAO,GAAG,UAAU,EAAE,CAAC;IAC7B,IAAI,CAAC,OAAO,EAAE,CAAC;QACd,MAAM,IAAI,KAAK,CACd,8CAA8C;YAC9C,gCAAgC,WAAW,IAAI;YAC/C,kDAAkD,CAClD,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC;AAAA,CACf;AAED,SAAS,aAAa,GAAwB;IAC7C,MAAM,gBAAgB,GAAG,OAAO,OAAO,CAAC,GAAG,CAAC,qBAAqB,KAAK,QAAQ;QAC7E,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,IAAI,EAAE;QAC1C,CAAC,CAAC,EAAE,CAAC;IACN,MAAM,GAAG,GAAG,gBAAgB,IAAI,UAAU,EAAE,CAAC,mBAAmB,CAAC;IACjE,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,IAAI;QAAE,OAAO,mBAAmB,CAAC;IAClE,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;QAC7B,MAAM,IAAI,KAAK,CAAC,0BAA0B,WAAW,kCAAkC,CAAC,CAAC;IAC1F,CAAC;IACD,MAAM,UAAU,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC5C,IAAI,CAAC,UAAU;QAAE,OAAO,mBAAmB,CAAC;IAC5C,IAAI,CAAC,sBAAsB,CAAC,QAAQ,CAAC,UAAiC,CAAC,EAAE,CAAC;QACzE,MAAM,IAAI,KAAK,CAAC,sCAAsC,GAAG,0BAA0B,sBAAsB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACzH,CAAC;IACD,OAAO,UAAiC,CAAC;AAAA,CACzC;AAED,SAAS,gBAAgB,GAAY;IACpC,MAAM,gBAAgB,GAAG,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC;IAC5D,IAAI,gBAAgB,KAAK,SAAS;QAAE,OAAO,gBAAgB,KAAK,GAAG,IAAI,gBAAgB,CAAC,WAAW,EAAE,KAAK,MAAM,CAAC;IACjH,MAAM,UAAU,GAAG,UAAU,EAAE,CAAC,oBAAoB,CAAC;IACrD,IAAI,UAAU,KAAK,SAAS,IAAI,UAAU,KAAK,IAAI;QAAE,OAAO,KAAK,CAAC;IAClE,IAAI,OAAO,UAAU,KAAK,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,2BAA2B,WAAW,oBAAoB,CAAC,CAAC;IACjH,OAAO,UAAU,CAAC;AAAA,CAClB;AAED,KAAK,UAAU,SAAS,CAAC,MAAoB,EAA0B;IACtE,MAAM,SAAS,GAAG,UAAU,EAAE,CAAC,eAAe,CAAC;IAC/C,OAAO,iBAAiB,CAAC;QACxB,QAAQ,EAAE,WAAW;QACrB,eAAe,EAAE,SAAS;QAC1B,gBAAgB,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB;QAC/C,MAAM;KACN,CAAC,CAAC;AAAA,CACH;AAED,SAAS,aAAa,CAAC,SAAiB,EAAE,MAAoB,EAAe;IAC5E,MAAM,OAAO,GAAG,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAC/C,OAAO,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;AAAA,CAC7D;AAED,SAAS,YAAY,CAAC,GAAY,EAAU;IAC3C,OAAO,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;AAAA,CACxD;AAED,SAAS,YAAY,CAAC,GAAY,EAAW;IAC5C,OAAO,YAAY,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;AAAA,CACzD;AAED,SAAS,WAAW,CAAC,OAAiC,EAAsE;IAC3H,OAAO;QACN,WAAW,EAAE,OAAO,EAAE,IAAI,EAAE,WAAW,IAAI,EAAE;QAC7C,aAAa,EAAE,OAAO,EAAE,IAAI,EAAE,aAAa,IAAI,KAAK;QACpD,GAAG,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACtD,CAAC;AAAA,CACF;AAED,SAAS,uBAAuB,CAAC,OAA+B,EAA0B;IACzF,MAAM,IAAI,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;IAC5B,OAAO,IAAI,CAAC,aAAa,CAAC;IAC1B,OAAO,IAAI,CAAC,aAAa,CAAC;IAC1B,OAAO,IAAI,CAAC,MAAM,CAAC;IACnB,OAAO,IAAI,CAAC,MAAM,CAAC;IACnB,OAAO,IAAI,CAAC,WAAW,CAAC,CAAC;IACzB,OAAO,IAAI,CAAC,WAAW,CAAC,CAAC;IACzB,OAAO,IAAI,CAAC;AAAA,CACZ;AAED,KAAK,UAAU,iBAAiB,CAC/B,GAAW,EACX,IAA4F,EAC5F,OAA4C,EACxB;IACpB,IAAI,OAAO,GAAG,MAAM,iBAAiB,CAAC,GAAG,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC;IACjE,IAAI,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;IAC3B,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,IAAI,qBAAqB,EAAE,SAAS,EAAE,EAAE,CAAC;QACzE,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,OAAO,EAAE,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAC;QAChF,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC;YAAE,OAAO,QAAQ,CAAC;QAE7D,MAAM,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAClD,IAAI,CAAC,QAAQ;YAAE,OAAO,QAAQ,CAAC;QAC/B,IAAI,SAAS,KAAK,qBAAqB;YAAE,MAAM,IAAI,KAAK,CAAC,+BAA+B,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QAE9G,MAAM,IAAI,GAAG,MAAM,iBAAiB,CAAC,IAAI,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC;QACvF,IAAI,IAAI,CAAC,MAAM,KAAK,OAAO,CAAC,MAAM;YAAE,OAAO,GAAG,uBAAuB,CAAC,OAAO,CAAC,CAAC;QAC/E,OAAO,GAAG,IAAI,CAAC;IAChB,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,+BAA+B,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;AAAA,CACrE;AAED,SAAS,UAAU,CAAC,GAAW,EAA2B;IACzD,OAAO;QACN,GAAG;QACH,OAAO,EAAE,CAAC,UAAU,CAAC;QACrB,eAAe,EAAE,IAAI;QACrB,GAAG,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;KACjD,CAAC;AAAA,CACF;AAED,KAAK,UAAU,cAAc,CAC5B,QAAgB,EAChB,IAA6B,EAC7B,MAA+B,EAC/B,OAA4C,EACT;IACnC,MAAM,OAAO,GAAG,cAAc,EAAE,CAAC;IACjC,MAAM,OAAO,GAAG,aAAa,EAAE,CAAC;IAChC,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,MAAM,CAAC,CAAC;IACvC,MAAM,OAAO,GAA2B,EAAE,cAAc,EAAE,kBAAkB,EAAE,CAAC;IAC/E,IAAI,MAAM;QAAE,OAAO,CAAC,aAAa,GAAG,UAAU,MAAM,EAAE,CAAC;IACvD,MAAM,UAAU,GAAG,GAAG,OAAO,IAAI,OAAO,IAAI,QAAQ,EAAE,CAAC;IACvD,MAAM,UAAU,GAAG,eAAe,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC,CAAC;IAChF,IAAI,CAAC;QACJ,MAAM,QAAQ,GAAG,MAAM,iBAAiB,CAAC,UAAU,EAAE;YACpD,MAAM,EAAE,MAAM;YACd,OAAO;YACP,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;YAC1B,MAAM,EAAE,aAAa,CAAC,OAAO,EAAE,SAAS,IAAI,kBAAkB,EAAE,MAAM,CAAC;SACvE,EAAE,OAAO,CAAC,CAAC;QACZ,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YAClB,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;YACnD,MAAM,IAAI,KAAK,CAAC,0BAA0B,QAAQ,CAAC,MAAM,KAAK,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;QAC/G,CAAC;QACD,IAAI,IAAa,CAAC;QAClB,IAAI,CAAC;YACJ,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QAC9B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACd,MAAM,IAAI,KAAK,CAAC,2CAA2C,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACjF,CAAC;QACD,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;YAC9D,MAAM,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAC;QAC3E,CAAC;QACD,MAAM,QAAQ,GAAG,IAA+B,CAAC;QACjD,IAAI,QAAQ,CAAC,OAAO,KAAK,KAAK,EAAE,CAAC;YAChC,MAAM,MAAM,GAAG,OAAO,QAAQ,CAAC,KAAK,KAAK,QAAQ,IAAI,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,eAAe,CAAC;YAC9G,MAAM,IAAI,KAAK,CAAC,kCAAkC,gBAAgB,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;QACvF,CAAC;QACD,IAAI,QAAQ,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;YAC/B,MAAM,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAC;QAC3E,CAAC;QACD,eAAe,CAAC,WAAW,CAAC,UAAU,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;QACzD,OAAO,QAAQ,CAAC;IACjB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACd,IAAI,YAAY,CAAC,GAAG,CAAC;YAAE,eAAe,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;;YAC7D,eAAe,CAAC,QAAQ,CAAC,UAAU,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC;QAC7D,MAAM,GAAG,CAAC;IACX,CAAC;AAAA,CACD;AAED,MAAM,UAAU,oBAAoB,GAAY;IAC/C,OAAO,UAAU,EAAE,KAAK,IAAI,CAAC;AAAA,CAC7B;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACzC,GAAW,EACX,MAAoB,EACpB,OAAiC,EACE;IACnC,cAAc,EAAE,CAAC;IACjB,MAAM,iBAAiB,CAAC,GAAG,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC;IACnD,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,QAAQ,EAAE,UAAU,CAAC,GAAG,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;IAClF,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC;IAC3B,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;QAC9D,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;IACvE,CAAC;IACD,MAAM,MAAM,GAAG,IAA2B,CAAC;IAC3C,IAAI,OAAO,MAAM,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QACzC,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;IAC9E,CAAC;IACD,MAAM,OAAO,GAAG,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;IACvC,IAAI,CAAC,OAAO;QAAE,OAAO,IAAI,CAAC;IAC1B,MAAM,aAAa,GAAG,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC;IAC7C,MAAM,KAAK,GAAG,OAAO,aAAa,KAAK,QAAQ,IAAI,aAAa,CAAC,IAAI,EAAE;QACtE,CAAC,CAAC,aAAa,CAAC,IAAI,EAAE;QACtB,CAAC,CAAC,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IAC/D,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;AAAA,CAC5C","sourcesContent":["import { existsSync, readFileSync } from \"node:fs\";\nimport { activityMonitor } from \"./activity.ts\";\nimport { redactCredential, resolveCredential } from \"./credential-source.ts\";\nimport type { ExtractedContent, ExtractOptions } from \"./extract.ts\";\nimport { validateRemoteUrl, type Lookup } from \"./ssrf-protection.ts\";\nimport { getWebSearchConfigPath } from \"./utils.ts\";\n\nconst CONFIG_PATH = getWebSearchConfigPath();\nconst DEFAULT_API_VERSION = \"v2\";\nconst EXTRACT_TIMEOUT_MS = 60_000;\nconst DEFAULT_MAX_REDIRECTS = 5;\nconst REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);\nconst SUPPORTED_API_VERSIONS = [\"v1\", \"v2\"] as const;\ntype FirecrawlApiVersion = (typeof SUPPORTED_API_VERSIONS)[number];\n\nexport interface FirecrawlSsrfOptions {\n\tallowRanges: string[];\n\ttrustEnvProxy: boolean;\n}\n\nexport interface FirecrawlExtractOptions extends Pick<ExtractOptions, \"timeoutMs\" | \"lookup\"> {\n\tssrf?: FirecrawlSsrfOptions;\n}\n\ninterface FirecrawlConfig {\n\tfirecrawlBaseUrl?: unknown;\n\tfirecrawlApiKey?: unknown;\n\tfirecrawlApiVersion?: unknown;\n\tfirecrawlFreshScrape?: unknown;\n}\n\ninterface FirecrawlScrapeData {\n\ttitle?: unknown;\n\tmarkdown?: unknown;\n\tmetadata?: { title?: unknown };\n}\n\nlet cachedConfig: FirecrawlConfig | null = null;\n\nfunction loadConfig(): FirecrawlConfig {\n\tif (cachedConfig) return cachedConfig;\n\tif (!existsSync(CONFIG_PATH)) {\n\t\tcachedConfig = {};\n\t\treturn cachedConfig;\n\t}\n\tconst raw = readFileSync(CONFIG_PATH, \"utf8\");\n\tlet parsed: unknown;\n\ttry {\n\t\tparsed = JSON.parse(raw);\n\t} catch (err) {\n\t\tconst message = err instanceof Error ? err.message : String(err);\n\t\tthrow new Error(`Failed to parse ${CONFIG_PATH}: ${message}`);\n\t}\n\tif (!parsed || typeof parsed !== \"object\" || Array.isArray(parsed)) {\n\t\tthrow new Error(`Invalid config in ${CONFIG_PATH}: expected a JSON object`);\n\t}\n\tcachedConfig = parsed as FirecrawlConfig;\n\treturn cachedConfig;\n}\n\nexport function clearFirecrawlConfigCache(): void {\n\tcachedConfig = null;\n}\n\nfunction normalizeBaseUrl(value: unknown): string | null {\n\tif (typeof value !== \"string\") return null;\n\tconst trimmed = value.trim();\n\tif (!trimmed) return null;\n\tlet parsed: URL;\n\ttry {\n\t\tparsed = new URL(trimmed);\n\t} catch {\n\t\tthrow new Error(`Invalid Firecrawl base URL in ${CONFIG_PATH}: expected an HTTP or HTTPS URL`);\n\t}\n\tif (parsed.protocol !== \"http:\" && parsed.protocol !== \"https:\") {\n\t\tthrow new Error(`Invalid Firecrawl base URL in ${CONFIG_PATH}: expected an HTTP or HTTPS URL`);\n\t}\n\tif (parsed.username || parsed.password) {\n\t\tthrow new Error(`Invalid Firecrawl base URL in ${CONFIG_PATH}: URL credentials are not allowed`);\n\t}\n\tparsed.pathname = parsed.pathname.replace(/\\/+$/, \"\");\n\tparsed.search = \"\";\n\tparsed.hash = \"\";\n\treturn parsed.toString().replace(/\\/+$/, \"\");\n}\n\nfunction getBaseUrl(): string | null {\n\treturn normalizeBaseUrl(process.env.FIRECRAWL_BASE_URL) ?? normalizeBaseUrl(loadConfig().firecrawlBaseUrl);\n}\n\nfunction requireBaseUrl(): string {\n\tconst baseUrl = getBaseUrl();\n\tif (!baseUrl) {\n\t\tthrow new Error(\n\t\t\t\"Firecrawl base URL not configured. Either:\\n\" +\n\t\t\t`  1. Set firecrawlBaseUrl in ${CONFIG_PATH}\\n` +\n\t\t\t\"  2. Set FIRECRAWL_BASE_URL environment variable\",\n\t\t);\n\t}\n\treturn baseUrl;\n}\n\nfunction getApiVersion(): FirecrawlApiVersion {\n\tconst environmentValue = typeof process.env.FIRECRAWL_API_VERSION === \"string\"\n\t\t? process.env.FIRECRAWL_API_VERSION.trim()\n\t\t: \"\";\n\tconst raw = environmentValue || loadConfig().firecrawlApiVersion;\n\tif (raw === undefined || raw === null) return DEFAULT_API_VERSION;\n\tif (typeof raw !== \"string\") {\n\t\tthrow new Error(`firecrawlApiVersion in ${CONFIG_PATH} must be a string (\"v1\" or \"v2\")`);\n\t}\n\tconst normalized = raw.trim().toLowerCase();\n\tif (!normalized) return DEFAULT_API_VERSION;\n\tif (!SUPPORTED_API_VERSIONS.includes(normalized as FirecrawlApiVersion)) {\n\t\tthrow new Error(`Unsupported Firecrawl API version \"${raw}\". Supported versions: ${SUPPORTED_API_VERSIONS.join(\", \")}`);\n\t}\n\treturn normalized as FirecrawlApiVersion;\n}\n\nfunction allowFreshScrape(): boolean {\n\tconst environmentValue = process.env.FIRECRAWL_FRESH_SCRAPE;\n\tif (environmentValue !== undefined) return environmentValue === \"1\" || environmentValue.toLowerCase() === \"true\";\n\tconst configured = loadConfig().firecrawlFreshScrape;\n\tif (configured === undefined || configured === null) return false;\n\tif (typeof configured !== \"boolean\") throw new Error(`firecrawlFreshScrape in ${CONFIG_PATH} must be a boolean`);\n\treturn configured;\n}\n\nasync function getApiKey(signal?: AbortSignal): Promise<string | null> {\n\tconst configKey = loadConfig().firecrawlApiKey;\n\treturn resolveCredential({\n\t\tprovider: \"Firecrawl\",\n\t\tconfiguredValue: configKey,\n\t\tenvironmentValue: process.env.FIRECRAWL_API_KEY,\n\t\tsignal,\n\t});\n}\n\nfunction requestSignal(timeoutMs: number, signal?: AbortSignal): AbortSignal {\n\tconst timeout = AbortSignal.timeout(timeoutMs);\n\treturn signal ? AbortSignal.any([timeout, signal]) : timeout;\n}\n\nfunction errorMessage(err: unknown): string {\n\treturn err instanceof Error ? err.message : String(err);\n}\n\nfunction isAbortError(err: unknown): boolean {\n\treturn errorMessage(err).toLowerCase().includes(\"abort\");\n}\n\nfunction ssrfOptions(options?: FirecrawlExtractOptions): { lookup?: Lookup; allowRanges: string[]; trustEnvProxy: boolean } {\n\treturn {\n\t\tallowRanges: options?.ssrf?.allowRanges ?? [],\n\t\ttrustEnvProxy: options?.ssrf?.trustEnvProxy ?? false,\n\t\t...(options?.lookup ? { lookup: options.lookup } : {}),\n\t};\n}\n\nfunction withoutSensitiveHeaders(headers: Record<string, string>): Record<string, string> {\n\tconst next = { ...headers };\n\tdelete next.Authorization;\n\tdelete next.authorization;\n\tdelete next.Cookie;\n\tdelete next.cookie;\n\tdelete next[\"X-API-Key\"];\n\tdelete next[\"x-api-key\"];\n\treturn next;\n}\n\nasync function fetchFirecrawlApi(\n\turl: string,\n\tinit: { method: string; headers: Record<string, string>; body: string; signal: AbortSignal },\n\toptions: FirecrawlExtractOptions | undefined,\n): Promise<Response> {\n\tlet current = await validateRemoteUrl(url, ssrfOptions(options));\n\tlet headers = init.headers;\n\tfor (let redirects = 0; redirects <= DEFAULT_MAX_REDIRECTS; redirects++) {\n\t\tconst response = await fetch(current, { ...init, headers, redirect: \"manual\" });\n\t\tif (!REDIRECT_STATUSES.has(response.status)) return response;\n\n\t\tconst location = response.headers.get(\"location\");\n\t\tif (!location) return response;\n\t\tif (redirects === DEFAULT_MAX_REDIRECTS) throw new Error(`Too many redirects fetching ${current.toString()}`);\n\n\t\tconst next = await validateRemoteUrl(new URL(location, current), ssrfOptions(options));\n\t\tif (next.origin !== current.origin) headers = withoutSensitiveHeaders(headers);\n\t\tcurrent = next;\n\t}\n\tthrow new Error(`Too many redirects fetching ${current.toString()}`);\n}\n\nfunction scrapeBody(url: string): Record<string, unknown> {\n\treturn {\n\t\turl,\n\t\tformats: [\"markdown\"],\n\t\tonlyMainContent: true,\n\t\t...(allowFreshScrape() ? {} : { lockdown: true }),\n\t};\n}\n\nasync function firecrawlFetch(\n\tendpoint: string,\n\tbody: Record<string, unknown>,\n\tsignal: AbortSignal | undefined,\n\toptions: FirecrawlExtractOptions | undefined,\n): Promise<Record<string, unknown>> {\n\tconst baseUrl = requireBaseUrl();\n\tconst version = getApiVersion();\n\tconst apiKey = await getApiKey(signal);\n\tconst headers: Record<string, string> = { \"Content-Type\": \"application/json\" };\n\tif (apiKey) headers.Authorization = `Bearer ${apiKey}`;\n\tconst requestUrl = `${baseUrl}/${version}/${endpoint}`;\n\tconst activityId = activityMonitor.logStart({ type: \"fetch\", url: requestUrl });\n\ttry {\n\t\tconst response = await fetchFirecrawlApi(requestUrl, {\n\t\t\tmethod: \"POST\",\n\t\t\theaders,\n\t\t\tbody: JSON.stringify(body),\n\t\t\tsignal: requestSignal(options?.timeoutMs ?? EXTRACT_TIMEOUT_MS, signal),\n\t\t}, options);\n\t\tif (!response.ok) {\n\t\t\tconst text = await response.text().catch(() => \"\");\n\t\t\tthrow new Error(`Firecrawl scrape error ${response.status}: ${redactCredential(text.slice(0, 300), apiKey)}`);\n\t\t}\n\t\tlet data: unknown;\n\t\ttry {\n\t\t\tdata = await response.json();\n\t\t} catch (err) {\n\t\t\tthrow new Error(`Firecrawl scrape returned invalid JSON: ${errorMessage(err)}`);\n\t\t}\n\t\tif (!data || typeof data !== \"object\" || Array.isArray(data)) {\n\t\t\tthrow new Error(\"Firecrawl scrape returned an unexpected response shape\");\n\t\t}\n\t\tconst envelope = data as Record<string, unknown>;\n\t\tif (envelope.success === false) {\n\t\t\tconst reason = typeof envelope.error === \"string\" && envelope.error.trim() ? envelope.error : \"unknown error\";\n\t\t\tthrow new Error(`Firecrawl scrape unsuccessful: ${redactCredential(reason, apiKey)}`);\n\t\t}\n\t\tif (envelope.success !== true) {\n\t\t\tthrow new Error(\"Firecrawl scrape returned an unexpected response shape\");\n\t\t}\n\t\tactivityMonitor.logComplete(activityId, response.status);\n\t\treturn envelope;\n\t} catch (err) {\n\t\tif (isAbortError(err)) activityMonitor.logComplete(activityId, 0);\n\t\telse activityMonitor.logError(activityId, errorMessage(err));\n\t\tthrow err;\n\t}\n}\n\nexport function isFirecrawlAvailable(): boolean {\n\treturn getBaseUrl() !== null;\n}\n\nexport async function extractWithFirecrawl(\n\turl: string,\n\tsignal?: AbortSignal,\n\toptions?: FirecrawlExtractOptions,\n): Promise<ExtractedContent | null> {\n\trequireBaseUrl();\n\tawait validateRemoteUrl(url, ssrfOptions(options));\n\tconst envelope = await firecrawlFetch(\"scrape\", scrapeBody(url), signal, options);\n\tconst data = envelope.data;\n\tif (!data || typeof data !== \"object\" || Array.isArray(data)) {\n\t\tthrow new Error(\"Firecrawl scrape returned an unexpected data shape\");\n\t}\n\tconst scrape = data as FirecrawlScrapeData;\n\tif (typeof scrape.markdown !== \"string\") {\n\t\tthrow new Error(\"Firecrawl scrape returned markdown in an unexpected shape\");\n\t}\n\tconst content = scrape.markdown.trim();\n\tif (!content) return null;\n\tconst metadataTitle = scrape.metadata?.title;\n\tconst title = typeof metadataTitle === \"string\" && metadataTitle.trim()\n\t\t? metadataTitle.trim()\n\t\t: typeof scrape.title === \"string\" ? scrape.title.trim() : \"\";\n\treturn { url, title, content, error: null };\n}\n"]}
+{"version":3,"file":"firecrawl.js","sourceRoot":"","sources":["../../../../src/humain/web-access/vendor/firecrawl.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAChD,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAG7E,OAAO,EAAE,cAAc,EAAE,iBAAiB,EAAe,MAAM,sBAAsB,CAAC;AACtF,OAAO,EAAE,sBAAsB,EAAE,MAAM,YAAY,CAAC;AAEpD,MAAM,WAAW,GAAG,sBAAsB,EAAE,CAAC;AAC7C,MAAM,mBAAmB,GAAG,IAAI,CAAC;AACjC,MAAM,kBAAkB,GAAG,MAAM,CAAC;AAClC,MAAM,iBAAiB,GAAG,MAAM,CAAC;AACjC,MAAM,qBAAqB,GAAG,CAAC,CAAC;AAChC,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAC7D,MAAM,sBAAsB,GAAG,CAAC,IAAI,EAAE,IAAI,CAAU,CAAC;AAiDrD,IAAI,YAAY,GAA2B,IAAI,CAAC;AAEhD,SAAS,UAAU,GAAoB;IACtC,IAAI,YAAY;QAAE,OAAO,YAAY,CAAC;IACtC,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;QAC9B,YAAY,GAAG,EAAE,CAAC;QAClB,OAAO,YAAY,CAAC;IACrB,CAAC;IACD,MAAM,GAAG,GAAG,YAAY,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;IAC9C,IAAI,MAAe,CAAC;IACpB,IAAI,CAAC;QACJ,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC1B,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACd,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACjE,MAAM,IAAI,KAAK,CAAC,mBAAmB,WAAW,KAAK,OAAO,EAAE,CAAC,CAAC;IAC/D,CAAC;IACD,IAAI,CAAC,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QACpE,MAAM,IAAI,KAAK,CAAC,qBAAqB,WAAW,0BAA0B,CAAC,CAAC;IAC7E,CAAC;IACD,YAAY,GAAG,MAAyB,CAAC;IACzC,OAAO,YAAY,CAAC;AAAA,CACpB;AAED,MAAM,UAAU,yBAAyB,GAAS;IACjD,YAAY,GAAG,IAAI,CAAC;AAAA,CACpB;AAED,SAAS,gBAAgB,CAAC,KAAc,EAAiB;IACxD,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IAC3C,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;IAC7B,IAAI,CAAC,OAAO;QAAE,OAAO,IAAI,CAAC;IAC1B,IAAI,MAAW,CAAC;IAChB,IAAI,CAAC;QACJ,MAAM,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC;IAC3B,CAAC;IAAC,MAAM,CAAC;QACR,MAAM,IAAI,KAAK,CAAC,iCAAiC,WAAW,iCAAiC,CAAC,CAAC;IAChG,CAAC;IACD,IAAI,MAAM,CAAC,QAAQ,KAAK,OAAO,IAAI,MAAM,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QACjE,MAAM,IAAI,KAAK,CAAC,iCAAiC,WAAW,iCAAiC,CAAC,CAAC;IAChG,CAAC;IACD,IAAI,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;QACxC,MAAM,IAAI,KAAK,CAAC,iCAAiC,WAAW,mCAAmC,CAAC,CAAC;IAClG,CAAC;IACD,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IACtD,MAAM,CAAC,MAAM,GAAG,EAAE,CAAC;IACnB,MAAM,CAAC,IAAI,GAAG,EAAE,CAAC;IACjB,OAAO,MAAM,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;AAAA,CAC7C;AAED,SAAS,UAAU,GAAkB;IACpC,OAAO,gBAAgB,CAAC,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,IAAI,gBAAgB,CAAC,UAAU,EAAE,CAAC,gBAAgB,CAAC,CAAC;AAAA,CAC3G;AAED,SAAS,cAAc,GAAW;IACjC,MAAM,OAAO,GAAG,UAAU,EAAE,CAAC;IAC7B,IAAI,CAAC,OAAO,EAAE,CAAC;QACd,MAAM,IAAI,KAAK,CACd,8CAA8C;YAC9C,gCAAgC,WAAW,IAAI;YAC/C,kDAAkD,CAClD,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC;AAAA,CACf;AAED,SAAS,aAAa,GAAwB;IAC7C,MAAM,gBAAgB,GAAG,OAAO,OAAO,CAAC,GAAG,CAAC,qBAAqB,KAAK,QAAQ;QAC7E,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,IAAI,EAAE;QAC1C,CAAC,CAAC,EAAE,CAAC;IACN,MAAM,GAAG,GAAG,gBAAgB,IAAI,UAAU,EAAE,CAAC,mBAAmB,CAAC;IACjE,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,IAAI;QAAE,OAAO,mBAAmB,CAAC;IAClE,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;QAC7B,MAAM,IAAI,KAAK,CAAC,0BAA0B,WAAW,kCAAkC,CAAC,CAAC;IAC1F,CAAC;IACD,MAAM,UAAU,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC5C,IAAI,CAAC,UAAU;QAAE,OAAO,mBAAmB,CAAC;IAC5C,IAAI,CAAC,sBAAsB,CAAC,QAAQ,CAAC,UAAiC,CAAC,EAAE,CAAC;QACzE,MAAM,IAAI,KAAK,CAAC,sCAAsC,GAAG,0BAA0B,sBAAsB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACzH,CAAC;IACD,OAAO,UAAiC,CAAC;AAAA,CACzC;AAED,SAAS,gBAAgB,GAAY;IACpC,MAAM,gBAAgB,GAAG,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC;IAC5D,IAAI,gBAAgB,KAAK,SAAS;QAAE,OAAO,gBAAgB,KAAK,GAAG,IAAI,gBAAgB,CAAC,WAAW,EAAE,KAAK,MAAM,CAAC;IACjH,MAAM,UAAU,GAAG,UAAU,EAAE,CAAC,oBAAoB,CAAC;IACrD,IAAI,UAAU,KAAK,SAAS,IAAI,UAAU,KAAK,IAAI;QAAE,OAAO,KAAK,CAAC;IAClE,IAAI,OAAO,UAAU,KAAK,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,2BAA2B,WAAW,oBAAoB,CAAC,CAAC;IACjH,OAAO,UAAU,CAAC;AAAA,CAClB;AAED,KAAK,UAAU,SAAS,CAAC,MAAoB,EAA0B;IACtE,MAAM,SAAS,GAAG,UAAU,EAAE,CAAC,eAAe,CAAC;IAC/C,OAAO,iBAAiB,CAAC;QACxB,QAAQ,EAAE,WAAW;QACrB,eAAe,EAAE,SAAS;QAC1B,gBAAgB,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB;QAC/C,MAAM;KACN,CAAC,CAAC;AAAA,CACH;AAED,SAAS,aAAa,CAAC,SAAiB,EAAE,MAAoB,EAAe;IAC5E,MAAM,OAAO,GAAG,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAC/C,OAAO,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;AAAA,CAC7D;AAED,SAAS,YAAY,CAAC,GAAY,EAAU;IAC3C,OAAO,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;AAAA,CACxD;AAED,SAAS,YAAY,CAAC,GAAY,EAAW;IAC5C,OAAO,YAAY,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;AAAA,CACzD;AAED,SAAS,WAAW,CAAC,OAA0D,EAAsE;IACpJ,MAAM,MAAM,GAAG,cAAc,EAAE,CAAC;IAChC,OAAO;QACN,WAAW,EAAE,OAAO,EAAE,IAAI,EAAE,WAAW,IAAI,MAAM,CAAC,WAAW;QAC7D,aAAa,EAAE,OAAO,EAAE,IAAI,EAAE,aAAa,IAAI,MAAM,CAAC,aAAa;QACnE,GAAG,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACtD,CAAC;AAAA,CACF;AAED,SAAS,uBAAuB,CAAC,OAA+B,EAA0B;IACzF,MAAM,IAAI,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;IAC5B,OAAO,IAAI,CAAC,aAAa,CAAC;IAC1B,OAAO,IAAI,CAAC,aAAa,CAAC;IAC1B,OAAO,IAAI,CAAC,MAAM,CAAC;IACnB,OAAO,IAAI,CAAC,MAAM,CAAC;IACnB,OAAO,IAAI,CAAC,WAAW,CAAC,CAAC;IACzB,OAAO,IAAI,CAAC,WAAW,CAAC,CAAC;IACzB,OAAO,IAAI,CAAC;AAAA,CACZ;AAED,KAAK,UAAU,iBAAiB,CAC/B,GAAW,EACX,IAA4F,EAC5F,OAAqE,EACjD;IACpB,IAAI,OAAO,GAAG,MAAM,iBAAiB,CAAC,GAAG,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC;IACjE,IAAI,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;IAC3B,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,IAAI,qBAAqB,EAAE,SAAS,EAAE,EAAE,CAAC;QACzE,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,OAAO,EAAE,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAC;QAChF,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC;YAAE,OAAO,QAAQ,CAAC;QAE7D,MAAM,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAClD,IAAI,CAAC,QAAQ;YAAE,OAAO,QAAQ,CAAC;QAC/B,IAAI,SAAS,KAAK,qBAAqB;YAAE,MAAM,IAAI,KAAK,CAAC,+BAA+B,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QAE9G,MAAM,IAAI,GAAG,MAAM,iBAAiB,CAAC,IAAI,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC;QACvF,IAAI,IAAI,CAAC,MAAM,KAAK,OAAO,CAAC,MAAM;YAAE,OAAO,GAAG,uBAAuB,CAAC,OAAO,CAAC,CAAC;QAC/E,OAAO,GAAG,IAAI,CAAC;IAChB,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,+BAA+B,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;AAAA,CACrE;AAED,SAAS,UAAU,CAAC,GAAW,EAA2B;IACzD,OAAO;QACN,GAAG;QACH,OAAO,EAAE,CAAC,UAAU,CAAC;QACrB,eAAe,EAAE,IAAI;QACrB,GAAG,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;KACjD,CAAC;AAAA,CACF;AAED,SAAS,cAAc,CAAC,KAAyB,EAAU;IAC1D,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACnE,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;AAAA,CACpD;AAED,SAAS,eAAe,CAAC,KAAa,EAAiB;IACtD,IAAI,KAAK,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACvC,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAC;IACxB,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC;QAAE,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACzD,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAC;IACxB,IAAI,CAAC;QACJ,MAAM,MAAM,GAAG,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,WAAW,KAAK,EAAE,CAAC,CAAC;QACpF,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC;IACzB,CAAC;IAAC,MAAM,CAAC;QACR,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAClD,CAAC;IACD,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;IACxC,OAAO,mCAAmC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;AAAA,CACtE;AAED,SAAS,iBAAiB,CAAC,YAAkC,EAAiB;IAC7E,MAAM,OAAO,GAAkB,EAAE,OAAO,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;IAC5D,KAAK,MAAM,GAAG,IAAI,YAAY,IAAI,EAAE,EAAE,CAAC;QACtC,MAAM,MAAM,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC;QACpC,IAAI,CAAC,MAAM;YAAE,SAAS;QACtB,MAAM,MAAM,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC;QAC9E,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC;YAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACnD,CAAC;IACD,OAAO,OAAO,CAAC;AAAA,CACf;AAED,SAAS,mBAAmB,CAAC,GAAW,EAAE,OAAsB,EAAW;IAC1E,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAC9E,IAAI,QAAgB,CAAC;IACrB,IAAI,CAAC;QACJ,QAAQ,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;IAChD,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,KAAK,CAAC;IACd,CAAC;IACD,MAAM,OAAO,GAAG,CAAC,MAAc,EAAE,EAAE,CAAC,QAAQ,KAAK,MAAM,IAAI,QAAQ,CAAC,QAAQ,CAAC,IAAI,MAAM,EAAE,CAAC,CAAC;IAC3F,IAAI,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC;QAAE,OAAO,KAAK,CAAC;IAChD,OAAO,OAAO,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;AAAA,CACrE;AAED,SAAS,gBAAgB,CAAC,KAAqC,EAAsB;IACpF,QAAQ,KAAK,EAAE,CAAC;QACf,KAAK,KAAK,EAAE,OAAO,OAAO,CAAC;QAC3B,KAAK,MAAM,EAAE,OAAO,OAAO,CAAC;QAC5B,KAAK,OAAO,EAAE,OAAO,OAAO,CAAC;QAC7B,KAAK,MAAM,EAAE,OAAO,OAAO,CAAC;QAC5B,SAAS,OAAO,SAAS,CAAC;IAC3B,CAAC;AAAA,CACD;AAED,SAAS,UAAU,CAAC,KAAa,EAAE,OAA+B,EAAE,UAAkB,EAAE,OAAsB,EAA2B;IACxI,OAAO;QACN,KAAK;QACL,KAAK,EAAE,UAAU;QACjB,OAAO,EAAE,CAAC,KAAK,CAAC;QAChB,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1E,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1G,GAAG,CAAC,gBAAgB,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,gBAAgB,CAAC,OAAO,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACpG,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC;YAC5B,aAAa,EAAE;gBACd,OAAO,EAAE,CAAC,UAAU,CAAC;gBACrB,eAAe,EAAE,IAAI;gBACrB,GAAG,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;aACjD;SACD,CAAC,CAAC,CAAC,EAAE,CAAC;KACP,CAAC;AAAA,CACF;AAED,SAAS,WAAW,CAAC,GAAG,MAAiB,EAAiB;IACzD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC5B,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE;YAAE,OAAO,KAAK,CAAC,IAAI,EAAE,CAAC;IACpE,CAAC;IACD,OAAO,IAAI,CAAC;AAAA,CACZ;AAED,SAAS,WAAW,CAAC,OAAkC,EAAU;IAChE,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;QAC9B,IAAI,MAAM,CAAC,OAAO;YAAE,OAAO,GAAG,MAAM,CAAC,OAAO,aAAa,MAAM,CAAC,KAAK,KAAK,MAAM,CAAC,GAAG,GAAG,CAAC;QACxF,OAAO,WAAW,MAAM,CAAC,KAAK,KAAK,MAAM,CAAC,GAAG,GAAG,CAAC;IAAA,CACjD,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AAAA,CAChB;AAED,SAAS,gBAAgB,CAAC,IAAa,EAAE,UAAkB,EAAE,OAAsB,EAGjF;IACD,MAAM,GAAG,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;QAC9B,CAAC,CAAC,IAAI;QACN,CAAC,CAAC,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAE,IAAgC,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;IACxF,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,0DAA0D,CAAC,CAAC;IACrG,MAAM,OAAO,GAA8B,EAAE,CAAC;IAC9C,MAAM,aAAa,GAAuB,EAAE,CAAC;IAC7C,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,OAAO,IAAI,GAAG,EAAE,CAAC;QAC3B,IAAI,CAAC,OAAO,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC;YAAE,SAAS;QAChF,MAAM,IAAI,GAAG,OAA8B,CAAC;QAC5C,MAAM,GAAG,GAAG,WAAW,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,SAAS,EAAE,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QAChF,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,mBAAmB,CAAC,GAAG,EAAE,OAAO,CAAC;YAAE,SAAS;QAC1E,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACd,MAAM,KAAK,GAAG,WAAW,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,QAAQ,EAAE,KAAK,CAAC,IAAI,GAAG,CAAC;QACnE,MAAM,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,QAAQ,EAAE,WAAW,CAAC,IAAI,EAAE,CAAC;QAC9F,OAAO,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,CAAC;QACtC,MAAM,QAAQ,GAAG,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC5C,IAAI,QAAQ;YAAE,aAAa,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACjF,IAAI,OAAO,CAAC,MAAM,IAAI,UAAU;YAAE,MAAM;IACzC,CAAC;IACD,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,CAAC;AAAA,CAClC;AAED,KAAK,UAAU,cAAc,CAC5B,QAAgB,EAChB,IAA6B,EAC7B,MAA+B,EAC/B,OAAqE,EACrE,KAAK,GAAG,QAAQ,EAChB,QAAQ,GAAgF,SAAS,EAC9D;IACnC,MAAM,OAAO,GAAG,cAAc,EAAE,CAAC;IACjC,MAAM,OAAO,GAAG,aAAa,EAAE,CAAC;IAChC,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,MAAM,CAAC,CAAC;IACvC,MAAM,OAAO,GAA2B,EAAE,cAAc,EAAE,kBAAkB,EAAE,CAAC;IAC/E,IAAI,MAAM;QAAE,OAAO,CAAC,aAAa,GAAG,UAAU,MAAM,EAAE,CAAC;IACvD,MAAM,UAAU,GAAG,GAAG,OAAO,IAAI,OAAO,IAAI,QAAQ,EAAE,CAAC;IACvD,MAAM,UAAU,GAAG,eAAe,CAAC,QAAQ,CAAC,QAAQ,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC,CAAC;IAC5F,IAAI,CAAC;QACJ,MAAM,QAAQ,GAAG,MAAM,iBAAiB,CAAC,UAAU,EAAE;YACpD,MAAM,EAAE,MAAM;YACd,OAAO;YACP,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;YAC1B,MAAM,EAAE,aAAa,CAAC,OAAO,EAAE,SAAS,IAAI,kBAAkB,EAAE,MAAM,CAAC;SACvE,EAAE,OAAO,CAAC,CAAC;QACZ,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YAClB,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;YACnD,MAAM,IAAI,KAAK,CAAC,aAAa,KAAK,UAAU,QAAQ,CAAC,MAAM,KAAK,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;QACjH,CAAC;QACD,IAAI,IAAa,CAAC;QAClB,IAAI,CAAC;YACJ,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QAC9B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACd,MAAM,IAAI,KAAK,CAAC,aAAa,KAAK,2BAA2B,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACnF,CAAC;QACD,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;YAC9D,MAAM,IAAI,KAAK,CAAC,aAAa,KAAK,wCAAwC,CAAC,CAAC;QAC7E,CAAC;QACD,MAAM,QAAQ,GAAG,IAA+B,CAAC;QACjD,IAAI,QAAQ,CAAC,OAAO,KAAK,KAAK,EAAE,CAAC;YAChC,MAAM,MAAM,GAAG,OAAO,QAAQ,CAAC,KAAK,KAAK,QAAQ,IAAI,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,eAAe,CAAC;YAC9G,MAAM,IAAI,KAAK,CAAC,aAAa,KAAK,kBAAkB,gBAAgB,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;QACzF,CAAC;QACD,IAAI,QAAQ,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;YAC/B,MAAM,IAAI,KAAK,CAAC,aAAa,KAAK,wCAAwC,CAAC,CAAC;QAC7E,CAAC;QACD,eAAe,CAAC,WAAW,CAAC,UAAU,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;QACzD,OAAO,QAAQ,CAAC;IACjB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACd,IAAI,YAAY,CAAC,GAAG,CAAC;YAAE,eAAe,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;;YAC7D,eAAe,CAAC,QAAQ,CAAC,UAAU,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC;QAC7D,MAAM,GAAG,CAAC;IACX,CAAC;AAAA,CACD;AAED,MAAM,UAAU,oBAAoB,GAAY;IAC/C,OAAO,UAAU,EAAE,KAAK,IAAI,CAAC;AAAA,CAC7B;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,KAAa,EAAE,OAAO,GAA2B,EAAE,EAA2B;IACvH,cAAc,EAAE,CAAC;IACjB,MAAM,UAAU,GAAG,cAAc,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IACtD,MAAM,OAAO,GAAG,iBAAiB,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;IACxD,MAAM,QAAQ,GAAG,MAAM,cAAc,CACpC,QAAQ,EACR,UAAU,CAAC,KAAK,EAAE,OAAO,EAAE,UAAU,EAAE,OAAO,CAAC,EAC/C,OAAO,CAAC,MAAM,EACd,EAAE,GAAG,OAAO,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,iBAAiB,EAAE,EACjE,QAAQ,EACR,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,CACtB,CAAC;IACF,MAAM,MAAM,GAAG,gBAAgB,CAAC,QAAQ,CAAC,IAAI,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;IACpE,MAAM,QAAQ,GAAmB,EAAE,MAAM,EAAE,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC;IAClG,IAAI,OAAO,CAAC,cAAc,IAAI,MAAM,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC;QAAE,QAAQ,CAAC,aAAa,GAAG,MAAM,CAAC,aAAa,CAAC;IAC7G,OAAO,QAAQ,CAAC;AAAA,CAChB;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACzC,GAAW,EACX,MAAoB,EACpB,OAAiC,EACE;IACnC,cAAc,EAAE,CAAC;IACjB,MAAM,iBAAiB,CAAC,GAAG,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC;IACnD,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,QAAQ,EAAE,UAAU,CAAC,GAAG,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;IAClF,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC;IAC3B,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;QAC9D,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;IACvE,CAAC;IACD,MAAM,MAAM,GAAG,IAA2B,CAAC;IAC3C,IAAI,OAAO,MAAM,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QACzC,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;IAC9E,CAAC;IACD,MAAM,OAAO,GAAG,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;IACvC,IAAI,CAAC,OAAO;QAAE,OAAO,IAAI,CAAC;IAC1B,MAAM,aAAa,GAAG,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC;IAC7C,MAAM,KAAK,GAAG,OAAO,aAAa,KAAK,QAAQ,IAAI,aAAa,CAAC,IAAI,EAAE;QACtE,CAAC,CAAC,aAAa,CAAC,IAAI,EAAE;QACtB,CAAC,CAAC,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IAC/D,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;AAAA,CAC5C","sourcesContent":["import { existsSync, readFileSync } from \"node:fs\";\nimport { activityMonitor } from \"./activity.ts\";\nimport { redactCredential, resolveCredential } from \"./credential-source.ts\";\nimport type { ExtractedContent, ExtractOptions } from \"./extract.ts\";\nimport type { SearchOptions, SearchResponse } from \"./perplexity.ts\";\nimport { loadSsrfConfig, validateRemoteUrl, type Lookup } from \"./ssrf-protection.ts\";\nimport { getWebSearchConfigPath } from \"./utils.ts\";\n\nconst CONFIG_PATH = getWebSearchConfigPath();\nconst DEFAULT_API_VERSION = \"v2\";\nconst EXTRACT_TIMEOUT_MS = 60_000;\nconst SEARCH_TIMEOUT_MS = 60_000;\nconst DEFAULT_MAX_REDIRECTS = 5;\nconst REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);\nconst SUPPORTED_API_VERSIONS = [\"v1\", \"v2\"] as const;\ntype FirecrawlApiVersion = (typeof SUPPORTED_API_VERSIONS)[number];\n\nexport interface FirecrawlSsrfOptions {\n\tallowRanges: string[];\n\ttrustEnvProxy: boolean;\n}\n\nexport interface FirecrawlExtractOptions extends Pick<ExtractOptions, \"timeoutMs\" | \"lookup\"> {\n\tssrf?: FirecrawlSsrfOptions;\n}\n\ninterface FirecrawlSearchOptions extends SearchOptions, Pick<ExtractOptions, \"timeoutMs\" | \"lookup\"> {\n\tincludeContent?: boolean;\n\tssrf?: FirecrawlSsrfOptions;\n}\n\ninterface FirecrawlConfig {\n\tfirecrawlBaseUrl?: unknown;\n\tfirecrawlApiKey?: unknown;\n\tfirecrawlApiVersion?: unknown;\n\tfirecrawlFreshScrape?: unknown;\n}\n\ninterface FirecrawlScrapeData {\n\ttitle?: unknown;\n\tmarkdown?: unknown;\n\tmetadata?: { title?: unknown };\n}\n\ninterface FirecrawlSearchItem {\n\ttitle?: unknown;\n\tdescription?: unknown;\n\tsnippet?: unknown;\n\turl?: unknown;\n\tmarkdown?: unknown;\n\tmetadata?: {\n\t\ttitle?: unknown;\n\t\tdescription?: unknown;\n\t\tsourceURL?: unknown;\n\t\turl?: unknown;\n\t};\n}\n\ninterface DomainFilters {\n\tinclude: string[];\n\texclude: string[];\n}\n\nlet cachedConfig: FirecrawlConfig | null = null;\n\nfunction loadConfig(): FirecrawlConfig {\n\tif (cachedConfig) return cachedConfig;\n\tif (!existsSync(CONFIG_PATH)) {\n\t\tcachedConfig = {};\n\t\treturn cachedConfig;\n\t}\n\tconst raw = readFileSync(CONFIG_PATH, \"utf8\");\n\tlet parsed: unknown;\n\ttry {\n\t\tparsed = JSON.parse(raw);\n\t} catch (err) {\n\t\tconst message = err instanceof Error ? err.message : String(err);\n\t\tthrow new Error(`Failed to parse ${CONFIG_PATH}: ${message}`);\n\t}\n\tif (!parsed || typeof parsed !== \"object\" || Array.isArray(parsed)) {\n\t\tthrow new Error(`Invalid config in ${CONFIG_PATH}: expected a JSON object`);\n\t}\n\tcachedConfig = parsed as FirecrawlConfig;\n\treturn cachedConfig;\n}\n\nexport function clearFirecrawlConfigCache(): void {\n\tcachedConfig = null;\n}\n\nfunction normalizeBaseUrl(value: unknown): string | null {\n\tif (typeof value !== \"string\") return null;\n\tconst trimmed = value.trim();\n\tif (!trimmed) return null;\n\tlet parsed: URL;\n\ttry {\n\t\tparsed = new URL(trimmed);\n\t} catch {\n\t\tthrow new Error(`Invalid Firecrawl base URL in ${CONFIG_PATH}: expected an HTTP or HTTPS URL`);\n\t}\n\tif (parsed.protocol !== \"http:\" && parsed.protocol !== \"https:\") {\n\t\tthrow new Error(`Invalid Firecrawl base URL in ${CONFIG_PATH}: expected an HTTP or HTTPS URL`);\n\t}\n\tif (parsed.username || parsed.password) {\n\t\tthrow new Error(`Invalid Firecrawl base URL in ${CONFIG_PATH}: URL credentials are not allowed`);\n\t}\n\tparsed.pathname = parsed.pathname.replace(/\\/+$/, \"\");\n\tparsed.search = \"\";\n\tparsed.hash = \"\";\n\treturn parsed.toString().replace(/\\/+$/, \"\");\n}\n\nfunction getBaseUrl(): string | null {\n\treturn normalizeBaseUrl(process.env.FIRECRAWL_BASE_URL) ?? normalizeBaseUrl(loadConfig().firecrawlBaseUrl);\n}\n\nfunction requireBaseUrl(): string {\n\tconst baseUrl = getBaseUrl();\n\tif (!baseUrl) {\n\t\tthrow new Error(\n\t\t\t\"Firecrawl base URL not configured. Either:\\n\" +\n\t\t\t`  1. Set firecrawlBaseUrl in ${CONFIG_PATH}\\n` +\n\t\t\t\"  2. Set FIRECRAWL_BASE_URL environment variable\",\n\t\t);\n\t}\n\treturn baseUrl;\n}\n\nfunction getApiVersion(): FirecrawlApiVersion {\n\tconst environmentValue = typeof process.env.FIRECRAWL_API_VERSION === \"string\"\n\t\t? process.env.FIRECRAWL_API_VERSION.trim()\n\t\t: \"\";\n\tconst raw = environmentValue || loadConfig().firecrawlApiVersion;\n\tif (raw === undefined || raw === null) return DEFAULT_API_VERSION;\n\tif (typeof raw !== \"string\") {\n\t\tthrow new Error(`firecrawlApiVersion in ${CONFIG_PATH} must be a string (\"v1\" or \"v2\")`);\n\t}\n\tconst normalized = raw.trim().toLowerCase();\n\tif (!normalized) return DEFAULT_API_VERSION;\n\tif (!SUPPORTED_API_VERSIONS.includes(normalized as FirecrawlApiVersion)) {\n\t\tthrow new Error(`Unsupported Firecrawl API version \"${raw}\". Supported versions: ${SUPPORTED_API_VERSIONS.join(\", \")}`);\n\t}\n\treturn normalized as FirecrawlApiVersion;\n}\n\nfunction allowFreshScrape(): boolean {\n\tconst environmentValue = process.env.FIRECRAWL_FRESH_SCRAPE;\n\tif (environmentValue !== undefined) return environmentValue === \"1\" || environmentValue.toLowerCase() === \"true\";\n\tconst configured = loadConfig().firecrawlFreshScrape;\n\tif (configured === undefined || configured === null) return false;\n\tif (typeof configured !== \"boolean\") throw new Error(`firecrawlFreshScrape in ${CONFIG_PATH} must be a boolean`);\n\treturn configured;\n}\n\nasync function getApiKey(signal?: AbortSignal): Promise<string | null> {\n\tconst configKey = loadConfig().firecrawlApiKey;\n\treturn resolveCredential({\n\t\tprovider: \"Firecrawl\",\n\t\tconfiguredValue: configKey,\n\t\tenvironmentValue: process.env.FIRECRAWL_API_KEY,\n\t\tsignal,\n\t});\n}\n\nfunction requestSignal(timeoutMs: number, signal?: AbortSignal): AbortSignal {\n\tconst timeout = AbortSignal.timeout(timeoutMs);\n\treturn signal ? AbortSignal.any([timeout, signal]) : timeout;\n}\n\nfunction errorMessage(err: unknown): string {\n\treturn err instanceof Error ? err.message : String(err);\n}\n\nfunction isAbortError(err: unknown): boolean {\n\treturn errorMessage(err).toLowerCase().includes(\"abort\");\n}\n\nfunction ssrfOptions(options?: FirecrawlExtractOptions | FirecrawlSearchOptions): { lookup?: Lookup; allowRanges: string[]; trustEnvProxy: boolean } {\n\tconst config = loadSsrfConfig();\n\treturn {\n\t\tallowRanges: options?.ssrf?.allowRanges ?? config.allowRanges,\n\t\ttrustEnvProxy: options?.ssrf?.trustEnvProxy ?? config.trustEnvProxy,\n\t\t...(options?.lookup ? { lookup: options.lookup } : {}),\n\t};\n}\n\nfunction withoutSensitiveHeaders(headers: Record<string, string>): Record<string, string> {\n\tconst next = { ...headers };\n\tdelete next.Authorization;\n\tdelete next.authorization;\n\tdelete next.Cookie;\n\tdelete next.cookie;\n\tdelete next[\"X-API-Key\"];\n\tdelete next[\"x-api-key\"];\n\treturn next;\n}\n\nasync function fetchFirecrawlApi(\n\turl: string,\n\tinit: { method: string; headers: Record<string, string>; body: string; signal: AbortSignal },\n\toptions: FirecrawlExtractOptions | FirecrawlSearchOptions | undefined,\n): Promise<Response> {\n\tlet current = await validateRemoteUrl(url, ssrfOptions(options));\n\tlet headers = init.headers;\n\tfor (let redirects = 0; redirects <= DEFAULT_MAX_REDIRECTS; redirects++) {\n\t\tconst response = await fetch(current, { ...init, headers, redirect: \"manual\" });\n\t\tif (!REDIRECT_STATUSES.has(response.status)) return response;\n\n\t\tconst location = response.headers.get(\"location\");\n\t\tif (!location) return response;\n\t\tif (redirects === DEFAULT_MAX_REDIRECTS) throw new Error(`Too many redirects fetching ${current.toString()}`);\n\n\t\tconst next = await validateRemoteUrl(new URL(location, current), ssrfOptions(options));\n\t\tif (next.origin !== current.origin) headers = withoutSensitiveHeaders(headers);\n\t\tcurrent = next;\n\t}\n\tthrow new Error(`Too many redirects fetching ${current.toString()}`);\n}\n\nfunction scrapeBody(url: string): Record<string, unknown> {\n\treturn {\n\t\turl,\n\t\tformats: [\"markdown\"],\n\t\tonlyMainContent: true,\n\t\t...(allowFreshScrape() ? {} : { lockdown: true }),\n\t};\n}\n\nfunction normalizeCount(value: number | undefined): number {\n\tif (typeof value !== \"number\" || !Number.isFinite(value)) return 5;\n\treturn Math.max(1, Math.min(Math.floor(value), 20));\n}\n\nfunction normalizeDomain(value: string): string | null {\n\tlet input = value.trim().toLowerCase();\n\tif (!input) return null;\n\tif (input.startsWith(\"-\")) input = input.slice(1).trim();\n\tif (!input) return null;\n\ttry {\n\t\tconst parsed = input.includes(\"://\") ? new URL(input) : new URL(`https://${input}`);\n\t\tinput = parsed.hostname;\n\t} catch {\n\t\tinput = input.split(\"/\")[0]?.split(\":\")[0] ?? \"\";\n\t}\n\tinput = input.replace(/^\\.+|\\.+$/g, \"\");\n\treturn /^[a-z0-9][a-z0-9.-]*\\.[a-z]{2,}$/i.test(input) ? input : null;\n}\n\nfunction parseDomainFilter(domainFilter: string[] | undefined): DomainFilters {\n\tconst filters: DomainFilters = { include: [], exclude: [] };\n\tfor (const raw of domainFilter ?? []) {\n\t\tconst domain = normalizeDomain(raw);\n\t\tif (!domain) continue;\n\t\tconst target = raw.trim().startsWith(\"-\") ? filters.exclude : filters.include;\n\t\tif (!target.includes(domain)) target.push(domain);\n\t}\n\treturn filters;\n}\n\nfunction passesDomainFilters(url: string, filters: DomainFilters): boolean {\n\tif (filters.include.length === 0 && filters.exclude.length === 0) return true;\n\tlet hostname: string;\n\ttry {\n\t\thostname = new URL(url).hostname.toLowerCase();\n\t} catch {\n\t\treturn false;\n\t}\n\tconst matches = (domain: string) => hostname === domain || hostname.endsWith(`.${domain}`);\n\tif (filters.exclude.some(matches)) return false;\n\treturn filters.include.length === 0 || filters.include.some(matches);\n}\n\nfunction mapRecencyFilter(value: SearchOptions[\"recencyFilter\"]): string | undefined {\n\tswitch (value) {\n\t\tcase \"day\": return \"qdr:d\";\n\t\tcase \"week\": return \"qdr:w\";\n\t\tcase \"month\": return \"qdr:m\";\n\t\tcase \"year\": return \"qdr:y\";\n\t\tdefault: return undefined;\n\t}\n}\n\nfunction searchBody(query: string, options: FirecrawlSearchOptions, numResults: number, filters: DomainFilters): Record<string, unknown> {\n\treturn {\n\t\tquery,\n\t\tlimit: numResults,\n\t\tsources: [\"web\"],\n\t\t...(filters.include.length > 0 ? { includeDomains: filters.include } : {}),\n\t\t...(filters.include.length === 0 && filters.exclude.length > 0 ? { excludeDomains: filters.exclude } : {}),\n\t\t...(mapRecencyFilter(options.recencyFilter) ? { tbs: mapRecencyFilter(options.recencyFilter) } : {}),\n\t\t...(options.includeContent ? {\n\t\t\tscrapeOptions: {\n\t\t\t\tformats: [\"markdown\"],\n\t\t\t\tonlyMainContent: true,\n\t\t\t\t...(allowFreshScrape() ? {} : { lockdown: true }),\n\t\t\t},\n\t\t} : {}),\n\t};\n}\n\nfunction firstString(...values: unknown[]): string | null {\n\tfor (const value of values) {\n\t\tif (typeof value === \"string\" && value.trim()) return value.trim();\n\t}\n\treturn null;\n}\n\nfunction buildAnswer(results: SearchResponse[\"results\"]): string {\n\treturn results.map((result) => {\n\t\tif (result.snippet) return `${result.snippet}\\nSource: ${result.title} (${result.url})`;\n\t\treturn `Source: ${result.title} (${result.url})`;\n\t}).join(\"\\n\\n\");\n}\n\nfunction mapSearchResults(data: unknown, numResults: number, filters: DomainFilters): {\n\tresults: SearchResponse[\"results\"];\n\tinlineContent: ExtractedContent[];\n} {\n\tconst web = Array.isArray(data)\n\t\t? data\n\t\t: data && typeof data === \"object\" ? (data as Record<string, unknown>).web : undefined;\n\tif (!Array.isArray(web)) throw new Error(\"Firecrawl search returned an unexpected web result shape\");\n\tconst results: SearchResponse[\"results\"] = [];\n\tconst inlineContent: ExtractedContent[] = [];\n\tconst seen = new Set<string>();\n\tfor (const rawItem of web) {\n\t\tif (!rawItem || typeof rawItem !== \"object\" || Array.isArray(rawItem)) continue;\n\t\tconst item = rawItem as FirecrawlSearchItem;\n\t\tconst url = firstString(item.url, item.metadata?.sourceURL, item.metadata?.url);\n\t\tif (!url || seen.has(url) || !passesDomainFilters(url, filters)) continue;\n\t\tseen.add(url);\n\t\tconst title = firstString(item.title, item.metadata?.title) ?? url;\n\t\tconst snippet = firstString(item.description, item.snippet, item.metadata?.description) ?? \"\";\n\t\tresults.push({ title, url, snippet });\n\t\tconst markdown = firstString(item.markdown);\n\t\tif (markdown) inlineContent.push({ url, title, content: markdown, error: null });\n\t\tif (results.length >= numResults) break;\n\t}\n\treturn { results, inlineContent };\n}\n\nasync function firecrawlFetch(\n\tendpoint: string,\n\tbody: Record<string, unknown>,\n\tsignal: AbortSignal | undefined,\n\toptions: FirecrawlExtractOptions | FirecrawlSearchOptions | undefined,\n\tlabel = endpoint,\n\tactivity: { type: \"api\"; query: string } | { type: \"fetch\"; url: string } | undefined = undefined,\n): Promise<Record<string, unknown>> {\n\tconst baseUrl = requireBaseUrl();\n\tconst version = getApiVersion();\n\tconst apiKey = await getApiKey(signal);\n\tconst headers: Record<string, string> = { \"Content-Type\": \"application/json\" };\n\tif (apiKey) headers.Authorization = `Bearer ${apiKey}`;\n\tconst requestUrl = `${baseUrl}/${version}/${endpoint}`;\n\tconst activityId = activityMonitor.logStart(activity ?? { type: \"fetch\", url: requestUrl });\n\ttry {\n\t\tconst response = await fetchFirecrawlApi(requestUrl, {\n\t\t\tmethod: \"POST\",\n\t\t\theaders,\n\t\t\tbody: JSON.stringify(body),\n\t\t\tsignal: requestSignal(options?.timeoutMs ?? EXTRACT_TIMEOUT_MS, signal),\n\t\t}, options);\n\t\tif (!response.ok) {\n\t\t\tconst text = await response.text().catch(() => \"\");\n\t\t\tthrow new Error(`Firecrawl ${label} error ${response.status}: ${redactCredential(text.slice(0, 300), apiKey)}`);\n\t\t}\n\t\tlet data: unknown;\n\t\ttry {\n\t\t\tdata = await response.json();\n\t\t} catch (err) {\n\t\t\tthrow new Error(`Firecrawl ${label} returned invalid JSON: ${errorMessage(err)}`);\n\t\t}\n\t\tif (!data || typeof data !== \"object\" || Array.isArray(data)) {\n\t\t\tthrow new Error(`Firecrawl ${label} returned an unexpected response shape`);\n\t\t}\n\t\tconst envelope = data as Record<string, unknown>;\n\t\tif (envelope.success === false) {\n\t\t\tconst reason = typeof envelope.error === \"string\" && envelope.error.trim() ? envelope.error : \"unknown error\";\n\t\t\tthrow new Error(`Firecrawl ${label} unsuccessful: ${redactCredential(reason, apiKey)}`);\n\t\t}\n\t\tif (envelope.success !== true) {\n\t\t\tthrow new Error(`Firecrawl ${label} returned an unexpected response shape`);\n\t\t}\n\t\tactivityMonitor.logComplete(activityId, response.status);\n\t\treturn envelope;\n\t} catch (err) {\n\t\tif (isAbortError(err)) activityMonitor.logComplete(activityId, 0);\n\t\telse activityMonitor.logError(activityId, errorMessage(err));\n\t\tthrow err;\n\t}\n}\n\nexport function isFirecrawlAvailable(): boolean {\n\treturn getBaseUrl() !== null;\n}\n\nexport async function searchWithFirecrawl(query: string, options: FirecrawlSearchOptions = {}): Promise<SearchResponse> {\n\trequireBaseUrl();\n\tconst numResults = normalizeCount(options.numResults);\n\tconst filters = parseDomainFilter(options.domainFilter);\n\tconst envelope = await firecrawlFetch(\n\t\t\"search\",\n\t\tsearchBody(query, options, numResults, filters),\n\t\toptions.signal,\n\t\t{ ...options, timeoutMs: options.timeoutMs ?? SEARCH_TIMEOUT_MS },\n\t\t\"search\",\n\t\t{ type: \"api\", query },\n\t);\n\tconst mapped = mapSearchResults(envelope.data, numResults, filters);\n\tconst response: SearchResponse = { answer: buildAnswer(mapped.results), results: mapped.results };\n\tif (options.includeContent && mapped.inlineContent.length > 0) response.inlineContent = mapped.inlineContent;\n\treturn response;\n}\n\nexport async function extractWithFirecrawl(\n\turl: string,\n\tsignal?: AbortSignal,\n\toptions?: FirecrawlExtractOptions,\n): Promise<ExtractedContent | null> {\n\trequireBaseUrl();\n\tawait validateRemoteUrl(url, ssrfOptions(options));\n\tconst envelope = await firecrawlFetch(\"scrape\", scrapeBody(url), signal, options);\n\tconst data = envelope.data;\n\tif (!data || typeof data !== \"object\" || Array.isArray(data)) {\n\t\tthrow new Error(\"Firecrawl scrape returned an unexpected data shape\");\n\t}\n\tconst scrape = data as FirecrawlScrapeData;\n\tif (typeof scrape.markdown !== \"string\") {\n\t\tthrow new Error(\"Firecrawl scrape returned markdown in an unexpected shape\");\n\t}\n\tconst content = scrape.markdown.trim();\n\tif (!content) return null;\n\tconst metadataTitle = scrape.metadata?.title;\n\tconst title = typeof metadataTitle === \"string\" && metadataTitle.trim()\n\t\t? metadataTitle.trim()\n\t\t: typeof scrape.title === \"string\" ? scrape.title.trim() : \"\";\n\treturn { url, title, content, error: null };\n}\n"]}

package/dist/humain/web-access/vendor/gemini-search.d.ts

@@ -1,7 +1,7 @@
 import type { ExtensionContext } from "../host.ts";
 import { type SearchResponse, type SearchOptions } from "./perplexity.ts";
-export declare const RESOLVED_SEARCH_PROVIDERS: readonly ["openai", "brave", "parallel", "tinyfish", "search1api", "searchinfinity", "querit", "tavily", "jina", "searxng", "duckduckgo", "perplexity", "gemini", "exa", "serpdive", "kagi", "ollama", "anysearch", "xai", "brightdata", "serpbase", "bocha"];
-export declare const SEARCH_PROVIDERS: readonly ["auto", "all", "openai", "brave", "parallel", "tinyfish", "search1api", "searchinfinity", "querit", "tavily", "jina", "searxng", "duckduckgo", "perplexity", "gemini", "exa", "serpdive", "kagi", "ollama", "anysearch", "xai", "brightdata", "serpbase", "bocha"];
+export declare const RESOLVED_SEARCH_PROVIDERS: readonly ["openai", "brave", "parallel", "tinyfish", "search1api", "searchinfinity", "querit", "tavily", "firecrawl", "jina", "searxng", "duckduckgo", "perplexity", "gemini", "exa", "serpdive", "kagi", "ollama", "anysearch", "xai", "brightdata", "serpbase", "bocha"];
+export declare const SEARCH_PROVIDERS: readonly ["auto", "all", "openai", "brave", "parallel", "tinyfish", "search1api", "searchinfinity", "querit", "tavily", "firecrawl", "jina", "searxng", "duckduckgo", "perplexity", "gemini", "exa", "serpdive", "kagi", "ollama", "anysearch", "xai", "brightdata", "serpbase", "bocha"];
 export type ResolvedSearchProvider = typeof RESOLVED_SEARCH_PROVIDERS[number];
 export type SearchProvider = typeof SEARCH_PROVIDERS[number];
 export type SearchProviderSelection = SearchProvider | ResolvedSearchProvider[];