@herbertgao/pi-extensions 2026.8.4 → 2026.8.6

package/node_modules/pi-web-access/extract.ts

@@ -21,9 +21,11 @@ import { extractWithFirecrawl, isFirecrawlAvailable } from "./firecrawl.ts";
 import { extractWithBrightDataUnlocker, isBrightDataUnlockerAvailable } from "./brightdata-unlocker.ts";
 import { isVideoFile, extractVideo, extractVideoFrame, getLocalVideoDuration } from "./video-extract.ts";
 import { appendDeclaredWebLinks, discoverDeclaredWebLinks, type DeclaredWebLink } from "./declared-web-links.ts";
-import { fetchRemoteUrl, loadFetchContentDomainPolicy, loadSsrfConfig, validateRemoteUrl, type Lookup } from "./ssrf-protection.ts";
+import { fetchRemoteUrl, loadFetchContentDomainPolicy, loadSsrfConfig, validateRemoteUrl, type DomainPolicy, type Lookup, type SsrfConfig } from "./ssrf-protection.ts";
 import { formatSeconds, getWebSearchConfigPath } from "./utils.ts";
 import { isImageEnabled } from "./feature-config.ts";
+import { assertAuthFetchUrl, authFetchRedirectGuard, type AuthFetchProfile } from "./auth-fetch.ts";
+import { getBrowserCookiesForHosts, getLastBrowserCookieDiagnostic } from "./chrome-cookies.ts";
 
 const DEFAULT_TIMEOUT_MS = 30000;
 const CONCURRENT_LIMIT = 3;
@@ -57,7 +59,8 @@ function isAbortError(err: unknown): boolean {
 }
 
 function isRedirectPolicyError(message: string): boolean {
-	return message.startsWith("Blocked internal ") ||
+	return message.startsWith("Authenticated fetch refused cross-origin redirect") ||
+		message.startsWith("Blocked internal ") ||
 		message.startsWith("Blocked hostname by fetch_content domain policy") ||
 		message.startsWith("Hostname not allowed by fetch_content domain policy") ||
 		message.startsWith("Too many redirects fetching ") ||
@@ -74,6 +77,56 @@ function imageGateError(): string | null {
 	}
 }
 
+async function resolveAuthCookieHeader(url: string | URL, profile: AuthFetchProfile): Promise<string> {
+	const parsed = assertAuthFetchUrl(profile, url.toString());
+	const result = await getBrowserCookiesForHosts({ hosts: [parsed.hostname], profile: profile.chromeProfile, requestUrl: parsed });
+	if (result?.cookieHeader) return result.cookieHeader;
+	if (!result) {
+		const diagnostic = getLastBrowserCookieDiagnostic();
+		throw new Error(`Authenticated fetch profile ${profile.name} could not read browser cookies${diagnostic ? `: ${diagnostic}` : ""}`);
+	}
+	throw new Error(`Authenticated fetch profile ${profile.name} could not build a cookie header`);
+}
+
+const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
+
+async function fetchAuthenticatedRemoteUrl(
+	url: string,
+	init: RequestInit,
+	validationOptions: { ssrf: SsrfConfig; domainPolicy: DomainPolicy; lookup?: Lookup },
+	profile: AuthFetchProfile,
+): Promise<Response> {
+	let current = await validateRemoteUrl(url, {
+		allowRanges: validationOptions.ssrf.allowRanges,
+		trustEnvProxy: validationOptions.ssrf.trustEnvProxy,
+		domainPolicy: validationOptions.domainPolicy,
+		...(validationOptions.lookup ? { lookup: validationOptions.lookup } : {}),
+	});
+	let requestInit = init;
+	for (let redirects = 0; redirects <= 5; redirects++) {
+		const cookieHeader = await resolveAuthCookieHeader(current, profile);
+		const headers = { ...(requestInit.headers as Record<string, string>), cookie: cookieHeader };
+		const response = await fetch(current, { ...requestInit, headers, redirect: "manual" });
+		if (!REDIRECT_STATUSES.has(response.status)) return response;
+		const location = response.headers.get("location");
+		if (!location) return response;
+		if (redirects === 5) throw new Error(`Too many redirects fetching ${current.toString()}`);
+		const from = current;
+		current = await validateRemoteUrl(new URL(location, current), {
+			allowRanges: validationOptions.ssrf.allowRanges,
+			trustEnvProxy: validationOptions.ssrf.trustEnvProxy,
+			domainPolicy: validationOptions.domainPolicy,
+			...(validationOptions.lookup ? { lookup: validationOptions.lookup } : {}),
+		});
+		authFetchRedirectGuard(profile, from, current);
+		if (response.status === 303 || ((response.status === 301 || response.status === 302) && requestInit.method?.toUpperCase() === "POST")) {
+			const { body: _body, ...nextInit } = requestInit;
+			requestInit = { ...nextInit, method: "GET" };
+		}
+	}
+	throw new Error(`Too many redirects fetching ${current.toString()}`);
+}
+
 function loadFetchRouting(): FetchRouting {
 	if (!existsSync(WEB_SEARCH_CONFIG_PATH)) {
 		return { providers: DEFAULT_FETCH_PROVIDER_ORDER, allowRemoteHostedProviders: false };
@@ -171,6 +224,7 @@ export interface ExtractOptions {
 	model?: string;
 	mode?: "readable" | "raw" | "answer";
 	answerModel?: string;
+	authFetchProfile?: AuthFetchProfile;
 	/** Custom DNS resolver used for SSRF validation. Primarily a test seam. */
 	lookup?: Lookup;
 }
@@ -313,11 +367,17 @@ async function extractLocalFrames(
 	return { frames, error: frames.length === 0 && firstError ? firstError.error : null };
 }
 
-function safeVideoInfo(url: string): { info: ReturnType<typeof isVideoFile>; error?: string } {
+type LocalVideoInfoResult =
+	| { status: "video"; info: NonNullable<ReturnType<typeof isVideoFile>> }
+	| { status: "not-video" }
+	| { status: "invalid"; error: string };
+
+function safeVideoInfo(url: string): LocalVideoInfoResult {
 	try {
-		return { info: isVideoFile(url) };
+		const info = isVideoFile(url);
+		return info ? { status: "video", info } : { status: "not-video" };
 	} catch (err) {
-		return { info: null, error: errorMessage(err) };
+		return { status: "invalid", error: errorMessage(err) };
 	}
 }
 
@@ -351,6 +411,14 @@ export async function extractContent(
 		}
 	}
 
+	if (options?.authFetchProfile) {
+		try {
+			return await extractViaHttp(url, signal, options);
+		} catch (err) {
+			return { url, title: "", content: "", error: errorMessage(err) };
+		}
+	}
+
 	if (options?.mode === "raw") {
 		return extractViaHttp(url, signal, options);
 	}
@@ -380,10 +448,10 @@ export async function extractContent(
 		}
 
 		const localVideo = safeVideoInfo(url);
-		if (localVideo.error) {
+		if (localVideo.status === "invalid") {
 			return { url, title: "", content: "", error: localVideo.error };
 		}
-		if (localVideo.info) {
+		if (localVideo.status === "video") {
 			const durationResult = await getLocalVideoDuration(localVideo.info.absolutePath);
 			if (typeof durationResult !== "number") {
 				return { url, title: "Frames", content: durationResult.error, error: durationResult.error };
@@ -463,10 +531,10 @@ export async function extractContent(
 		}
 
 		const localVideo = safeVideoInfo(url);
-		if (localVideo.error) {
+		if (localVideo.status === "invalid") {
 			return { url, title: "", content: "", error: localVideo.error };
 		}
-		if (localVideo.info) {
+		if (localVideo.status === "video") {
 			if (spec.type === "range") {
 				const timestamps = frameCount
 					? computeRangeTimestamps(spec.start, spec.end, frameCount)
@@ -495,10 +563,10 @@ export async function extractContent(
 	}
 
 	const localVideo = safeVideoInfo(url);
-	if (localVideo.error) {
+	if (localVideo.status === "invalid") {
 		return { url, title: "", content: "", error: localVideo.error };
 	}
-	if (localVideo.info) {
+	if (localVideo.status === "video") {
 		try {
 			const result = await extractVideo(localVideo.info, signal, options);
 			if (signal?.aborted) return abortedResult(url);
@@ -548,7 +616,7 @@ export async function extractContent(
 			url,
 			title: "",
 			content: "",
-			error: "Could not extract YouTube video content. Sign into Google in Chrome for automatic access, or set GEMINI_API_KEY.",
+			error: "Could not extract YouTube video content. Sign into Google in a supported Chromium browser for automatic access, or set GEMINI_API_KEY.",
 		};
 	}
 
@@ -914,29 +982,33 @@ async function extractViaHttp(
 	try {
 		const ssrf = loadSsrfConfig();
 		const domainPolicy = loadFetchContentDomainPolicy();
-		const response = await fetchRemoteUrl(
-			url,
-			{
-				signal: controller.signal,
-				headers: {
-					"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36",
-					"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
-					"Accept-Language": "en-US,en;q=0.9",
-					"Cache-Control": "no-cache",
-					"Sec-Fetch-Dest": "document",
-					"Sec-Fetch-Mode": "navigate",
-					"Sec-Fetch-Site": "none",
-					"Sec-Fetch-User": "?1",
-					"Upgrade-Insecure-Requests": "1",
-				},
-			},
-			{
-				allowRanges: ssrf.allowRanges,
-				trustEnvProxy: ssrf.trustEnvProxy,
-				domainPolicy,
-				...(options?.lookup ? { lookup: options.lookup } : {}),
+		const authProfile = options?.authFetchProfile;
+		const requestInit = {
+			signal: controller.signal,
+			headers: {
+				"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36",
+				"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
+				"Accept-Language": "en-US,en;q=0.9",
+				"Cache-Control": "no-cache",
+				"Sec-Fetch-Dest": "document",
+				"Sec-Fetch-Mode": "navigate",
+				"Sec-Fetch-Site": "none",
+				"Sec-Fetch-User": "?1",
+				"Upgrade-Insecure-Requests": "1",
 			},
-		);
+		};
+		const response = authProfile
+			? await fetchAuthenticatedRemoteUrl(url, requestInit, { ssrf, domainPolicy, ...(options?.lookup ? { lookup: options.lookup } : {}) }, authProfile)
+			: await fetchRemoteUrl(
+				url,
+				requestInit,
+				{
+					allowRanges: ssrf.allowRanges,
+					trustEnvProxy: ssrf.trustEnvProxy,
+					domainPolicy,
+					...(options?.lookup ? { lookup: options.lookup } : {}),
+				},
+			);
 
 		if (!response.ok && options?.mode !== "raw") {
 			activityMonitor.logComplete(activityId, response.status);

package/node_modules/pi-web-access/fetch-params.ts

@@ -8,6 +8,7 @@ export interface FetchContentParams {
 	model?: unknown;
 	mode?: unknown;
 	answerModel?: unknown;
+	auth?: unknown;
 }
 
 export interface NormalizedFetchContentParams {
@@ -20,6 +21,7 @@ export interface NormalizedFetchContentParams {
 		model?: string;
 		mode?: "readable" | "raw" | "answer";
 		answerModel?: string;
+		auth?: true | string;
 	};
 }
 
@@ -28,7 +30,7 @@ export function normalizeFetchContentParams(params: FetchContentParams): Normali
 	const urlList = normalizedUrls.length > 0 ? normalizedUrls : normalizeSingleUrl(params.url);
 	const prompt = normalizeOptionalString(params.prompt);
 	const timestamp = normalizeOptionalString(params.timestamp);
-	const frames = normalizeOptionalInteger(params.frames);
+	const frames = normalizeOptionalFrameCount(params.frames);
 
 	const shouldIncludeFrames = frames !== undefined && (timestamp !== undefined || frames > 1);
 
@@ -36,6 +38,7 @@ export function normalizeFetchContentParams(params: FetchContentParams): Normali
 	const model = normalizeOptionalString(params.model);
 	const mode = normalizeMode(params.mode);
 	const answerModel = normalizeOptionalString(params.answerModel);
+	const auth = normalizeAuth(params.auth);
 
 	return {
 		urlList,
@@ -47,6 +50,7 @@ export function normalizeFetchContentParams(params: FetchContentParams): Normali
 			...(model !== undefined ? { model } : {}),
 			...(mode !== undefined ? { mode } : {}),
 			...(answerModel !== undefined ? { answerModel } : {}),
+			...(auth !== undefined ? { auth } : {}),
 		},
 	};
 }
@@ -74,8 +78,18 @@ function normalizeMode(value: unknown): "readable" | "raw" | "answer" | undefine
 	throw new Error('mode must be "readable", "raw", or "answer"');
 }
 
-function normalizeOptionalInteger(value: unknown): number | undefined {
-	if (typeof value !== "number" || !Number.isInteger(value) || value < 1) return undefined;
+function normalizeAuth(value: unknown): true | string | undefined {
+	if (value === undefined || value === false) return undefined;
+	if (value === true) return true;
+	if (typeof value === "string") {
+		const trimmed = value.trim();
+		if (trimmed) return trimmed;
+	}
+	throw new Error("auth must be a profile name, true, or false");
+}
+
+function normalizeOptionalFrameCount(value: unknown): number | undefined {
+	if (typeof value !== "number" || !Number.isInteger(value) || value < 1 || value > 12) return undefined;
 	return value;
 }
 

package/node_modules/pi-web-access/firecrawl.ts

@@ -2,12 +2,14 @@ import { existsSync, readFileSync } from "node:fs";
 import { activityMonitor } from "./activity.ts";
 import { redactCredential, resolveCredential } from "./credential-source.ts";
 import type { ExtractedContent, ExtractOptions } from "./extract.ts";
-import { validateRemoteUrl, type Lookup } from "./ssrf-protection.ts";
+import type { SearchOptions, SearchResponse } from "./perplexity.ts";
+import { loadSsrfConfig, validateRemoteUrl, type Lookup } from "./ssrf-protection.ts";
 import { getWebSearchConfigPath } from "./utils.ts";
 
 const CONFIG_PATH = getWebSearchConfigPath();
 const DEFAULT_API_VERSION = "v2";
 const EXTRACT_TIMEOUT_MS = 60_000;
+const SEARCH_TIMEOUT_MS = 60_000;
 const DEFAULT_MAX_REDIRECTS = 5;
 const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
 const SUPPORTED_API_VERSIONS = ["v1", "v2"] as const;
@@ -22,6 +24,11 @@ export interface FirecrawlExtractOptions extends Pick<ExtractOptions, "timeoutMs
 	ssrf?: FirecrawlSsrfOptions;
 }
 
+interface FirecrawlSearchOptions extends SearchOptions, Pick<ExtractOptions, "timeoutMs" | "lookup"> {
+	includeContent?: boolean;
+	ssrf?: FirecrawlSsrfOptions;
+}
+
 interface FirecrawlConfig {
 	firecrawlBaseUrl?: unknown;
 	firecrawlApiKey?: unknown;
@@ -35,6 +42,25 @@ interface FirecrawlScrapeData {
 	metadata?: { title?: unknown };
 }
 
+interface FirecrawlSearchItem {
+	title?: unknown;
+	description?: unknown;
+	snippet?: unknown;
+	url?: unknown;
+	markdown?: unknown;
+	metadata?: {
+		title?: unknown;
+		description?: unknown;
+		sourceURL?: unknown;
+		url?: unknown;
+	};
+}
+
+interface DomainFilters {
+	include: string[];
+	exclude: string[];
+}
+
 let cachedConfig: FirecrawlConfig | null = null;
 
 function loadConfig(): FirecrawlConfig {
@@ -149,10 +175,11 @@ function isAbortError(err: unknown): boolean {
 	return errorMessage(err).toLowerCase().includes("abort");
 }
 
-function ssrfOptions(options?: FirecrawlExtractOptions): { lookup?: Lookup; allowRanges: string[]; trustEnvProxy: boolean } {
+function ssrfOptions(options?: FirecrawlExtractOptions | FirecrawlSearchOptions): { lookup?: Lookup; allowRanges: string[]; trustEnvProxy: boolean } {
+	const config = loadSsrfConfig();
 	return {
-		allowRanges: options?.ssrf?.allowRanges ?? [],
-		trustEnvProxy: options?.ssrf?.trustEnvProxy ?? false,
+		allowRanges: options?.ssrf?.allowRanges ?? config.allowRanges,
+		trustEnvProxy: options?.ssrf?.trustEnvProxy ?? config.trustEnvProxy,
 		...(options?.lookup ? { lookup: options.lookup } : {}),
 	};
 }
@@ -171,7 +198,7 @@ function withoutSensitiveHeaders(headers: Record<string, string>): Record<string
 async function fetchFirecrawlApi(
 	url: string,
 	init: { method: string; headers: Record<string, string>; body: string; signal: AbortSignal },
-	options: FirecrawlExtractOptions | undefined,
+	options: FirecrawlExtractOptions | FirecrawlSearchOptions | undefined,
 ): Promise<Response> {
 	let current = await validateRemoteUrl(url, ssrfOptions(options));
 	let headers = init.headers;
@@ -199,11 +226,126 @@ function scrapeBody(url: string): Record<string, unknown> {
 	};
 }
 
+function normalizeCount(value: number | undefined): number {
+	if (typeof value !== "number" || !Number.isFinite(value)) return 5;
+	return Math.max(1, Math.min(Math.floor(value), 20));
+}
+
+function normalizeDomain(value: string): string | null {
+	let input = value.trim().toLowerCase();
+	if (!input) return null;
+	if (input.startsWith("-")) input = input.slice(1).trim();
+	if (!input) return null;
+	try {
+		const parsed = input.includes("://") ? new URL(input) : new URL(`https://${input}`);
+		input = parsed.hostname;
+	} catch {
+		input = input.split("/")[0]?.split(":")[0] ?? "";
+	}
+	input = input.replace(/^\.+|\.+$/g, "");
+	return /^[a-z0-9][a-z0-9.-]*\.[a-z]{2,}$/i.test(input) ? input : null;
+}
+
+function parseDomainFilter(domainFilter: string[] | undefined): DomainFilters {
+	const filters: DomainFilters = { include: [], exclude: [] };
+	for (const raw of domainFilter ?? []) {
+		const domain = normalizeDomain(raw);
+		if (!domain) continue;
+		const target = raw.trim().startsWith("-") ? filters.exclude : filters.include;
+		if (!target.includes(domain)) target.push(domain);
+	}
+	return filters;
+}
+
+function passesDomainFilters(url: string, filters: DomainFilters): boolean {
+	if (filters.include.length === 0 && filters.exclude.length === 0) return true;
+	let hostname: string;
+	try {
+		hostname = new URL(url).hostname.toLowerCase();
+	} catch {
+		return false;
+	}
+	const matches = (domain: string) => hostname === domain || hostname.endsWith(`.${domain}`);
+	if (filters.exclude.some(matches)) return false;
+	return filters.include.length === 0 || filters.include.some(matches);
+}
+
+function mapRecencyFilter(value: SearchOptions["recencyFilter"]): string | undefined {
+	switch (value) {
+		case "day": return "qdr:d";
+		case "week": return "qdr:w";
+		case "month": return "qdr:m";
+		case "year": return "qdr:y";
+		default: return undefined;
+	}
+}
+
+function searchBody(query: string, options: FirecrawlSearchOptions, numResults: number, filters: DomainFilters): Record<string, unknown> {
+	return {
+		query,
+		limit: numResults,
+		sources: ["web"],
+		...(filters.include.length > 0 ? { includeDomains: filters.include } : {}),
+		...(filters.include.length === 0 && filters.exclude.length > 0 ? { excludeDomains: filters.exclude } : {}),
+		...(mapRecencyFilter(options.recencyFilter) ? { tbs: mapRecencyFilter(options.recencyFilter) } : {}),
+		...(options.includeContent ? {
+			scrapeOptions: {
+				formats: ["markdown"],
+				onlyMainContent: true,
+				...(allowFreshScrape() ? {} : { lockdown: true }),
+			},
+		} : {}),
+	};
+}
+
+function firstString(...values: unknown[]): string | null {
+	for (const value of values) {
+		if (typeof value === "string" && value.trim()) return value.trim();
+	}
+	return null;
+}
+
+function buildAnswer(results: SearchResponse["results"]): string {
+	return results.map((result) => {
+		if (result.snippet) return `${result.snippet}\nSource: ${result.title} (${result.url})`;
+		return `Source: ${result.title} (${result.url})`;
+	}).join("\n\n");
+}
+
+function mapSearchResults(data: unknown, numResults: number, filters: DomainFilters): {
+	results: SearchResponse["results"];
+	inlineContent: ExtractedContent[];
+} {
+	const web = Array.isArray(data)
+		? data
+		: data && typeof data === "object" ? (data as Record<string, unknown>).web : undefined;
+	if (!Array.isArray(web)) throw new Error("Firecrawl search returned an unexpected web result shape");
+	const results: SearchResponse["results"] = [];
+	const inlineContent: ExtractedContent[] = [];
+	const seen = new Set<string>();
+	for (const rawItem of web) {
+		if (!rawItem || typeof rawItem !== "object" || Array.isArray(rawItem)) continue;
+		const item = rawItem as FirecrawlSearchItem;
+		const url = firstString(item.url, item.metadata?.sourceURL, item.metadata?.url);
+		if (!url || seen.has(url) || !passesDomainFilters(url, filters)) continue;
+		seen.add(url);
+		const title = firstString(item.title, item.metadata?.title) ?? url;
+		const snippet = firstString(item.description, item.snippet, item.metadata?.description) ?? "";
+		results.push({ title, url, snippet });
+		const markdown = firstString(item.markdown);
+		if (markdown) inlineContent.push({ url, title, content: markdown, error: null });
+		if (results.length >= numResults) break;
+	}
+	return { results, inlineContent };
+}
+
 async function firecrawlFetch(
 	endpoint: string,
 	body: Record<string, unknown>,
 	signal: AbortSignal | undefined,
-	options: FirecrawlExtractOptions | undefined,
+	options: FirecrawlExtractOptions | FirecrawlSearchOptions | undefined,
+	label = endpoint,
+	activity: { type: "api"; query: string } | { type: "fetch"; url: string } | undefined = undefined,
 ): Promise<Record<string, unknown>> {
 	const baseUrl = requireBaseUrl();
 	const version = getApiVersion();
@@ -211,7 +353,7 @@ async function firecrawlFetch(
 	const headers: Record<string, string> = { "Content-Type": "application/json" };
 	if (apiKey) headers.Authorization = `Bearer ${apiKey}`;
 	const requestUrl = `${baseUrl}/${version}/${endpoint}`;
-	const activityId = activityMonitor.logStart({ type: "fetch", url: requestUrl });
+	const activityId = activityMonitor.logStart(activity ?? { type: "fetch", url: requestUrl });
 	try {
 		const response = await fetchFirecrawlApi(requestUrl, {
 			method: "POST",
@@ -221,24 +363,24 @@ async function firecrawlFetch(
 		}, options);
 		if (!response.ok) {
 			const text = await response.text().catch(() => "");
-			throw new Error(`Firecrawl scrape error ${response.status}: ${redactCredential(text.slice(0, 300), apiKey)}`);
+			throw new Error(`Firecrawl ${label} error ${response.status}: ${redactCredential(text.slice(0, 300), apiKey)}`);
 		}
 		let data: unknown;
 		try {
 			data = await response.json();
 		} catch (err) {
-			throw new Error(`Firecrawl scrape returned invalid JSON: ${errorMessage(err)}`);
+			throw new Error(`Firecrawl ${label} returned invalid JSON: ${errorMessage(err)}`);
 		}
 		if (!data || typeof data !== "object" || Array.isArray(data)) {
-			throw new Error("Firecrawl scrape returned an unexpected response shape");
+			throw new Error(`Firecrawl ${label} returned an unexpected response shape`);
 		}
 		const envelope = data as Record<string, unknown>;
 		if (envelope.success === false) {
 			const reason = typeof envelope.error === "string" && envelope.error.trim() ? envelope.error : "unknown error";
-			throw new Error(`Firecrawl scrape unsuccessful: ${redactCredential(reason, apiKey)}`);
+			throw new Error(`Firecrawl ${label} unsuccessful: ${redactCredential(reason, apiKey)}`);
 		}
 		if (envelope.success !== true) {
-			throw new Error("Firecrawl scrape returned an unexpected response shape");
+			throw new Error(`Firecrawl ${label} returned an unexpected response shape`);
 		}
 		activityMonitor.logComplete(activityId, response.status);
 		return envelope;
@@ -253,6 +395,24 @@ export function isFirecrawlAvailable(): boolean {
 	return getBaseUrl() !== null;
 }
 
+export async function searchWithFirecrawl(query: string, options: FirecrawlSearchOptions = {}): Promise<SearchResponse> {
+	requireBaseUrl();
+	const numResults = normalizeCount(options.numResults);
+	const filters = parseDomainFilter(options.domainFilter);
+	const envelope = await firecrawlFetch(
+		"search",
+		searchBody(query, options, numResults, filters),
+		options.signal,
+		{ ...options, timeoutMs: options.timeoutMs ?? SEARCH_TIMEOUT_MS },
+		"search",
+		{ type: "api", query },
+	);
+	const mapped = mapSearchResults(envelope.data, numResults, filters);
+	const response: SearchResponse = { answer: buildAnswer(mapped.results), results: mapped.results };
+	if (options.includeContent && mapped.inlineContent.length > 0) response.inlineContent = mapped.inlineContent;
+	return response;
+}
+
 export async function extractWithFirecrawl(
 	url: string,
 	signal?: AbortSignal,

package/node_modules/pi-web-access/gemini-search.ts

@@ -14,6 +14,7 @@ import { isSearch1APIAvailable, searchWithSearch1API } from "./search1api.ts";
 import { isSearchinfinityAvailable, searchWithSearchinfinity } from "./searchinfinity.ts";
 import { isQueritAvailable, searchWithQuerit } from "./querit.ts";
 import { isTavilyAvailable, searchWithTavily } from "./tavily.ts";
+import { isFirecrawlAvailable, searchWithFirecrawl } from "./firecrawl.ts";
 import { isJinaSearchAvailable, searchWithJina } from "./jina-search.ts";
 import { isSerpdiveAvailable, searchWithSerpdive } from "./serpdive.ts";
 import { isKagiAvailable, searchWithKagi } from "./kagi.ts";
@@ -27,7 +28,7 @@ import { isBrightDataAvailable, searchWithBrightData } from "./brightdata.ts";
 import { isSerpBaseAvailable, searchWithSerpBase } from "./serpbase.ts";
 import { getWebSearchConfigPath } from "./utils.ts";
 
-export const RESOLVED_SEARCH_PROVIDERS = ["openai", "brave", "parallel", "tinyfish", "search1api", "searchinfinity", "querit", "tavily", "jina", "searxng", "duckduckgo", "perplexity", "gemini", "exa", "serpdive", "kagi", "ollama", "anysearch", "xai", "brightdata", "serpbase", "bocha"] as const;
+export const RESOLVED_SEARCH_PROVIDERS = ["openai", "brave", "parallel", "tinyfish", "search1api", "searchinfinity", "querit", "tavily", "firecrawl", "jina", "searxng", "duckduckgo", "perplexity", "gemini", "exa", "serpdive", "kagi", "ollama", "anysearch", "xai", "brightdata", "serpbase", "bocha"] as const;
 export const SEARCH_PROVIDERS = ["auto", "all", ...RESOLVED_SEARCH_PROVIDERS] as const;
 
 export type ResolvedSearchProvider = typeof RESOLVED_SEARCH_PROVIDERS[number];
@@ -91,7 +92,7 @@ const CONFIG_PATH = getWebSearchConfigPath();
 const DEFAULT_SEARCH_MODEL = "gemini-3.6-flash";
 // Explicit-only providers (DuckDuckGo, AnySearch, xAI, Bright Data, SerpBase) are deliberately absent:
 // `all` must never fan out to an opt-in or paid provider without the user asking for it.
-const ALL_SEARCH_PROVIDERS: ResolvedSearchProvider[] = ["searxng", "openai", "exa", "brave", "parallel", "tinyfish", "search1api", "searchinfinity", "querit", "tavily", "jina", "serpdive", "kagi", "ollama", "perplexity", "gemini", "bocha"];
+const ALL_SEARCH_PROVIDERS: ResolvedSearchProvider[] = ["searxng", "openai", "exa", "brave", "parallel", "tinyfish", "search1api", "searchinfinity", "querit", "tavily", "firecrawl", "jina", "serpdive", "kagi", "ollama", "perplexity", "gemini", "bocha"];
 const VALID_ROUTING_KINDS = ["transient", "quota", "network", "invalid-response"] as const;
 
 type SearchConfig = {
@@ -304,6 +305,7 @@ async function searchWithResolvedProvider(
 	if (provider === "searchinfinity") return { ...(await searchWithSearchinfinity(query, options)), provider };
 	if (provider === "querit") return { ...(await searchWithQuerit(query, options)), provider };
 	if (provider === "tavily") return { ...(await searchWithTavily(query, options)), provider };
+	if (provider === "firecrawl") return { ...(await searchWithFirecrawl(query, options)), provider };
 	if (provider === "jina") return { ...(await searchWithJina(query, options)), provider };
 	if (provider === "serpdive") return { ...(await searchWithSerpdive(query, options)), provider };
 	if (provider === "kagi") return { ...(await searchWithKagi(query, options)), provider };
@@ -340,6 +342,7 @@ async function isResolvedProviderAvailable(provider: ResolvedSearchProvider, opt
 	if (provider === "searchinfinity") return isSearchinfinityAvailable();
 	if (provider === "querit") return isQueritAvailable();
 	if (provider === "tavily") return isTavilyAvailable();
+	if (provider === "firecrawl") return isFirecrawlAvailable();
 	if (provider === "jina") return isJinaSearchAvailable();
 	if (provider === "serpdive") return isSerpdiveAvailable();
 	if (provider === "kagi") return isKagiAvailable();
@@ -362,6 +365,7 @@ function providerLabel(provider: ResolvedSearchProvider): string {
 	if (provider === "search1api") return "Search1API";
 	if (provider === "searchinfinity") return "Searchinfinity";
 	if (provider === "querit") return "Querit";
+	if (provider === "firecrawl") return "Firecrawl";
 	if (provider === "serpdive") return "SERPdive";
 	if (provider === "searxng") return "SearXNG";
 	if (provider === "duckduckgo") return "DuckDuckGo";
@@ -600,6 +604,16 @@ export async function search(query: string, options: FullSearchOptions = {}): Pr
 		}
 	}
 
+	if (isFirecrawlAvailable()) {
+		try {
+			const result = await searchWithFirecrawl(query, options);
+			return { ...result, provider: "firecrawl" };
+		} catch (err) {
+			if (isAbortError(err)) throw err;
+			fallbackErrors.push(`Firecrawl: ${errorMessage(err)}`);
+		}
+	}
+
 	if (isJinaSearchAvailable()) {
 		try {
 			const result = await searchWithJina(query, options);
@@ -675,8 +689,8 @@ export async function search(query: string, options: FullSearchOptions = {}): Pr
 	throw new Error(
 		"No search provider available. Either:\n" +
 		"  1. Use /login to sign in with a Codex subscription for OpenAI web search\n" +
-		`  2. Set openaiApiKey, braveApiKey, parallelApiKey, tinyfishApiKey, search1apiApiKey, searchinfinityApiKey, queritApiKey, tavilyApiKey, jinaApiKey, serpdiveApiKey, kagiApiKey, ollamaApiKey, searxngBaseUrl, perplexityApiKey, exaApiKey, geminiApiKey, bochaApiKey, or cloudflareApiKey in ${CONFIG_PATH}\n` +
-		"  3. Set OPENAI_API_KEY, BRAVE_API_KEY, PARALLEL_API_KEY, TINYFISH_API_KEY, SEARCH1API_KEY, SEARCHINFINITY_API_KEY, QUERIT_API_KEY, TAVILY_API_KEY, JINA_API_KEY, SERPDIVE_API_KEY, KAGI_API_KEY, BOCHA_API_KEY, OLLAMA_API_KEY, SEARXNG_BASE_URL, EXA_API_KEY, PERPLEXITY_API_KEY, GEMINI_API_KEY, or CLOUDFLARE_API_KEY env vars\n" +
+		`  2. Set openaiApiKey, braveApiKey, parallelApiKey, tinyfishApiKey, search1apiApiKey, searchinfinityApiKey, queritApiKey, tavilyApiKey, firecrawlBaseUrl, jinaApiKey, serpdiveApiKey, kagiApiKey, ollamaApiKey, searxngBaseUrl, perplexityApiKey, exaApiKey, geminiApiKey, bochaApiKey, or cloudflareApiKey in ${CONFIG_PATH}\n` +
+		"  3. Set OPENAI_API_KEY, BRAVE_API_KEY, PARALLEL_API_KEY, TINYFISH_API_KEY, SEARCH1API_KEY, SEARCHINFINITY_API_KEY, QUERIT_API_KEY, TAVILY_API_KEY, FIRECRAWL_BASE_URL, JINA_API_KEY, SERPDIVE_API_KEY, KAGI_API_KEY, BOCHA_API_KEY, OLLAMA_API_KEY, SEARXNG_BASE_URL, EXA_API_KEY, PERPLEXITY_API_KEY, GEMINI_API_KEY, or CLOUDFLARE_API_KEY env vars\n" +
 		"  4. Set GOOGLE_GEMINI_BASE_URL with CLOUDFLARE_API_KEY for Cloudflare AI Gateway routing\n" +
 		"  5. Sign into gemini.google.com in a supported Chromium-based browser\n" +
 		"  6. Explicitly select provider: \"anysearch\" for anonymous AnySearch, \"xai\" for Grok, \"brightdata\" with brightdataSerpZone for paid Bright Data SERP, or \"serpbase\" with serpbaseApiKey for paid Google SERP"