@herbertgao/pi-extensions 2026.8.1 → 2026.8.3

package/node_modules/pi-web-access/gemini-search.ts

@@ -14,17 +14,20 @@ import { isSearch1APIAvailable, searchWithSearch1API } from "./search1api.ts";
 import { isSearchinfinityAvailable, searchWithSearchinfinity } from "./searchinfinity.ts";
 import { isQueritAvailable, searchWithQuerit } from "./querit.ts";
 import { isTavilyAvailable, searchWithTavily } from "./tavily.ts";
+import { isJinaSearchAvailable, searchWithJina } from "./jina-search.ts";
 import { isSerpdiveAvailable, searchWithSerpdive } from "./serpdive.ts";
 import { isKagiAvailable, searchWithKagi } from "./kagi.ts";
+import { isBochaAvailable, searchWithBocha } from "./bocha.ts";
 import { isOllamaAvailable, searchWithOllama } from "./ollama.ts";
 import { isSearXNGAvailable, searchWithSearXNG } from "./searxng.ts";
+import { isDuckDuckGoAvailable, searchWithDuckDuckGo } from "./duckduckgo.ts";
 import { isAnySearchAvailable, searchWithAnySearch } from "./anysearch.ts";
 import { isXaiSearchAvailable, searchWithXai } from "./xai-search.ts";
 import { isBrightDataAvailable, searchWithBrightData } from "./brightdata.ts";
 import { isSerpBaseAvailable, searchWithSerpBase } from "./serpbase.ts";
 import { getWebSearchConfigPath } from "./utils.ts";
 
-export const RESOLVED_SEARCH_PROVIDERS = ["openai", "brave", "parallel", "tinyfish", "search1api", "searchinfinity", "querit", "tavily", "searxng", "perplexity", "gemini", "exa", "serpdive", "kagi", "ollama", "anysearch", "xai", "brightdata", "serpbase"] as const;
+export const RESOLVED_SEARCH_PROVIDERS = ["openai", "brave", "parallel", "tinyfish", "search1api", "searchinfinity", "querit", "tavily", "jina", "searxng", "duckduckgo", "perplexity", "gemini", "exa", "serpdive", "kagi", "ollama", "anysearch", "xai", "brightdata", "serpbase", "bocha"] as const;
 export const SEARCH_PROVIDERS = ["auto", "all", ...RESOLVED_SEARCH_PROVIDERS] as const;
 
 export type ResolvedSearchProvider = typeof RESOLVED_SEARCH_PROVIDERS[number];
@@ -44,7 +47,7 @@ export type SearchProviderErrorKind =
 
 export interface SearchRoutingConfig {
 	providers: ResolvedSearchProvider[];
-	fallbackOn: Array<Extract<SearchProviderErrorKind, "transient" | "quota" | "network">>;
+	fallbackOn: Array<Extract<SearchProviderErrorKind, "transient" | "quota" | "network" | "invalid-response">>;
 }
 
 export class SearchProviderError extends Error {
@@ -86,10 +89,10 @@ export interface AttributedSearchResponse extends SearchResponse {
 
 const CONFIG_PATH = getWebSearchConfigPath();
 const DEFAULT_SEARCH_MODEL = "gemini-3.6-flash";
-// Explicit-only providers (AnySearch, xAI, Bright Data, SerpBase) are deliberately absent:
-// `all` must never fan out to a paid/surprising provider without the user asking for it.
-const ALL_SEARCH_PROVIDERS: ResolvedSearchProvider[] = ["searxng", "openai", "exa", "brave", "parallel", "tinyfish", "search1api", "searchinfinity", "querit", "tavily", "serpdive", "kagi", "ollama", "perplexity", "gemini"];
-const VALID_ROUTING_KINDS = ["transient", "quota", "network"] as const;
+// Explicit-only providers (DuckDuckGo, AnySearch, xAI, Bright Data, SerpBase) are deliberately absent:
+// `all` must never fan out to an opt-in or paid provider without the user asking for it.
+const ALL_SEARCH_PROVIDERS: ResolvedSearchProvider[] = ["searxng", "openai", "exa", "brave", "parallel", "tinyfish", "search1api", "searchinfinity", "querit", "tavily", "jina", "serpdive", "kagi", "ollama", "perplexity", "gemini", "bocha"];
+const VALID_ROUTING_KINDS = ["transient", "quota", "network", "invalid-response"] as const;
 
 type SearchConfig = {
 	searchProvider: SearchProviderSelection;
@@ -143,7 +146,7 @@ function normalizeSearchRouting(value: unknown): SearchRoutingConfig {
 	const fallbackOn: SearchRoutingConfig["fallbackOn"] = [];
 	for (const kind of raw.fallbackOn) {
 		if (typeof kind !== "string" || !VALID_ROUTING_KINDS.includes(kind as typeof VALID_ROUTING_KINDS[number])) {
-			throw new Error(`searchRouting.fallbackOn in ${CONFIG_PATH} may only contain transient, quota, or network`);
+			throw new Error(`searchRouting.fallbackOn in ${CONFIG_PATH} may only contain transient, quota, network, or invalid-response`);
 		}
 		if (!fallbackOn.includes(kind as SearchRoutingConfig["fallbackOn"][number])) {
 			fallbackOn.push(kind as SearchRoutingConfig["fallbackOn"][number]);
@@ -257,6 +260,8 @@ function classifyProviderError(provider: ResolvedSearchProvider, err: unknown):
 		kind = "credential";
 	} else if (isAbortError(err)) {
 		kind = "aborted";
+	} else if (provider === "xai" && status === 403 && /spending[- ]limit|(?:no|out of) credits?|insufficient quota|quota (?:exceeded|exhausted)|credits? (?:exhausted|depleted|used up)/.test(lower)) {
+		kind = "quota";
 	} else if (status === 401 || status === 403) {
 		kind = "auth";
 	} else if (status === 400 || status === 422) {
@@ -271,7 +276,7 @@ function classifyProviderError(provider: ResolvedSearchProvider, err: unknown):
 		kind = "auth";
 	} else if (/bad request|invalid request/.test(lower)) {
 		kind = "invalid-request";
-	} else if (/invalid json|no parseable response|returned invalid response|returned empty response/.test(lower)) {
+	} else if (/invalid json|no parseable response|no parseable results|invalid response|returned empty response/.test(lower)) {
 		kind = "invalid-response";
 	} else if (/temporar|service unavailable|server error/.test(lower)) {
 		kind = "transient";
@@ -299,8 +304,10 @@ async function searchWithResolvedProvider(
 	if (provider === "searchinfinity") return { ...(await searchWithSearchinfinity(query, options)), provider };
 	if (provider === "querit") return { ...(await searchWithQuerit(query, options)), provider };
 	if (provider === "tavily") return { ...(await searchWithTavily(query, options)), provider };
+	if (provider === "jina") return { ...(await searchWithJina(query, options)), provider };
 	if (provider === "serpdive") return { ...(await searchWithSerpdive(query, options)), provider };
 	if (provider === "kagi") return { ...(await searchWithKagi(query, options)), provider };
+	if (provider === "bocha") return { ...(await searchWithBocha(query, options)), provider };
 	if (provider === "ollama") return { ...(await searchWithOllama(query, options)), provider };
 	if (provider === "anysearch") return { ...(await searchWithAnySearch(query, options)), provider };
 	if (provider === "xai") return { ...(await searchWithXai(query, options, options.extensionContext)), provider };
@@ -308,6 +315,7 @@ async function searchWithResolvedProvider(
 	if (provider === "serpbase") return { ...(await searchWithSerpBase(query, options)), provider };
 	if (provider === "perplexity") return { ...(await searchWithPerplexity(query, options)), provider };
 	if (provider === "searxng") return { ...(await searchWithSearXNG(query, options)), provider };
+	if (provider === "duckduckgo") return { ...(await searchWithDuckDuckGo(query, options)), provider };
 	if (provider === "gemini") {
 		const result = await searchWithGemini(query, options, true);
 		if (result) return { ...result, provider };
@@ -332,8 +340,10 @@ async function isResolvedProviderAvailable(provider: ResolvedSearchProvider, opt
 	if (provider === "searchinfinity") return isSearchinfinityAvailable();
 	if (provider === "querit") return isQueritAvailable();
 	if (provider === "tavily") return isTavilyAvailable();
+	if (provider === "jina") return isJinaSearchAvailable();
 	if (provider === "serpdive") return isSerpdiveAvailable();
 	if (provider === "kagi") return isKagiAvailable();
+	if (provider === "bocha") return isBochaAvailable();
 	if (provider === "ollama") return isOllamaAvailable();
 	if (provider === "anysearch") return isAnySearchAvailable();
 	if (provider === "xai") return isXaiSearchAvailable(options.extensionContext);
@@ -341,6 +351,7 @@ async function isResolvedProviderAvailable(provider: ResolvedSearchProvider, opt
 	if (provider === "serpbase") return isSerpBaseAvailable();
 	if (provider === "perplexity") return isPerplexityAvailable();
 	if (provider === "searxng") return isSearXNGAvailable();
+	if (provider === "duckduckgo") return isDuckDuckGoAvailable();
 	if (provider === "gemini") return isGeminiApiAvailable() || !!(await isGeminiWebAvailable());
 	return isExaAvailable();
 }
@@ -353,7 +364,9 @@ function providerLabel(provider: ResolvedSearchProvider): string {
 	if (provider === "querit") return "Querit";
 	if (provider === "serpdive") return "SERPdive";
 	if (provider === "searxng") return "SearXNG";
+	if (provider === "duckduckgo") return "DuckDuckGo";
 	if (provider === "kagi") return "Kagi";
+	if (provider === "bocha") return "Bocha";
 	if (provider === "ollama") return "Ollama";
 	if (provider === "xai") return "xAI";
 	if (provider === "brightdata") return "Bright Data";
@@ -587,6 +600,16 @@ export async function search(query: string, options: FullSearchOptions = {}): Pr
 		}
 	}
 
+	if (isJinaSearchAvailable()) {
+		try {
+			const result = await searchWithJina(query, options);
+			return { ...result, provider: "jina" };
+		} catch (err) {
+			if (isAbortError(err)) throw err;
+			fallbackErrors.push(`Jina: ${errorMessage(err)}`);
+		}
+	}
+
 	if (isSerpdiveAvailable()) {
 		try {
 			const result = await searchWithSerpdive(query, options);
@@ -607,6 +630,16 @@ export async function search(query: string, options: FullSearchOptions = {}): Pr
 		}
 	}
 
+	if (isBochaAvailable()) {
+		try {
+			const result = await searchWithBocha(query, options);
+			return { ...result, provider: "bocha" };
+		} catch (err) {
+			if (isAbortError(err)) throw err;
+			fallbackErrors.push(`Bocha: ${errorMessage(err)}`);
+		}
+	}
+
 	if (isOllamaAvailable()) {
 		try {
 			const result = await searchWithOllama(query, options);
@@ -642,8 +675,8 @@ export async function search(query: string, options: FullSearchOptions = {}): Pr
 	throw new Error(
 		"No search provider available. Either:\n" +
 		"  1. Use /login to sign in with a Codex subscription for OpenAI web search\n" +
-		`  2. Set openaiApiKey, braveApiKey, parallelApiKey, tinyfishApiKey, search1apiApiKey, searchinfinityApiKey, queritApiKey, tavilyApiKey, serpdiveApiKey, kagiApiKey, ollamaApiKey, searxngBaseUrl, perplexityApiKey, exaApiKey, geminiApiKey, or cloudflareApiKey in ${CONFIG_PATH}\n` +
-		"  3. Set OPENAI_API_KEY, BRAVE_API_KEY, PARALLEL_API_KEY, TINYFISH_API_KEY, SEARCH1API_KEY, SEARCHINFINITY_API_KEY, QUERIT_API_KEY, TAVILY_API_KEY, SERPDIVE_API_KEY, KAGI_API_KEY, OLLAMA_API_KEY, SEARXNG_BASE_URL, EXA_API_KEY, PERPLEXITY_API_KEY, GEMINI_API_KEY, or CLOUDFLARE_API_KEY env vars\n" +
+		`  2. Set openaiApiKey, braveApiKey, parallelApiKey, tinyfishApiKey, search1apiApiKey, searchinfinityApiKey, queritApiKey, tavilyApiKey, jinaApiKey, serpdiveApiKey, kagiApiKey, ollamaApiKey, searxngBaseUrl, perplexityApiKey, exaApiKey, geminiApiKey, bochaApiKey, or cloudflareApiKey in ${CONFIG_PATH}\n` +
+		"  3. Set OPENAI_API_KEY, BRAVE_API_KEY, PARALLEL_API_KEY, TINYFISH_API_KEY, SEARCH1API_KEY, SEARCHINFINITY_API_KEY, QUERIT_API_KEY, TAVILY_API_KEY, JINA_API_KEY, SERPDIVE_API_KEY, KAGI_API_KEY, BOCHA_API_KEY, OLLAMA_API_KEY, SEARXNG_BASE_URL, EXA_API_KEY, PERPLEXITY_API_KEY, GEMINI_API_KEY, or CLOUDFLARE_API_KEY env vars\n" +
 		"  4. Set GOOGLE_GEMINI_BASE_URL with CLOUDFLARE_API_KEY for Cloudflare AI Gateway routing\n" +
 		"  5. Sign into gemini.google.com in a supported Chromium-based browser\n" +
 		"  6. Explicitly select provider: \"anysearch\" for anonymous AnySearch, \"xai\" for Grok, \"brightdata\" with brightdataSerpZone for paid Bright Data SERP, or \"serpbase\" with serpbaseApiKey for paid Google SERP"
@@ -652,7 +685,7 @@ export async function search(query: string, options: FullSearchOptions = {}): Pr
 
 async function searchWithGeminiApi(query: string, options: SearchOptions = {}): Promise<SearchResponse | null> {
 	const requestSignal = AbortSignal.any([
-		AbortSignal.timeout(60000),
+		AbortSignal.timeout(120000),
 		...(options.signal ? [options.signal] : []),
 	]);
 	const apiKey = await getApiKey(requestSignal);
@@ -711,7 +744,7 @@ async function searchWithGeminiWeb(query: string, options: SearchOptions = {}):
 	try {
 		const text = await queryWithCookies(prompt, cookies, {
 			signal: options.signal,
-			timeoutMs: 60000,
+			timeoutMs: 120000,
 		});
 
 		activityMonitor.logComplete(activityId, 200);

package/node_modules/pi-web-access/gemini-web.ts

@@ -24,6 +24,71 @@ const MODEL_HEADERS: Record<string, string> = {
 
 const REQUIRED_COOKIES = ["__Secure-1PSID", "__Secure-1PSIDTS"];
 
+// Gemini Web talks to Google's frontend inside the host agent (pi), whose
+// global undici dispatcher runs HTTP/1.1 with undici's default 16 KiB
+// maxHeaderSize. Google's gemini.google.com /app response headers routinely
+// exceed that budget over HTTP/1.1, so cookie-bearing requests fail with
+// UND_ERR_HEADERS_OVERFLOW surfaced as "fetch failed". Route these requests
+// through a dedicated undici Agent with a larger header budget instead; fall
+// back to the ambient global fetch when undici is not resolvable (e.g.
+// standalone test environments).
+let geminiFetchImpl: typeof fetch | null = null;
+
+// Test-only hook: lets tests inject a simulated transport instead of patching
+// globalThis.fetch. Gemini Web deliberately bypasses the ambient global fetch
+// (inside the pi host agent it is constrained to undici's 16 KiB
+// maxHeaderSize and cannot serve Google's /app page over HTTP/1.1), so
+// globalThis.fetch patching is no longer a reliable way to intercept these
+// requests.
+export function setGeminiFetchOverrideForTests(fetchImpl: typeof fetch | null): void {
+	geminiFetchOverride = fetchImpl;
+}
+
+let geminiFetchOverride: typeof fetch | null = null;
+
+// The header budget Gemini Web needs for Google frontend responses.
+// Google's /app page exceeds undici's default 16 KiB maxHeaderSize when
+// served over HTTP/1.1, which is how pi's global dispatcher operates.
+export const GEMINI_MAX_HEADER_SIZE = 4 * 1024 * 1024;
+
+// Build a fetch that routes through a dedicated undici Agent with a larger
+// header budget than the host agent's global dispatcher allows. Exported for
+// tests so the agent configuration can be asserted without network access.
+export function createGeminiFetch(undiciImpl: typeof import("undici")): typeof fetch {
+	const agent = new undiciImpl.EnvHttpProxyAgent({
+		allowH2: false,
+		connectTimeout: 30_000,
+		maxHeaderSize: GEMINI_MAX_HEADER_SIZE,
+		pipelining: 1,
+	});
+	// undici 8 types its fetch request/response structurally differently from
+	// the DOM lib types (duplex/textStream on Request, Symbol.dispose on
+	// Headers); the shape this module uses (status, ok, headers.get, text) is
+	// identical, so bridge the two.
+	type UndiciFetch = typeof undiciImpl.fetch;
+	return (input, init) =>
+		undiciImpl.fetch(
+			input as unknown as Parameters<UndiciFetch>[0],
+			{ ...init, dispatcher: agent } as unknown as Parameters<UndiciFetch>[1],
+		) as unknown as Promise<Response>;
+}
+
+export async function resolveGeminiFetch(): Promise<typeof fetch> {
+	if (geminiFetchOverride) return geminiFetchOverride;
+	if (geminiFetchImpl) return geminiFetchImpl;
+
+	let undici: typeof import("undici");
+	try {
+		undici = await import("undici");
+	} catch {
+		geminiFetchImpl = fetch;
+		return geminiFetchImpl;
+	}
+
+	geminiFetchImpl = createGeminiFetch(undici);
+	return geminiFetchImpl;
+}
+
 export interface GeminiWebOptions {
 	youtubeUrl?: string;
 	model?: string;
@@ -129,7 +194,7 @@ async function runGeminiWebOnce(
 	params.set("at", accessToken);
 	params.set("f.req", fReq);
 
-	const res = await fetch(GEMINI_STREAM_GENERATE_URL, {
+	const res = await (await resolveGeminiFetch())(GEMINI_STREAM_GENERATE_URL, {
 		method: "POST",
 		redirect: "error",
 		headers: {
@@ -192,7 +257,7 @@ async function fetchWithCookieRedirects(
 	let current = url;
 	const allowedOrigin = new URL(url).origin;
 	for (let i = 0; i <= maxRedirects; i++) {
-		const res = await fetch(current, {
+		const res = await (await resolveGeminiFetch())(current, {
 			headers: { "user-agent": USER_AGENT, cookie: cookieHeader },
 			redirect: "manual",
 			signal,
@@ -302,7 +367,7 @@ async function uploadFile(
 		Buffer.from(footer, "utf-8"),
 	]);
 
-	const res = await fetch(GEMINI_UPLOAD_URL, {
+	const res = await (await resolveGeminiFetch())(GEMINI_UPLOAD_URL, {
 		method: "POST",
 		redirect: "error",
 		headers: {