@oh-my-pi/pi-coding-agent 16.4.2 → 16.4.3

package/src/vibe/state.ts

@@ -0,0 +1,4 @@
+/** Vibe mode session-level state, mirroring {@link ../plan-mode/state.ts}. */
+export interface VibeModeState {
+	enabled: boolean;
+}

package/src/web/search/index.ts

@@ -231,12 +231,20 @@ export async function runSearchQuery(
 	params: SearchQueryParams,
 	options: { authStorage?: AuthStorage; sessionId?: string; signal?: AbortSignal } = {},
 ): Promise<{ content: Array<{ type: "text"; text: string }>; details: SearchRenderDetails }> {
-	const authStorage = options.authStorage ?? (await discoverAuthStorage());
-	return executeSearch("cli-web-search", params, {
-		authStorage,
-		sessionId: options.sessionId,
-		signal: options.signal,
-	});
+	const createdAuthStorage = options.authStorage ? undefined : await discoverAuthStorage();
+	const authStorage = options.authStorage ?? createdAuthStorage;
+	if (!authStorage) {
+		throw new Error("Failed to initialize authentication storage");
+	}
+	try {
+		return await executeSearch("cli-web-search", params, {
+			authStorage,
+			sessionId: options.sessionId,
+			signal: options.signal,
+		});
+	} finally {
+		createdAuthStorage?.close();
+	}
 }
 
 /**

package/src/web/search/provider.ts

@@ -114,6 +114,41 @@ const PROVIDER_META: Record<SearchProviderId, ProviderMeta> = {
 		label: SEARCH_PROVIDER_LABELS.duckduckgo,
 		load: async () => new (await import("./providers/duckduckgo")).DuckDuckGoProvider(),
 	},
+	google: {
+		id: "google",
+		label: SEARCH_PROVIDER_LABELS.google,
+		load: async () => new (await import("./providers/google")).GoogleProvider(),
+	},
+	bing: {
+		id: "bing",
+		label: SEARCH_PROVIDER_LABELS.bing,
+		load: async () => new (await import("./providers/bing")).BingProvider(),
+	},
+	yahoo: {
+		id: "yahoo",
+		label: SEARCH_PROVIDER_LABELS.yahoo,
+		load: async () => new (await import("./providers/yahoo")).YahooProvider(),
+	},
+	ecosia: {
+		id: "ecosia",
+		label: SEARCH_PROVIDER_LABELS.ecosia,
+		load: async () => new (await import("./providers/ecosia")).EcosiaProvider(),
+	},
+	startpage: {
+		id: "startpage",
+		label: SEARCH_PROVIDER_LABELS.startpage,
+		load: async () => new (await import("./providers/startpage")).StartpageProvider(),
+	},
+	mojeek: {
+		id: "mojeek",
+		label: SEARCH_PROVIDER_LABELS.mojeek,
+		load: async () => new (await import("./providers/mojeek")).MojeekProvider(),
+	},
+	public: {
+		id: "public",
+		label: SEARCH_PROVIDER_LABELS.public,
+		load: async () => new (await import("./providers/public")).PublicWebProvider(),
+	},
 };
 
 const instanceCache = new Map<SearchProviderId, SearchProvider>();
@@ -180,7 +215,8 @@ export function setExcludedSearchProviders(providers: readonly SearchProviderId[
 	excludedProvIds = new Set(providers);
 }
 
-function isSearchProviderExcluded(id: SearchProviderId): boolean {
+/** `true` when settings exclude `id` from web search (auto chain and the Public Web fan-out). */
+export function isSearchProviderExcluded(id: SearchProviderId): boolean {
 	return excludedProvIds.has(id);
 }
 

package/src/web/search/providers/bing.ts

@@ -0,0 +1,197 @@
+import type { AuthStorage } from "@oh-my-pi/pi-ai";
+import { parseHTML } from "linkedom";
+import type { SearchResponse, SearchSource } from "../../../web/search/types";
+import { SearchProviderError } from "../../../web/search/types";
+import { clampNumResults } from "../utils";
+import type { SearchParams } from "./base";
+import { SearchProvider } from "./base";
+import { browserFetch } from "./browser-page";
+import { classifyProviderHttpError, withHardTimeout } from "./utils";
+
+/**
+ * Bing's HTML search frontend. A plain GET with browser navigation headers
+ * returns a fully server-rendered results page — no JavaScript challenge on
+ * the organic path — so we parse it directly without a real browser.
+ */
+const BING_HOME_URL = "https://www.bing.com/";
+const BING_SEARCH_URL = "https://www.bing.com/search";
+const DEFAULT_NUM_RESULTS = 10;
+const MAX_NUM_RESULTS = 20;
+const MS_PER_DAY = 86_400_000;
+
+/**
+ * Recency → Bing `filters=ex1:"…"` freshness codes, as emitted by Bing's own
+ * "Any time" dropdown. `year` has no fixed code; the dropdown emits a custom
+ * epoch-day range (`ez5_<start>_<end>`, days since 1970-01-01) which
+ * {@link recencyToFilters} computes. Bing parses the parameter (the SERP
+ * filter UI reflects it) but enforcement is server-side and vantage-dependent.
+ */
+const RECENCY_TO_BING_EZ: Record<Exclude<NonNullable<SearchParams["recency"]>, "year">, string> = {
+	day: "ez1",
+	week: "ez2",
+	month: "ez3",
+};
+
+/** Snippet containers observed on Bing result blocks, in preference order. */
+const BING_SNIPPET_SELECTORS: readonly string[] = [".b_caption p", "p[class*='b_lineclamp']", ".b_algoSlug"];
+
+interface ParsedResult {
+	title: string;
+	url: string;
+	snippet?: string;
+}
+
+/** Build the `filters` value for a recency window, mirroring Bing's dropdown URLs. */
+function recencyToFilters(recency: NonNullable<SearchParams["recency"]>): string {
+	if (recency === "year") {
+		const epochDay = Math.floor(Date.now() / MS_PER_DAY);
+		return `ex1:"ez5_${epochDay - 365}_${epochDay}"`;
+	}
+	return `ex1:"${RECENCY_TO_BING_EZ[recency]}"`;
+}
+
+/**
+ * Resolve a Bing result href to the underlying target URL.
+ *
+ * Organic hrefs are usually wrapped as `https://www.bing.com/ck/a?…&u=a1<payload>`
+ * where the payload after the literal `a1` prefix is the unpadded base64url
+ * encoding of the target URL. Direct external hrefs also occur; Bing-internal
+ * links (vertical tabs, ads plumbing) and non-http(s) schemes are rejected.
+ */
+function unwrapResultUrl(href: string): string | undefined {
+	let url: URL;
+	try {
+		url = new URL(href, BING_HOME_URL);
+	} catch {
+		return undefined;
+	}
+
+	if (url.hostname === "bing.com" || url.hostname.endsWith(".bing.com")) {
+		if (url.pathname !== "/ck/a") return undefined;
+		const wrapped = url.searchParams.get("u");
+		if (!wrapped?.startsWith("a1")) return undefined;
+		try {
+			url = new URL(Buffer.from(wrapped.slice(2), "base64url").toString("utf-8"));
+		} catch {
+			return undefined;
+		}
+	}
+
+	if (url.protocol !== "http:" && url.protocol !== "https:") return undefined;
+	return url.href;
+}
+
+function findSnippet(item: Element): string | undefined {
+	for (const selector of BING_SNIPPET_SELECTORS) {
+		const text = (item.querySelector(selector)?.textContent ?? "").replace(/\s+/g, " ").trim();
+		if (text) return text;
+	}
+	return undefined;
+}
+
+/**
+ * Pull organic result blocks out of the page in document order.
+ *
+ * Each organic hit is an `<li class="b_algo">` with the title link in
+ * `h2 > a[href]` (sitelink/attribution anchors live outside the `h2`) and the
+ * preview text in one of {@link BING_SNIPPET_SELECTORS}. Ads, answer cards,
+ * and the "no results" row use other classes and fall out naturally.
+ */
+function parseHtmlResults(html: string): ParsedResult[] {
+	const { document } = parseHTML(html);
+	const results: ParsedResult[] = [];
+	for (const item of document.querySelectorAll("li.b_algo")) {
+		const anchor = item.querySelector("h2 a[href]");
+		const href = anchor?.getAttribute("href");
+		if (!href) continue;
+		const url = unwrapResultUrl(href);
+		if (!url) continue;
+		const title = (anchor?.textContent ?? "").replace(/\s+/g, " ").trim();
+		if (!title) continue;
+		results.push({ title, url, snippet: findSnippet(item) });
+	}
+	return results;
+}
+
+/**
+ * `true` when Bing answered with its CAPTCHA/consent interstitial instead of
+ * a results page. The challenge redirects to `/turing/captcha/…`; body
+ * markers are only trusted when no organic result block is present so a
+ * search *about* CAPTCHAs never trips the detector.
+ */
+function isChallengeResponse(html: string, finalUrl: string): boolean {
+	if (finalUrl.includes("/turing/captcha")) return true;
+	if (html.includes('class="b_algo"')) return false;
+	return /turing\/captcha|b_captcha|px-captcha|verify (?:that )?you are (?:a )?human/i.test(html);
+}
+
+function buildSearchUrl(params: SearchParams, numResults: number): string {
+	const url = new URL(BING_SEARCH_URL);
+	url.searchParams.set("q", params.query);
+	url.searchParams.set("count", String(numResults));
+	url.searchParams.set("mkt", "en-US");
+	url.searchParams.set("setlang", "en");
+	if (params.recency) url.searchParams.set("filters", recencyToFilters(params.recency));
+	return url.href;
+}
+
+async function callBingHtml(params: SearchParams, numResults: number): Promise<string> {
+	const url = buildSearchUrl(params, numResults);
+	const page = await browserFetch(url, {
+		fetch: params.fetch ?? fetch,
+		signal: withHardTimeout(params.signal),
+		referer: BING_HOME_URL,
+	});
+
+	const body = page.html;
+	if (isChallengeResponse(body, page.url)) {
+		throw new SearchProviderError(
+			"bing",
+			"Bing blocked the request with a CAPTCHA challenge. Bing throttles automated searches from datacenter/shared-egress IPs; try the duckduckgo or mojeek provider, or configure a credentialed provider such as Brave, Tavily, Exa, or Kagi.",
+			429,
+		);
+	}
+	if (page.status < 200 || page.status >= 300) {
+		const classified = classifyProviderHttpError("bing", page.status, body);
+		if (classified) throw classified;
+		throw new SearchProviderError("bing", `Bing HTML error (${page.status})`, page.status);
+	}
+
+	return body;
+}
+
+/** Execute a Bing web search via the server-rendered HTML results page. */
+export async function searchBing(params: SearchParams): Promise<SearchResponse> {
+	const numResults = clampNumResults(params.numSearchResults ?? params.limit, DEFAULT_NUM_RESULTS, MAX_NUM_RESULTS);
+	const html = await callBingHtml(params, numResults);
+	const parsed = parseHtmlResults(html);
+
+	const sources: SearchSource[] = [];
+	const seen = new Set<string>();
+	for (const result of parsed) {
+		if (seen.has(result.url)) continue;
+		seen.add(result.url);
+		sources.push({ title: result.title, url: result.url, snippet: result.snippet });
+		if (sources.length >= numResults) break;
+	}
+
+	return { provider: "bing", sources };
+}
+
+/** Search provider for Bing (no API key required). */
+export class BingProvider extends SearchProvider {
+	readonly id = "bing";
+	readonly label = "Bing";
+
+	isAvailable(_authStorage: AuthStorage): boolean {
+		return true;
+	}
+
+	isExplicitlyAvailable(_authStorage: AuthStorage): boolean {
+		return true;
+	}
+
+	search(params: SearchParams): Promise<SearchResponse> {
+		return searchBing(params);
+	}
+}

package/src/web/search/providers/browser-headers.ts

@@ -0,0 +1,92 @@
+import { HeaderGenerator } from "header-generator";
+
+// Instantiate the singleton header generator.
+// This matches modern browsers from real-world query statistics
+// and randomizes between Chrome, Firefox, Safari, and Edge.
+const generator = new HeaderGenerator({
+	browserListQuery: "last 3 versions",
+	devices: ["desktop"],
+	operatingSystems: ["windows", "macos", "linux"],
+	locales: ["en-US", "en"],
+	httpVersion: "2",
+	strict: false,
+});
+
+// A fallback desktop Mac Chrome navigation fingerprint matching
+// the previous static default setup for deterministic or non-randomized calls.
+const CHROME_FALLBACK_HEADERS: Record<string, string> = {
+	Accept:
+		"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+	"Accept-Language": "en-US,en;q=0.9",
+	"Cache-Control": "max-age=0",
+	Priority: "u=0, i",
+	"Sec-Ch-Ua": '"Google Chrome";v="149", "Chromium";v="149", ";Not A Brand";v="99"',
+	"Sec-Ch-Ua-Mobile": "?0",
+	"Sec-Ch-Ua-Platform": '"macOS"',
+	"Sec-Fetch-Dest": "document",
+	"Sec-Fetch-Mode": "navigate",
+	"Sec-Fetch-Site": "none",
+	"Sec-Fetch-User": "?1",
+	"Upgrade-Insecure-Requests": "1",
+	"User-Agent":
+		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/149.0.0.0 Safari/537.36",
+};
+
+function canonicalizeHeaderNames(headers: Record<string, string>): Record<string, string> {
+	const canonicalized: Record<string, string> = {};
+
+	for (const key in headers) {
+		const value = headers[key];
+		if (value === undefined) continue;
+
+		// Retain Client Hints (sec-ch-ua*) in their standard lower-case representation
+		if (key.startsWith("sec-ch-ua")) {
+			canonicalized[key] = value;
+			continue;
+		}
+
+		// Retain diagnostics or other HTTP/2 custom lower-case keys
+		if (["dnt", "rtt", "ect"].includes(key)) {
+			canonicalized[key.toUpperCase()] = value;
+			continue;
+		}
+
+		// Retain HTTP/2 specific pseudo headers if any, or general standard casing overrides
+		if (key === "te") {
+			canonicalized.TE = value;
+			continue;
+		}
+
+		// Pascalize words separated by hyphens (e.g. accept-language -> Accept-Language)
+		const pascalized = key
+			.split("-")
+			.map(part => (part[0] ? part[0].toUpperCase() + part.slice(1).toLowerCase() : ""))
+			.join("-");
+
+		canonicalized[pascalized] = value;
+	}
+
+	return canonicalized;
+}
+
+/**
+ * Build a fresh, internally consistent desktop navigation fingerprint for one HTTP request.
+ * By default, this randomizes across valid modern versions of Chrome, Firefox, Edge, and Safari
+ * using real-world traffic data. Set `randomized` to `false` when a fetch must preserve a
+ * stable Mac Chrome identity.
+ */
+export function buildBrowserNavigationHeaders(options?: { randomized?: boolean }): Record<string, string> {
+	const randomized = options?.randomized !== false;
+	if (!randomized) {
+		return { ...CHROME_FALLBACK_HEADERS };
+	}
+
+	try {
+		// Generate realistic, consistent headers with the Bayesian generator
+		const generated = generator.getHeaders();
+		return canonicalizeHeaderNames(generated);
+	} catch {
+		// Gracefully recover to the robust default profile on unexpected generator errors
+		return { ...CHROME_FALLBACK_HEADERS };
+	}
+}

package/src/web/search/providers/browser-page.ts

@@ -0,0 +1,123 @@
+import type { FetchImpl } from "@oh-my-pi/pi-ai";
+import { untilAborted } from "@oh-my-pi/pi-utils";
+import type { Page } from "puppeteer-core";
+import { applyStealthPatches, applyViewport } from "../../../tools/browser/launch";
+import { acquireBrowser, holdBrowser, releaseBrowser } from "../../../tools/browser/registry";
+import { buildBrowserNavigationHeaders } from "./browser-headers";
+import { SEARCH_HARD_TIMEOUT_MS } from "./utils";
+
+/** HTML plus the response status and final URL after redirects or browser navigation. */
+export interface LoadedHtmlPage {
+	html: string;
+	status: number;
+	url: string;
+}
+
+interface BrowserFallbackOptions {
+	homeUrl?: string;
+	ready?: { selector: string; timeoutMs: number };
+	afterNavigation?: (page: Page, signal: AbortSignal) => Promise<void>;
+	shouldFallback: (page: LoadedHtmlPage) => boolean;
+	attempts?: number;
+	retryDelayMs?: number;
+}
+
+/** Controls a browser-profiled fetch and its optional headless-browser fallback. */
+export interface BrowserFetchOptions {
+	fetch?: FetchImpl;
+	signal: AbortSignal;
+	randomizeHeaders?: boolean;
+	referer?: string;
+	init?: Omit<RequestInit, "headers" | "signal">;
+	headers?: Readonly<Record<string, string>>;
+	browser?: BrowserFallbackOptions;
+}
+
+async function fetchHtmlPage(url: string, options: BrowserFetchOptions, fetchImpl: FetchImpl): Promise<LoadedHtmlPage> {
+	const response = await fetchImpl(url, {
+		...options.init,
+		headers: {
+			...buildBrowserNavigationHeaders({ randomized: options.randomizeHeaders }),
+			...(options.referer ? { Referer: options.referer, "Sec-Fetch-Site": "same-origin" } : {}),
+			...options.headers,
+		},
+		signal: options.signal,
+	});
+	return { html: await response.text(), status: response.status, url: response.url || url };
+}
+
+async function browseHtmlPage(
+	url: string,
+	options: BrowserFallbackOptions,
+	signal: AbortSignal,
+): Promise<LoadedHtmlPage> {
+	const { homeUrl, ready } = options;
+	const attempts = Math.max(1, options.attempts ?? 1);
+	const handle = await untilAborted(signal, () =>
+		acquireBrowser(
+			{ kind: "headless", headless: true },
+			{
+				cwd: process.cwd(),
+				signal,
+			},
+		),
+	);
+	if (!("browser" in handle)) {
+		await releaseBrowser(handle, { kill: false });
+		throw new Error("Headless browser acquisition returned a non-Puppeteer browser");
+	}
+
+	holdBrowser(handle);
+	let page: Page | undefined;
+	try {
+		const activePage = await untilAborted(signal, () => handle.browser.newPage());
+		page = activePage;
+		await applyViewport(activePage);
+		await applyStealthPatches(handle.browser, activePage, handle.stealth);
+		if (homeUrl) {
+			await untilAborted(signal, () =>
+				activePage.goto(homeUrl, { waitUntil: "domcontentloaded", timeout: SEARCH_HARD_TIMEOUT_MS }),
+			);
+		}
+		for (let attempt = 0; attempt < attempts; attempt++) {
+			if (attempt > 0 && options.retryDelayMs) await Bun.sleep(options.retryDelayMs);
+
+			const response = await untilAborted(signal, () =>
+				activePage.goto(url, { waitUntil: "domcontentloaded", timeout: SEARCH_HARD_TIMEOUT_MS }),
+			);
+			if (options.afterNavigation) await options.afterNavigation(activePage, signal);
+			if (ready) {
+				await untilAborted(signal, () =>
+					activePage.waitForSelector(ready.selector, { timeout: ready.timeoutMs }).catch(() => null),
+				);
+			}
+			const loaded = {
+				html: await untilAborted(signal, () => activePage.content()),
+				status: response?.status() ?? 200,
+				url: activePage.url(),
+			};
+			if (!options.shouldFallback(loaded) || attempt === attempts - 1) return loaded;
+		}
+		throw new Error("Browser fallback exhausted without a response");
+	} finally {
+		await page?.close().catch(() => undefined);
+		await releaseBrowser(handle, { kill: false });
+	}
+}
+
+/** Fetch with a fresh browser profile, escalating rejected production responses to the stealth browser. */
+export async function browserFetch(url: string, options: BrowserFetchOptions): Promise<LoadedHtmlPage> {
+	const fetchImpl = options.fetch ?? fetch;
+	let page: LoadedHtmlPage;
+	try {
+		page = await fetchHtmlPage(url, options, fetchImpl);
+	} catch (error) {
+		if (options.fetch || !options.browser) throw error;
+		return browseHtmlPage(url, options.browser, options.signal);
+	}
+
+	if (!options.browser || options.fetch) return page;
+	const isSuccessful = page.status >= 200 && page.status < 300;
+	if (isSuccessful && !options.browser.shouldFallback(page)) return page;
+	return browseHtmlPage(url, options.browser, options.signal);
+}

package/src/web/search/providers/duckduckgo.ts

@@ -4,6 +4,7 @@ import { SearchProviderError } from "../../../web/search/types";
 import { clampNumResults } from "../utils";
 import type { SearchParams } from "./base";
 import { SearchProvider } from "./base";
+import { browserFetch } from "./browser-page";
 import { classifyProviderHttpError, withHardTimeout } from "./utils";
 
 /**
@@ -28,15 +29,6 @@ const RECENCY_TO_DDG_DF: Record<NonNullable<SearchParams["recency"]>, string> =
 	year: "y",
 };
 
-/**
- * Browser-like UA so DDG serves the standard results page instead of the
- * mobile-only or noscript variants. DDG returns HTTP 202 plus an anomaly
- * modal when it suspects automation; we surface that as a clear error so
- * the orchestrator can fall through to the next provider with context.
- */
-const BROWSER_USER_AGENT =
-	"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/149.0.0.0 Safari/537.36";
-
 interface ParsedResult {
 	title: string;
 	url: string;
@@ -133,35 +125,22 @@ async function callDuckDuckGoHtml(params: SearchParams): Promise<string> {
 	// Add b: "" parameter as specified in the browser fetch template to match real browser form submission
 	form.set("b", "");
 
-	const response = await (params.fetch ?? fetch)(DUCKDUCKGO_HTML_URL, {
-		method: "POST",
-		body: form.toString(),
-		headers: {
-			Accept:
-				"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
-			"Accept-Language": "en,en-US;q=0.9",
-			"Cache-Control": "max-age=0",
-			"Content-Type": "application/x-www-form-urlencoded",
-			Priority: "u=0, i",
-			"Sec-Ch-Ua": '"Google Chrome";v="149", "Chromium";v="149", "Not)A;Brand";v="24"',
-			"Sec-Ch-Ua-Mobile": "?0",
-			"Sec-Ch-Ua-Platform": '"macOS"',
-			"Sec-Fetch-Dest": "document",
-			"Sec-Fetch-Mode": "navigate",
-			"Sec-Fetch-Site": "same-origin",
-			"Sec-Fetch-User": "?1",
-			"Upgrade-Insecure-Requests": "1",
-			"User-Agent": BROWSER_USER_AGENT,
-			Referer: "https://html.duckduckgo.com/",
-		},
+	const page = await browserFetch(DUCKDUCKGO_HTML_URL, {
+		fetch: params.fetch ?? fetch,
 		signal: withHardTimeout(params.signal),
+		referer: "https://html.duckduckgo.com/",
+		init: {
+			method: "POST",
+			body: form.toString(),
+		},
+		headers: { "Content-Type": "application/x-www-form-urlencoded" },
 	});
 
-	const body = await response.text();
-	if (!response.ok && response.status !== 202) {
-		const classified = classifyProviderHttpError("duckduckgo", response.status, body);
+	const body = page.html;
+	if (page.status < 200 || page.status >= 300) {
+		const classified = classifyProviderHttpError("duckduckgo", page.status, body);
 		if (classified) throw classified;
-		throw new SearchProviderError("duckduckgo", `DuckDuckGo HTML error (${response.status})`, response.status);
+		throw new SearchProviderError("duckduckgo", `DuckDuckGo HTML error (${page.status})`, page.status);
 	}
 
 	if (isAnomalyResponse(body)) {