@oh-my-pi/pi-coding-agent 16.4.2 → 16.4.3

package/src/web/search/providers/ecosia.ts

@@ -0,0 +1,178 @@
+import type { AuthStorage } from "@oh-my-pi/pi-ai";
+import { parseHTML } from "linkedom";
+import type { SearchResponse, SearchSource } from "../../../web/search/types";
+import { SearchProviderError } from "../../../web/search/types";
+import { clampNumResults } from "../utils";
+import type { SearchParams } from "./base";
+import { SearchProvider } from "./base";
+import type { LoadedHtmlPage } from "./browser-page";
+import { browserFetch } from "./browser-page";
+import { classifyProviderHttpError, withHardTimeout } from "./utils";
+
+/**
+ * Ecosia serves a server-rendered Vue/Nuxt results page (no `__NUXT_DATA__`
+ * JSON island — probed 2026-07), so both load paths parse the same markup:
+ * `<article data-test-id="organic-result">` blocks whose title anchor carries
+ * the final target URL directly (no redirect wrapper). The site fronts search
+ * with Cloudflare. Requests start with a browser-profiled fetch and escalate
+ * to the shared stealth browser only when the response is blocked or fails.
+ *
+ * Recency is ignored: Ecosia's web results expose no date filter in the UI
+ * and the legacy Bing-era `freshness` param is a server-side no-op (verified
+ * live), so per the {@link SearchParams.recency} contract the field must not
+ * be approximated.
+ */
+const ECOSIA_HOME_URL = "https://www.ecosia.org/";
+const ECOSIA_SEARCH_URL = "https://www.ecosia.org/search";
+const DEFAULT_NUM_RESULTS = 10;
+const MAX_NUM_RESULTS = 20;
+const RESULT_RENDER_TIMEOUT_MS = 10_000;
+
+interface ParsedResult {
+	title: string;
+	url: string;
+	snippet?: string;
+}
+
+/**
+ * Validate a result anchor href. Organic anchors carry the target URL
+ * directly; anything non-http(s) or pointing back at ecosia.org (internal
+ * navigation such as the images/news verticals) is rejected.
+ */
+function resolveResultUrl(href: string): string | undefined {
+	let url: URL;
+	try {
+		url = new URL(href, ECOSIA_HOME_URL);
+	} catch {
+		return undefined;
+	}
+	if (url.protocol !== "http:" && url.protocol !== "https:") return undefined;
+	if (url.hostname === "ecosia.org" || url.hostname === "www.ecosia.org") return undefined;
+	return url.href;
+}
+
+/**
+ * Pull organic results out of the page in document order. Each result is an
+ * `<article data-test-id="organic-result">` with the title inside
+ * `<h2 data-test-id="result-title">` wrapped by the target link, and the
+ * preview text inside `<p data-test-id="web-result-description">`. The inner
+ * paragraph is preferred over its `result-description` container because the
+ * container also holds screen-reader-only thumbnail captions on video rows.
+ * Ad slots (`data-test-id="ad-google"`) and entity/infobox cards use
+ * different test-ids and never match.
+ */
+function parseHtmlResults(html: string): ParsedResult[] {
+	const { document } = parseHTML(html);
+	const results: ParsedResult[] = [];
+	for (const article of document.querySelectorAll('article[data-test-id="organic-result"]')) {
+		const heading = article.querySelector('[data-test-id="result-title"]');
+		const href = heading?.closest("a")?.getAttribute("href");
+		if (!heading || !href) continue;
+		const url = resolveResultUrl(href);
+		if (!url) continue;
+		const title = (heading.textContent ?? "").replace(/\s+/g, " ").trim();
+		if (!title) continue;
+		const description =
+			article.querySelector('[data-test-id="web-result-description"]') ??
+			article.querySelector('[data-test-id="result-description"]');
+		const snippet = (description?.textContent ?? "").replace(/\s+/g, " ").trim();
+		results.push({ title, url, snippet: snippet || undefined });
+	}
+	return results;
+}
+
+/**
+ * `true` when Ecosia's Cloudflare front answered with the managed challenge
+ * instead of results. The observed page is a 403 titled "Ecosia Firewall"
+ * carrying the `_cf_chl_opt` bootstrap and the challenge-platform loader.
+ */
+function isBlockedPage(page: LoadedHtmlPage): boolean {
+	return (
+		page.status === 403 ||
+		page.status === 429 ||
+		page.html.includes("Ecosia Firewall") ||
+		page.html.includes("_cf_chl_opt") ||
+		page.html.includes("/cdn-cgi/challenge-platform/") ||
+		/confirm you.{0,3}re not a robot/i.test(page.html)
+	);
+}
+
+async function callEcosiaHtml(params: SearchParams): Promise<string> {
+	const signal = withHardTimeout(params.signal);
+	const url = new URL(ECOSIA_SEARCH_URL);
+	url.searchParams.set("q", params.query);
+
+	let page: LoadedHtmlPage;
+	try {
+		page = await browserFetch(url.href, {
+			fetch: params.fetch,
+			signal,
+			referer: ECOSIA_HOME_URL,
+			browser: {
+				homeUrl: ECOSIA_HOME_URL,
+				ready: {
+					selector: 'article[data-test-id="organic-result"]',
+					timeoutMs: RESULT_RENDER_TIMEOUT_MS,
+				},
+				shouldFallback: isBlockedPage,
+			},
+		});
+	} catch (error) {
+		if (error instanceof SearchProviderError || params.signal?.aborted) throw error;
+		if (signal.aborted) {
+			throw new SearchProviderError("ecosia", "Ecosia search timed out.", 504);
+		}
+		const message = error instanceof Error ? error.message : String(error);
+		throw new SearchProviderError("ecosia", `Ecosia search failed: ${message}`, 503);
+	}
+
+	if (isBlockedPage(page)) {
+		throw new SearchProviderError(
+			"ecosia",
+			"Ecosia blocked the request with a Cloudflare bot challenge. Ecosia's firewall throttles automated searches from datacenter/shared-egress IPs; try another web search provider such as DuckDuckGo, Brave, or Tavily.",
+			429,
+		);
+	}
+	if (page.status < 200 || page.status >= 300) {
+		const classified = classifyProviderHttpError("ecosia", page.status, page.html);
+		if (classified) throw classified;
+		throw new SearchProviderError("ecosia", `Ecosia HTML error (${page.status})`, page.status);
+	}
+	return page.html;
+}
+
+/** Execute an Ecosia web search and parse the server-rendered result page. */
+export async function searchEcosia(params: SearchParams): Promise<SearchResponse> {
+	const numResults = clampNumResults(params.numSearchResults ?? params.limit, DEFAULT_NUM_RESULTS, MAX_NUM_RESULTS);
+	const html = await callEcosiaHtml(params);
+	const parsed = parseHtmlResults(html);
+
+	const sources: SearchSource[] = [];
+	const seen = new Set<string>();
+	for (const result of parsed) {
+		if (seen.has(result.url)) continue;
+		seen.add(result.url);
+		sources.push({ title: result.title, url: result.url, snippet: result.snippet });
+		if (sources.length >= numResults) break;
+	}
+
+	return { provider: "ecosia", sources };
+}
+
+/** Search provider for Ecosia (no API key required). */
+export class EcosiaProvider extends SearchProvider {
+	readonly id = "ecosia";
+	readonly label = "Ecosia";
+
+	isAvailable(_authStorage: AuthStorage): boolean {
+		return true;
+	}
+
+	isExplicitlyAvailable(_authStorage: AuthStorage): boolean {
+		return true;
+	}
+
+	search(params: SearchParams): Promise<SearchResponse> {
+		return searchEcosia(params);
+	}
+}

package/src/web/search/providers/google.ts

@@ -0,0 +1,193 @@
+import type { AuthStorage } from "@oh-my-pi/pi-ai";
+import { parseHTML } from "linkedom";
+import type { SearchResponse, SearchSource } from "../../../web/search/types";
+import { SearchProviderError } from "../../../web/search/types";
+import { clampNumResults } from "../utils";
+import type { SearchParams } from "./base";
+import { SearchProvider } from "./base";
+import type { LoadedHtmlPage } from "./browser-page";
+import { browserFetch } from "./browser-page";
+import { withHardTimeout } from "./utils";
+
+const GOOGLE_HOME_URL = "https://www.google.com/";
+const GOOGLE_SEARCH_URL = "https://www.google.com/search";
+const DEFAULT_NUM_RESULTS = 10;
+const MAX_NUM_RESULTS = 20;
+const RESULT_RENDER_TIMEOUT_MS = 10_000;
+
+const RECENCY_TO_GOOGLE_TBS: Record<NonNullable<SearchParams["recency"]>, string> = {
+	day: "qdr:d",
+	week: "qdr:w",
+	month: "qdr:m",
+	year: "qdr:y",
+};
+const GOOGLE_SNIPPET_SELECTORS: readonly string[] = [
+	"[data-sncf='1'] .VwiC3b",
+	".VwiC3b",
+	".IsZvec",
+	".BNeawe.s3v9rd",
+	"[data-sncf='1']",
+];
+
+interface ParsedResult {
+	title: string;
+	url: string;
+	snippet?: string;
+}
+
+function normalizeText(value: string | null | undefined): string {
+	return (value ?? "").replace(/\s+/g, " ").trim();
+}
+
+function unwrapResultUrl(href: string): string | undefined {
+	let url: URL;
+	try {
+		url = new URL(href, GOOGLE_HOME_URL);
+	} catch {
+		return undefined;
+	}
+
+	if ((url.hostname === "google.com" || url.hostname === "www.google.com") && url.pathname === "/url") {
+		const target = url.searchParams.get("q") || url.searchParams.get("url");
+		if (!target) return undefined;
+		try {
+			url = new URL(target);
+		} catch {
+			return undefined;
+		}
+	}
+
+	if (url.protocol !== "http:" && url.protocol !== "https:") return undefined;
+	if (url.hostname === "google.com" || url.hostname === "www.google.com") return undefined;
+	return url.href;
+}
+
+function findSnippet(heading: Element): string | undefined {
+	const container = heading.closest(".tF2Cxc, .MjjYud, .Gx5Zad") ?? heading.parentElement?.parentElement;
+	if (!container) return undefined;
+
+	for (const selector of GOOGLE_SNIPPET_SELECTORS) {
+		const text = normalizeText(container.querySelector(selector)?.textContent).replace(/\s*Read more$/i, "");
+		if (text) return text;
+	}
+	return undefined;
+}
+
+function parseHtmlResults(html: string): ParsedResult[] {
+	const { document } = parseHTML(html);
+	const results: ParsedResult[] = [];
+	for (const heading of document.querySelectorAll("h3")) {
+		const anchor = heading.closest("a");
+		const href = anchor?.getAttribute("href");
+		if (!href) continue;
+		const url = unwrapResultUrl(href);
+		if (!url) continue;
+		const title = normalizeText(heading.textContent);
+		if (!title) continue;
+		results.push({ title, url, snippet: findSnippet(heading) });
+	}
+	return results;
+}
+
+function buildSearchUrl(params: SearchParams, numResults: number): string {
+	const url = new URL(GOOGLE_SEARCH_URL);
+	url.searchParams.set("q", params.query);
+	url.searchParams.set("num", String(numResults));
+	url.searchParams.set("hl", "en");
+	url.searchParams.set("gl", "us");
+	url.searchParams.set("udm", "14");
+	url.searchParams.set("pws", "0");
+	const tbs = params.recency ? RECENCY_TO_GOOGLE_TBS[params.recency] : undefined;
+	if (tbs) url.searchParams.set("tbs", tbs);
+	return url.href;
+}
+
+function blockReason(page: LoadedHtmlPage): "javascript" | "traffic" | undefined {
+	if (page.html.includes("/httpservice/retry/enablejs") && !/<h3\b/i.test(page.html)) return "javascript";
+	if (
+		page.status === 403 ||
+		page.status === 429 ||
+		page.url.includes("/sorry/") ||
+		/unusual traffic|detected unusual traffic|g-recaptcha/i.test(page.html)
+	) {
+		return "traffic";
+	}
+	return undefined;
+}
+
+async function callGoogleHtml(params: SearchParams, numResults: number): Promise<string> {
+	const signal = withHardTimeout(params.signal);
+	const url = buildSearchUrl(params, numResults);
+	let page: LoadedHtmlPage;
+	try {
+		page = await browserFetch(url, {
+			fetch: params.fetch,
+			signal,
+			referer: GOOGLE_HOME_URL,
+			browser: {
+				homeUrl: GOOGLE_HOME_URL,
+				ready: { selector: "a h3", timeoutMs: RESULT_RENDER_TIMEOUT_MS },
+				shouldFallback: candidate => blockReason(candidate) !== undefined,
+			},
+		});
+	} catch (error) {
+		if (error instanceof SearchProviderError || params.signal?.aborted) throw error;
+		if (signal.aborted) {
+			throw new SearchProviderError("google", "Google browser search timed out.", 504);
+		}
+		const message = error instanceof Error ? error.message : String(error);
+		throw new SearchProviderError("google", `Google browser search failed: ${message}`, 503);
+	}
+
+	const blocked = blockReason(page);
+	if (blocked === "traffic") {
+		throw new SearchProviderError(
+			"google",
+			"Google blocked the browser search with an automated-traffic challenge. Try another web search provider or retry later.",
+			429,
+		);
+	}
+	if (page.status < 200 || page.status >= 300) {
+		throw new SearchProviderError("google", `Google HTML error (${page.status})`, page.status);
+	}
+	if (blocked === "javascript") {
+		throw new SearchProviderError(
+			"google",
+			"Google returned its JavaScript challenge instead of rendered search results.",
+			429,
+		);
+	}
+	return page.html;
+}
+
+/** Execute a Google web search with fetch-first loading and a headless-browser fallback. */
+export async function searchGoogle(params: SearchParams): Promise<SearchResponse> {
+	const numResults = clampNumResults(params.numSearchResults ?? params.limit, DEFAULT_NUM_RESULTS, MAX_NUM_RESULTS);
+	const html = await callGoogleHtml(params, numResults);
+	const parsed = parseHtmlResults(html);
+
+	const sources: SearchSource[] = [];
+	const seen = new Set<string>();
+	for (const result of parsed) {
+		if (seen.has(result.url)) continue;
+		seen.add(result.url);
+		sources.push({ title: result.title, url: result.url, snippet: result.snippet });
+		if (sources.length >= numResults) break;
+	}
+
+	return { provider: "google", sources };
+}
+
+/** Fetch-first Google Search provider with a headless-browser fallback; no API key is required. */
+export class GoogleProvider extends SearchProvider {
+	readonly id = "google";
+	readonly label = "Google";
+
+	isAvailable(_authStorage: AuthStorage): boolean {
+		return true;
+	}
+
+	search(params: SearchParams): Promise<SearchResponse> {
+		return searchGoogle(params);
+	}
+}

package/src/web/search/providers/mojeek.ts

@@ -0,0 +1,206 @@
+import type { AuthStorage } from "@oh-my-pi/pi-ai";
+import { untilAborted } from "@oh-my-pi/pi-utils";
+import { parseHTML } from "linkedom";
+import type { Page } from "puppeteer-core";
+import type { SearchResponse, SearchSource } from "../../../web/search/types";
+import { SearchProviderError } from "../../../web/search/types";
+import { clampNumResults } from "../utils";
+import type { SearchParams } from "./base";
+import { SearchProvider } from "./base";
+import type { LoadedHtmlPage } from "./browser-page";
+import { browserFetch } from "./browser-page";
+import { classifyProviderHttpError, withHardTimeout } from "./utils";
+
+const MOJEEK_ORIGIN = "https://www.mojeek.de";
+const MOJEEK_HOME_URL = `${MOJEEK_ORIGIN}/?arc=none&lang=en&lb=en&theme=dark`;
+const MOJEEK_SEARCH_URL = `${MOJEEK_ORIGIN}/search`;
+const DEFAULT_NUM_RESULTS = 10;
+const MAX_NUM_RESULTS = 20;
+/**
+ * ALTCHA can complete quickly, but its verified redirect is occasionally
+ * delayed by queueing on the challenge backend.
+ */
+const CAPTCHA_SOLVE_TIMEOUT_MS = 45_000;
+
+interface ParsedResult {
+	title: string;
+	url: string;
+	snippet?: string;
+}
+
+/**
+ * Validate a result href. Mojeek links results directly to the target site
+ * (no redirect wrapper), so this only filters out non-HTTP schemes and
+ * intra-Mojeek navigation rows (verticals, paging) that share the markup.
+ */
+function normalizeResultUrl(href: string): string | undefined {
+	let url: URL;
+	try {
+		url = new URL(href, MOJEEK_HOME_URL);
+	} catch {
+		return undefined;
+	}
+	if (url.protocol !== "http:" && url.protocol !== "https:") return undefined;
+	if (
+		url.hostname === "mojeek.com" ||
+		url.hostname.endsWith(".mojeek.com") ||
+		url.hostname === "mojeek.co.uk" ||
+		url.hostname.endsWith(".mojeek.co.uk") ||
+		url.hostname === "mojeek.fr" ||
+		url.hostname.endsWith(".mojeek.fr") ||
+		url.hostname === "mojeek.de" ||
+		url.hostname.endsWith(".mojeek.de")
+	) {
+		return undefined;
+	}
+	return url.href;
+}
+
+/**
+ * Pull result blocks out of a Mojeek results page in document order.
+ *
+ * Each organic result renders as `ul.results-standard > li` with the title in
+ * `h2 > a.title` (href is the direct target URL) and the preview text in
+ * `p.s`. Clustered sub-results (`li.clu-result`) share the same shape; rows
+ * without a title anchor (infoboxes, spelling suggestions) are skipped.
+ */
+function parseHtmlResults(html: string): ParsedResult[] {
+	const { document } = parseHTML(html);
+	const results: ParsedResult[] = [];
+	for (const item of document.querySelectorAll("ul.results-standard > li")) {
+		const anchor = item.querySelector("h2 a.title") ?? item.querySelector("a.title");
+		const href = anchor?.getAttribute("href");
+		if (!href) continue;
+		const url = normalizeResultUrl(href);
+		if (!url) continue;
+		const title = (anchor?.textContent ?? "").replace(/\s+/g, " ").trim();
+		if (!title) continue;
+		const snippet = (item.querySelector("p.s")?.textContent ?? "").replace(/\s+/g, " ").trim();
+		results.push({ title, url, snippet: snippet || undefined });
+	}
+	return results;
+}
+
+function buildSearchUrl(params: SearchParams, numResults: number): string {
+	const url = new URL(MOJEEK_SEARCH_URL);
+	url.searchParams.set("q", params.query);
+	url.searchParams.set("t", String(numResults));
+	url.searchParams.set("arc", "none");
+	url.searchParams.set("lang", "en");
+	url.searchParams.set("lb", "en");
+	url.searchParams.set("theme", "dark");
+	// Mojeek's `since` filter accepts the relative tokens day/week/month/year
+	// verbatim — the same vocabulary as `recency` (verified live: each window
+	// returns a near-disjoint, fresher result set). Dates reflect crawl or
+	// last-modification time per Mojeek's operator docs.
+	if (params.recency) url.searchParams.set("since", params.recency);
+	return url.href;
+}
+
+/** Solve Mojeek's ALTCHA interstitial and wait for its verified redirect to populate results. */
+async function solveCaptcha(page: Page, signal: AbortSignal): Promise<void> {
+	if (await untilAborted(signal, () => page.$("ul.results-standard li"))) return;
+
+	const checkbox = await untilAborted(signal, () => page.$("altcha-widget input[type=checkbox]"));
+	if (!checkbox) return;
+
+	const navigation = page
+		.waitForNavigation({ waitUntil: "domcontentloaded", timeout: CAPTCHA_SOLVE_TIMEOUT_MS })
+		.catch(() => null);
+	await untilAborted(signal, () => checkbox.click());
+	await untilAborted(signal, () => navigation);
+	await untilAborted(signal, () =>
+		page.waitForSelector("ul.results-standard li", { timeout: CAPTCHA_SOLVE_TIMEOUT_MS }).catch(() => null),
+	);
+}
+
+function isRobotPage(page: LoadedHtmlPage): boolean {
+	return (
+		(page.html.includes("altcha-widget") ||
+			page.html.includes("captcha-wrap") ||
+			/sending automated queries/i.test(page.html)) &&
+		!page.html.includes("results-standard")
+	);
+}
+
+async function callMojeekHtml(params: SearchParams, numResults: number): Promise<string> {
+	const signal = withHardTimeout(params.signal);
+	const url = buildSearchUrl(params, numResults);
+	let page: LoadedHtmlPage;
+	try {
+		page = await browserFetch(url, {
+			fetch: params.fetch,
+			signal,
+			randomizeHeaders: false,
+			referer: MOJEEK_HOME_URL,
+			browser: {
+				homeUrl: MOJEEK_HOME_URL,
+				afterNavigation: solveCaptcha,
+				shouldFallback: isRobotPage,
+				attempts: 2,
+				retryDelayMs: 1_000,
+			},
+		});
+	} catch (error) {
+		if (error instanceof SearchProviderError || params.signal?.aborted) throw error;
+		if (signal.aborted) {
+			throw new SearchProviderError("mojeek", "Mojeek search timed out.", 504);
+		}
+		const message = error instanceof Error ? error.message : String(error);
+		throw new SearchProviderError("mojeek", `Mojeek search failed: ${message}`, 503);
+	}
+
+	// Robot walls: the ALTCHA proof-of-work captcha page arrives as HTTP 200
+	// (`<title>Captcha</title>`, `altcha-widget`) and the "automated queries"
+	// refusal as HTTP 403. Both bodies are more actionable than their raw
+	// statuses, so check them before the generic status handling.
+	if (isRobotPage(page)) {
+		throw new SearchProviderError(
+			"mojeek",
+			"Mojeek blocked the request with its automated-queries wall. Mojeek rate-limits scripted searches from datacenter/shared-egress IPs; retry later or configure another provider such as Brave, Tavily, Exa, or Kagi.",
+			429,
+		);
+	}
+	if (page.status < 200 || page.status >= 300) {
+		const classified = classifyProviderHttpError("mojeek", page.status, page.html);
+		if (classified) throw classified;
+		throw new SearchProviderError("mojeek", `Mojeek HTML error (${page.status})`, page.status);
+	}
+	return page.html;
+}
+
+/** Execute a Mojeek web search against the standard HTML results page. */
+export async function searchMojeek(params: SearchParams): Promise<SearchResponse> {
+	const numResults = clampNumResults(params.numSearchResults ?? params.limit, DEFAULT_NUM_RESULTS, MAX_NUM_RESULTS);
+	const html = await callMojeekHtml(params, numResults);
+	const parsed = parseHtmlResults(html);
+
+	const sources: SearchSource[] = [];
+	const seen = new Set<string>();
+	for (const result of parsed) {
+		if (seen.has(result.url)) continue;
+		seen.add(result.url);
+		sources.push({ title: result.title, url: result.url, snippet: result.snippet });
+		if (sources.length >= numResults) break;
+	}
+
+	return { provider: "mojeek", sources };
+}
+
+/** Search provider for Mojeek (independent index, no API key required). */
+export class MojeekProvider extends SearchProvider {
+	readonly id = "mojeek";
+	readonly label = "Mojeek";
+
+	isAvailable(_authStorage: AuthStorage): boolean {
+		return true;
+	}
+
+	isExplicitlyAvailable(_authStorage: AuthStorage): boolean {
+		return true;
+	}
+
+	search(params: SearchParams): Promise<SearchResponse> {
+		return searchMojeek(params);
+	}
+}