@herbertgao/pi-extensions 2026.9.2 → 2026.9.4

package/node_modules/pi-web-access/crawl4ai.ts

@@ -0,0 +1,204 @@
+import { existsSync, readFileSync } from "node:fs";
+import net from "node:net";
+import { activityMonitor } from "./activity.ts";
+import { redactCredential, resolveCredential } from "./credential-source.ts";
+import type { ExtractedContent, ExtractOptions } from "./extract.ts";
+import { fetchRemoteUrl, loadSsrfConfig, validateRemoteUrl, type Lookup, type SsrfConfig } from "./ssrf-protection.ts";
+import { getWebSearchConfigPath } from "./utils.ts";
+
+const CONFIG_PATH = getWebSearchConfigPath();
+const EXTRACT_TIMEOUT_MS = 60_000;
+const MARKDOWN_FILTER = "fit";
+
+export interface Crawl4aiExtractOptions extends Pick<ExtractOptions, "timeoutMs" | "lookup"> {
+	ssrf?: SsrfConfig;
+}
+
+interface Crawl4aiConfig {
+	crawl4aiBaseUrl?: unknown;
+	crawl4aiApiToken?: unknown;
+}
+
+let cachedConfig: Crawl4aiConfig | null = null;
+
+function loadConfig(): Crawl4aiConfig {
+	if (cachedConfig) return cachedConfig;
+	if (!existsSync(CONFIG_PATH)) {
+		cachedConfig = {};
+		return cachedConfig;
+	}
+	const raw = readFileSync(CONFIG_PATH, "utf8");
+	let parsed: unknown;
+	try {
+		parsed = JSON.parse(raw);
+	} catch (err) {
+		const message = err instanceof Error ? err.message : String(err);
+		throw new Error(`Failed to parse ${CONFIG_PATH}: ${message}`);
+	}
+	if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
+		throw new Error(`Invalid config in ${CONFIG_PATH}: expected a JSON object`);
+	}
+	cachedConfig = parsed as Crawl4aiConfig;
+	return cachedConfig;
+}
+
+export function clearCrawl4aiConfigCache(): void {
+	cachedConfig = null;
+}
+
+function normalizeBaseUrl(value: unknown): string | null {
+	if (typeof value !== "string") return null;
+	const trimmed = value.trim();
+	if (!trimmed) return null;
+	let parsed: URL;
+	try {
+		parsed = new URL(trimmed);
+	} catch {
+		throw new Error(`Invalid Crawl4AI base URL in ${CONFIG_PATH}: expected an HTTP or HTTPS URL`);
+	}
+	if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
+		throw new Error(`Invalid Crawl4AI base URL in ${CONFIG_PATH}: expected an HTTP or HTTPS URL`);
+	}
+	if (parsed.username || parsed.password) {
+		throw new Error(`Invalid Crawl4AI base URL in ${CONFIG_PATH}: URL credentials are not allowed`);
+	}
+	parsed.pathname = parsed.pathname.replace(/\/+$/, "");
+	parsed.search = "";
+	parsed.hash = "";
+	return parsed.toString().replace(/\/+$/, "");
+}
+
+function getBaseUrl(): string | null {
+	return normalizeBaseUrl(process.env.CRAWL4AI_BASE_URL) ?? normalizeBaseUrl(loadConfig().crawl4aiBaseUrl);
+}
+
+function requireBaseUrl(): string {
+	const baseUrl = getBaseUrl();
+	if (!baseUrl) {
+		throw new Error(
+			"Crawl4AI base URL not configured. Either:\n" +
+			`  1. Set crawl4aiBaseUrl in ${CONFIG_PATH}\n` +
+			"  2. Set CRAWL4AI_BASE_URL environment variable",
+		);
+	}
+	return baseUrl;
+}
+
+async function getApiToken(signal?: AbortSignal): Promise<string | null> {
+	return resolveCredential({
+		provider: "Crawl4AI",
+		configuredValue: loadConfig().crawl4aiApiToken,
+		environmentValue: process.env.CRAWL4AI_API_TOKEN,
+		signal,
+	});
+}
+
+function requestSignal(timeoutMs: number, signal?: AbortSignal): AbortSignal {
+	const timeout = AbortSignal.timeout(timeoutMs);
+	return signal ? AbortSignal.any([timeout, signal]) : timeout;
+}
+
+function errorMessage(err: unknown): string {
+	return err instanceof Error ? err.message : String(err);
+}
+
+function isAbortException(err: unknown): boolean {
+	return err instanceof DOMException && (err.name === "AbortError" || err.name === "TimeoutError");
+}
+
+function ssrfOptions(options?: Crawl4aiExtractOptions): SsrfConfig & { lookup?: Lookup } {
+	return {
+		...(options?.ssrf ?? loadSsrfConfig()),
+		...(options?.lookup ? { lookup: options.lookup } : {}),
+	};
+}
+
+function isLoopbackApiUrl(url: URL): boolean {
+	const hostname = url.hostname.toLowerCase().replace(/^\[|\]$/g, "").replace(/\.$/, "");
+	if (hostname === "localhost" || hostname === "::1") return true;
+	if (net.isIP(hostname) !== 4) return false;
+	return hostname.split(".")[0] === "127";
+}
+
+function firstHeadingTitle(markdown: string): string {
+	return /^[ \t]*#[ \t]+(\S.*?)[ \t\r]*$/m.exec(markdown)?.[1] ?? "";
+}
+
+export function isCrawl4aiAvailable(): boolean {
+	return getBaseUrl() !== null;
+}
+
+export async function extractWithCrawl4ai(
+	url: string,
+	signal?: AbortSignal,
+	options?: Crawl4aiExtractOptions,
+): Promise<ExtractedContent | null> {
+	const baseUrl = requireBaseUrl();
+	const ssrf = ssrfOptions(options);
+	await validateRemoteUrl(url, ssrf);
+	const token = await getApiToken(signal);
+	const headers: Record<string, string> = { "Content-Type": "application/json" };
+	if (token) headers.Authorization = `Bearer ${token}`;
+	const requestUrl = new URL(`${baseUrl}/md`);
+	const init = {
+		method: "POST",
+		headers,
+		body: JSON.stringify({ url, f: MARKDOWN_FILTER }),
+		signal: requestSignal(options?.timeoutMs ?? EXTRACT_TIMEOUT_MS, signal),
+	};
+	let seeOther = false;
+	const activityId = activityMonitor.logStart({ type: "fetch", url: requestUrl.toString() });
+	try {
+		const response = await fetchRemoteUrl(requestUrl, init, {
+			...ssrf,
+			allowLoopback: isLoopbackApiUrl(requestUrl),
+			onRedirect: ({ to, init: redirectInit, response }) => {
+				if (to.origin !== requestUrl.origin) {
+					throw new Error(`Crawl4AI refused cross-origin redirect to ${to.origin}`);
+				}
+				// 303 genuinely means "GET the other resource", and it is final: once a hop has turned the
+				// chain into a GET, no later hop may resurrect the extraction body.
+				if (response.status === 303) seeOther = true;
+				// 301/302 are the hops a reverse proxy in front of a self-hosted instance actually emits, and
+				// fetchRemoteUrl turns those into a bodyless GET that /md cannot serve, so replay the POST.
+				// 307/308 already keep the method, so their init needs no help.
+				const replayPost = !seeOther && (response.status === 301 || response.status === 302);
+				return replayPost ? init : redirectInit;
+			},
+		});
+		if (!response.ok) {
+			const text = await response.text().catch(() => "");
+			throw new Error(`Crawl4AI md error ${response.status}: ${redactCredential(text, token).slice(0, 300)}`);
+		}
+		let data: unknown;
+		try {
+			data = await response.json();
+		} catch {
+			// The parser message quotes the offending body, which can echo the configured token, so it never
+			// reaches the thrown error or the activity log.
+			throw new Error("Crawl4AI md returned invalid JSON");
+		}
+		if (!data || typeof data !== "object" || Array.isArray(data)) {
+			throw new Error("Crawl4AI md returned an unexpected response shape");
+		}
+		const envelope = data as Record<string, unknown>;
+		if (envelope.success === false) {
+			const detail = typeof envelope.error === "string" ? envelope.error : typeof envelope.detail === "string" ? envelope.detail : "";
+			throw new Error(`Crawl4AI md unsuccessful: ${redactCredential(detail.trim() || "unknown error", token)}`);
+		}
+		if (envelope.success !== true) {
+			throw new Error("Crawl4AI md returned an unexpected response shape");
+		}
+		if (typeof envelope.markdown !== "string") {
+			throw new Error("Crawl4AI md returned markdown in an unexpected shape");
+		}
+		activityMonitor.logComplete(activityId, response.status);
+		const content = envelope.markdown.trim();
+		if (!content) return null;
+		return { url, title: firstHeadingTitle(content), content, error: null };
+	} catch (err) {
+		if (signal?.aborted || isAbortException(err)) activityMonitor.logComplete(activityId, 0);
+		else activityMonitor.logError(activityId, errorMessage(err));
+		throw err;
+	}
+}

package/node_modules/pi-web-access/credential-source.ts

@@ -10,6 +10,7 @@ const COMMAND_ENVIRONMENT_NAMES = [
 	"HOME",
 	"USER",
 	"LOGNAME",
+	"OP_SERVICE_ACCOUNT_TOKEN",
 	"PATH",
 	"LANG",
 	"LC_ALL",

package/node_modules/pi-web-access/curator-page.ts

@@ -43,6 +43,7 @@ function buildProviderButtons(
 		{ value: "mistral", label: "Mistral", available: available.mistral },
 		{ value: "brightdata", label: "Bright Data", available: available.brightdata },
 		{ value: "serpbase", label: "SerpBase", available: available.serpbase },
+		{ value: "serpapi", label: "SerpApi", available: available.serpapi },
 		{ value: "serper", label: "Serper", available: available.serper },
 		{ value: "valyu", label: "Valyu", available: available.valyu },
 	];
@@ -149,6 +150,7 @@ ${CSS}
 <button class="btn btn-secondary" id="btn-summary-regenerate">Regenerate</button>
 <button class="btn btn-secondary" id="btn-summary-preview" title="Preview rendered summary">Preview</button>
 <button class="btn btn-submit" id="btn-summary-approve">Approve</button>
+<button class="btn btn-submit" id="btn-summary-approve-remaining">Approve + auto-summary remaining searches for this prompt</button>
 </div>
 </section>
 </main>
@@ -1197,6 +1199,7 @@ main {
 }
 .summary-actions {
   display: flex;
+  flex-wrap: wrap;
   align-items: center;
   justify-content: flex-end;
   gap: 8px;
@@ -1469,7 +1472,7 @@ const SCRIPT = `(function() {
   var token = DATA.sessionToken;
   var timeoutSec = DATA.timeout;
   var queries = Array.isArray(DATA.queries) ? DATA.queries : [];
-  var providers = ["all", "openai", "exa", "brave", "parallel", "parallel-mcp", "tinyfish", "search1api", "searchinfinity", "querit", "tavily", "firecrawl", "jina", "serpdive", "kagi", "bocha", "ollama", "searxng", "duckduckgo", "perplexity", "gemini", "kimi", "anysearch", "xcrawl", "xai", "mistral", "brightdata", "serpbase", "serper", "valyu"];
+  var providers = ["all", "openai", "exa", "brave", "parallel", "parallel-mcp", "tinyfish", "search1api", "searchinfinity", "querit", "tavily", "firecrawl", "jina", "serpdive", "kagi", "bocha", "ollama", "searxng", "duckduckgo", "perplexity", "gemini", "kimi", "anysearch", "xcrawl", "xai", "mistral", "brightdata", "serpbase", "serpapi", "serper", "valyu"];
   var availProviders = DATA.availableProviders && typeof DATA.availableProviders === "object" ? DATA.availableProviders : {};
   var workflow = "summary-review";
   var initialDefaultProvider = typeof DATA.defaultProvider === "string" ? DATA.defaultProvider : "exa";
@@ -1534,6 +1537,7 @@ const SCRIPT = `(function() {
   var btnSummaryRegenerate = document.getElementById("btn-summary-regenerate");
   var btnSummaryPreview = document.getElementById("btn-summary-preview");
   var btnSummaryApprove = document.getElementById("btn-summary-approve");
+  var btnSummaryApproveRemaining = document.getElementById("btn-summary-approve-remaining");
   var successOverlay = document.getElementById("success-overlay");
   var successText = document.getElementById("success-text");
   var expiredOverlay = document.getElementById("expired-overlay");
@@ -1699,6 +1703,9 @@ const SCRIPT = `(function() {
     if (provider === "mistral") return "Mistral";
     if (provider === "brightdata") return "Bright Data";
     if (provider === "serpbase") return "SerpBase";
+    if (provider === "serpapi") return "SerpApi";
+    if (provider === "serper") return "Serper";
+    if (provider === "valyu") return "Valyu";
     return "Unknown";
   }
 
@@ -2114,6 +2121,9 @@ const SCRIPT = `(function() {
     if (btnSummaryApprove) {
       btnSummaryApprove.disabled = submitted || timerExpired || submitInFlight || stage === "generating-summary" || isRegenerating || !hasSelection || !hasDraft;
     }
+    if (btnSummaryApproveRemaining) {
+      btnSummaryApproveRemaining.disabled = submitted || timerExpired || submitInFlight || stage === "generating-summary" || isRegenerating || !hasSelection || !hasDraft;
+    }
 
     applyProviderInterlocks();
   }
@@ -3205,7 +3215,7 @@ const SCRIPT = `(function() {
     requestSummary(selected);
   }
 
-  function doApprove() {
+  function doApprove(autoApproveRemainingSearches) {
     if (submitted || timerExpired || submitInFlight || stage !== "summary-review") return;
 
     var selected = getSelectedIndices();
@@ -3217,6 +3227,7 @@ const SCRIPT = `(function() {
 
     var draft = getSummaryDraftText();
     var payload = { selected: selected };
+    if (autoApproveRemainingSearches === true) payload.autoApproveRemainingSearches = true;
     if (draft.length > 0) {
       payload.summary = draft;
       payload.summaryMeta = normalizeSummaryMeta(summaryMeta, summaryMeta && summaryMeta.edited === true);
@@ -3455,6 +3466,13 @@ const SCRIPT = `(function() {
     });
   }
 
+  if (btnSummaryApproveRemaining) {
+    btnSummaryApproveRemaining.addEventListener("click", function() {
+      doApprove(true);
+      resetTimer();
+    });
+  }
+
   if (summaryInput) {
     summaryInput.addEventListener("input", function() {
       if (!summaryMeta || typeof summaryMeta !== "object") {

package/node_modules/pi-web-access/curator-run.ts

@@ -0,0 +1,44 @@
+import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+
+export type WebSearchWorkflow = "none" | "summary-review" | "auto-summary";
+
+export function resolveWebSearchWorkflow(input: unknown, hasUI: boolean): WebSearchWorkflow {
+	const normalized = typeof input === "string" ? input.trim().toLowerCase() : "";
+	if (normalized === "auto-summary") return "auto-summary";
+	if (!hasUI) return "none";
+	if (normalized === "none") return "none";
+	return "summary-review";
+}
+
+export class CuratorRunState {
+	private autoApproveRemaining = false;
+
+	approveRemainingSearches(): void {
+		this.autoApproveRemaining = true;
+	}
+
+	reset(): void {
+		this.autoApproveRemaining = false;
+	}
+
+	resolve(requestedWorkflow: unknown, configuredWorkflow: unknown, hasUI: boolean): WebSearchWorkflow {
+		const inherited = requestedWorkflow === undefined;
+		const workflow = resolveWebSearchWorkflow(
+			inherited ? configuredWorkflow : requestedWorkflow,
+			hasUI,
+		);
+		return inherited && this.autoApproveRemaining && workflow === "summary-review"
+			? "auto-summary"
+			: workflow;
+	}
+}
+
+export function registerCuratorRunLifecycle(pi: Pick<ExtensionAPI, "on">): CuratorRunState {
+	const state = new CuratorRunState();
+	pi.on("before_agent_start", () => state.reset());
+	pi.on("agent_settled", () => state.reset());
+	pi.on("session_start", () => state.reset());
+	pi.on("session_tree", () => state.reset());
+	pi.on("session_shutdown", () => state.reset());
+	return state;
+}

package/node_modules/pi-web-access/curator-server.ts

@@ -41,7 +41,7 @@ export interface IndexedCuratorSearchEntry extends CuratorSearchEntry {
 }
 
 export interface CuratorServerCallbacks {
-	onSubmit: (payload: { selectedQueryIndices: number[]; summary?: string; summaryMeta?: SummaryMeta; rawResults?: boolean }) => void;
+	onSubmit: (payload: { selectedQueryIndices: number[]; summary?: string; summaryMeta?: SummaryMeta; rawResults?: boolean; autoApproveRemainingSearches?: boolean }) => void;
 	onCancel: (reason: "user" | "timeout" | "stale") => void;
 	onProviderChange: (provider: string) => void;
 	onAddSearch: (query: string, provider?: string) => Promise<CuratorSearchEntry[]>;
@@ -642,12 +642,14 @@ export function startCuratorServer(
 					return;
 				}
 				const rawResults = (body as { rawResults?: unknown }).rawResults === true;
+				const autoApproveRemainingSearches = (body as { autoApproveRemainingSearches?: unknown }).autoApproveRemainingSearches === true;
 				sendJson(res, 200, { ok: true });
 				setImmediate(() => callbacks.onSubmit({
 					selectedQueryIndices: parsed.indices,
 					...(summary !== undefined ? { summary } : {}),
 					...(summaryMeta !== undefined ? { summaryMeta } : {}),
 					rawResults,
+					...(autoApproveRemainingSearches ? { autoApproveRemainingSearches: true } : {}),
 				}));
 				return;
 			}

package/node_modules/pi-web-access/duckduckgo.ts

@@ -1,4 +1,3 @@
-import { parseHTML } from "linkedom";
 import { activityMonitor } from "./activity.ts";
 import type { SearchOptions, SearchResult, SearchResponse } from "./perplexity.ts";
 
@@ -88,6 +87,7 @@ export async function searchWithDuckDuckGo(query: string, options: SearchOptions
 			throw new Error(`DuckDuckGo search error ${response.status}: ${body.slice(0, 300)}`);
 		}
 
+		const { parseHTML } = await import("linkedom");
 		const { document } = parseHTML(await response.text());
 		const filters = normalizeDomainFilters(options.domainFilter);
 		const results: SearchResult[] = [];

package/node_modules/pi-web-access/extract.ts

@@ -1,8 +1,5 @@
 import { existsSync, readFileSync } from "node:fs";
-import { Readability } from "@mozilla/readability";
-import { resizeImage } from "@earendil-works/pi-coding-agent";
-import { parseHTML } from "linkedom";
-import TurndownService from "turndown";
+import type TurndownService from "turndown";
 import pLimit from "p-limit";
 import { activityMonitor } from "./activity.ts";
 import { extractRSCContent } from "./rsc-extract.ts";
@@ -20,6 +17,7 @@ import { extractWithQuerit, isQueritAvailable } from "./querit.ts";
 import { extractWithKagi, isKagiExtractAvailable } from "./kagi.ts";
 import { extractWithOllama, isOllamaFetchAvailable } from "./ollama.ts";
 import { extractWithFirecrawl, isFirecrawlAvailable } from "./firecrawl.ts";
+import { extractWithCrawl4ai, isCrawl4aiAvailable } from "./crawl4ai.ts";
 import { extractWithBrightDataUnlocker, isBrightDataUnlockerAvailable } from "./brightdata-unlocker.ts";
 import { isVideoFile, extractVideo, extractVideoFrame, getLocalVideoDuration } from "./video-extract.ts";
 import { appendDeclaredWebLinks, discoverDeclaredWebLinks, type DeclaredWebLink } from "./declared-web-links.ts";
@@ -70,10 +68,10 @@ function loadFetchTimeoutMs(): number {
 const NON_RECOVERABLE_ERRORS = ["Unsupported content type", "Response too large", "PDF extraction is disabled", "Image fetching is disabled"];
 const MIN_USEFUL_CONTENT = 500;
 const SUPPORTED_IMAGE_TYPES = new Set(["image/png", "image/jpeg", "image/webp", "image/gif"]);
-const FETCH_PROVIDERS = ["http", "firecrawl", "jina", "tinyfish", "search1api", "querit", "kagi", "ollama", "parallel", "parallel-mcp", "brightdata", "gemini"] as const;
+const FETCH_PROVIDERS = ["http", "firecrawl", "crawl4ai", "jina", "tinyfish", "search1api", "querit", "kagi", "ollama", "parallel", "parallel-mcp", "brightdata", "gemini"] as const;
 type FetchProvider = typeof FETCH_PROVIDERS[number];
 type FetchRouting = { providers: FetchProvider[]; allowRemoteHostedProviders: boolean };
-const DEFAULT_FETCH_PROVIDER_ORDER: FetchProvider[] = ["http", "firecrawl", "jina", "tinyfish", "search1api", "querit", "kagi", "ollama", "parallel", "brightdata", "gemini"];
+const DEFAULT_FETCH_PROVIDER_ORDER: FetchProvider[] = ["http", "firecrawl", "crawl4ai", "jina", "tinyfish", "search1api", "querit", "kagi", "ollama", "parallel", "brightdata", "gemini"];
 const REMOTE_HOSTED_FETCH_PROVIDERS = new Set<FetchProvider>(["jina", "tinyfish", "search1api", "querit", "kagi", "ollama", "parallel", "parallel-mcp", "brightdata", "gemini"]);
 
 function isDefuddleConsoleError(args: Parameters<typeof console.error>): boolean {
@@ -83,6 +81,7 @@ function isDefuddleConsoleError(args: Parameters<typeof console.error>): boolean
 
 async function extractWithDefuddle(text: string, url: string): Promise<{ title: string; content: string } | null> {
 	const { Defuddle } = await import("defuddle/node");
+	const { parseHTML } = await import("linkedom");
 	const { document } = parseHTML(text);
 	Object.defineProperty(document, "location", {
 		value: new URL(url),
@@ -135,6 +134,10 @@ function isAbortError(err: unknown): boolean {
 	return errorMessage(err).toLowerCase().includes("abort");
 }
 
+function isAbortException(err: unknown): boolean {
+	return err instanceof DOMException && (err.name === "AbortError" || err.name === "TimeoutError");
+}
+
 function isRedirectPolicyError(message: string): boolean {
 	return message.startsWith("Authenticated fetch refused cross-origin redirect") ||
 		message.startsWith("Blocked internal ") ||
@@ -288,10 +291,19 @@ function abortedResult(url: string): ExtractedContent {
 	return { url, title: "", content: "", error: "Aborted" };
 }
 
-const turndown = new TurndownService({
-	headingStyle: "atx",
-	codeBlockStyle: "fenced",
-});
+// Share first-use initialization without loading Turndown at extension startup.
+let turndownInstance: Promise<TurndownService> | undefined;
+async function loadTurndown(): Promise<TurndownService> {
+	const { default: TurndownService } = await import("turndown");
+	return new TurndownService({
+		headingStyle: "atx",
+		codeBlockStyle: "fenced",
+	});
+}
+function getTurndown(): Promise<TurndownService> {
+	turndownInstance ??= loadTurndown();
+	return turndownInstance;
+}
 
 const fetchLimit = pLimit(CONCURRENT_LIMIT);
 
@@ -329,7 +341,7 @@ export interface ExtractOptions {
 	answerModel?: string;
 	authFetchProfile?: AuthFetchProfile;
 	toolNames?: RegisteredToolNames;
-	/** Optional http(s) proxy URL; routed through the curl-backed transport. */
+	/** Optional HTTP(S) or SOCKS proxy URL; routed through the curl-backed transport. */
 	proxy?: string;
 	/** Custom DNS resolver used for SSRF validation. Primarily a test seam. */
 	lookup?: Lookup;
@@ -788,6 +800,7 @@ export async function extractContent(
 	};
 
 	let firecrawlError: string | null = null;
+	let crawl4aiError: string | null = null;
 	let tinyfishError: string | null = null;
 	let search1apiError: string | null = null;
 	let queritError: string | null = null;
@@ -830,6 +843,25 @@ export async function extractContent(
 			continue;
 		}
 
+		if (provider === "crawl4ai") {
+			try {
+				if (isCrawl4aiAvailable()) {
+					const ssrf = loadSsrfConfig();
+					const crawl4aiResult = await extractWithCrawl4ai(url, signal, {
+						timeoutMs: options?.timeoutMs,
+						...(options?.lookup ? { lookup: options.lookup } : {}),
+						ssrf,
+					});
+					if (crawl4aiResult) return withDeclaredLinks(crawl4aiResult);
+				}
+			} catch (err) {
+				if (signal?.aborted || isAbortException(err)) return abortedResult(url);
+				crawl4aiError = errorMessage(err);
+				if (isConfigParseError(err)) return parseErrorResult(crawl4aiError);
+			}
+			continue;
+		}
+
 		if (provider === "jina") {
 			const jinaResult = await extractWithJinaReader(url, fetchTimeoutMs, signal, options?.lookup);
 			if (jinaResult) return withDeclaredLinks(jinaResult);
@@ -991,6 +1023,7 @@ export async function extractContent(
 	const guidance = [
 		finalHttpResult?.error ?? "No fetch_content provider returned content",
 		...(firecrawlError ? [`Firecrawl fallback failed: ${firecrawlError}`] : []),
+		...(crawl4aiError ? [`Crawl4AI fallback failed: ${crawl4aiError}`] : []),
 		...(tinyfishError ? [`TinyFish fallback failed: ${tinyfishError}`] : []),
 		...(search1apiError ? [`Search1API fallback failed: ${search1apiError}`] : []),
 		...(queritError ? [`Querit fallback failed: ${queritError}`] : []),
@@ -1002,6 +1035,7 @@ export async function extractContent(
 		"",
 		"Fallback options:",
 		`  • Set firecrawlBaseUrl in ${WEB_SEARCH_CONFIG_PATH} to a self-hosted Firecrawl instance`,
+		`  • Set crawl4aiBaseUrl in ${WEB_SEARCH_CONFIG_PATH} to a self-hosted Crawl4AI instance`,
 		`  • Set tinyfishApiKey in ${WEB_SEARCH_CONFIG_PATH} or TINYFISH_API_KEY`,
 		`  • Set search1apiApiKey in ${WEB_SEARCH_CONFIG_PATH} or SEARCH1API_KEY`,
 		`  • Set queritApiKey in ${WEB_SEARCH_CONFIG_PATH} or QUERIT_API_KEY`,
@@ -1118,6 +1152,7 @@ async function extractViaHttp(
 	const activityId = activityMonitor.logStart({ type: "fetch", url });
 
 	const controller = new AbortController();
+	const startedAt = Date.now();
 	const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
 
 	const onAbort = () => controller.abort();
@@ -1210,6 +1245,7 @@ async function extractViaHttp(
 			}
 			try {
 				const buffer = await readResponseBufferWithLimit(response, maxResponseSize, () => responseSizeLimitError(maxResponseSize));
+				const { resizeImage } = await import("@earendil-works/pi-coding-agent");
 				const resized = await resizeImage(new Uint8Array(buffer), mimeType, { maxWidth: 2000, maxHeight: 2000 });
 				activityMonitor.logComplete(activityId, response.status);
 				if (!resized) return { url, title: "", content: "", error: `Could not decode image: ${mimeType}`, mimeType, status: response.status };
@@ -1278,6 +1314,7 @@ async function extractViaHttp(
 			return { url, title, content: text, error: null };
 		}
 
+		const { parseHTML } = await import("linkedom");
 		const { document } = parseHTML(text);
 		const documentTitle = document.title?.trim() ?? "";
 		const declaredLinks = discoverDeclaredWebLinks(
@@ -1285,6 +1322,7 @@ async function extractViaHttp(
 			response.headers.get("link"),
 			response.url || url,
 		);
+		const { Readability } = await import("@mozilla/readability");
 		const reader = new Readability(document as unknown as Document);
 		const article = reader.parse();
 
@@ -1332,7 +1370,7 @@ async function extractViaHttp(
 		if (typeof article.content !== "string") {
 			throw new Error("Readability returned invalid article content");
 		}
-		const markdown = turndown.turndown(article.content);
+		const markdown = (await getTurndown()).turndown(article.content);
 		activityMonitor.logComplete(activityId, response.status);
 
 		if (markdown.length < MIN_USEFUL_CONTENT) {
@@ -1387,6 +1425,13 @@ async function extractViaHttp(
 	} finally {
 		clearTimeout(timeoutId);
 		signal?.removeEventListener("abort", onAbort);
+		// Imports and CPU-bound processing need not observe the fetch signal, and
+		// can finish before an expired timer gets a turn. Guard every exit, with
+		// caller cancellation taking precedence over the internal deadline.
+		if (signal?.aborted) return abortedResult(url);
+		if (controller.signal.aborted || Date.now() - startedAt >= timeoutMs) {
+			return { url, title: "", content: "", error: "The operation was aborted." };
+		}
 	}
 }
 

package/node_modules/pi-web-access/fetch-params.ts

@@ -96,7 +96,7 @@ function normalizeAuth(value: unknown): true | string | undefined {
 
 function normalizeProxy(value: unknown): string | undefined {
 	if (value === undefined || value === false) return undefined;
-	if (value === null) throw new Error("proxy must be an http(s) proxy URL string");
+	if (value === null) throw new Error("proxy must be an http(s) or socks proxy URL string");
 	const normalized = normalizeProxyUrl(value, "proxy");
 	return normalized ?? "";
 }

package/node_modules/pi-web-access/gemini-search.ts

@@ -34,13 +34,14 @@ import { isXcrawlAvailable, searchWithXCrawl } from "./xcrawl.ts";
 import { isXaiSearchAvailable, searchWithXai } from "./xai-search.ts";
 import { isBrightDataAvailable, searchWithBrightData } from "./brightdata.ts";
 import { isSerpBaseAvailable, searchWithSerpBase } from "./serpbase.ts";
+import { isSerpApiAvailable, searchWithSerpApi } from "./serpapi.ts";
 import { isSerperAvailable, searchWithSerper } from "./serper.ts";
 import { isValyuAvailable, searchWithValyu } from "./valyu.ts";
 import { isKimiSearchAvailable, searchWithKimi } from "./kimi-search.ts";
 import { isMistralAvailable, searchWithMistral } from "./mistral-search.ts";
 import { getWebSearchConfigPath } from "./utils.ts";
 
-export const RESOLVED_SEARCH_PROVIDERS = ["openai", "brave", "parallel", "parallel-mcp", "tinyfish", "search1api", "searchinfinity", "querit", "tavily", "firecrawl", "jina", "searxng", "duckduckgo", "perplexity", "gemini", "kimi", "exa", "serpdive", "kagi", "ollama", "anysearch", "xai", "mistral", "brightdata", "serpbase", "serper", "valyu", "bocha", "xcrawl"] as const;
+export const RESOLVED_SEARCH_PROVIDERS = ["openai", "brave", "parallel", "parallel-mcp", "tinyfish", "search1api", "searchinfinity", "querit", "tavily", "firecrawl", "jina", "searxng", "duckduckgo", "perplexity", "gemini", "kimi", "exa", "serpdive", "kagi", "ollama", "anysearch", "xai", "mistral", "brightdata", "serpbase", "serpapi", "serper", "valyu", "bocha", "xcrawl"] as const;
 export const SEARCH_PROVIDERS = ["auto", "all", ...RESOLVED_SEARCH_PROVIDERS] as const;
 
 export type ResolvedSearchProvider = typeof RESOLVED_SEARCH_PROVIDERS[number];
@@ -105,7 +106,7 @@ export interface AttributedSearchResponse extends SearchResponse {
 
 const CONFIG_PATH = getWebSearchConfigPath();
 const DEFAULT_SEARCH_MODEL = "gemini-3.6-flash";
-// Explicit-only providers (Parallel MCP, DuckDuckGo, Kimi, AnySearch, XCrawl, xAI, Mistral, Bright Data, SerpBase, Serper, Valyu) are deliberately absent:
+// Explicit-only providers (Parallel MCP, DuckDuckGo, Kimi, AnySearch, XCrawl, xAI, Mistral, Bright Data, SerpBase, SerpApi, Serper, Valyu) are deliberately absent:
 // `all` must never fan out to an opt-in or paid provider without the user asking for it.
 export const ALL_SEARCH_PROVIDERS: ResolvedSearchProvider[] = ["searxng", "openai", "exa", "brave", "parallel", "tinyfish", "search1api", "searchinfinity", "querit", "tavily", "firecrawl", "jina", "serpdive", "kagi", "ollama", "perplexity", "gemini", "bocha"];
 const VALID_ROUTING_KINDS = ["transient", "quota", "network", "invalid-response", "unsupported"] as const;
@@ -311,7 +312,7 @@ function classifyProviderError(provider: ResolvedSearchProvider, err: unknown):
 		kind = "unsupported";
 	} else if (status === 400 || status === 422) {
 		kind = "invalid-request";
-	} else if (status === 402 || status === 429) {
+	} else if (status === 402 || status === 429 || (provider === "tavily" && status === 432)) {
 		kind = "quota";
 	} else if (status !== undefined && (status === 408 || status === 425 || status >= 500)) {
 		kind = "transient";
@@ -364,6 +365,7 @@ async function searchWithResolvedProvider(
 	if (provider === "mistral") return { ...(await searchWithMistral(query, options)), provider };
 	if (provider === "brightdata") return { ...(await searchWithBrightData(query, options)), provider };
 	if (provider === "serpbase") return { ...(await searchWithSerpBase(query, options)), provider };
+	if (provider === "serpapi") return { ...(await searchWithSerpApi(query, options)), provider };
 	if (provider === "serper") return { ...(await searchWithSerper(query, options)), provider };
 	if (provider === "valyu") return { ...(await searchWithValyu(query, options)), provider };
 	if (provider === "xcrawl") return { ...(await searchWithXCrawl(query, options)), provider };
@@ -412,6 +414,7 @@ async function isResolvedProviderAvailable(provider: ResolvedSearchProvider, opt
 	if (provider === "mistral") return isMistralAvailable();
 	if (provider === "brightdata") return isBrightDataAvailable();
 	if (provider === "serpbase") return isSerpBaseAvailable();
+	if (provider === "serpapi") return isSerpApiAvailable();
 	if (provider === "serper") return isSerperAvailable();
 	if (provider === "valyu") return isValyuAvailable();
 	if (provider === "xcrawl") return isXcrawlAvailable();
@@ -451,6 +454,7 @@ function providerLabel(provider: ResolvedSearchProvider): string {
 	if (provider === "mistral") return "Mistral";
 	if (provider === "brightdata") return "Bright Data";
 	if (provider === "serpbase") return "SerpBase";
+	if (provider === "serpapi") return "SerpApi";
 	if (provider === "serper") return "Serper";
 	if (provider === "valyu") return "Valyu";
 	return provider.charAt(0).toUpperCase() + provider.slice(1);
@@ -479,7 +483,7 @@ async function searchWithProviders(
 			: await isResolvedProviderAvailable(provider, options),
 	})))).filter((entry) => entry.available).map((entry) => entry.provider);
 	if (providers.length === 0) {
-		throw new Error("No configured search provider available for provider \"all\". Parallel MCP, DuckDuckGo, Kimi, AnySearch, xAI, Mistral, Bright Data, SerpBase, Serper, Valyu, and XCrawl are excluded.");
+		throw new Error("No configured search provider available for provider \"all\". Parallel MCP, DuckDuckGo, Kimi, AnySearch, xAI, Mistral, Bright Data, SerpBase, SerpApi, Serper, Valyu, and XCrawl are excluded.");
 	}
 
 	const settled = await Promise.allSettled(
@@ -772,7 +776,7 @@ export async function search(query: string, options: FullSearchOptions = {}): Pr
 		"  3. Set OPENAI_API_KEY, BRAVE_API_KEY, PARALLEL_API_KEY, TINYFISH_API_KEY, SEARCH1API_KEY, SEARCHINFINITY_API_KEY, QUERIT_API_KEY, TAVILY_API_KEY, FIRECRAWL_BASE_URL, JINA_API_KEY, SERPDIVE_API_KEY, KAGI_API_KEY, BOCHA_API_KEY, OLLAMA_API_KEY, SEARXNG_BASE_URL, EXA_API_KEY, PERPLEXITY_API_KEY, GEMINI_API_KEY, or CLOUDFLARE_API_KEY env vars\n" +
 		"  4. Set GOOGLE_GEMINI_BASE_URL with CLOUDFLARE_API_KEY for Cloudflare AI Gateway routing\n" +
 		"  5. Sign into gemini.google.com in a supported Chromium-based browser\n" +
-		"  6. Explicitly select provider: \"anysearch\" for anonymous AnySearch, \"xcrawl\" for XCrawl, \"xai\" for Grok, \"mistral\" for Mistral Conversations web search, \"brightdata\" with brightdataSerpZone for paid Bright Data SERP, \"serpbase\" or \"serper\" for Google SERP, or \"valyu\" for research search"
+		"  6. Explicitly select provider: \"anysearch\" for anonymous AnySearch, \"xcrawl\" for XCrawl, \"xai\" for Grok, \"mistral\" for Mistral Conversations web search, \"brightdata\" with brightdataSerpZone for paid Bright Data SERP, \"serpbase\", \"serpapi\", or \"serper\" for Google SERP, or \"valyu\" for research search"
 	);
 }