@bacnh85/pi-web 0.3.0 → 0.4.3

package/extensions/lib/extract.ts

@@ -0,0 +1,185 @@
+// Unified content extraction orchestrator.
+// Modes:
+//   "static"  → JSDOM+Readability (no external API)
+//   "dynamic" → Firecrawl Scrape (JS rendering/structured JSON)
+//   "full"    → Crawl4AI markdown endpoint
+//   "auto"    → static → dynamic → full
+
+import { cwdFromContext, includeProjectEnv } from "./config";
+import { loadFirecrawlConfig, loadCrawl4aiConfig, type FirecrawlConfig, type Crawl4aiConfig } from "./config";
+import { fetchReadableContent } from "./content";
+import { firecrawlRequest, type FirecrawlResult } from "./firecrawl";
+import { fetchCrawl4aiMarkdown } from "./crawl4ai";
+import { formatFirecrawlScrape } from "./format";
+
+export type ExtractMode = "auto" | "static" | "dynamic" | "full";
+export type ExtractAttemptStatus = "success" | "empty" | "error";
+
+export interface ExtractParams {
+	url: string;
+	mode?: ExtractMode;
+	prompt?: string;
+	schema?: unknown;
+	content_chars?: number;
+	timeout_ms?: number;
+	wait_for?: number;
+	mobile?: boolean;
+	signal?: AbortSignal;
+	/** Internal: caller-provided ctx for env lookup. */
+	_ctx?: Record<string, unknown>;
+}
+
+export interface ExtractResult {
+	title: string;
+	markdown: string;
+	backend: string;
+	links?: string[];
+	structured?: unknown;
+}
+
+export interface ExtractAttempt {
+	mode: Exclude<ExtractMode, "auto">;
+	backend: string;
+	status: ExtractAttemptStatus;
+	message: string;
+	contentLength: number;
+}
+
+export interface ExtractDiagnostics {
+	result: ExtractResult;
+	attempts: ExtractAttempt[];
+	selectedMode: Exclude<ExtractMode, "auto">;
+	fallbackUsed: boolean;
+}
+
+const MIN_USEFUL_MARKDOWN_CHARS = 120;
+
+function sanitizeError(error: unknown): string {
+	const raw = error instanceof Error ? error.message : String(error);
+	return raw
+		.replace(/Bearer\s+[A-Za-z0-9._~+/=-]+/gi, "Bearer [REDACTED]")
+		.replace(/api[_-]?key[=:][^\s&]+/gi, "api_key=[REDACTED]")
+		.slice(0, 500);
+}
+
+function isUseful(result: ExtractResult | null, mode: ExtractMode, explicit: boolean): result is ExtractResult {
+	if (!result) return false;
+	const length = result.markdown.trim().length;
+	if (length === 0) return false;
+	return explicit || mode !== "static" || length >= MIN_USEFUL_MARKDOWN_CHARS;
+}
+
+function structuredSection(value: unknown): string {
+	if (value === undefined) return "";
+	return `\n\n## Structured extraction\n\n\`\`\`json\n${JSON.stringify(value, null, 2).slice(0, 10000)}\n\`\`\``;
+}
+
+async function extractStatic(params: ExtractParams): Promise<ExtractResult | null> {
+	const article = await fetchReadableContent(params.url, params.timeout_ms ?? 15000, params.signal);
+	return {
+		title: article.title ?? "",
+		markdown: article.markdown.slice(0, params.content_chars ?? 20000),
+		backend: "static",
+	};
+}
+
+function findStructuredPayload(result: FirecrawlResult): unknown {
+	const data = result.data as Record<string, unknown> | undefined;
+	if (!data) return undefined;
+	return data.json ?? data.extract ?? data.structuredData ?? data.llm_extraction;
+}
+
+async function extractDynamic(params: ExtractParams, ctx?: Record<string, unknown>): Promise<ExtractResult | null> {
+	let fcConfig: FirecrawlConfig;
+	try {
+		fcConfig = loadFirecrawlConfig({}, cwdFromContext(ctx ?? {}), includeProjectEnv(ctx ?? {}));
+	} catch (e) {
+		throw new Error(`Firecrawl configuration unavailable: ${sanitizeError(e)}`);
+	}
+
+	const formats: unknown[] = ["markdown"];
+	if (params.prompt || params.schema) {
+		const jsonEntry: Record<string, unknown> = { type: "json" };
+		if (params.prompt) jsonEntry.prompt = params.prompt;
+		if (params.schema) jsonEntry.schema = params.schema;
+		formats.push(jsonEntry);
+	}
+	const body: Record<string, unknown> = {
+		url: params.url,
+		formats,
+		onlyMainContent: true,
+		...(params.wait_for ? { waitFor: params.wait_for } : {}),
+		...(params.mobile ? { mobile: true } : {}),
+	};
+	const result = await firecrawlRequest(fcConfig, "POST", "/scrape", body, params.signal) as FirecrawlResult;
+	const structured = findStructuredPayload(result);
+	const raw = `${formatFirecrawlScrape(result as Record<string, unknown>, params.content_chars ?? 20000)}${structuredSection(structured)}`;
+	const titleMatch = raw.match(/^# (.+)$/m);
+	return {
+		title: titleMatch ? titleMatch[1] : "",
+		markdown: raw,
+		backend: "dynamic",
+		structured,
+	};
+}
+
+async function extractFull(params: ExtractParams, ctx?: Record<string, unknown>): Promise<ExtractResult | null> {
+	let c4aiConfig: Crawl4aiConfig;
+	try {
+		c4aiConfig = loadCrawl4aiConfig({}, cwdFromContext(ctx ?? {}), includeProjectEnv(ctx ?? {}));
+	} catch (e) {
+		throw new Error(`Crawl4AI configuration unavailable: ${sanitizeError(e)}`);
+	}
+	const result = await fetchCrawl4aiMarkdown(c4aiConfig, params.url, "fit", undefined, params.signal);
+	return {
+		title: "",
+		markdown: result.markdown.slice(0, params.content_chars ?? 20000),
+		backend: "full",
+	};
+}
+
+function modesFor(params: ExtractParams): Array<Exclude<ExtractMode, "auto">> {
+	const mode = params.mode ?? "auto";
+	if (mode !== "auto") return [mode];
+	return ["static", "dynamic", "full"];
+}
+
+async function runExtractor(mode: Exclude<ExtractMode, "auto">, params: ExtractParams, ctx?: Record<string, unknown>): Promise<ExtractResult | null> {
+	if (mode === "static") return extractStatic(params);
+	if (mode === "dynamic") return extractDynamic(params, ctx);
+	return extractFull(params, ctx);
+}
+
+export async function extractWithDiagnostics(params: ExtractParams): Promise<ExtractDiagnostics> {
+	const ctx = params._ctx;
+	const explicit = (params.mode ?? "auto") !== "auto";
+	const modes = modesFor(params);
+	const attempts: ExtractAttempt[] = [];
+
+	for (const mode of modes) {
+		try {
+			const result = await runExtractor(mode, params, ctx);
+			if (isUseful(result, mode, explicit)) {
+				if (!explicit && attempts.length > 0) {
+					result.markdown = `[Extraction fell back to ${mode === "dynamic" ? "Firecrawl Scrape (dynamic mode)" : "Crawl4AI (full browser mode)"}]\n\n${result.markdown}`;
+				}
+				attempts.push({ mode, backend: result.backend, status: "success", message: `Selected ${mode}`, contentLength: result.markdown.length });
+				return { result, attempts, selectedMode: mode, fallbackUsed: attempts.length > 1 };
+			}
+			const emptyResult = result as ExtractResult | null;
+			const length = emptyResult?.markdown.trim().length ?? 0;
+			attempts.push({ mode, backend: emptyResult?.backend ?? mode, status: "empty", message: `${mode} returned insufficient content`, contentLength: length });
+		} catch (e) {
+			attempts.push({ mode, backend: mode, status: "error", message: sanitizeError(e), contentLength: 0 });
+			if (explicit) break;
+		}
+	}
+
+	const diagnostics = attempts.map((a) => `${a.mode}: ${a.status}${a.message ? ` (${a.message})` : ""}`).join("\n");
+	const mode = params.mode ?? "auto";
+	throw new Error(
+		`Extraction failed for ${params.url} with mode="${mode}".\n${diagnostics}\n` +
+		"Try a different mode or use web_screenshot for a visual snapshot.",
+	);
+}
+

package/{lib → extensions/lib}/firecrawl.ts

@@ -1,7 +1,7 @@
 // Firecrawl API client (search, scrape, map, crawl).
 
 import type { FirecrawlConfig } from "./config";
-import { signalWithTimeout, withRetry } from "./retry";
+import { signalWithTimeout } from "./retry";
 
 // ---------------------------------------------------------------------------
 // Types
@@ -53,34 +53,22 @@ async function firecrawlRequestJson(
 }
 
 /**
- * Firecrawl API request with optional v2→v1 fallback on 404.
- * Returns the response data and optionally logs fallback via console.warn.
+ * Firecrawl API request with retry.
  */
 export async function firecrawlRequest(
 	config: FirecrawlConfig,
 	method: string,
 	endpoint: string,
 	body?: unknown,
-	allowFallback = true,
 	signal?: AbortSignal,
 ): Promise<Record<string, unknown>> {
-	return withRetry(async () => {
+	for (let attempt = 0; attempt < 3; attempt++) {
 		try {
 			return await firecrawlRequestJson(config, method, endpoint, body, signal);
 		} catch (e) {
-			if (
-				!allowFallback ||
-				config.isHosted ||
-				!config.baseUrl.endsWith("/v2") ||
-				!(e instanceof FirecrawlHttpError) ||
-				e.status !== 404
-			)
-				throw e;
-			const v1Config = { ...config, baseUrl: config.baseUrl.replace(/\/v2$/, "/v1") };
-			console.warn(
-				`[pi-web] Firecrawl v2 endpoint returned 404, falling back to v1 at ${v1Config.baseUrl}`,
-			);
-			return await firecrawlRequestJson(v1Config, method, endpoint, body, signal);
+			if (attempt === 2) throw e;
+			await new Promise((r) => setTimeout(r, 1000 * Math.pow(2, attempt)));
 		}
-	}, { maxRetries: 2, retryableErrors: ["timeout", "econn", "etimedout", "network", "socket"] });
+	}
+	throw new Error("unreachable");
 }

package/{lib → extensions/lib}/format.ts

@@ -1,6 +1,7 @@
 // Text formatting and result formatting helpers for pi-web.
 
-import { OUTPUT_MAX_BYTES, OUTPUT_MAX_LINES } from "./config";
+const OUTPUT_MAX_BYTES = 50 * 1024;
+const OUTPUT_MAX_LINES = 2_000;
 
 // ---------------------------------------------------------------------------
 // Search result item types
@@ -47,28 +48,6 @@ export function truncateText(text: string): string {
 	return output;
 }
 
-// ---------------------------------------------------------------------------
-// Search result formatting
-// ---------------------------------------------------------------------------
-
-export function formatSearchResults(results: SearchResultItem[]): string {
-	if (!results.length) return "No results found.";
-	return results
-		.map((r, i) =>
-			[
-				`--- Result ${i + 1} ---`,
-				`Title: ${r.title || ""}`,
-				`Link: ${r.url || ""}`,
-				r.age ? `Age: ${r.age}` : "",
-				r.snippet || r.description ? `Snippet: ${r.snippet || r.description}` : "",
-				r.content || r.markdown ? `Content:\n${r.content || r.markdown}` : "",
-			]
-				.filter(Boolean)
-				.join("\n"),
-		)
-		.join("\n\n");
-}
-
 // ---------------------------------------------------------------------------
 // Firecrawl scrape formatting
 // ---------------------------------------------------------------------------
@@ -175,51 +154,3 @@ export function formatCrawl4aiResult(data: Record<string, unknown>, maxChars = 2
 		.join("\n\n---\n\n") || "(No data)";
 }
 
-export function formatCrawl4aiStream(lines: string, _maxChars = 20000): string {
-	const results: string[] = [];
-	for (const line of lines.split("\n")) {
-		const trimmed = line.trim();
-		if (!trimmed) continue;
-		try {
-			const parsed = JSON.parse(trimmed);
-			if (parsed.status === "completed") {
-				results.push("[Stream complete]");
-				break;
-			}
-			results.push(formatCrawl4aiResult(parsed, _maxChars));
-		} catch {
-			results.push(trimmed.slice(0, 1000));
-		}
-	}
-	return results.join("\n\n---\n\n");
-}
-
-export function formatFirecrawlSearch(data: Record<string, unknown>, maxChars = 5000): string {
-	const rootData = data.data as Record<string, unknown> | undefined;
-	const web = (rootData?.web as unknown[]) || [];
-	const news = (rootData?.news as unknown[]) || [];
-	const images = (rootData?.images as unknown[]) || [];
-	const results: SearchResultItem[] = [...web, ...news].map((r: any) => ({
-		title: r.title || r.metadata?.title || "",
-		url: r.url || r.metadata?.sourceURL || "",
-		snippet: r.description || r.snippet || "",
-		markdown: r.markdown ? String(r.markdown).slice(0, maxChars) : "",
-	}));
-	let text = formatSearchResults(results);
-	if (images.length) {
-		text +=
-			"\n\n" +
-			images
-				.map((img: any, i: number) =>
-					[
-						`--- Image ${i + 1} ---`,
-						`Title: ${img.title || ""}`,
-						`Image: ${img.imageUrl || ""}`,
-						`Page: ${img.url || ""}`,
-					].join("\n"),
-				)
-				.join("\n\n");
-	}
-	if (data.warning) text += `\n\nWarning: ${data.warning}`;
-	return text;
-}

package/extensions/lib/retry.ts

@@ -0,0 +1,29 @@
+// Retry utility: signal/timeout helpers.
+
+/**
+ * Combine a timeout signal with an optional external signal.
+ */
+export function signalWithTimeout(timeoutMs: number, signal?: AbortSignal): AbortSignal {
+	const timeoutSignal = AbortSignal.timeout(timeoutMs);
+	if (!signal) return timeoutSignal;
+	return AbortSignal.any([signal, timeoutSignal]);
+}
+
+/**
+ * Sleep that is abortable via an AbortSignal.
+ */
+export function abortableSleep(ms: number, signal?: AbortSignal): Promise<void> {
+	if (signal?.aborted) return Promise.reject(signal.reason ?? new Error("Operation aborted."));
+	return new Promise((resolve, reject) => {
+		const timer = setTimeout(cleanupResolve, ms);
+		function cleanupResolve() {
+			signal?.removeEventListener("abort", cleanupReject);
+			resolve();
+		}
+		function cleanupReject() {
+			clearTimeout(timer);
+			reject(signal?.reason ?? new Error("Operation aborted."));
+		}
+		signal?.addEventListener("abort", cleanupReject, { once: true });
+	});
+}

package/extensions/lib/search.ts

@@ -0,0 +1,240 @@
+// Unified search orchestrator.
+// Probes backend availability and adaptively tries backends:
+//   SearXNG (self-hosted) for broad discovery,
+//   Brave (hosted API) for precision/inline content,
+//   Firecrawl as last-resort search.
+
+import { findEnvValue, cwdFromContext, includeProjectEnv, normalizeSearxngBaseUrl } from "./config";
+import { fetchBraveResults } from "./brave";
+import { fetchSearxngResults } from "./searxng";
+import { firecrawlRequest, type FirecrawlResult } from "./firecrawl";
+import { loadFirecrawlConfig, loadSearxngConfig, type FirecrawlConfig } from "./config";
+import { fetchReadableContent } from "./content";
+import { sanitizeSnippet } from "./format";
+
+export type SearchBackend = "searxng" | "brave" | "firecrawl";
+export type SearchAttemptStatus = "skipped" | "success" | "empty" | "error";
+
+export interface SearchParams {
+	query: string;
+	count?: number;
+	freshness?: string;
+	country?: string;
+	backend?: "auto" | SearchBackend;
+	engines?: string;
+	include_content?: boolean;
+	content_chars?: number;
+	timeout_ms?: number;
+	signal?: AbortSignal;
+	/** Internal: caller-provided ctx for env lookup. */
+	_ctx?: Record<string, unknown>;
+}
+
+export interface SearchResult {
+	title: string;
+	url: string;
+	snippet: string;
+	age: string;
+	content: string;
+	backend: string;
+}
+
+export interface SearchAttempt {
+	backend: SearchBackend;
+	status: SearchAttemptStatus;
+	message: string;
+	resultCount: number;
+}
+
+export interface SearchDiagnostics {
+	results: SearchResult[];
+	attempts: SearchAttempt[];
+	selectedBackend: SearchBackend | "";
+	backendOrder: SearchBackend[];
+}
+
+interface BackendConfig {
+	searxng: { configured: boolean; baseUrl: string };
+	brave: { configured: boolean; apiKey: string };
+	firecrawl: { configured: boolean; config: FirecrawlConfig | null };
+}
+
+function probeBackends(ctx?: Record<string, unknown>): BackendConfig {
+	const cwd = cwdFromContext(ctx ?? {});
+	const trusted = includeProjectEnv(ctx ?? {});
+
+	let searxngBaseUrl: string;
+	try {
+		const searxngConfig = loadSearxngConfig({}, cwd, trusted);
+		searxngBaseUrl = searxngConfig.baseUrl;
+	} catch {
+		searxngBaseUrl = normalizeSearxngBaseUrl();
+	}
+
+	const braveKey = findEnvValue("BRAVE_API_KEY", cwd, trusted);
+
+	let fcConfig: FirecrawlConfig | null = null;
+	try {
+		fcConfig = loadFirecrawlConfig({}, cwd, trusted);
+	} catch {
+		// Not configured or unsafe to use from this context.
+	}
+
+	return {
+		searxng: { configured: true, baseUrl: searxngBaseUrl },
+		brave: { configured: Boolean(braveKey.value), apiKey: braveKey.value ?? "" },
+		firecrawl: { configured: fcConfig !== null, config: fcConfig },
+	};
+}
+
+async function searchSearxng(params: SearchParams): Promise<SearchResult[]> {
+	const cfg = loadSearxngConfig(
+		{ ...(params.engines ? { engines: params.engines } : {}) } as Record<string, unknown>,
+		cwdFromContext(params._ctx ?? {}),
+		includeProjectEnv(params._ctx ?? {}),
+	);
+	const raw = await fetchSearxngResults(
+		{ query: params.query, count: params.count ?? 5, engines: params.engines, timeout_ms: params.timeout_ms } as Record<string, unknown>,
+		cfg.baseUrl,
+		params.signal,
+	);
+	return (raw.results || []).map((r) => ({
+		title: r.title ?? "",
+		url: r.url ?? "",
+		snippet: r.snippet ?? "",
+		age: r.publishedDate ?? "",
+		content: "",
+		backend: "searxng",
+	}));
+}
+
+async function searchBrave(params: SearchParams, backends: BackendConfig): Promise<SearchResult[]> {
+	if (!backends.brave.configured) throw new Error("Brave not configured (no BRAVE_API_KEY)");
+	const raw = await fetchBraveResults(
+		params.query,
+		params.count ?? 5,
+		params.country ?? "US",
+		params.freshness ?? "",
+		backends.brave.apiKey,
+		params.signal,
+	);
+	const results = raw.map((r) => ({
+		title: r.title ?? "",
+		url: r.url ?? "",
+		snippet: r.snippet ?? "",
+		age: r.age ?? "",
+		content: "",
+		backend: "brave",
+	}));
+	if (params.include_content && results.length > 0) {
+		const maxChars = params.content_chars ?? 5000;
+		await Promise.all(
+			results.map(async (r) => {
+				try {
+					const article = await fetchReadableContent(r.url, params.timeout_ms ?? 10000, params.signal);
+					r.content = article.markdown.slice(0, maxChars);
+				} catch (e: any) {
+					r.content = `(Fetch error for ${r.url}: ${sanitizeError(e)})`;
+				}
+			}),
+		);
+	}
+	return results;
+}
+
+async function searchFirecrawl(params: SearchParams, backends: BackendConfig): Promise<SearchResult[]> {
+	if (!backends.firecrawl.configured || !backends.firecrawl.config) {
+		throw new Error("Firecrawl not configured (no FIRECRAWL_API_URL)");
+	}
+	const body: Record<string, unknown> = {
+		query: params.query,
+		limit: Math.min(100, Math.max(1, params.count ?? 5)),
+		sources: [{ type: "web" }],
+		...(params.country ? { country: String(params.country).toUpperCase() } : {}),
+	};
+	const result = await firecrawlRequest(backends.firecrawl.config, "POST", "/search", body, params.signal) as FirecrawlResult;
+	const rootData = result.data as Record<string, unknown> | undefined;
+	const web = (rootData?.web as unknown[]) || [];
+	const news = (rootData?.news as unknown[]) || [];
+	const items = [...web, ...news];
+	return items.map((r: any) => ({
+		title: sanitizeSnippet(r.title ?? r.metadata?.title ?? ""),
+		url: r.url ?? r.metadata?.sourceURL ?? "",
+		snippet: sanitizeSnippet(r.description ?? r.snippet ?? ""),
+		age: "",
+		content: r.markdown ? String(r.markdown).slice(0, params.content_chars ?? 5000) : "",
+		backend: "firecrawl",
+	}));
+}
+
+function isPrecisionQuery(params: SearchParams): boolean {
+	const query = params.query.trim();
+	const lower = query.toLowerCase();
+	if (params.include_content) return true;
+	if (/\bsite:[^\s]+/i.test(query)) return true;
+	if (/"[^"]+"/.test(query)) return true;
+	if (/\b(docs?|documentation|api|github|repo|repository|changelog|release|issue|bug|sdk|package)\b/i.test(query)) return true;
+	const terms = query.split(/\s+/).filter(Boolean);
+	if (terms.length > 0 && terms.length <= 4 && /\b[A-Z][A-Za-z0-9_-]{2,}\b/.test(query)) return true;
+	if (/\b[a-z0-9-]+\.(com|dev|io|org|net|app|ai|tv)\b/i.test(lower)) return true;
+	return false;
+}
+
+export function selectSearchBackendOrder(params: SearchParams): SearchBackend[] {
+	if (params.backend && params.backend !== "auto") return [params.backend];
+	if (params.engines && !params.include_content) return ["searxng", "brave", "firecrawl"];
+	if (isPrecisionQuery(params)) return ["brave", "searxng", "firecrawl"];
+	return ["searxng", "brave", "firecrawl"];
+}
+
+function sanitizeError(error: unknown): string {
+	const raw = error instanceof Error ? error.message : String(error);
+	return raw
+		.replace(/Bearer\s+[A-Za-z0-9._~+/=-]+/gi, "Bearer [REDACTED]")
+		.replace(/X-Subscription-Token\s*[:=]\s*[^\s]+/gi, "X-Subscription-Token: [REDACTED]")
+		.replace(/api[_-]?key[=:][^\s&]+/gi, "api_key=[REDACTED]")
+		.slice(0, 500);
+}
+
+function isConfigured(backend: SearchBackend, backends: BackendConfig): boolean {
+	if (backend === "searxng") return backends.searxng.configured;
+	if (backend === "brave") return backends.brave.configured;
+	return backends.firecrawl.configured;
+}
+
+async function runBackend(backend: SearchBackend, params: SearchParams, backends: BackendConfig): Promise<SearchResult[]> {
+	if (backend === "searxng") return searchSearxng(params);
+	if (backend === "brave") return searchBrave(params, backends);
+	return searchFirecrawl(params, backends);
+}
+
+export async function searchWithDiagnostics(params: SearchParams): Promise<SearchDiagnostics> {
+	const backends = probeBackends(params._ctx);
+	const backendOrder = selectSearchBackendOrder(params);
+	const attempts: SearchAttempt[] = [];
+
+	for (const backend of backendOrder) {
+		if (!isConfigured(backend, backends)) {
+			attempts.push({ backend, status: "skipped", message: `${backend} is not configured`, resultCount: 0 });
+			continue;
+		}
+		try {
+			const results = await runBackend(backend, params, backends);
+			if (results.length > 0) {
+				attempts.push({ backend, status: "success", message: `Selected ${backend}`, resultCount: results.length });
+				return { results, attempts, selectedBackend: backend, backendOrder };
+			}
+			attempts.push({ backend, status: "empty", message: `${backend} returned 0 results`, resultCount: 0 });
+		} catch (e) {
+			attempts.push({ backend, status: "error", message: sanitizeError(e), resultCount: 0 });
+			if (params.backend && params.backend !== "auto") break;
+		}
+	}
+
+	const diagnostics = attempts.map((a) => `${a.backend}: ${a.status}${a.message ? ` (${a.message})` : ""}`).join("\n");
+	throw new Error(
+		`All web search backends failed or returned no results.\n${diagnostics}\n` +
+		"Use backend to force a provider or check configuration with web_status.",
+	);
+}
+

package/extensions/lib/searxng.ts

@@ -0,0 +1,69 @@
+// SearXNG metasearch client.
+
+import { sanitizeSnippet } from "./format";
+import { signalWithTimeout } from "./retry";
+
+export interface SearxngResultItem {
+	title: string;
+	url: string;
+	snippet: string;
+	engine: string;
+	category: string;
+	score?: number;
+	publishedDate: string;
+}
+
+export interface SearxngResponse {
+	results: SearxngResultItem[];
+	[key: string]: unknown;
+}
+
+export async function fetchSearxngResults(
+	params: Record<string, unknown>,
+	baseUrl: string,
+	signal?: AbortSignal,
+): Promise<SearxngResponse> {
+	for (let attempt = 0; attempt < 3; attempt++) {
+		try {
+			const limit = Math.min(50, Math.max(1, (params.count as number) ?? 5));
+			const url = new URL(`${baseUrl}/search`);
+			url.searchParams.set("q", params.query as string);
+			url.searchParams.set("format", "json");
+			url.searchParams.set("pageno", String(Math.max(1, (params.pageno as number) ?? 1)));
+			for (const key of ["categories", "engines", "language", "time_range", "safesearch"] as const) {
+				const val = params[key];
+				if (val !== undefined && val !== "") url.searchParams.set(key, String(val));
+			}
+			const timeoutMs = (params.timeout_ms as number) || 15000;
+			const response = await fetch(url, {
+				headers: { Accept: "application/json", "User-Agent": "pi-web/0.1 SearXNG" },
+				signal: signalWithTimeout(timeoutMs, signal),
+			});
+			const text = await response.text();
+			if (!response.ok) {
+				const hint =
+					response.status === 403
+						? "\nHint: enable JSON output in SearXNG settings.yml with search.formats including json."
+						: "";
+				throw new Error(`HTTP ${response.status}: ${response.statusText}${hint}${text ? `\n${text}` : ""}`);
+			}
+			const data: any = text ? JSON.parse(text) : {};
+			const results = (data.results || [])
+				.slice(0, limit)
+				.map((r: any) => ({
+					title: sanitizeSnippet(r.title || ""),
+					url: r.url || "",
+					snippet: sanitizeSnippet(r.content || r.snippet || ""),
+					engine: r.engine || "",
+					category: r.category || "",
+					score: r.score,
+					publishedDate: r.publishedDate || r.published_date || "",
+				}));
+			return { ...data, results };
+		} catch (e) {
+			if (attempt === 2) throw e;
+			await new Promise((r) => setTimeout(r, 1000 * Math.pow(2, attempt)));
+		}
+	}
+	throw new Error("unreachable");
+}

package/extensions/package.json

@@ -0,0 +1,3 @@
+{
+  "type": "module"
+}