@bacnh85/pi-web 0.5.2 → 0.5.4

package/extensions/lib/search.ts

@@ -16,212 +16,218 @@ export type SearchBackend = "searxng" | "brave" | "firecrawl";
 export type SearchAttemptStatus = "skipped" | "success" | "empty" | "error";
 
 export interface SearchParams {
-	query: string;
-	count?: number;
-	freshness?: string;
-	country?: string;
-	backend?: "auto" | SearchBackend;
-	engines?: string;
-	include_content?: boolean;
-	content_chars?: number;
-	timeout_ms?: number;
-	signal?: AbortSignal;
-	/** Internal: caller-provided ctx for env lookup. */
-	_ctx?: Record<string, unknown>;
+  query: string;
+  count?: number;
+  freshness?: string;
+  country?: string;
+  backend?: "auto" | SearchBackend;
+  engines?: string;
+  include_content?: boolean;
+  content_chars?: number;
+  timeout_ms?: number;
+  signal?: AbortSignal;
+  /** Internal: caller-provided ctx for env lookup. */
+  _ctx?: Record<string, unknown>;
 }
 
 export interface SearchResult {
-	title: string;
-	url: string;
-	snippet: string;
-	age: string;
-	content: string;
-	backend: string;
+  title: string;
+  url: string;
+  snippet: string;
+  age: string;
+  content: string;
+  backend: string;
 }
 
 export interface SearchAttempt {
-	backend: SearchBackend;
-	status: SearchAttemptStatus;
-	message: string;
-	resultCount: number;
+  backend: SearchBackend;
+  status: SearchAttemptStatus;
+  message: string;
+  resultCount: number;
 }
 
 export interface SearchDiagnostics {
-	results: SearchResult[];
-	attempts: SearchAttempt[];
-	selectedBackend: SearchBackend | "";
-	backendOrder: SearchBackend[];
+  results: SearchResult[];
+  attempts: SearchAttempt[];
+  selectedBackend: SearchBackend | "";
+  backendOrder: SearchBackend[];
 }
 
 interface BackendConfig {
-	searxng: { configured: boolean; baseUrl: string };
-	brave: { configured: boolean; apiKey: string };
-	firecrawl: { configured: boolean; config: FirecrawlConfig | null };
+  searxng: { configured: boolean; baseUrl: string };
+  brave: { configured: boolean; apiKey: string };
+  firecrawl: { configured: boolean; config: FirecrawlConfig | null };
 }
 
 function probeBackends(ctx?: Record<string, unknown>): BackendConfig {
-	const cwd = cwdFromContext(ctx ?? {});
-	const trusted = includeProjectEnv(ctx ?? {});
-
-	let searxngBaseUrl: string;
-	try {
-		const searxngConfig = loadSearxngConfig({}, cwd, trusted);
-		searxngBaseUrl = searxngConfig.baseUrl;
-	} catch {
-		searxngBaseUrl = normalizeSearxngBaseUrl();
-	}
-
-	const braveKey = findEnvValue("BRAVE_API_KEY", cwd, trusted);
-
-	let fcConfig: FirecrawlConfig | null = null;
-	try {
-		fcConfig = loadFirecrawlConfig({}, cwd, trusted);
-	} catch {
-		// Not configured or unsafe to use from this context.
-	}
-
-	return {
-		searxng: { configured: true, baseUrl: searxngBaseUrl },
-		brave: { configured: Boolean(braveKey.value), apiKey: braveKey.value ?? "" },
-		firecrawl: { configured: fcConfig !== null, config: fcConfig },
-	};
+  const cwd = cwdFromContext(ctx ?? {});
+  const trusted = includeProjectEnv(ctx ?? {});
+
+  let searxngBaseUrl: string;
+  try {
+    const searxngConfig = loadSearxngConfig({}, cwd, trusted);
+    searxngBaseUrl = searxngConfig.baseUrl;
+  } catch {
+    searxngBaseUrl = normalizeSearxngBaseUrl();
+  }
+
+  const braveKey = findEnvValue("BRAVE_API_KEY", cwd, trusted);
+
+  let fcConfig: FirecrawlConfig | null = null;
+  try {
+    fcConfig = loadFirecrawlConfig({}, cwd, trusted);
+  } catch {
+    // Not configured or unsafe to use from this context.
+  }
+
+  return {
+    searxng: { configured: true, baseUrl: searxngBaseUrl },
+    brave: { configured: Boolean(braveKey.value), apiKey: braveKey.value ?? "" },
+    firecrawl: { configured: fcConfig !== null, config: fcConfig },
+  };
 }
 
 async function searchSearxng(params: SearchParams): Promise<SearchResult[]> {
-	const cfg = loadSearxngConfig(
-		{ ...(params.engines ? { engines: params.engines } : {}) } as Record<string, unknown>,
-		cwdFromContext(params._ctx ?? {}),
-		includeProjectEnv(params._ctx ?? {}),
-	);
-	const raw = await fetchSearxngResults(
-		{ query: params.query, count: params.count ?? 5, engines: params.engines, timeout_ms: params.timeout_ms } as Record<string, unknown>,
-		cfg.baseUrl,
-		params.signal,
-	);
-	return (raw.results || []).map((r) => ({
-		title: r.title ?? "",
-		url: r.url ?? "",
-		snippet: r.snippet ?? "",
-		age: r.publishedDate ?? "",
-		content: "",
-		backend: "searxng",
-	}));
+  const cfg = loadSearxngConfig(
+    { ...(params.engines ? { engines: params.engines } : {}) } as Record<string, unknown>,
+    cwdFromContext(params._ctx ?? {}),
+    includeProjectEnv(params._ctx ?? {}),
+  );
+  const raw = await fetchSearxngResults(
+    { query: params.query, count: params.count ?? 5, engines: params.engines, timeout_ms: params.timeout_ms } as Record<string, unknown>,
+    cfg.baseUrl,
+    params.signal,
+  );
+  return (raw.results || []).map((r) => ({
+    title: r.title ?? "",
+    url: r.url ?? "",
+    snippet: r.snippet ?? "",
+    age: r.publishedDate ?? "",
+    content: "",
+    backend: "searxng",
+  }));
 }
 
 async function searchBrave(params: SearchParams, backends: BackendConfig): Promise<SearchResult[]> {
-	if (!backends.brave.configured) throw new Error("Brave not configured (no BRAVE_API_KEY)");
-	const raw = await fetchBraveResults(
-		params.query,
-		params.count ?? 5,
-		params.country ?? "US",
-		params.freshness ?? "",
-		backends.brave.apiKey,
-		params.signal,
-	);
-	const results = raw.map((r) => ({
-		title: r.title ?? "",
-		url: r.url ?? "",
-		snippet: r.snippet ?? "",
-		age: r.age ?? "",
-		content: "",
-		backend: "brave",
-	}));
-	if (params.include_content && results.length > 0) {
-		const maxChars = params.content_chars ?? 5000;
-		await Promise.all(
-			results.map(async (r) => {
-				try {
-					const article = await fetchReadableContent(r.url, params.timeout_ms ?? 10000, params.signal);
-					r.content = article.markdown.slice(0, maxChars);
-				} catch (e: any) {
-					r.content = `(Fetch error for ${r.url}: ${sanitizeError(e)})`;
-				}
-			}),
-		);
-	}
-	return results;
+  if (!backends.brave.configured) throw new Error("Brave not configured (no BRAVE_API_KEY)");
+  const raw = await fetchBraveResults(
+    params.query,
+    params.count ?? 5,
+    params.country ?? "US",
+    params.freshness ?? "",
+    backends.brave.apiKey,
+    params.signal,
+  );
+  const results = raw.map((r) => ({
+    title: r.title ?? "",
+    url: r.url ?? "",
+    snippet: r.snippet ?? "",
+    age: r.age ?? "",
+    content: "",
+    backend: "brave",
+  }));
+  if (params.include_content && results.length > 0) {
+    const maxChars = params.content_chars ?? 5000;
+    // Cap concurrent content fetches to avoid OOM from parallel JSDOM parsing
+    const MAX_CONCURRENT_FETCHES = 5;
+    const batchSize = Math.min(results.length, MAX_CONCURRENT_FETCHES);
+    for (let i = 0; i < results.length; i += batchSize) {
+      const batch = results.slice(i, i + batchSize);
+      await Promise.all(
+        batch.map(async (r) => {
+          try {
+            const article = await fetchReadableContent(r.url, params.timeout_ms ?? 10000, params.signal);
+            r.content = article.markdown.slice(0, maxChars);
+          } catch (e: any) {
+            r.content = `(Fetch error for ${r.url}: ${sanitizeError(e)})`;
+          }
+        }),
+      );
+    }
+  }
+  return results;
 }
 
 async function searchFirecrawl(params: SearchParams, backends: BackendConfig): Promise<SearchResult[]> {
-	if (!backends.firecrawl.configured || !backends.firecrawl.config) {
-		throw new Error("Firecrawl not configured (no FIRECRAWL_API_URL)");
-	}
-	const body: Record<string, unknown> = {
-		query: params.query,
-		limit: Math.min(100, Math.max(1, params.count ?? 5)),
-		sources: [{ type: "web" }],
-		...(params.country ? { country: String(params.country).toUpperCase() } : {}),
-	};
-	const result = await firecrawlRequest(backends.firecrawl.config, "POST", "/search", body, params.signal) as FirecrawlResult;
-	const rootData = result.data as Record<string, unknown> | undefined;
-	const web = (rootData?.web as unknown[]) || [];
-	const news = (rootData?.news as unknown[]) || [];
-	const items = [...web, ...news];
-	return items.map((r: any) => ({
-		title: sanitizeSnippet(r.title ?? r.metadata?.title ?? ""),
-		url: r.url ?? r.metadata?.sourceURL ?? "",
-		snippet: sanitizeSnippet(r.description ?? r.snippet ?? ""),
-		age: "",
-		content: r.markdown ? String(r.markdown).slice(0, params.content_chars ?? 5000) : "",
-		backend: "firecrawl",
-	}));
+  if (!backends.firecrawl.configured || !backends.firecrawl.config) {
+    throw new Error("Firecrawl not configured (no FIRECRAWL_API_URL)");
+  }
+  const body: Record<string, unknown> = {
+    query: params.query,
+    limit: Math.min(100, Math.max(1, params.count ?? 5)),
+    sources: [{ type: "web" }],
+    ...(params.country ? { country: String(params.country).toUpperCase() } : {}),
+  };
+  const result = await firecrawlRequest(backends.firecrawl.config, "POST", "/search", body, params.signal) as FirecrawlResult;
+  const rootData = result.data as Record<string, unknown> | undefined;
+  const web = (rootData?.web as unknown[]) || [];
+  const news = (rootData?.news as unknown[]) || [];
+  const items = [...web, ...news];
+  return items.map((r: any) => ({
+    title: sanitizeSnippet(r.title ?? r.metadata?.title ?? ""),
+    url: r.url ?? r.metadata?.sourceURL ?? "",
+    snippet: sanitizeSnippet(r.description ?? r.snippet ?? ""),
+    age: "",
+    content: r.markdown ? String(r.markdown).slice(0, params.content_chars ?? 5000) : "",
+    backend: "firecrawl",
+  }));
 }
 
 function isPrecisionQuery(params: SearchParams): boolean {
-	const query = params.query.trim();
-	const lower = query.toLowerCase();
-	if (params.include_content) return true;
-	if (/\bsite:[^\s]+/i.test(query)) return true;
-	if (/"[^"]+"/.test(query)) return true;
-	if (/\b(docs?|documentation|api|github|repo|repository|changelog|release|issue|bug|sdk|package)\b/i.test(query)) return true;
-	const terms = query.split(/\s+/).filter(Boolean);
-	if (terms.length > 0 && terms.length <= 4 && /\b[A-Z][A-Za-z0-9_-]{2,}\b/.test(query)) return true;
-	if (/\b[a-z0-9-]+\.(com|dev|io|org|net|app|ai|tv)\b/i.test(lower)) return true;
-	return false;
+  const query = params.query.trim();
+  const lower = query.toLowerCase();
+  if (params.include_content) return true;
+  if (/\bsite:[^\s]+/i.test(query)) return true;
+  if (/"[^"]+"/.test(query)) return true;
+  if (/\b(docs?|documentation|api|github|repo|repository|changelog|release|issue|bug|sdk|package)\b/i.test(query)) return true;
+  const terms = query.split(/\s+/).filter(Boolean);
+  if (terms.length > 0 && terms.length <= 4 && /\b[A-Z][A-Za-z0-9_-]{2,}\b/.test(query)) return true;
+  if (/\b[a-z0-9-]+\.(com|dev|io|org|net|app|ai|tv)\b/i.test(lower)) return true;
+  return false;
 }
 
 export function selectSearchBackendOrder(params: SearchParams): SearchBackend[] {
-	if (params.backend && params.backend !== "auto") return [params.backend];
-	if (params.engines && !params.include_content) return ["searxng", "brave", "firecrawl"];
-	if (isPrecisionQuery(params)) return ["brave", "searxng", "firecrawl"];
-	return ["searxng", "brave", "firecrawl"];
+  if (params.backend && params.backend !== "auto") return [params.backend];
+  if (params.engines && !params.include_content) return ["searxng", "brave", "firecrawl"];
+  if (isPrecisionQuery(params)) return ["brave", "searxng", "firecrawl"];
+  return ["searxng", "brave", "firecrawl"];
 }
 
 function isConfigured(backend: SearchBackend, backends: BackendConfig): boolean {
-	if (backend === "searxng") return backends.searxng.configured;
-	if (backend === "brave") return backends.brave.configured;
-	return backends.firecrawl.configured;
+  if (backend === "searxng") return backends.searxng.configured;
+  if (backend === "brave") return backends.brave.configured;
+  return backends.firecrawl.configured;
 }
 
 async function runBackend(backend: SearchBackend, params: SearchParams, backends: BackendConfig): Promise<SearchResult[]> {
-	if (backend === "searxng") return searchSearxng(params);
-	if (backend === "brave") return searchBrave(params, backends);
-	return searchFirecrawl(params, backends);
+  if (backend === "searxng") return searchSearxng(params);
+  if (backend === "brave") return searchBrave(params, backends);
+  return searchFirecrawl(params, backends);
 }
 
 export async function searchWithDiagnostics(params: SearchParams): Promise<SearchDiagnostics> {
-	const backends = probeBackends(params._ctx);
-	const backendOrder = selectSearchBackendOrder(params);
-	const attempts: SearchAttempt[] = [];
-
-	for (const backend of backendOrder) {
-		if (!isConfigured(backend, backends)) {
-			attempts.push({ backend, status: "skipped", message: `${backend} is not configured`, resultCount: 0 });
-			continue;
-		}
-		try {
-			const results = await runBackend(backend, params, backends);
-			if (results.length > 0) {
-				attempts.push({ backend, status: "success", message: `Selected ${backend}`, resultCount: results.length });
-				return { results, attempts, selectedBackend: backend, backendOrder };
-			}
-			attempts.push({ backend, status: "empty", message: `${backend} returned 0 results`, resultCount: 0 });
-		} catch (e) {
-			attempts.push({ backend, status: "error", message: sanitizeError(e), resultCount: 0 });
-			if (params.backend && params.backend !== "auto") break;
-		}
-	}
-
-	return { results: [], attempts, selectedBackend: "" as const, backendOrder };
+  const backends = probeBackends(params._ctx);
+  const backendOrder = selectSearchBackendOrder(params);
+  const attempts: SearchAttempt[] = [];
+
+  for (const backend of backendOrder) {
+    if (!isConfigured(backend, backends)) {
+      attempts.push({ backend, status: "skipped", message: `${backend} is not configured`, resultCount: 0 });
+      continue;
+    }
+    try {
+      const results = await runBackend(backend, params, backends);
+      if (results.length > 0) {
+        attempts.push({ backend, status: "success", message: `Selected ${backend}`, resultCount: results.length });
+        return { results, attempts, selectedBackend: backend, backendOrder };
+      }
+      attempts.push({ backend, status: "empty", message: `${backend} returned 0 results`, resultCount: 0 });
+    } catch (e) {
+      attempts.push({ backend, status: "error", message: sanitizeError(e), resultCount: 0 });
+      if (params.backend && params.backend !== "auto") break;
+    }
+  }
+
+  return { results: [], attempts, selectedBackend: "" as const, backendOrder };
 }
 

package/extensions/lib/searxng.ts

@@ -4,60 +4,60 @@ import { sanitizeSnippet } from "./format";
 import { signalWithTimeout, withRetry } from "./retry";
 
 export interface SearxngResultItem {
-	title: string;
-	url: string;
-	snippet: string;
-	engine: string;
-	category: string;
-	score?: number;
-	publishedDate: string;
+  title: string;
+  url: string;
+  snippet: string;
+  engine: string;
+  category: string;
+  score?: number;
+  publishedDate: string;
 }
 
 export interface SearxngResponse {
-	results: SearxngResultItem[];
-	[key: string]: unknown;
+  results: SearxngResultItem[];
+  [key: string]: unknown;
 }
 
 export async function fetchSearxngResults(
-	params: Record<string, unknown>,
-	baseUrl: string,
-	signal?: AbortSignal,
+  params: Record<string, unknown>,
+  baseUrl: string,
+  signal?: AbortSignal,
 ): Promise<SearxngResponse> {
-	return withRetry(async () => {
-		const limit = Math.min(50, Math.max(1, (params.count as number) ?? 5));
-		const url = new URL(`${baseUrl}/search`);
-		url.searchParams.set("q", params.query as string);
-		url.searchParams.set("format", "json");
-		url.searchParams.set("pageno", String(Math.max(1, (params.pageno as number) ?? 1)));
-		for (const key of ["categories", "engines", "language", "time_range", "safesearch"] as const) {
-			const val = params[key];
-			if (val !== undefined && val !== "") url.searchParams.set(key, String(val));
-		}
-		const timeoutMs = (params.timeout_ms as number) || 15000;
-		const response = await fetch(url, {
-			headers: { Accept: "application/json", "User-Agent": "pi-web/0.1 SearXNG" },
-			signal: signalWithTimeout(timeoutMs, signal),
-		});
-		const text = await response.text();
-		if (!response.ok) {
-			const hint =
-				response.status === 403
-					? "\nHint: enable JSON output in SearXNG settings.yml with search.formats including json."
-					: "";
-			throw new Error(`HTTP ${response.status}: ${response.statusText}${hint}${text ? `\n${text}` : ""}`);
-		}
-		const data: any = text ? JSON.parse(text) : {};
-		const results = (data.results || [])
-			.slice(0, limit)
-			.map((r: any) => ({
-				title: sanitizeSnippet(r.title || ""),
-				url: r.url || "",
-				snippet: sanitizeSnippet(r.content || r.snippet || ""),
-				engine: r.engine || "",
-				category: r.category || "",
-				score: r.score,
-				publishedDate: r.publishedDate || r.published_date || "",
-			}));
-		return { ...data, results };
-	}, undefined, signal);
+  return withRetry(async () => {
+    const limit = Math.min(50, Math.max(1, (params.count as number) ?? 5));
+    const url = new URL(`${baseUrl}/search`);
+    url.searchParams.set("q", params.query as string);
+    url.searchParams.set("format", "json");
+    url.searchParams.set("pageno", String(Math.max(1, (params.pageno as number) ?? 1)));
+    for (const key of ["categories", "engines", "language", "time_range", "safesearch"] as const) {
+      const val = params[key];
+      if (val !== undefined && val !== "") url.searchParams.set(key, String(val));
+    }
+    const timeoutMs = (params.timeout_ms as number) || 15000;
+    const response = await fetch(url, {
+      headers: { Accept: "application/json", "User-Agent": "pi-web/0.1 SearXNG" },
+      signal: signalWithTimeout(timeoutMs, signal),
+    });
+    const text = await response.text();
+    if (!response.ok) {
+      const hint =
+        response.status === 403
+          ? "\nHint: enable JSON output in SearXNG settings.yml with search.formats including json."
+          : "";
+      throw new Error(`HTTP ${response.status}: ${response.statusText}${hint}${text ? `\n${text}` : ""}`);
+    }
+    const data: any = text ? JSON.parse(text) : {};
+    const results = (data.results || [])
+      .slice(0, limit)
+      .map((r: any) => ({
+        title: sanitizeSnippet(r.title || ""),
+        url: r.url || "",
+        snippet: sanitizeSnippet(r.content || r.snippet || ""),
+        engine: r.engine || "",
+        category: r.category || "",
+        score: r.score,
+        publishedDate: r.publishedDate || r.published_date || "",
+      }));
+    return { ...data, results };
+  }, undefined, signal);
 }

package/extensions/types.d.ts

@@ -1,3 +1,3 @@
 declare module "turndown-plugin-gfm" {
-	export const gfm: unknown;
+  export const gfm: unknown;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bacnh85/pi-web",
-  "version": "0.5.2",
+  "version": "0.5.4",
   "description": "Pi extension for web search, page extraction, Firecrawl scraping/crawling, and Crawl4AI headless browser crawling.",
   "type": "module",
   "license": "MIT",