@bacnh85/pi-web 0.5.1 → 0.5.3

package/extensions/index.ts

@@ -18,8 +18,9 @@ import {
 	truncateText,
 	formatFirecrawlScrape,
 	formatCrawl4aiResult,
+	formatUnifiedSearchResults,
 } from "./lib/format";
-import { searchWithDiagnostics, type SearchResult as UnifiedSearchResult } from "./lib/search";
+import { searchWithDiagnostics } from "./lib/search";
 import { extractWithDiagnostics, type ExtractMode } from "./lib/extract";
 import { firecrawlRequest, type FirecrawlResult } from "./lib/firecrawl";
 import {
@@ -48,28 +49,6 @@ const crawl4aiControlSchema = {
 };
 
 
-// ---------------------------------------------------------------------------
-// Helper: format unified search results for text output
-// ---------------------------------------------------------------------------
-
-function formatUnifiedSearchResults(results: UnifiedSearchResult[]): string {
-	if (!results.length) return "No results found.";
-	return results
-		.map((r, i) =>
-			[
-				`--- Result ${i + 1} (backend: ${r.backend}) ---`,
-				`Title: ${r.title || ""}`,
-				`Link: ${r.url || ""}`,
-				r.age ? `Age: ${r.age}` : "",
-				r.snippet ? `Snippet: ${r.snippet}` : "",
-				r.content ? `Content:\n${r.content}` : "",
-			]
-				.filter(Boolean)
-				.join("\n"),
-		)
-		.join("\n\n");
-}
-
 // ---------------------------------------------------------------------------
 // Extension entry point
 // ---------------------------------------------------------------------------

package/extensions/lib/brave.ts

@@ -16,6 +16,7 @@ export async function fetchBraveResults(
 	signal?: AbortSignal,
 ): Promise<BraveResult[]> {
 	return withRetry(async () => {
+		if (signal?.aborted) throw signal.reason ?? new Error("Operation aborted.");
 		const params = new URLSearchParams({ q: query, count: String(count), country });
 		if (freshness) params.append("freshness", freshness);
 		const response = await fetch(`https://api.search.brave.com/res/v1/web/search?${params}`, {
@@ -36,5 +37,5 @@ export async function fetchBraveResults(
 				snippet: sanitizeSnippet(r.description || ""),
 				age: sanitizeSnippet(r.age || r.page_age || ""),
 			}));
-	});
+	}, undefined, signal);
 }

package/extensions/lib/config.ts

@@ -9,8 +9,8 @@ import path from "node:path";
 // ---------------------------------------------------------------------------
 
 export const HOSTED_FIRECRAWL_BASE_URL = "https://api.firecrawl.dev/v2";
-export const DEFAULT_SEARXNG_BASE_URL = "http://172.30.55.22:8888";
-export const DEFAULT_CRAWL4AI_API_URL = "http://172.30.55.22:11235";
+export const DEFAULT_SEARXNG_BASE_URL = "http://127.0.0.1:8888";
+export const DEFAULT_CRAWL4AI_API_URL = "http://127.0.0.1:11235";
 
 // ---------------------------------------------------------------------------
 // Environment loading
@@ -54,7 +54,7 @@ export function parseDotenvValue(rawValue: string): string {
 
 export function parseDotenvFile(file: string): Record<string, string> | null {
 	let text: string;
-	try { text = fs.readFileSync(file, "utf8"); } catch (e: any) { if (e?.code === "ENOENT") return null; throw e; }
+	try { text = fs.readFileSync(file, "utf8"); } catch { return null; }
 	const values: Record<string, string> = {};
 	for (const rawLine of text.split(/\r?\n/)) {
 		const line = rawLine.trim();
@@ -142,7 +142,10 @@ export function loadCrawl4aiConfig(params: Record<string, unknown> = {}, cwd = p
 	const explicitApiUrl = params.crawl4ai_api_url as string | undefined;
 	const explicitApiToken = params.crawl4ai_api_token as string | undefined;
 	const baseUrl = normalizeCrawl4aiApiUrl(explicitApiUrl || apiUrlLookup.value);
-	const apiToken = explicitApiToken || apiTokenLookup.value || "";
+	// Security: only use env-var API token when URL is default or also from env var.
+	// If the URL is overridden via params but no explicit token was provided, don't send
+	// the env-var token to a potentially attacker-controlled URL.
+	const apiToken = explicitApiToken || (explicitApiUrl ? "" : apiTokenLookup.value) || "";
 	const timeoutValue = (params.timeout_ms as number) || findEnvValue("CRAWL4AI_API_TIMEOUT_MS", cwd, includeCwdEnv).value;
 	const timeoutMs = timeoutValue ? Number.parseInt(String(timeoutValue), 10) : 60000;
 	if (!Number.isInteger(timeoutMs) || timeoutMs < 1000) throw new Error("CRAWL4AI_API_TIMEOUT_MS/timeout_ms must be an integer >= 1000.");
@@ -171,8 +174,11 @@ export function loadFirecrawlConfig(params: Record<string, unknown> = {}, cwd =
 	const apiKeyLookup = findEnvValue("FIRECRAWL_API_KEY", cwd, includeCwdEnv);
 	const explicitApiKey = (params.firecrawl_api_key as string) || (params.api_key as string);
 	const apiUrl = (params.firecrawl_api_url as string) || (params.api_url as string) || apiUrlLookup.value;
-	const apiKey = explicitApiKey || apiKeyLookup.value || "";
-	const timeoutValue = (params.timeout_ms as number) || findEnvValue("FIRECRAWL_TIMEOUT_MS", cwd).value;
+	// Security: only use env-var API key when URL is default or also from env var.
+	// If the URL is overridden via params but no explicit key was provided, don't send
+	// the env-var key to a potentially attacker-controlled URL.
+	const apiKey = explicitApiKey || (apiUrl !== apiUrlLookup.value && apiUrl ? "" : apiKeyLookup.value) || "";
+	const timeoutValue = (params.timeout_ms as number) || findEnvValue("FIRECRAWL_TIMEOUT_MS", cwd, includeCwdEnv).value;
 	const baseUrl = normalizeFirecrawlBaseUrl(apiUrl);
 	const isHosted = !apiUrl || baseUrl.startsWith(HOSTED_FIRECRAWL_BASE_URL);
 	const timeoutMs = timeoutValue ? Number.parseInt(String(timeoutValue), 10) : 60000;

package/extensions/lib/content.ts

@@ -62,17 +62,25 @@ export async function fetchReadableContent(
 	const html = await response.text();
 	const deps = loadReadableContentDependencies();
 	const dom = new deps.JSDOM(html, { url });
-	const article = new deps.Readability(dom.window.document).parse();
-	if (article?.content) {
-		return { title: article.title || "", markdown: htmlToMarkdown(article.content, deps) };
+	try {
+		const article = new deps.Readability(dom.window.document).parse();
+		if (article?.content) {
+			return { title: article.title || "", markdown: htmlToMarkdown(article.content, deps) };
+		}
+	} finally {
+		dom.window.close();
 	}
 	// Fallback: strip non-content elements and use main/article/body
 	const fallbackDoc = new deps.JSDOM(html, { url });
-	const doc = fallbackDoc.window.document;
-	doc.querySelectorAll("script, style, noscript, nav, header, footer, aside").forEach((el: any) => el.remove());
-	const title = doc.querySelector("title")?.textContent?.trim() || "";
-	const main = doc.querySelector("main, article, [role='main'], .content, #content") || doc.body;
-	const fallbackHtml = main?.innerHTML || "";
-	if (fallbackHtml.trim().length <= 100) throw new Error("Could not extract readable content from this page.");
-	return { title, markdown: htmlToMarkdown(fallbackHtml, deps) };
+	try {
+		const doc = fallbackDoc.window.document;
+		doc.querySelectorAll("script, style, noscript, nav, header, footer, aside").forEach((el: any) => el.remove());
+		const title = doc.querySelector("title")?.textContent?.trim() || "";
+		const main = doc.querySelector("main, article, [role='main'], .content, #content") || doc.body;
+		const fallbackHtml = main?.innerHTML || "";
+		if (fallbackHtml.trim().length <= 100) throw new Error("Could not extract readable content from this page.");
+		return { title, markdown: htmlToMarkdown(fallbackHtml, deps) };
+	} finally {
+		fallbackDoc.window.close();
+	}
 }

package/extensions/lib/crawl4ai.ts

@@ -3,19 +3,7 @@
 // Endpoints: /crawl, /crawl/stream, /md, /screenshot, /pdf, /health
 
 import type { Crawl4aiConfig } from "./config";
-import { signalWithTimeout } from "./retry";
-
-// ---------------------------------------------------------------------------
-// Error
-// ---------------------------------------------------------------------------
-
-export class Crawl4aiHttpError extends Error {
-	status: number;
-	constructor(status: number, statusText: string, text: string) {
-		super(`HTTP ${status}: ${statusText}${text ? `\n${text}` : ""}`);
-		this.status = status;
-	}
-}
+import { signalWithTimeout, withRetry, HttpError } from "./retry";
 
 // ---------------------------------------------------------------------------
 // Low-level HTTP helpers
@@ -38,7 +26,7 @@ async function crawl4aiRequestJson(
 		signal: signalWithTimeout(config.timeoutMs, signal),
 	});
 	const text = await response.text();
-	if (!response.ok) throw new Crawl4aiHttpError(response.status, response.statusText, text);
+	if (!response.ok) throw new HttpError(response.status, response.statusText, text);
 	return text ? JSON.parse(text) : { success: true };
 }
 
@@ -52,15 +40,7 @@ export async function crawl4aiRequest(
 	body?: unknown,
 	signal?: AbortSignal,
 ): Promise<Record<string, unknown>> {
-	for (let attempt = 0; attempt < 3; attempt++) {
-		try {
-			return await crawl4aiRequestJson(config, method, endpoint, body, signal);
-		} catch (e) {
-			if (attempt === 2) throw e;
-			await new Promise((r) => setTimeout(r, 1000 * Math.pow(2, attempt)));
-		}
-	}
-	throw new Error("unreachable");
+	return withRetry(() => crawl4aiRequestJson(config, method, endpoint, body, signal), 3, signal);
 }
 
 // ---------------------------------------------------------------------------

package/extensions/lib/extract.ts

@@ -10,7 +10,7 @@ import { loadFirecrawlConfig, loadCrawl4aiConfig, type FirecrawlConfig, type Cra
 import { fetchReadableContent } from "./content";
 import { firecrawlRequest, type FirecrawlResult } from "./firecrawl";
 import { fetchCrawl4aiMarkdown } from "./crawl4ai";
-import { formatFirecrawlScrape } from "./format";
+import { formatFirecrawlScrape, sanitizeError } from "./format";
 
 export type ExtractMode = "auto" | "static" | "dynamic" | "full";
 export type ExtractAttemptStatus = "success" | "empty" | "error";
@@ -56,14 +56,6 @@ export interface ExtractDiagnostics {
 
 const MIN_USEFUL_MARKDOWN_CHARS = 120;
 
-function sanitizeError(error: unknown): string {
-	const raw = error instanceof Error ? error.message : String(error);
-	return raw
-		.replace(/Bearer\s+[A-Za-z0-9._~+/=-]+/gi, "Bearer [REDACTED]")
-		.replace(/api[_-]?key[=:][^\s&]+/gi, "api_key=[REDACTED]")
-		.slice(0, 500);
-}
-
 function isUseful(result: ExtractResult | null, mode: ExtractMode, explicit: boolean): result is ExtractResult {
 	if (!result) return false;
 	const length = result.markdown.trim().length;
@@ -99,12 +91,9 @@ async function extractDynamic(params: ExtractParams, ctx?: Record<string, unknow
 		throw new Error(`Firecrawl configuration unavailable: ${sanitizeError(e)}`);
 	}
 
-	const formats: unknown[] = ["markdown"];
+	const formats: string[] = ["markdown"];
 	if (params.prompt || params.schema) {
-		const jsonEntry: Record<string, unknown> = { type: "json" };
-		if (params.prompt) jsonEntry.prompt = params.prompt;
-		if (params.schema) jsonEntry.schema = params.schema;
-		formats.push(jsonEntry);
+		formats.push("json");
 	}
 	const body: Record<string, unknown> = {
 		url: params.url,
@@ -113,6 +102,12 @@ async function extractDynamic(params: ExtractParams, ctx?: Record<string, unknow
 		...(params.wait_for ? { waitFor: params.wait_for } : {}),
 		...(params.mobile ? { mobile: true } : {}),
 	};
+	if (params.prompt || params.schema) {
+		const jsonOptions: Record<string, unknown> = {};
+		if (params.prompt) jsonOptions.prompt = params.prompt;
+		if (params.schema) jsonOptions.schema = params.schema;
+		body.jsonOptions = jsonOptions;
+	}
 	const result = await firecrawlRequest(fcConfig, "POST", "/scrape", body, params.signal) as FirecrawlResult;
 	const structured = findStructuredPayload(result);
 	const raw = `${formatFirecrawlScrape(result as Record<string, unknown>, params.content_chars ?? 20000)}${structuredSection(structured)}`;
@@ -129,7 +124,7 @@ async function extractFull(params: ExtractParams, ctx?: Record<string, unknown>)
 	let c4aiConfig: Crawl4aiConfig;
 	try {
 		c4aiConfig = loadCrawl4aiConfig(
-		{ crawl4ai_api_token: params.crawl4ai_api_token, crawl4ai_api_url: params.crawl4ai_api_url },
+		{ crawl4ai_api_token: params.crawl4ai_api_token, crawl4ai_api_url: params.crawl4ai_api_url, timeout_ms: params.timeout_ms },
 		cwdFromContext(ctx ?? {}),
 		includeProjectEnv(ctx ?? {}),
 	);

package/extensions/lib/firecrawl.ts

@@ -1,7 +1,7 @@
 // Firecrawl API client (search, scrape, map, crawl).
 
 import type { FirecrawlConfig } from "./config";
-import { signalWithTimeout, withRetry } from "./retry";
+import { signalWithTimeout, withRetry, HttpError } from "./retry";
 
 // ---------------------------------------------------------------------------
 // Types
@@ -15,18 +15,6 @@ export interface FirecrawlResult {
 	[key: string]: unknown;
 }
 
-// ---------------------------------------------------------------------------
-// Error
-// ---------------------------------------------------------------------------
-
-export class FirecrawlHttpError extends Error {
-	status: number;
-	constructor(status: number, statusText: string, text: string) {
-		super(`HTTP ${status}: ${statusText}${text ? `\n${text}` : ""}`);
-		this.status = status;
-	}
-}
-
 // ---------------------------------------------------------------------------
 // Low-level HTTP helpers
 // ---------------------------------------------------------------------------
@@ -48,7 +36,7 @@ async function firecrawlRequestJson(
 		signal: signalWithTimeout(config.timeoutMs, signal),
 	});
 	const text = await response.text();
-	if (!response.ok) throw new FirecrawlHttpError(response.status, response.statusText, text);
+	if (!response.ok) throw new HttpError(response.status, response.statusText, text);
 	return text ? JSON.parse(text) : { success: true };
 }
 
@@ -62,5 +50,5 @@ export async function firecrawlRequest(
 	body?: unknown,
 	signal?: AbortSignal,
 ): Promise<Record<string, unknown>> {
-	return withRetry(() => firecrawlRequestJson(config, method, endpoint, body, signal));
+	return withRetry(() => firecrawlRequestJson(config, method, endpoint, body, signal), undefined, signal);
 }

package/extensions/lib/format.ts

@@ -36,16 +36,56 @@ export function sanitizeSnippet(text = ""): string {
 		.trim();
 }
 
+/**
+ * Sanitize an error message — redact sensitive tokens and truncate length.
+ */
+export function sanitizeError(error: unknown): string {
+	const raw = error instanceof Error ? error.message : String(error);
+	return raw
+		.replace(/Bearer\s+[A-Za-z0-9._~+/=-]+/gi, "Bearer [REDACTED]")
+		.replace(/X-Subscription-Token\s*[:=]\s*[^\s]+/gi, "X-Subscription-Token: [REDACTED]")
+		.replace(/api[_-]?key[=:][^\s&]+/gi, "api_key=[REDACTED]")
+		.slice(0, 500);
+}
+
 export function truncateText(text: string): string {
 	const lines = text.split("\n");
-	let output = lines.slice(0, OUTPUT_MAX_LINES).join("\n");
-	while (Buffer.byteLength(output, "utf8") > OUTPUT_MAX_BYTES) {
-		output = output.slice(0, Math.max(0, output.length - 1024));
-	}
-	if (lines.length > OUTPUT_MAX_LINES || output.length < text.length) {
-		output += `\n\n[Web output truncated to ${OUTPUT_MAX_LINES} lines / ${OUTPUT_MAX_BYTES} bytes.]`;
-	}
-	return output;
+	const lineTruncated = lines.slice(0, OUTPUT_MAX_LINES).join("\n");
+	const buf = Buffer.from(lineTruncated, "utf8");
+	if (buf.length <= OUTPUT_MAX_BYTES && lines.length <= OUTPUT_MAX_LINES) return text;
+	const truncated = buf.slice(0, OUTPUT_MAX_BYTES).toString("utf8").replace(/\uFFFD+$/g, "");
+	return truncated + `\n\n[Web output truncated to ${OUTPUT_MAX_LINES} lines / ${OUTPUT_MAX_BYTES} bytes.]`;
+}
+
+// ---------------------------------------------------------------------------
+// Unified search result formatting
+// ---------------------------------------------------------------------------
+
+export interface UnifiedSearchResult {
+	title: string;
+	url: string;
+	snippet: string;
+	age: string;
+	content: string;
+	backend: string;
+}
+
+export function formatUnifiedSearchResults(results: UnifiedSearchResult[]): string {
+	if (!results.length) return "No results found.";
+	return results
+		.map((r, i) =>
+			[
+				`--- Result ${i + 1} (backend: ${r.backend}) ---`,
+				`Title: ${r.title || ""}`,
+				`Link: ${r.url || ""}`,
+				r.age ? `Age: ${r.age}` : "",
+				r.snippet ? `Snippet: ${r.snippet}` : "",
+				r.content ? `Content:\n${r.content}` : "",
+			]
+				.filter(Boolean)
+				.join("\n"),
+		)
+		.join("\n\n");
 }
 
 // ---------------------------------------------------------------------------

package/extensions/lib/retry.ts

@@ -1,13 +1,30 @@
-// Retry utility: signal/timeout helpers.
+// Retry utility: signal/timeout helpers and shared HTTP error class.
 
-export async function withRetry<T>(fn: () => Promise<T>, attempts = 3): Promise<T> {
+// ---------------------------------------------------------------------------
+// Shared HTTP error
+// ---------------------------------------------------------------------------
+
+export class HttpError extends Error {
+	status: number;
+	constructor(status: number, statusText: string, text: string) {
+		super(`HTTP ${status}: ${statusText}${text ? `\n${text}` : ""}`);
+		this.status = status;
+	}
+}
+
+// ---------------------------------------------------------------------------
+// Retry
+// ---------------------------------------------------------------------------
+
+export async function withRetry<T>(fn: () => Promise<T>, attempts = 3, signal?: AbortSignal): Promise<T> {
 	let last: unknown;
 	for (let attempt = 0; attempt < attempts; attempt++) {
+		if (signal?.aborted) throw signal.reason ?? new Error("Operation aborted.");
 		try { return await fn(); }
 		catch (error) {
 			last = error;
 			if (attempt === attempts - 1) break;
-			await abortableSleep(1000 * 2 ** attempt);
+			await abortableSleep(1000 * 2 ** attempt, signal);
 		}
 	}
 	throw last ?? new Error("retry failed");

package/extensions/lib/search.ts

@@ -10,7 +10,7 @@ import { fetchSearxngResults } from "./searxng";
 import { firecrawlRequest, type FirecrawlResult } from "./firecrawl";
 import { loadFirecrawlConfig, loadSearxngConfig, type FirecrawlConfig } from "./config";
 import { fetchReadableContent } from "./content";
-import { sanitizeSnippet } from "./format";
+import { sanitizeSnippet, sanitizeError } from "./format";
 
 export type SearchBackend = "searxng" | "brave" | "firecrawl";
 export type SearchAttemptStatus = "skipped" | "success" | "empty" | "error";
@@ -128,16 +128,22 @@ async function searchBrave(params: SearchParams, backends: BackendConfig): Promi
 	}));
 	if (params.include_content && results.length > 0) {
 		const maxChars = params.content_chars ?? 5000;
-		await Promise.all(
-			results.map(async (r) => {
-				try {
-					const article = await fetchReadableContent(r.url, params.timeout_ms ?? 10000, params.signal);
-					r.content = article.markdown.slice(0, maxChars);
-				} catch (e: any) {
-					r.content = `(Fetch error for ${r.url}: ${sanitizeError(e)})`;
-				}
-			}),
-		);
+		// Cap concurrent content fetches to avoid OOM from parallel JSDOM parsing
+		const MAX_CONCURRENT_FETCHES = 5;
+		const batchSize = Math.min(results.length, MAX_CONCURRENT_FETCHES);
+		for (let i = 0; i < results.length; i += batchSize) {
+			const batch = results.slice(i, i + batchSize);
+			await Promise.all(
+				batch.map(async (r) => {
+					try {
+						const article = await fetchReadableContent(r.url, params.timeout_ms ?? 10000, params.signal);
+						r.content = article.markdown.slice(0, maxChars);
+					} catch (e: any) {
+						r.content = `(Fetch error for ${r.url}: ${sanitizeError(e)})`;
+					}
+				}),
+			);
+		}
 	}
 	return results;
 }
@@ -187,15 +193,6 @@ export function selectSearchBackendOrder(params: SearchParams): SearchBackend[]
 	return ["searxng", "brave", "firecrawl"];
 }
 
-function sanitizeError(error: unknown): string {
-	const raw = error instanceof Error ? error.message : String(error);
-	return raw
-		.replace(/Bearer\s+[A-Za-z0-9._~+/=-]+/gi, "Bearer [REDACTED]")
-		.replace(/X-Subscription-Token\s*[:=]\s*[^\s]+/gi, "X-Subscription-Token: [REDACTED]")
-		.replace(/api[_-]?key[=:][^\s&]+/gi, "api_key=[REDACTED]")
-		.slice(0, 500);
-}
-
 function isConfigured(backend: SearchBackend, backends: BackendConfig): boolean {
 	if (backend === "searxng") return backends.searxng.configured;
 	if (backend === "brave") return backends.brave.configured;
@@ -231,10 +228,6 @@ export async function searchWithDiagnostics(params: SearchParams): Promise<Searc
 		}
 	}
 
-	const diagnostics = attempts.map((a) => `${a.backend}: ${a.status}${a.message ? ` (${a.message})` : ""}`).join("\n");
-	throw new Error(
-		`All web search backends failed or returned no results.\n${diagnostics}\n` +
-		"Use backend to force a provider or check configuration with web_status.",
-	);
+	return { results: [], attempts, selectedBackend: "" as const, backendOrder };
 }
 

package/extensions/lib/searxng.ts

@@ -59,5 +59,5 @@ export async function fetchSearxngResults(
 				publishedDate: r.publishedDate || r.published_date || "",
 			}));
 		return { ...data, results };
-	});
+	}, undefined, signal);
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bacnh85/pi-web",
-  "version": "0.5.1",
+  "version": "0.5.3",
   "description": "Pi extension for web search, page extraction, Firecrawl scraping/crawling, and Crawl4AI headless browser crawling.",
   "type": "module",
   "license": "MIT",