@bacnh85/pi-web 0.5.2 → 0.5.3

package/extensions/lib/config.ts

@@ -9,8 +9,8 @@ import path from "node:path";
 // ---------------------------------------------------------------------------
 
 export const HOSTED_FIRECRAWL_BASE_URL = "https://api.firecrawl.dev/v2";
-export const DEFAULT_SEARXNG_BASE_URL = "http://172.30.55.22:8888";
-export const DEFAULT_CRAWL4AI_API_URL = "http://172.30.55.22:11235";
+export const DEFAULT_SEARXNG_BASE_URL = "http://127.0.0.1:8888";
+export const DEFAULT_CRAWL4AI_API_URL = "http://127.0.0.1:11235";
 
 // ---------------------------------------------------------------------------
 // Environment loading
@@ -54,7 +54,7 @@ export function parseDotenvValue(rawValue: string): string {
 
 export function parseDotenvFile(file: string): Record<string, string> | null {
 	let text: string;
-	try { text = fs.readFileSync(file, "utf8"); } catch (e: any) { if (e?.code === "ENOENT") return null; throw e; }
+	try { text = fs.readFileSync(file, "utf8"); } catch { return null; }
 	const values: Record<string, string> = {};
 	for (const rawLine of text.split(/\r?\n/)) {
 		const line = rawLine.trim();
@@ -142,7 +142,10 @@ export function loadCrawl4aiConfig(params: Record<string, unknown> = {}, cwd = p
 	const explicitApiUrl = params.crawl4ai_api_url as string | undefined;
 	const explicitApiToken = params.crawl4ai_api_token as string | undefined;
 	const baseUrl = normalizeCrawl4aiApiUrl(explicitApiUrl || apiUrlLookup.value);
-	const apiToken = explicitApiToken || apiTokenLookup.value || "";
+	// Security: only use env-var API token when URL is default or also from env var.
+	// If the URL is overridden via params but no explicit token was provided, don't send
+	// the env-var token to a potentially attacker-controlled URL.
+	const apiToken = explicitApiToken || (explicitApiUrl ? "" : apiTokenLookup.value) || "";
 	const timeoutValue = (params.timeout_ms as number) || findEnvValue("CRAWL4AI_API_TIMEOUT_MS", cwd, includeCwdEnv).value;
 	const timeoutMs = timeoutValue ? Number.parseInt(String(timeoutValue), 10) : 60000;
 	if (!Number.isInteger(timeoutMs) || timeoutMs < 1000) throw new Error("CRAWL4AI_API_TIMEOUT_MS/timeout_ms must be an integer >= 1000.");
@@ -171,7 +174,10 @@ export function loadFirecrawlConfig(params: Record<string, unknown> = {}, cwd =
 	const apiKeyLookup = findEnvValue("FIRECRAWL_API_KEY", cwd, includeCwdEnv);
 	const explicitApiKey = (params.firecrawl_api_key as string) || (params.api_key as string);
 	const apiUrl = (params.firecrawl_api_url as string) || (params.api_url as string) || apiUrlLookup.value;
-	const apiKey = explicitApiKey || apiKeyLookup.value || "";
+	// Security: only use env-var API key when URL is default or also from env var.
+	// If the URL is overridden via params but no explicit key was provided, don't send
+	// the env-var key to a potentially attacker-controlled URL.
+	const apiKey = explicitApiKey || (apiUrl !== apiUrlLookup.value && apiUrl ? "" : apiKeyLookup.value) || "";
 	const timeoutValue = (params.timeout_ms as number) || findEnvValue("FIRECRAWL_TIMEOUT_MS", cwd, includeCwdEnv).value;
 	const baseUrl = normalizeFirecrawlBaseUrl(apiUrl);
 	const isHosted = !apiUrl || baseUrl.startsWith(HOSTED_FIRECRAWL_BASE_URL);

package/extensions/lib/content.ts

@@ -62,17 +62,25 @@ export async function fetchReadableContent(
 	const html = await response.text();
 	const deps = loadReadableContentDependencies();
 	const dom = new deps.JSDOM(html, { url });
-	const article = new deps.Readability(dom.window.document).parse();
-	if (article?.content) {
-		return { title: article.title || "", markdown: htmlToMarkdown(article.content, deps) };
+	try {
+		const article = new deps.Readability(dom.window.document).parse();
+		if (article?.content) {
+			return { title: article.title || "", markdown: htmlToMarkdown(article.content, deps) };
+		}
+	} finally {
+		dom.window.close();
 	}
 	// Fallback: strip non-content elements and use main/article/body
 	const fallbackDoc = new deps.JSDOM(html, { url });
-	const doc = fallbackDoc.window.document;
-	doc.querySelectorAll("script, style, noscript, nav, header, footer, aside").forEach((el: any) => el.remove());
-	const title = doc.querySelector("title")?.textContent?.trim() || "";
-	const main = doc.querySelector("main, article, [role='main'], .content, #content") || doc.body;
-	const fallbackHtml = main?.innerHTML || "";
-	if (fallbackHtml.trim().length <= 100) throw new Error("Could not extract readable content from this page.");
-	return { title, markdown: htmlToMarkdown(fallbackHtml, deps) };
+	try {
+		const doc = fallbackDoc.window.document;
+		doc.querySelectorAll("script, style, noscript, nav, header, footer, aside").forEach((el: any) => el.remove());
+		const title = doc.querySelector("title")?.textContent?.trim() || "";
+		const main = doc.querySelector("main, article, [role='main'], .content, #content") || doc.body;
+		const fallbackHtml = main?.innerHTML || "";
+		if (fallbackHtml.trim().length <= 100) throw new Error("Could not extract readable content from this page.");
+		return { title, markdown: htmlToMarkdown(fallbackHtml, deps) };
+	} finally {
+		fallbackDoc.window.close();
+	}
 }

package/extensions/lib/format.ts

@@ -50,14 +50,11 @@ export function sanitizeError(error: unknown): string {
 
 export function truncateText(text: string): string {
 	const lines = text.split("\n");
-	let output = lines.slice(0, OUTPUT_MAX_LINES).join("\n");
-	while (Buffer.byteLength(output, "utf8") > OUTPUT_MAX_BYTES) {
-		output = output.slice(0, Math.max(0, output.length - 1024));
-	}
-	if (lines.length > OUTPUT_MAX_LINES || output.length < text.length) {
-		output += `\n\n[Web output truncated to ${OUTPUT_MAX_LINES} lines / ${OUTPUT_MAX_BYTES} bytes.]`;
-	}
-	return output;
+	const lineTruncated = lines.slice(0, OUTPUT_MAX_LINES).join("\n");
+	const buf = Buffer.from(lineTruncated, "utf8");
+	if (buf.length <= OUTPUT_MAX_BYTES && lines.length <= OUTPUT_MAX_LINES) return text;
+	const truncated = buf.slice(0, OUTPUT_MAX_BYTES).toString("utf8").replace(/\uFFFD+$/g, "");
+	return truncated + `\n\n[Web output truncated to ${OUTPUT_MAX_LINES} lines / ${OUTPUT_MAX_BYTES} bytes.]`;
 }
 
 // ---------------------------------------------------------------------------

package/extensions/lib/search.ts

@@ -128,16 +128,22 @@ async function searchBrave(params: SearchParams, backends: BackendConfig): Promi
 	}));
 	if (params.include_content && results.length > 0) {
 		const maxChars = params.content_chars ?? 5000;
-		await Promise.all(
-			results.map(async (r) => {
-				try {
-					const article = await fetchReadableContent(r.url, params.timeout_ms ?? 10000, params.signal);
-					r.content = article.markdown.slice(0, maxChars);
-				} catch (e: any) {
-					r.content = `(Fetch error for ${r.url}: ${sanitizeError(e)})`;
-				}
-			}),
-		);
+		// Cap concurrent content fetches to avoid OOM from parallel JSDOM parsing
+		const MAX_CONCURRENT_FETCHES = 5;
+		const batchSize = Math.min(results.length, MAX_CONCURRENT_FETCHES);
+		for (let i = 0; i < results.length; i += batchSize) {
+			const batch = results.slice(i, i + batchSize);
+			await Promise.all(
+				batch.map(async (r) => {
+					try {
+						const article = await fetchReadableContent(r.url, params.timeout_ms ?? 10000, params.signal);
+						r.content = article.markdown.slice(0, maxChars);
+					} catch (e: any) {
+						r.content = `(Fetch error for ${r.url}: ${sanitizeError(e)})`;
+					}
+				}),
+			);
+		}
 	}
 	return results;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bacnh85/pi-web",
-  "version": "0.5.2",
+  "version": "0.5.3",
   "description": "Pi extension for web search, page extraction, Firecrawl scraping/crawling, and Crawl4AI headless browser crawling.",
   "type": "module",
   "license": "MIT",