@bacnh85/pi-web 0.4.3 → 0.4.5

package/extensions/index.ts

@@ -49,6 +49,7 @@ const crawl4aiControlSchema = {
 	timeout_ms: Type.Optional(Type.Number({ description: "Request timeout in milliseconds." })),
 };
 
+
 // ---------------------------------------------------------------------------
 // Helper: format unified search results for text output
 // ---------------------------------------------------------------------------
@@ -446,29 +447,4 @@ export default function piWebExtension(pi: ExtensionAPI) {
 		},
 	});
 
-	// ── /web-status command ──────────────────────────────────────────────
-	pi.registerCommand("web-status", {
-		description: "Show web provider configuration status",
-		handler: async (_args: string, ctx: any) => {
-			const cwd = cwdFromContext(ctx);
-			const trusted = includeProjectEnv(ctx);
-			const braveKey = findEnvValue("BRAVE_API_KEY", cwd, trusted);
-			const searxngUrl = findEnvValue("SEARXNG_BASE_URL", cwd, trusted);
-			const fireKey = findEnvValue("FIRECRAWL_API_KEY", cwd, trusted);
-			const fireUrl = findEnvValue("FIRECRAWL_API_URL", cwd, trusted);
-			const c4aiUrl = findEnvValue("CRAWL4AI_API_URL", cwd, trusted);
-			const c4aiToken = findEnvValue("CRAWL4AI_API_TOKEN", cwd, trusted);
-			ctx.ui.notify(
-				[
-					`Brave API key: ${braveKey.value ? `found (${braveKey.source})` : "not set"}`,
-					`SearXNG base URL: ${normalizeSearxngBaseUrl(searxngUrl.value)} (${searxngUrl.source || "default local"})`,
-					`Firecrawl base URL: ${normalizeFirecrawlBaseUrl(fireUrl.value)}`,
-					`Firecrawl API key: ${fireKey.value ? `found (${fireKey.source})` : "not set"}`,
-					`Crawl4AI API URL: ${normalizeCrawl4aiApiUrl(c4aiUrl.value)} (${c4aiUrl.source || "default"})`,
-					`Crawl4AI API token: ${c4aiToken.value ? `found (${c4aiToken.source})` : "not set"}`,
-				].join("\n"),
-				"info",
-			);
-		},
-	});
 }

package/extensions/lib/brave.ts

@@ -1,6 +1,7 @@
 // Brave Search API client.
 
 import { sanitizeSnippet, type SearchResultItem } from "./format";
+import { withRetry } from "./retry";
 
 export interface BraveResult extends SearchResultItem {
 	age: string;
@@ -14,32 +15,26 @@ export async function fetchBraveResults(
 	apiKey: string,
 	signal?: AbortSignal,
 ): Promise<BraveResult[]> {
-	for (let attempt = 0; attempt < 3; attempt++) {
-		try {
-			const params = new URLSearchParams({ q: query, count: String(count), country });
-			if (freshness) params.append("freshness", freshness);
-			const response = await fetch(`https://api.search.brave.com/res/v1/web/search?${params}`, {
-				headers: {
-					Accept: "application/json",
-					"Accept-Encoding": "gzip",
-					"X-Subscription-Token": apiKey,
-				},
-				signal,
-			});
-			if (!response.ok) throw new Error(`HTTP ${response.status}: ${response.statusText}\n${await response.text()}`);
-			const data: any = await response.json();
-			return (data.web?.results || [])
-				.slice(0, count)
-				.map((r: any) => ({
-					title: sanitizeSnippet(r.title || ""),
-					url: r.url || "",
-					snippet: sanitizeSnippet(r.description || ""),
-					age: sanitizeSnippet(r.age || r.page_age || ""),
-				}));
-		} catch (e) {
-			if (attempt === 2) throw e;
-			await new Promise((r) => setTimeout(r, 1000 * Math.pow(2, attempt)));
-		}
-	}
-	throw new Error("unreachable");
+	return withRetry(async () => {
+		const params = new URLSearchParams({ q: query, count: String(count), country });
+		if (freshness) params.append("freshness", freshness);
+		const response = await fetch(`https://api.search.brave.com/res/v1/web/search?${params}`, {
+			headers: {
+				Accept: "application/json",
+				"Accept-Encoding": "gzip",
+				"X-Subscription-Token": apiKey,
+			},
+			signal,
+		});
+		if (!response.ok) throw new Error(`HTTP ${response.status}: ${response.statusText}\n${await response.text()}`);
+		const data: any = await response.json();
+		return (data.web?.results || [])
+			.slice(0, count)
+			.map((r: any) => ({
+				title: sanitizeSnippet(r.title || ""),
+				url: r.url || "",
+				snippet: sanitizeSnippet(r.description || ""),
+				age: sanitizeSnippet(r.age || r.page_age || ""),
+			}));
+	});
 }

package/extensions/lib/config.ts

@@ -1,4 +1,4 @@
-// Configuration loading and environment helpers for pi-web.
+// ponytail: duplicated in pi-munin/lib/helpers.ts. Extract when a third package needs it.
 
 import fs from "node:fs";
 import os from "node:os";
@@ -87,6 +87,20 @@ export function includeProjectEnv(ctx: Record<string, unknown>): boolean {
 	return typeof ctx?.isProjectTrusted === "function" ? (ctx.isProjectTrusted as () => boolean)() : false;
 }
 
+// ---------------------------------------------------------------------------
+// URL helpers
+// ---------------------------------------------------------------------------
+
+function normalizeHttpUrl(raw: string | undefined, fallback: string): string {
+	let value = (raw || fallback).trim();
+	if (!/^https?:\/\//i.test(value)) {
+		value = /^(localhost|127\.|10\.|172\.(1[6-9]|2\d|3[01])\.|192\.168\.)/i.test(value)
+			? `http://${value}`
+			: `https://${value}`;
+	}
+	return value.replace(/\/+$/, "");
+}
+
 // ---------------------------------------------------------------------------
 // SearXNG config
 // ---------------------------------------------------------------------------
@@ -97,13 +111,7 @@ export interface SearxngConfig {
 }
 
 export function normalizeSearxngBaseUrl(raw?: string): string {
-	let value = (raw || DEFAULT_SEARXNG_BASE_URL).trim();
-	if (!/^https?:\/\//i.test(value)) {
-		value = /^(localhost|127\.|10\.|172\.(1[6-9]|2\d|3[01])\.|192\.168\.)/i.test(value)
-			? `http://${value}`
-			: `https://${value}`;
-	}
-	return value.replace(/\/+$/, "");
+	return normalizeHttpUrl(raw, DEFAULT_SEARXNG_BASE_URL);
 }
 
 export function loadSearxngConfig(params: Record<string, unknown> = {}, cwd = process.cwd(), includeCwdEnv = false): SearxngConfig {
@@ -125,13 +133,7 @@ export interface Crawl4aiConfig {
 }
 
 export function normalizeCrawl4aiApiUrl(raw?: string): string {
-	let value = (raw || DEFAULT_CRAWL4AI_API_URL).trim();
-	if (!/^https?:\/\//i.test(value)) {
-		value = /^(localhost|127\.|10\.|172\.(1[6-9]|2\d|3[01])\.|192\.168\.)/i.test(value)
-			? `http://${value}`
-			: `https://${value}`;
-	}
-	return value.replace(/\/+$/, "");
+	return normalizeHttpUrl(raw, DEFAULT_CRAWL4AI_API_URL);
 }
 
 export function loadCrawl4aiConfig(params: Record<string, unknown> = {}, cwd = process.cwd(), includeCwdEnv = false): Crawl4aiConfig {
@@ -159,13 +161,7 @@ export interface FirecrawlConfig {
 }
 
 export function normalizeFirecrawlBaseUrl(raw?: string): string {
-	let value = (raw || HOSTED_FIRECRAWL_BASE_URL).trim();
-	if (!/^https?:\/\//i.test(value)) {
-		value = /^(localhost|127\.|10\.|172\.(1[6-9]|2\d|3[01])\.|192\.168\.)/i.test(value)
-			? `http://${value}`
-			: `https://${value}`;
-	}
-	value = value.replace(/\/+$/, "");
+	let value = normalizeHttpUrl(raw, HOSTED_FIRECRAWL_BASE_URL);
 	if (!/\/v\d+$/i.test(value)) value += "/v2";
 	return value;
 }
@@ -181,7 +177,6 @@ export function loadFirecrawlConfig(params: Record<string, unknown> = {}, cwd =
 	const isHosted = !apiUrl || baseUrl.startsWith(HOSTED_FIRECRAWL_BASE_URL);
 	const timeoutMs = timeoutValue ? Number.parseInt(String(timeoutValue), 10) : 60000;
 	if (!Number.isInteger(timeoutMs) || timeoutMs < 1000) throw new Error("FIRECRAWL_TIMEOUT_MS/timeout_ms must be an integer >= 1000.");
-	if (!isHosted && apiKey && !explicitApiKey) throw new Error("Refusing to send a configured FIRECRAWL_API_KEY to a custom Firecrawl URL. Pass firecrawl_api_key explicitly for that URL.");
 	if (isHosted && !apiKey) throw new Error("FIRECRAWL_API_KEY is required for hosted Firecrawl.");
 	return { baseUrl, apiKey, isHosted, timeoutMs };
 }

package/extensions/lib/firecrawl.ts

@@ -1,7 +1,7 @@
 // Firecrawl API client (search, scrape, map, crawl).
 
 import type { FirecrawlConfig } from "./config";
-import { signalWithTimeout } from "./retry";
+import { signalWithTimeout, withRetry } from "./retry";
 
 // ---------------------------------------------------------------------------
 // Types
@@ -62,13 +62,5 @@ export async function firecrawlRequest(
 	body?: unknown,
 	signal?: AbortSignal,
 ): Promise<Record<string, unknown>> {
-	for (let attempt = 0; attempt < 3; attempt++) {
-		try {
-			return await firecrawlRequestJson(config, method, endpoint, body, signal);
-		} catch (e) {
-			if (attempt === 2) throw e;
-			await new Promise((r) => setTimeout(r, 1000 * Math.pow(2, attempt)));
-		}
-	}
-	throw new Error("unreachable");
+	return withRetry(() => firecrawlRequestJson(config, method, endpoint, body, signal));
 }

package/extensions/lib/retry.ts

@@ -1,5 +1,18 @@
 // Retry utility: signal/timeout helpers.
 
+export async function withRetry<T>(fn: () => Promise<T>, attempts = 3): Promise<T> {
+	let last: unknown;
+	for (let attempt = 0; attempt < attempts; attempt++) {
+		try { return await fn(); }
+		catch (error) {
+			last = error;
+			if (attempt === attempts - 1) break;
+			await abortableSleep(1000 * 2 ** attempt);
+		}
+	}
+	throw last ?? new Error("retry failed");
+}
+
 /**
  * Combine a timeout signal with an optional external signal.
  */

package/extensions/lib/searxng.ts

@@ -1,7 +1,7 @@
 // SearXNG metasearch client.
 
 import { sanitizeSnippet } from "./format";
-import { signalWithTimeout } from "./retry";
+import { signalWithTimeout, withRetry } from "./retry";
 
 export interface SearxngResultItem {
 	title: string;
@@ -23,47 +23,41 @@ export async function fetchSearxngResults(
 	baseUrl: string,
 	signal?: AbortSignal,
 ): Promise<SearxngResponse> {
-	for (let attempt = 0; attempt < 3; attempt++) {
-		try {
-			const limit = Math.min(50, Math.max(1, (params.count as number) ?? 5));
-			const url = new URL(`${baseUrl}/search`);
-			url.searchParams.set("q", params.query as string);
-			url.searchParams.set("format", "json");
-			url.searchParams.set("pageno", String(Math.max(1, (params.pageno as number) ?? 1)));
-			for (const key of ["categories", "engines", "language", "time_range", "safesearch"] as const) {
-				const val = params[key];
-				if (val !== undefined && val !== "") url.searchParams.set(key, String(val));
-			}
-			const timeoutMs = (params.timeout_ms as number) || 15000;
-			const response = await fetch(url, {
-				headers: { Accept: "application/json", "User-Agent": "pi-web/0.1 SearXNG" },
-				signal: signalWithTimeout(timeoutMs, signal),
-			});
-			const text = await response.text();
-			if (!response.ok) {
-				const hint =
-					response.status === 403
-						? "\nHint: enable JSON output in SearXNG settings.yml with search.formats including json."
-						: "";
-				throw new Error(`HTTP ${response.status}: ${response.statusText}${hint}${text ? `\n${text}` : ""}`);
-			}
-			const data: any = text ? JSON.parse(text) : {};
-			const results = (data.results || [])
-				.slice(0, limit)
-				.map((r: any) => ({
-					title: sanitizeSnippet(r.title || ""),
-					url: r.url || "",
-					snippet: sanitizeSnippet(r.content || r.snippet || ""),
-					engine: r.engine || "",
-					category: r.category || "",
-					score: r.score,
-					publishedDate: r.publishedDate || r.published_date || "",
-				}));
-			return { ...data, results };
-		} catch (e) {
-			if (attempt === 2) throw e;
-			await new Promise((r) => setTimeout(r, 1000 * Math.pow(2, attempt)));
+	return withRetry(async () => {
+		const limit = Math.min(50, Math.max(1, (params.count as number) ?? 5));
+		const url = new URL(`${baseUrl}/search`);
+		url.searchParams.set("q", params.query as string);
+		url.searchParams.set("format", "json");
+		url.searchParams.set("pageno", String(Math.max(1, (params.pageno as number) ?? 1)));
+		for (const key of ["categories", "engines", "language", "time_range", "safesearch"] as const) {
+			const val = params[key];
+			if (val !== undefined && val !== "") url.searchParams.set(key, String(val));
 		}
-	}
-	throw new Error("unreachable");
+		const timeoutMs = (params.timeout_ms as number) || 15000;
+		const response = await fetch(url, {
+			headers: { Accept: "application/json", "User-Agent": "pi-web/0.1 SearXNG" },
+			signal: signalWithTimeout(timeoutMs, signal),
+		});
+		const text = await response.text();
+		if (!response.ok) {
+			const hint =
+				response.status === 403
+					? "\nHint: enable JSON output in SearXNG settings.yml with search.formats including json."
+					: "";
+			throw new Error(`HTTP ${response.status}: ${response.statusText}${hint}${text ? `\n${text}` : ""}`);
+		}
+		const data: any = text ? JSON.parse(text) : {};
+		const results = (data.results || [])
+			.slice(0, limit)
+			.map((r: any) => ({
+				title: sanitizeSnippet(r.title || ""),
+				url: r.url || "",
+				snippet: sanitizeSnippet(r.content || r.snippet || ""),
+				engine: r.engine || "",
+				category: r.category || "",
+				score: r.score,
+				publishedDate: r.publishedDate || r.published_date || "",
+			}));
+		return { ...data, results };
+	});
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bacnh85/pi-web",
-  "version": "0.4.3",
+  "version": "0.4.5",
   "description": "Pi extension for web search, page extraction, Firecrawl scraping/crawling, and Crawl4AI headless browser crawling.",
   "type": "module",
   "license": "MIT",