@bacnh85/pi-web 0.5.0 → 0.5.2

package/extensions/index.ts

@@ -18,8 +18,9 @@ import {
 	truncateText,
 	formatFirecrawlScrape,
 	formatCrawl4aiResult,
+	formatUnifiedSearchResults,
 } from "./lib/format";
-import { searchWithDiagnostics, type SearchResult as UnifiedSearchResult } from "./lib/search";
+import { searchWithDiagnostics } from "./lib/search";
 import { extractWithDiagnostics, type ExtractMode } from "./lib/extract";
 import { firecrawlRequest, type FirecrawlResult } from "./lib/firecrawl";
 import {
@@ -48,28 +49,6 @@ const crawl4aiControlSchema = {
 };
 
 
-// ---------------------------------------------------------------------------
-// Helper: format unified search results for text output
-// ---------------------------------------------------------------------------
-
-function formatUnifiedSearchResults(results: UnifiedSearchResult[]): string {
-	if (!results.length) return "No results found.";
-	return results
-		.map((r, i) =>
-			[
-				`--- Result ${i + 1} (backend: ${r.backend}) ---`,
-				`Title: ${r.title || ""}`,
-				`Link: ${r.url || ""}`,
-				r.age ? `Age: ${r.age}` : "",
-				r.snippet ? `Snippet: ${r.snippet}` : "",
-				r.content ? `Content:\n${r.content}` : "",
-			]
-				.filter(Boolean)
-				.join("\n"),
-		)
-		.join("\n\n");
-}
-
 // ---------------------------------------------------------------------------
 // Extension entry point
 // ---------------------------------------------------------------------------

package/extensions/lib/brave.ts

@@ -16,6 +16,7 @@ export async function fetchBraveResults(
 	signal?: AbortSignal,
 ): Promise<BraveResult[]> {
 	return withRetry(async () => {
+		if (signal?.aborted) throw signal.reason ?? new Error("Operation aborted.");
 		const params = new URLSearchParams({ q: query, count: String(count), country });
 		if (freshness) params.append("freshness", freshness);
 		const response = await fetch(`https://api.search.brave.com/res/v1/web/search?${params}`, {
@@ -36,5 +37,5 @@ export async function fetchBraveResults(
 				snippet: sanitizeSnippet(r.description || ""),
 				age: sanitizeSnippet(r.age || r.page_age || ""),
 			}));
-	});
+	}, undefined, signal);
 }

package/extensions/lib/config.ts

@@ -172,7 +172,7 @@ export function loadFirecrawlConfig(params: Record<string, unknown> = {}, cwd =
 	const explicitApiKey = (params.firecrawl_api_key as string) || (params.api_key as string);
 	const apiUrl = (params.firecrawl_api_url as string) || (params.api_url as string) || apiUrlLookup.value;
 	const apiKey = explicitApiKey || apiKeyLookup.value || "";
-	const timeoutValue = (params.timeout_ms as number) || findEnvValue("FIRECRAWL_TIMEOUT_MS", cwd).value;
+	const timeoutValue = (params.timeout_ms as number) || findEnvValue("FIRECRAWL_TIMEOUT_MS", cwd, includeCwdEnv).value;
 	const baseUrl = normalizeFirecrawlBaseUrl(apiUrl);
 	const isHosted = !apiUrl || baseUrl.startsWith(HOSTED_FIRECRAWL_BASE_URL);
 	const timeoutMs = timeoutValue ? Number.parseInt(String(timeoutValue), 10) : 60000;

package/extensions/lib/crawl4ai.ts

@@ -3,19 +3,7 @@
 // Endpoints: /crawl, /crawl/stream, /md, /screenshot, /pdf, /health
 
 import type { Crawl4aiConfig } from "./config";
-import { signalWithTimeout } from "./retry";
-
-// ---------------------------------------------------------------------------
-// Error
-// ---------------------------------------------------------------------------
-
-export class Crawl4aiHttpError extends Error {
-	status: number;
-	constructor(status: number, statusText: string, text: string) {
-		super(`HTTP ${status}: ${statusText}${text ? `\n${text}` : ""}`);
-		this.status = status;
-	}
-}
+import { signalWithTimeout, withRetry, HttpError } from "./retry";
 
 // ---------------------------------------------------------------------------
 // Low-level HTTP helpers
@@ -38,7 +26,7 @@ async function crawl4aiRequestJson(
 		signal: signalWithTimeout(config.timeoutMs, signal),
 	});
 	const text = await response.text();
-	if (!response.ok) throw new Crawl4aiHttpError(response.status, response.statusText, text);
+	if (!response.ok) throw new HttpError(response.status, response.statusText, text);
 	return text ? JSON.parse(text) : { success: true };
 }
 
@@ -52,15 +40,7 @@ export async function crawl4aiRequest(
 	body?: unknown,
 	signal?: AbortSignal,
 ): Promise<Record<string, unknown>> {
-	for (let attempt = 0; attempt < 3; attempt++) {
-		try {
-			return await crawl4aiRequestJson(config, method, endpoint, body, signal);
-		} catch (e) {
-			if (attempt === 2) throw e;
-			await new Promise((r) => setTimeout(r, 1000 * Math.pow(2, attempt)));
-		}
-	}
-	throw new Error("unreachable");
+	return withRetry(() => crawl4aiRequestJson(config, method, endpoint, body, signal), 3, signal);
 }
 
 // ---------------------------------------------------------------------------

package/extensions/lib/extract.ts

@@ -10,7 +10,7 @@ import { loadFirecrawlConfig, loadCrawl4aiConfig, type FirecrawlConfig, type Cra
 import { fetchReadableContent } from "./content";
 import { firecrawlRequest, type FirecrawlResult } from "./firecrawl";
 import { fetchCrawl4aiMarkdown } from "./crawl4ai";
-import { formatFirecrawlScrape } from "./format";
+import { formatFirecrawlScrape, sanitizeError } from "./format";
 
 export type ExtractMode = "auto" | "static" | "dynamic" | "full";
 export type ExtractAttemptStatus = "success" | "empty" | "error";
@@ -56,14 +56,6 @@ export interface ExtractDiagnostics {
 
 const MIN_USEFUL_MARKDOWN_CHARS = 120;
 
-function sanitizeError(error: unknown): string {
-	const raw = error instanceof Error ? error.message : String(error);
-	return raw
-		.replace(/Bearer\s+[A-Za-z0-9._~+/=-]+/gi, "Bearer [REDACTED]")
-		.replace(/api[_-]?key[=:][^\s&]+/gi, "api_key=[REDACTED]")
-		.slice(0, 500);
-}
-
 function isUseful(result: ExtractResult | null, mode: ExtractMode, explicit: boolean): result is ExtractResult {
 	if (!result) return false;
 	const length = result.markdown.trim().length;
@@ -99,12 +91,9 @@ async function extractDynamic(params: ExtractParams, ctx?: Record<string, unknow
 		throw new Error(`Firecrawl configuration unavailable: ${sanitizeError(e)}`);
 	}
 
-	const formats: unknown[] = ["markdown"];
+	const formats: string[] = ["markdown"];
 	if (params.prompt || params.schema) {
-		const jsonEntry: Record<string, unknown> = { type: "json" };
-		if (params.prompt) jsonEntry.prompt = params.prompt;
-		if (params.schema) jsonEntry.schema = params.schema;
-		formats.push(jsonEntry);
+		formats.push("json");
 	}
 	const body: Record<string, unknown> = {
 		url: params.url,
@@ -113,6 +102,12 @@ async function extractDynamic(params: ExtractParams, ctx?: Record<string, unknow
 		...(params.wait_for ? { waitFor: params.wait_for } : {}),
 		...(params.mobile ? { mobile: true } : {}),
 	};
+	if (params.prompt || params.schema) {
+		const jsonOptions: Record<string, unknown> = {};
+		if (params.prompt) jsonOptions.prompt = params.prompt;
+		if (params.schema) jsonOptions.schema = params.schema;
+		body.jsonOptions = jsonOptions;
+	}
 	const result = await firecrawlRequest(fcConfig, "POST", "/scrape", body, params.signal) as FirecrawlResult;
 	const structured = findStructuredPayload(result);
 	const raw = `${formatFirecrawlScrape(result as Record<string, unknown>, params.content_chars ?? 20000)}${structuredSection(structured)}`;
@@ -129,7 +124,7 @@ async function extractFull(params: ExtractParams, ctx?: Record<string, unknown>)
 	let c4aiConfig: Crawl4aiConfig;
 	try {
 		c4aiConfig = loadCrawl4aiConfig(
-		{ crawl4ai_api_token: params.crawl4ai_api_token, crawl4ai_api_url: params.crawl4ai_api_url },
+		{ crawl4ai_api_token: params.crawl4ai_api_token, crawl4ai_api_url: params.crawl4ai_api_url, timeout_ms: params.timeout_ms },
 		cwdFromContext(ctx ?? {}),
 		includeProjectEnv(ctx ?? {}),
 	);
@@ -181,11 +176,20 @@ export async function extractWithDiagnostics(params: ExtractParams): Promise<Ext
 		}
 	}
 
-	const diagnostics = attempts.map((a) => `${a.mode}: ${a.status}${a.message ? ` (${a.message})` : ""}`).join("\n");
-	const mode = params.mode ?? "auto";
-	throw new Error(
-		`Extraction failed for ${params.url} with mode="${mode}".\n${diagnostics}\n` +
-		"Try a different mode or use web_screenshot for a visual snapshot.",
-	);
+	// All modes exhausted — return the best attempt with diagnostics instead of throwing
+	const lastAttempt = attempts[attempts.length - 1];
+	const bestResult: ExtractResult = {
+		title: "",
+		markdown: `[All extraction modes failed for ${params.url}]\n\n` +
+			attempts.map((a) => `  ${a.mode}: ${a.status}${a.message ? ` (${a.message})` : ""}`).join("\n") +
+			"\n\nTry web_screenshot for a visual snapshot, or verify the URL is accessible.",
+		backend: lastAttempt?.backend ?? "none",
+	};
+	return {
+		result: bestResult,
+		attempts,
+		selectedMode: lastAttempt?.mode ?? "static",
+		fallbackUsed: true,
+	};
 }
 

package/extensions/lib/firecrawl.ts

@@ -1,7 +1,7 @@
 // Firecrawl API client (search, scrape, map, crawl).
 
 import type { FirecrawlConfig } from "./config";
-import { signalWithTimeout, withRetry } from "./retry";
+import { signalWithTimeout, withRetry, HttpError } from "./retry";
 
 // ---------------------------------------------------------------------------
 // Types
@@ -15,18 +15,6 @@ export interface FirecrawlResult {
 	[key: string]: unknown;
 }
 
-// ---------------------------------------------------------------------------
-// Error
-// ---------------------------------------------------------------------------
-
-export class FirecrawlHttpError extends Error {
-	status: number;
-	constructor(status: number, statusText: string, text: string) {
-		super(`HTTP ${status}: ${statusText}${text ? `\n${text}` : ""}`);
-		this.status = status;
-	}
-}
-
 // ---------------------------------------------------------------------------
 // Low-level HTTP helpers
 // ---------------------------------------------------------------------------
@@ -48,7 +36,7 @@ async function firecrawlRequestJson(
 		signal: signalWithTimeout(config.timeoutMs, signal),
 	});
 	const text = await response.text();
-	if (!response.ok) throw new FirecrawlHttpError(response.status, response.statusText, text);
+	if (!response.ok) throw new HttpError(response.status, response.statusText, text);
 	return text ? JSON.parse(text) : { success: true };
 }
 
@@ -62,5 +50,5 @@ export async function firecrawlRequest(
 	body?: unknown,
 	signal?: AbortSignal,
 ): Promise<Record<string, unknown>> {
-	return withRetry(() => firecrawlRequestJson(config, method, endpoint, body, signal));
+	return withRetry(() => firecrawlRequestJson(config, method, endpoint, body, signal), undefined, signal);
 }

package/extensions/lib/format.ts

@@ -36,6 +36,18 @@ export function sanitizeSnippet(text = ""): string {
 		.trim();
 }
 
+/**
+ * Sanitize an error message — redact sensitive tokens and truncate length.
+ */
+export function sanitizeError(error: unknown): string {
+	const raw = error instanceof Error ? error.message : String(error);
+	return raw
+		.replace(/Bearer\s+[A-Za-z0-9._~+/=-]+/gi, "Bearer [REDACTED]")
+		.replace(/X-Subscription-Token\s*[:=]\s*[^\s]+/gi, "X-Subscription-Token: [REDACTED]")
+		.replace(/api[_-]?key[=:][^\s&]+/gi, "api_key=[REDACTED]")
+		.slice(0, 500);
+}
+
 export function truncateText(text: string): string {
 	const lines = text.split("\n");
 	let output = lines.slice(0, OUTPUT_MAX_LINES).join("\n");
@@ -48,6 +60,37 @@ export function truncateText(text: string): string {
 	return output;
 }
 
+// ---------------------------------------------------------------------------
+// Unified search result formatting
+// ---------------------------------------------------------------------------
+
+export interface UnifiedSearchResult {
+	title: string;
+	url: string;
+	snippet: string;
+	age: string;
+	content: string;
+	backend: string;
+}
+
+export function formatUnifiedSearchResults(results: UnifiedSearchResult[]): string {
+	if (!results.length) return "No results found.";
+	return results
+		.map((r, i) =>
+			[
+				`--- Result ${i + 1} (backend: ${r.backend}) ---`,
+				`Title: ${r.title || ""}`,
+				`Link: ${r.url || ""}`,
+				r.age ? `Age: ${r.age}` : "",
+				r.snippet ? `Snippet: ${r.snippet}` : "",
+				r.content ? `Content:\n${r.content}` : "",
+			]
+				.filter(Boolean)
+				.join("\n"),
+		)
+		.join("\n\n");
+}
+
 // ---------------------------------------------------------------------------
 // Firecrawl scrape formatting
 // ---------------------------------------------------------------------------

package/extensions/lib/retry.ts

@@ -1,13 +1,30 @@
-// Retry utility: signal/timeout helpers.
+// Retry utility: signal/timeout helpers and shared HTTP error class.
 
-export async function withRetry<T>(fn: () => Promise<T>, attempts = 3): Promise<T> {
+// ---------------------------------------------------------------------------
+// Shared HTTP error
+// ---------------------------------------------------------------------------
+
+export class HttpError extends Error {
+	status: number;
+	constructor(status: number, statusText: string, text: string) {
+		super(`HTTP ${status}: ${statusText}${text ? `\n${text}` : ""}`);
+		this.status = status;
+	}
+}
+
+// ---------------------------------------------------------------------------
+// Retry
+// ---------------------------------------------------------------------------
+
+export async function withRetry<T>(fn: () => Promise<T>, attempts = 3, signal?: AbortSignal): Promise<T> {
 	let last: unknown;
 	for (let attempt = 0; attempt < attempts; attempt++) {
+		if (signal?.aborted) throw signal.reason ?? new Error("Operation aborted.");
 		try { return await fn(); }
 		catch (error) {
 			last = error;
 			if (attempt === attempts - 1) break;
-			await abortableSleep(1000 * 2 ** attempt);
+			await abortableSleep(1000 * 2 ** attempt, signal);
 		}
 	}
 	throw last ?? new Error("retry failed");

package/extensions/lib/search.ts

@@ -10,7 +10,7 @@ import { fetchSearxngResults } from "./searxng";
 import { firecrawlRequest, type FirecrawlResult } from "./firecrawl";
 import { loadFirecrawlConfig, loadSearxngConfig, type FirecrawlConfig } from "./config";
 import { fetchReadableContent } from "./content";
-import { sanitizeSnippet } from "./format";
+import { sanitizeSnippet, sanitizeError } from "./format";
 
 export type SearchBackend = "searxng" | "brave" | "firecrawl";
 export type SearchAttemptStatus = "skipped" | "success" | "empty" | "error";
@@ -187,15 +187,6 @@ export function selectSearchBackendOrder(params: SearchParams): SearchBackend[]
 	return ["searxng", "brave", "firecrawl"];
 }
 
-function sanitizeError(error: unknown): string {
-	const raw = error instanceof Error ? error.message : String(error);
-	return raw
-		.replace(/Bearer\s+[A-Za-z0-9._~+/=-]+/gi, "Bearer [REDACTED]")
-		.replace(/X-Subscription-Token\s*[:=]\s*[^\s]+/gi, "X-Subscription-Token: [REDACTED]")
-		.replace(/api[_-]?key[=:][^\s&]+/gi, "api_key=[REDACTED]")
-		.slice(0, 500);
-}
-
 function isConfigured(backend: SearchBackend, backends: BackendConfig): boolean {
 	if (backend === "searxng") return backends.searxng.configured;
 	if (backend === "brave") return backends.brave.configured;
@@ -231,10 +222,6 @@ export async function searchWithDiagnostics(params: SearchParams): Promise<Searc
 		}
 	}
 
-	const diagnostics = attempts.map((a) => `${a.backend}: ${a.status}${a.message ? ` (${a.message})` : ""}`).join("\n");
-	throw new Error(
-		`All web search backends failed or returned no results.\n${diagnostics}\n` +
-		"Use backend to force a provider or check configuration with web_status.",
-	);
+	return { results: [], attempts, selectedBackend: "" as const, backendOrder };
 }
 

package/extensions/lib/searxng.ts

@@ -59,5 +59,5 @@ export async function fetchSearxngResults(
 				publishedDate: r.publishedDate || r.published_date || "",
 			}));
 		return { ...data, results };
-	});
+	}, undefined, signal);
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bacnh85/pi-web",
-  "version": "0.5.0",
+  "version": "0.5.2",
   "description": "Pi extension for web search, page extraction, Firecrawl scraping/crawling, and Crawl4AI headless browser crawling.",
   "type": "module",
   "license": "MIT",
@@ -24,7 +24,7 @@
     "crawling"
   ],
   "scripts": {
-    "test": "cd extensions && mocha"
+    "test": "cd extensions && npx mocha"
   },
   "files": [
     "README.md",