@bacnh85/pi-web 0.3.0 → 0.4.1

package/lib/config.ts

@@ -12,7 +12,7 @@ export const OUTPUT_MAX_BYTES = 50 * 1024;
 export const OUTPUT_MAX_LINES = 2_000;
 export const HOSTED_FIRECRAWL_BASE_URL = "https://api.firecrawl.dev/v2";
 export const DEFAULT_SEARXNG_BASE_URL = "http://172.30.55.22:8888";
-export const DEFAULT_CRAWL4AI_BASE_URL = "http://172.30.55.22:11235";
+export const DEFAULT_CRAWL4AI_API_URL = "http://172.30.55.22:11235";
 
 // ---------------------------------------------------------------------------
 // Environment loading
@@ -126,8 +126,8 @@ export interface Crawl4aiConfig {
 	timeoutMs: number;
 }
 
-export function normalizeCrawl4aiBaseUrl(raw?: string): string {
-	let value = (raw || DEFAULT_CRAWL4AI_BASE_URL).trim();
+export function normalizeCrawl4aiApiUrl(raw?: string): string {
+	let value = (raw || DEFAULT_CRAWL4AI_API_URL).trim();
 	if (!/^https?:\/\//i.test(value)) {
 		value = /^(localhost|127\.|10\.|172\.(1[6-9]|2\d|3[01])\.|192\.168\.)/i.test(value)
 			? `http://${value}`
@@ -137,15 +137,15 @@ export function normalizeCrawl4aiBaseUrl(raw?: string): string {
 }
 
 export function loadCrawl4aiConfig(params: Record<string, unknown> = {}, cwd = process.cwd(), includeCwdEnv = false): Crawl4aiConfig {
-	const baseUrlLookup = findEnvValue("CRAWL4AI_BASE_URL", cwd, includeCwdEnv);
+	const apiUrlLookup = findEnvValue("CRAWL4AI_API_URL", cwd, includeCwdEnv);
 	const apiTokenLookup = findEnvValue("CRAWL4AI_API_TOKEN", cwd, includeCwdEnv);
-	const explicitBaseUrl = params.crawl4ai_base_url as string | undefined;
+	const explicitApiUrl = params.crawl4ai_api_url as string | undefined;
 	const explicitApiToken = params.crawl4ai_api_token as string | undefined;
-	const baseUrl = normalizeCrawl4aiBaseUrl(explicitBaseUrl || baseUrlLookup.value);
+	const baseUrl = normalizeCrawl4aiApiUrl(explicitApiUrl || apiUrlLookup.value);
 	const apiToken = explicitApiToken || apiTokenLookup.value || "";
-	const timeoutValue = (params.timeout_ms as number) || findEnvValue("CRAWL4AI_TIMEOUT_MS", cwd, includeCwdEnv).value;
+	const timeoutValue = (params.timeout_ms as number) || findEnvValue("CRAWL4AI_API_TIMEOUT_MS", cwd, includeCwdEnv).value;
 	const timeoutMs = timeoutValue ? Number.parseInt(String(timeoutValue), 10) : 60000;
-	if (!Number.isInteger(timeoutMs) || timeoutMs < 1000) throw new Error("CRAWL4AI_TIMEOUT_MS/timeout_ms must be an integer >= 1000.");
+	if (!Number.isInteger(timeoutMs) || timeoutMs < 1000) throw new Error("CRAWL4AI_API_TIMEOUT_MS/timeout_ms must be an integer >= 1000.");
 	return { baseUrl, apiToken, timeoutMs };
 }
 

package/lib/extract.ts

@@ -0,0 +1,189 @@
+// Unified content extraction orchestrator.
+// Modes:
+//   "static"  → JSDOM+Readability (no external API)
+//   "dynamic" → Firecrawl Scrape (JS rendering/structured JSON)
+//   "full"    → Crawl4AI markdown endpoint
+//   "auto"    → static → dynamic → full
+
+import { cwdFromContext, includeProjectEnv } from "./config";
+import { loadFirecrawlConfig, loadCrawl4aiConfig, type FirecrawlConfig, type Crawl4aiConfig } from "./config";
+import { fetchReadableContent } from "./content";
+import { firecrawlRequest, type FirecrawlResult } from "./firecrawl";
+import { fetchCrawl4aiMarkdown } from "./crawl4ai";
+import { formatFirecrawlScrape } from "./format";
+
+export type ExtractMode = "auto" | "static" | "dynamic" | "full";
+export type ExtractAttemptStatus = "success" | "empty" | "error";
+
+export interface ExtractParams {
+	url: string;
+	mode?: ExtractMode;
+	prompt?: string;
+	schema?: unknown;
+	content_chars?: number;
+	timeout_ms?: number;
+	wait_for?: number;
+	mobile?: boolean;
+	signal?: AbortSignal;
+	/** Internal: caller-provided ctx for env lookup. */
+	_ctx?: Record<string, unknown>;
+}
+
+export interface ExtractResult {
+	title: string;
+	markdown: string;
+	backend: string;
+	links?: string[];
+	structured?: unknown;
+}
+
+export interface ExtractAttempt {
+	mode: Exclude<ExtractMode, "auto">;
+	backend: string;
+	status: ExtractAttemptStatus;
+	message: string;
+	contentLength: number;
+}
+
+export interface ExtractDiagnostics {
+	result: ExtractResult;
+	attempts: ExtractAttempt[];
+	selectedMode: Exclude<ExtractMode, "auto">;
+	fallbackUsed: boolean;
+}
+
+const MIN_USEFUL_MARKDOWN_CHARS = 120;
+
+function sanitizeError(error: unknown): string {
+	const raw = error instanceof Error ? error.message : String(error);
+	return raw
+		.replace(/Bearer\s+[A-Za-z0-9._~+/=-]+/gi, "Bearer [REDACTED]")
+		.replace(/api[_-]?key[=:][^\s&]+/gi, "api_key=[REDACTED]")
+		.slice(0, 500);
+}
+
+function isUseful(result: ExtractResult | null, mode: ExtractMode, explicit: boolean): result is ExtractResult {
+	if (!result) return false;
+	const length = result.markdown.trim().length;
+	if (length === 0) return false;
+	return explicit || mode !== "static" || length >= MIN_USEFUL_MARKDOWN_CHARS;
+}
+
+function structuredSection(value: unknown): string {
+	if (value === undefined) return "";
+	return `\n\n## Structured extraction\n\n\`\`\`json\n${JSON.stringify(value, null, 2).slice(0, 10000)}\n\`\`\``;
+}
+
+async function extractStatic(params: ExtractParams): Promise<ExtractResult | null> {
+	const article = await fetchReadableContent(params.url, params.timeout_ms ?? 15000, params.signal);
+	return {
+		title: article.title ?? "",
+		markdown: article.markdown.slice(0, params.content_chars ?? 20000),
+		backend: "static",
+	};
+}
+
+function findStructuredPayload(result: FirecrawlResult): unknown {
+	const data = result.data as Record<string, unknown> | undefined;
+	if (!data) return undefined;
+	return data.json ?? data.extract ?? data.structuredData ?? data.llm_extraction;
+}
+
+async function extractDynamic(params: ExtractParams, ctx?: Record<string, unknown>): Promise<ExtractResult | null> {
+	let fcConfig: FirecrawlConfig;
+	try {
+		fcConfig = loadFirecrawlConfig({}, cwdFromContext(ctx ?? {}), includeProjectEnv(ctx ?? {}));
+	} catch (e) {
+		throw new Error(`Firecrawl configuration unavailable: ${sanitizeError(e)}`);
+	}
+
+	const formats: unknown[] = ["markdown"];
+	if (params.prompt || params.schema) {
+		const jsonEntry: Record<string, unknown> = { type: "json" };
+		if (params.prompt) jsonEntry.prompt = params.prompt;
+		if (params.schema) jsonEntry.schema = params.schema;
+		formats.push(jsonEntry);
+	}
+	const body: Record<string, unknown> = {
+		url: params.url,
+		formats,
+		onlyMainContent: true,
+		...(params.wait_for ? { waitFor: params.wait_for } : {}),
+		...(params.mobile ? { mobile: true } : {}),
+	};
+	const result = await firecrawlRequest(fcConfig, "POST", "/scrape", body, true, params.signal) as FirecrawlResult;
+	const structured = findStructuredPayload(result);
+	const raw = `${formatFirecrawlScrape(result as Record<string, unknown>, params.content_chars ?? 20000)}${structuredSection(structured)}`;
+	const titleMatch = raw.match(/^# (.+)$/m);
+	return {
+		title: titleMatch ? titleMatch[1] : "",
+		markdown: raw,
+		backend: "dynamic",
+		structured,
+	};
+}
+
+async function extractFull(params: ExtractParams, ctx?: Record<string, unknown>): Promise<ExtractResult | null> {
+	let c4aiConfig: Crawl4aiConfig;
+	try {
+		c4aiConfig = loadCrawl4aiConfig({}, cwdFromContext(ctx ?? {}), includeProjectEnv(ctx ?? {}));
+	} catch (e) {
+		throw new Error(`Crawl4AI configuration unavailable: ${sanitizeError(e)}`);
+	}
+	const result = await fetchCrawl4aiMarkdown(c4aiConfig, params.url, "fit", undefined, params.signal);
+	return {
+		title: "",
+		markdown: result.markdown.slice(0, params.content_chars ?? 20000),
+		backend: "full",
+	};
+}
+
+function modesFor(params: ExtractParams): Array<Exclude<ExtractMode, "auto">> {
+	const mode = params.mode ?? "auto";
+	if (mode !== "auto") return [mode];
+	return ["static", "dynamic", "full"];
+}
+
+async function runExtractor(mode: Exclude<ExtractMode, "auto">, params: ExtractParams, ctx?: Record<string, unknown>): Promise<ExtractResult | null> {
+	if (mode === "static") return extractStatic(params);
+	if (mode === "dynamic") return extractDynamic(params, ctx);
+	return extractFull(params, ctx);
+}
+
+export async function extractWithDiagnostics(params: ExtractParams): Promise<ExtractDiagnostics> {
+	const ctx = params._ctx;
+	const explicit = (params.mode ?? "auto") !== "auto";
+	const modes = modesFor(params);
+	const attempts: ExtractAttempt[] = [];
+
+	for (const mode of modes) {
+		try {
+			const result = await runExtractor(mode, params, ctx);
+			if (isUseful(result, mode, explicit)) {
+				if (!explicit && attempts.length > 0) {
+					result.markdown = `[Extraction fell back to ${mode === "dynamic" ? "Firecrawl Scrape (dynamic mode)" : "Crawl4AI (full browser mode)"}]\n\n${result.markdown}`;
+				}
+				attempts.push({ mode, backend: result.backend, status: "success", message: `Selected ${mode}`, contentLength: result.markdown.length });
+				return { result, attempts, selectedMode: mode, fallbackUsed: attempts.length > 1 };
+			}
+			const emptyResult = result as ExtractResult | null;
+			const length = emptyResult?.markdown.trim().length ?? 0;
+			attempts.push({ mode, backend: emptyResult?.backend ?? mode, status: "empty", message: `${mode} returned insufficient content`, contentLength: length });
+		} catch (e) {
+			attempts.push({ mode, backend: mode, status: "error", message: sanitizeError(e), contentLength: 0 });
+			if (explicit) break;
+		}
+	}
+
+	const diagnostics = attempts.map((a) => `${a.mode}: ${a.status}${a.message ? ` (${a.message})` : ""}`).join("\n");
+	const mode = params.mode ?? "auto";
+	throw new Error(
+		`Extraction failed for ${params.url} with mode="${mode}".\n${diagnostics}\n` +
+		"Try a different mode or use web_screenshot for a visual snapshot.",
+	);
+}
+
+/** Compatibility wrapper returning only the selected extraction result. */
+export async function universalExtract(params: ExtractParams): Promise<ExtractResult> {
+	return (await extractWithDiagnostics(params)).result;
+}

package/lib/search.ts

@@ -0,0 +1,244 @@
+// Unified search orchestrator.
+// Probes backend availability and adaptively tries backends:
+//   SearXNG (self-hosted) for broad discovery,
+//   Brave (hosted API) for precision/inline content,
+//   Firecrawl as last-resort search.
+
+import { findEnvValue, cwdFromContext, includeProjectEnv, normalizeSearxngBaseUrl } from "./config";
+import { fetchBraveResults } from "./brave";
+import { fetchSearxngResults } from "./searxng";
+import { firecrawlRequest, type FirecrawlResult } from "./firecrawl";
+import { loadFirecrawlConfig, loadSearxngConfig, type FirecrawlConfig } from "./config";
+import { fetchReadableContent } from "./content";
+import { sanitizeSnippet } from "./format";
+
+export type SearchBackend = "searxng" | "brave" | "firecrawl";
+export type SearchAttemptStatus = "skipped" | "success" | "empty" | "error";
+
+export interface SearchParams {
+	query: string;
+	count?: number;
+	freshness?: string;
+	country?: string;
+	backend?: "auto" | SearchBackend;
+	engines?: string;
+	include_content?: boolean;
+	content_chars?: number;
+	timeout_ms?: number;
+	signal?: AbortSignal;
+	/** Internal: caller-provided ctx for env lookup. */
+	_ctx?: Record<string, unknown>;
+}
+
+export interface SearchResult {
+	title: string;
+	url: string;
+	snippet: string;
+	age: string;
+	content: string;
+	backend: string;
+}
+
+export interface SearchAttempt {
+	backend: SearchBackend;
+	status: SearchAttemptStatus;
+	message: string;
+	resultCount: number;
+}
+
+export interface SearchDiagnostics {
+	results: SearchResult[];
+	attempts: SearchAttempt[];
+	selectedBackend: SearchBackend | "";
+	backendOrder: SearchBackend[];
+}
+
+interface BackendConfig {
+	searxng: { configured: boolean; baseUrl: string };
+	brave: { configured: boolean; apiKey: string };
+	firecrawl: { configured: boolean; config: FirecrawlConfig | null };
+}
+
+function probeBackends(ctx?: Record<string, unknown>): BackendConfig {
+	const cwd = cwdFromContext(ctx ?? {});
+	const trusted = includeProjectEnv(ctx ?? {});
+
+	let searxngBaseUrl: string;
+	try {
+		const searxngConfig = loadSearxngConfig({}, cwd, trusted);
+		searxngBaseUrl = searxngConfig.baseUrl;
+	} catch {
+		searxngBaseUrl = normalizeSearxngBaseUrl();
+	}
+
+	const braveKey = findEnvValue("BRAVE_API_KEY", cwd, trusted);
+
+	let fcConfig: FirecrawlConfig | null = null;
+	try {
+		fcConfig = loadFirecrawlConfig({}, cwd, trusted);
+	} catch {
+		// Not configured or unsafe to use from this context.
+	}
+
+	return {
+		searxng: { configured: true, baseUrl: searxngBaseUrl },
+		brave: { configured: Boolean(braveKey.value), apiKey: braveKey.value ?? "" },
+		firecrawl: { configured: fcConfig !== null, config: fcConfig },
+	};
+}
+
+async function searchSearxng(params: SearchParams): Promise<SearchResult[]> {
+	const cfg = loadSearxngConfig(
+		{ ...(params.engines ? { engines: params.engines } : {}) } as Record<string, unknown>,
+		cwdFromContext(params._ctx ?? {}),
+		includeProjectEnv(params._ctx ?? {}),
+	);
+	const raw = await fetchSearxngResults(
+		{ query: params.query, count: params.count ?? 5, engines: params.engines, timeout_ms: params.timeout_ms } as Record<string, unknown>,
+		cfg.baseUrl,
+		params.signal,
+	);
+	return (raw.results || []).map((r) => ({
+		title: r.title ?? "",
+		url: r.url ?? "",
+		snippet: r.snippet ?? "",
+		age: r.publishedDate ?? "",
+		content: "",
+		backend: "searxng",
+	}));
+}
+
+async function searchBrave(params: SearchParams, backends: BackendConfig): Promise<SearchResult[]> {
+	if (!backends.brave.configured) throw new Error("Brave not configured (no BRAVE_API_KEY)");
+	const raw = await fetchBraveResults(
+		params.query,
+		params.count ?? 5,
+		params.country ?? "US",
+		params.freshness ?? "",
+		backends.brave.apiKey,
+		params.signal,
+	);
+	const results = raw.map((r) => ({
+		title: r.title ?? "",
+		url: r.url ?? "",
+		snippet: r.snippet ?? "",
+		age: r.age ?? "",
+		content: "",
+		backend: "brave",
+	}));
+	if (params.include_content && results.length > 0) {
+		const maxChars = params.content_chars ?? 5000;
+		await Promise.all(
+			results.map(async (r) => {
+				try {
+					const article = await fetchReadableContent(r.url, params.timeout_ms ?? 10000, params.signal);
+					r.content = article.markdown.slice(0, maxChars);
+				} catch (e: any) {
+					r.content = `(Fetch error for ${r.url}: ${sanitizeError(e)})`;
+				}
+			}),
+		);
+	}
+	return results;
+}
+
+async function searchFirecrawl(params: SearchParams, backends: BackendConfig): Promise<SearchResult[]> {
+	if (!backends.firecrawl.configured || !backends.firecrawl.config) {
+		throw new Error("Firecrawl not configured (no FIRECRAWL_API_KEY/FIRECRAWL_API_URL)");
+	}
+	const body: Record<string, unknown> = {
+		query: params.query,
+		limit: Math.min(100, Math.max(1, params.count ?? 5)),
+		sources: [{ type: "web" }],
+		...(params.country ? { country: String(params.country).toUpperCase() } : {}),
+	};
+	const result = await firecrawlRequest(backends.firecrawl.config, "POST", "/search", body, true, params.signal) as FirecrawlResult;
+	const rootData = result.data as Record<string, unknown> | undefined;
+	const web = (rootData?.web as unknown[]) || [];
+	const news = (rootData?.news as unknown[]) || [];
+	const items = [...web, ...news];
+	return items.map((r: any) => ({
+		title: sanitizeSnippet(r.title ?? r.metadata?.title ?? ""),
+		url: r.url ?? r.metadata?.sourceURL ?? "",
+		snippet: sanitizeSnippet(r.description ?? r.snippet ?? ""),
+		age: "",
+		content: r.markdown ? String(r.markdown).slice(0, params.content_chars ?? 5000) : "",
+		backend: "firecrawl",
+	}));
+}
+
+function isPrecisionQuery(params: SearchParams): boolean {
+	const query = params.query.trim();
+	const lower = query.toLowerCase();
+	if (params.include_content) return true;
+	if (/\bsite:[^\s]+/i.test(query)) return true;
+	if (/"[^"]+"/.test(query)) return true;
+	if (/\b(docs?|documentation|api|github|repo|repository|changelog|release|issue|bug|sdk|package)\b/i.test(query)) return true;
+	const terms = query.split(/\s+/).filter(Boolean);
+	if (terms.length > 0 && terms.length <= 4 && /\b[A-Z][A-Za-z0-9_-]{2,}\b/.test(query)) return true;
+	if (/\b[a-z0-9-]+\.(com|dev|io|org|net|app|ai|tv)\b/i.test(lower)) return true;
+	return false;
+}
+
+export function selectSearchBackendOrder(params: SearchParams): SearchBackend[] {
+	if (params.backend && params.backend !== "auto") return [params.backend];
+	if (params.engines && !params.include_content) return ["searxng", "brave", "firecrawl"];
+	if (isPrecisionQuery(params)) return ["brave", "searxng", "firecrawl"];
+	return ["searxng", "brave", "firecrawl"];
+}
+
+function sanitizeError(error: unknown): string {
+	const raw = error instanceof Error ? error.message : String(error);
+	return raw
+		.replace(/Bearer\s+[A-Za-z0-9._~+/=-]+/gi, "Bearer [REDACTED]")
+		.replace(/X-Subscription-Token\s*[:=]\s*[^\s]+/gi, "X-Subscription-Token: [REDACTED]")
+		.replace(/api[_-]?key[=:][^\s&]+/gi, "api_key=[REDACTED]")
+		.slice(0, 500);
+}
+
+function isConfigured(backend: SearchBackend, backends: BackendConfig): boolean {
+	if (backend === "searxng") return backends.searxng.configured;
+	if (backend === "brave") return backends.brave.configured;
+	return backends.firecrawl.configured;
+}
+
+async function runBackend(backend: SearchBackend, params: SearchParams, backends: BackendConfig): Promise<SearchResult[]> {
+	if (backend === "searxng") return searchSearxng(params);
+	if (backend === "brave") return searchBrave(params, backends);
+	return searchFirecrawl(params, backends);
+}
+
+export async function searchWithDiagnostics(params: SearchParams): Promise<SearchDiagnostics> {
+	const backends = probeBackends(params._ctx);
+	const backendOrder = selectSearchBackendOrder(params);
+	const attempts: SearchAttempt[] = [];
+
+	for (const backend of backendOrder) {
+		if (!isConfigured(backend, backends)) {
+			attempts.push({ backend, status: "skipped", message: `${backend} is not configured`, resultCount: 0 });
+			continue;
+		}
+		try {
+			const results = await runBackend(backend, params, backends);
+			if (results.length > 0) {
+				attempts.push({ backend, status: "success", message: `Selected ${backend}`, resultCount: results.length });
+				return { results, attempts, selectedBackend: backend, backendOrder };
+			}
+			attempts.push({ backend, status: "empty", message: `${backend} returned 0 results`, resultCount: 0 });
+		} catch (e) {
+			attempts.push({ backend, status: "error", message: sanitizeError(e), resultCount: 0 });
+			if (params.backend && params.backend !== "auto") break;
+		}
+	}
+
+	const diagnostics = attempts.map((a) => `${a.backend}: ${a.status}${a.message ? ` (${a.message})` : ""}`).join("\n");
+	throw new Error(
+		`All web search backends failed or returned no results.\n${diagnostics}\n` +
+		"Use backend to force a provider or check configuration with web_status.",
+	);
+}
+
+/** Compatibility wrapper returning only results. */
+export async function universalSearch(params: SearchParams): Promise<SearchResult[]> {
+	return (await searchWithDiagnostics(params)).results;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bacnh85/pi-web",
-  "version": "0.3.0",
+  "version": "0.4.1",
   "description": "Pi extension for web search, page extraction, Firecrawl scraping/crawling, and Crawl4AI headless browser crawling.",
   "type": "module",
   "license": "MIT",