@bacnh85/pi-web 0.2.1 → 0.4.1

package/index.ts

@@ -9,22 +9,31 @@ import {
 	includeProjectEnv,
 	normalizeSearxngBaseUrl,
 	normalizeFirecrawlBaseUrl,
-	loadSearxngConfig,
+	normalizeCrawl4aiApiUrl,
 	loadFirecrawlConfig,
+	loadCrawl4aiConfig,
 	HOSTED_FIRECRAWL_BASE_URL,
 } from "./lib/config";
-import { truncateText, formatSearchResults, formatFirecrawlScrape, formatFirecrawlSearch } from "./lib/format";
-import { fetchReadableContent } from "./lib/content";
-import { fetchBraveResults } from "./lib/brave";
-import { fetchSearxngResults } from "./lib/searxng";
+import {
+	truncateText,
+	formatFirecrawlScrape,
+	formatCrawl4aiResult,
+} from "./lib/format";
+import { searchWithDiagnostics, type SearchResult as UnifiedSearchResult } from "./lib/search";
+import { extractWithDiagnostics, type ExtractMode } from "./lib/extract";
 import { firecrawlRequest, type FirecrawlResult } from "./lib/firecrawl";
+import {
+	fetchCrawl4aiCrawl,
+	fetchCrawl4aiScreenshot,
+	fetchCrawl4aiPdf,
+	fetchCrawl4aiHealth,
+} from "./lib/crawl4ai";
 
 // ---------------------------------------------------------------------------
-// Schema fragments shared across tool registrations
+// Shared schema fragment
 // ---------------------------------------------------------------------------
 
-const controlSchema = {
-	api_key: Type.Optional(Type.String({ description: "Provider API key override. Prefer env vars." })),
+const sharedControlSchema = {
 	timeout_ms: Type.Optional(Type.Number({ description: "Request timeout in milliseconds." })),
 };
 
@@ -34,230 +43,156 @@ const firecrawlControlSchema = {
 	timeout_ms: Type.Optional(Type.Number({ description: "Request timeout in milliseconds." })),
 };
 
-const searxngControlSchema = {
-	searxng_base_url: Type.Optional(Type.String({ description: "SearXNG base URL override. Defaults to SEARXNG_BASE_URL or http://172.30.55.22:8888." })),
+const crawl4aiControlSchema = {
+	crawl4ai_api_url: Type.Optional(Type.String({ description: "Crawl4AI API URL override. Defaults to CRAWL4AI_API_URL or http://172.30.55.22:11235." })),
+	crawl4ai_api_token: Type.Optional(Type.String({ description: "Crawl4AI API token override. Prefer CRAWL4AI_API_TOKEN." })),
 	timeout_ms: Type.Optional(Type.Number({ description: "Request timeout in milliseconds." })),
 };
 
+// ---------------------------------------------------------------------------
+// Helper: format unified search results for text output
+// ---------------------------------------------------------------------------
+
+function formatUnifiedSearchResults(results: UnifiedSearchResult[]): string {
+	if (!results.length) return "No results found.";
+	return results
+		.map((r, i) =>
+			[
+				`--- Result ${i + 1} (backend: ${r.backend}) ---`,
+				`Title: ${r.title || ""}`,
+				`Link: ${r.url || ""}`,
+				r.age ? `Age: ${r.age}` : "",
+				r.snippet ? `Snippet: ${r.snippet}` : "",
+				r.content ? `Content:\n${r.content}` : "",
+			]
+				.filter(Boolean)
+				.join("\n"),
+		)
+		.join("\n\n");
+}
+
 // ---------------------------------------------------------------------------
 // Extension entry point
 // ---------------------------------------------------------------------------
 
 export default function piWebExtension(pi: ExtensionAPI) {
-	// ── Brave Search ──────────────────────────────────────────────────────
+	// ── web_search ────────────────────────────────────────────────────────
 	pi.registerTool({
-		name: "brave_search",
-		label: "Brave Search",
+		name: "web_search",
+		label: "Web Search",
 		description:
-			"Search the web with Brave Search API, optionally fetching readable page content. Best as a fast hosted fallback or independent index for source discovery, current facts, docs lookup, and general web search.",
-		promptSnippet: "Search current web results with Brave",
+			"Search the web with adaptive backend selection. Uses SearXNG (self-hosted) for broad discovery, Brave (hosted API) for precision-sensitive queries and include_content, and Firecrawl as a last resort. Use backend to force a provider.",
+		promptSnippet: "Search current web results",
 		promptGuidelines: [
-			"Use SearXNG first for general web search when the self-hosted instance is available; use Brave as a fast hosted fallback or independent second source.",
-			"Use include_content only for a small number of results to avoid rate limits and extra page fetches.",
-			"Cite result URLs when web results materially support the answer; if Brave is rate-limited, try SearXNG or Firecrawl search.",
+			"Use web_search for source discovery, documentation lookup, current facts, and general web search.",
+			"Backend auto-selection is adaptive: broad discovery uses SearXNG first; precision queries, site: searches, docs/source lookups, and include_content use Brave first; Firecrawl is last resort.",
+			"If auto-selection returns poor-quality results, force backend: 'brave' or backend: 'searxng' for a different search index.",
+			"Use the backend parameter to force a specific search engine (e.g., backend: 'brave').",
+			"Use engines parameter for SearXNG tuning (e.g., engines: 'google,github' for technical queries).",
+			"include_content prefers Brave automatically because only Brave search fetches inline page content; for other result URLs, use web_extract.",
+			"Note: Firecrawl Search has poor semantic accuracy on domain-specific/ambiguous queries; prefer SearXNG or Brave for precision.",
+			"Cite result URLs when web results materially support the answer.",
 		],
 		parameters: Type.Object({
-			...controlSchema,
 			query: Type.String(),
 			count: Type.Optional(Type.Number({ default: 5 })),
+			freshness: Type.Optional(Type.String({ description: "Time filter: e.g., 'pw' (past week), 'pm' (past month), 'py' (past year), or a date range like '2024-01-01to2024-06-30'." })),
 			country: Type.Optional(Type.String({ default: "US" })),
-			freshness: Type.Optional(Type.String()),
-			include_content: Type.Optional(Type.Boolean({ default: false })),
+			backend: Type.Optional(Type.Union(
+				[Type.Literal("auto"), Type.Literal("searxng"), Type.Literal("brave"), Type.Literal("firecrawl")],
+				{ default: "auto", description: "Search backend: auto (default), searxng, brave, or firecrawl." },
+			)),
+			engines: Type.Optional(Type.String({ description: "SearXNG engine override (e.g., 'google,github' for technical queries). Only used when backend is searxng or auto." })),
+			include_content: Type.Optional(Type.Boolean({ default: false, description: "Fetch page content alongside search results. Slower but provides inline content." })),
 			content_chars: Type.Optional(Type.Number({ default: 5000 })),
+			...sharedControlSchema,
 		}),
 		async execute(_id: string, params: Record<string, unknown>, signal: AbortSignal, _onUpdate: unknown, ctx: any) {
-			const cwd = cwdFromContext(ctx);
-			const apiKey = (params.api_key as string) || findEnvValue("BRAVE_API_KEY", cwd, includeProjectEnv(ctx)).value;
-			if (!apiKey) throw new Error("BRAVE_API_KEY is required.");
-			const count = Math.min(20, Math.max(1, (params.count as number) ?? 5));
-			const results = await fetchBraveResults(
-				params.query as string,
-				count,
-				((params.country as string) || "US").toUpperCase(),
-				(params.freshness as string) || "",
-				apiKey,
+			const diagnostics = await searchWithDiagnostics({
+				query: params.query as string,
+				count: params.count as number | undefined,
+				freshness: params.freshness as string | undefined,
+				country: params.country as string | undefined,
+				backend: params.backend as "auto" | "searxng" | "brave" | "firecrawl" | undefined,
+				engines: params.engines as string | undefined,
+				include_content: params.include_content as boolean | undefined,
+				content_chars: params.content_chars as number | undefined,
+				timeout_ms: params.timeout_ms as number | undefined,
 				signal,
-			);
-			if (params.include_content) {
-				await Promise.all(
-					results.map(async (r) => {
-						try {
-							const content = await fetchReadableContent(r.url, (params.timeout_ms as number) || 10000, signal);
-							r.content = content.markdown.slice(0, (params.content_chars as number) ?? 5000);
-						} catch (e: any) {
-							r.content = `(Fetch error for ${r.url}: ${e?.message || String(e)})`;
-						}
-					}),
-				);
-			}
-			return { content: [{ type: "text" as const, text: truncateText(formatSearchResults(results)) }], details: results };
+				_ctx: ctx,
+			});
+			const attempts = diagnostics.attempts.map((a) => `${a.backend}: ${a.status}${a.message ? ` (${a.message})` : ""}`).join("\n");
+			const text = `${formatUnifiedSearchResults(diagnostics.results)}\n\n--- Search diagnostics ---\nSelected backend: ${diagnostics.selectedBackend}\n${attempts}`;
+			return { content: [{ type: "text" as const, text: truncateText(text) }], details: diagnostics };
 		},
 	});
 
-	// ── Web Content (readable extraction) ─────────────────────────────────
+	// ── web_extract ──────────────────────────────────────────────────────
 	pi.registerTool({
-		name: "web_content",
-		label: "Web Content",
+		name: "web_extract",
+		label: "Web Content Extraction",
 		description:
-			"Fetch a URL and extract readable markdown content using JSDOM + Readability + Turndown (does not use the Brave API). Best for fast known-URL article/docs extraction.",
+			"Extract readable content from a URL with automatic backend selection. Tries static extraction (JSDOM+Readability, no API key) first, then dynamic extraction (Firecrawl Scrape, JS rendering), then full browser extraction (Crawl4AI, headless browser). Use the mode parameter for explicit control. If all modes fail, try web_screenshot for a visual snapshot.",
 		promptSnippet: "Extract readable webpage content as markdown",
 		promptGuidelines: [
-			"Use for known URLs when simple readable markdown extraction is enough.",
-			"Prefer Firecrawl scrape for dynamic pages, pages that block direct fetching, or when links/structured extraction are needed.",
+			"Use web_extract to get clean markdown content from a known URL.",
+			"Use mode: 'static' for simple static pages (fast, no API key needed, uses JSDOM+Readability).",
+			"Use mode: 'dynamic' for JavaScript-rendered pages (uses Firecrawl Scrape).",
+			"Use mode: 'full' for pages that need Crawl4AI rendering when other modes fail (resource-intensive).",
+			"Default mode 'auto' tries static → dynamic → full. Successive fallbacks are noted in output.",
+			"⚠️ Dynamic mode (Firecrawl Scrape) may fail on bot-protected sites (Ansible docs, CDN-backed doc sites). Full mode (Crawl4AI) handles more sites but is heavier.",
+			"If all extraction modes fail, try web_screenshot for a visual snapshot, or the page may require interactive login.",
+			"Use the prompt and schema parameters for structured JSON extraction (dynamic mode only); structured JSON is returned in details and text when available.",
 			"Cite the source URL when using extracted content in an answer.",
 		],
 		parameters: Type.Object({
 			url: Type.String(),
-			timeout_ms: Type.Optional(Type.Number({ default: 15000 })),
-			content_chars: Type.Optional(Type.Number({ default: 20000 })),
-		}),
-		async execute(_id: string, params: Record<string, unknown>, signal: AbortSignal) {
-			const parsed = new URL(params.url as string);
-			if (!["http:", "https:"].includes(parsed.protocol)) throw new Error("URL must use http or https.");
-			const article = await fetchReadableContent(parsed.href, (params.timeout_ms as number) || 15000, signal);
-			const text = `${article.title ? `# ${article.title}\n\n` : ""}${article.markdown.slice(0, (params.content_chars as number) ?? 20000)}`;
-			return { content: [{ type: "text" as const, text: truncateText(text) }], details: { url: parsed.href, ...article } };
-		},
-	});
-
-	// ── SearXNG Search ───────────────────────────────────────────────────
-	pi.registerTool({
-		name: "searxng_search",
-		label: "SearXNG Search",
-		description:
-			"Search the web through the configured self-hosted SearXNG metasearch instance. Best for free, self-hosted source discovery without hosted API keys.",
-		promptSnippet: "Search web results with self-hosted SearXNG",
-		promptGuidelines: [
-			"Use SearXNG for self-hosted metasearch and source discovery, especially when Brave is unavailable or avoiding hosted search APIs.",
-			"Use SearXNG before browser/scrape tools when the task starts with finding URLs or sources.",
-			"Cite result URLs when SearXNG results materially support the answer.",
-		],
-		parameters: Type.Object({
-			...searxngControlSchema,
-			query: Type.String(),
-			count: Type.Optional(Type.Number({ default: 5 })),
-			pageno: Type.Optional(Type.Number({ default: 1 })),
-			categories: Type.Optional(Type.String({ description: "Comma-separated SearXNG categories, e.g. general,science." })),
-			engines: Type.Optional(Type.String({ description: "Comma-separated SearXNG engines, e.g. brave,wikipedia." })),
-			language: Type.Optional(Type.String()),
-			time_range: Type.Optional(Type.Union([Type.Literal("day"), Type.Literal("month"), Type.Literal("year")])),
-			safesearch: Type.Optional(Type.Number({ description: "SearXNG safesearch level: 0, 1, or 2." })),
-		}),
-		async execute(_id: string, params: Record<string, unknown>, signal: AbortSignal, _onUpdate: unknown, ctx: any) {
-			const config = loadSearxngConfig(params, cwdFromContext(ctx), includeProjectEnv(ctx));
-			const result = await fetchSearxngResults(params, config.baseUrl, signal);
-			return { content: [{ type: "text" as const, text: truncateText(formatSearchResults(result.results || [])) }], details: { ...result, baseUrl: config.baseUrl } };
-		},
-	});
-
-	// ── Firecrawl Search ─────────────────────────────────────────────────
-	pi.registerTool({
-		name: "firecrawl_search",
-		label: "Firecrawl Search",
-		description:
-			"Search web/news/images through Firecrawl, optionally scraping result markdown. Best when SearXNG/Brave are unavailable or when Firecrawl-backed search/scrape is preferred.",
-		promptSnippet: "Search with Firecrawl",
-		promptGuidelines: [
-			"Use when SearXNG and Brave are missing/rate-limited/insufficient, when Firecrawl search is preferred, or when you need scraped markdown from results.",
-			"Keep limits conservative; scraped search is slower and heavier than plain search.",
-			"Cite result URLs and verify scraped content before relying on it.",
-		],
-		parameters: Type.Object({
-			...firecrawlControlSchema,
-			query: Type.String(),
-			limit: Type.Optional(Type.Number({ default: 5 })),
-			sources: Type.Optional(Type.String({ description: "Comma-separated sources: web,news,images", default: "web" })),
-			country: Type.Optional(Type.String()),
-			location: Type.Optional(Type.String()),
-			tbs: Type.Optional(Type.String()),
-			scrape: Type.Optional(Type.Boolean({ default: false })),
-			content_chars: Type.Optional(Type.Number({ default: 5000 })),
-		}),
-		async execute(_id: string, params: Record<string, unknown>, signal: AbortSignal, _onUpdate: unknown, ctx: any) {
-			const config = loadFirecrawlConfig(params, cwdFromContext(ctx), includeProjectEnv(ctx));
-			const sources = String(params.sources || "web")
-				.split(",")
-				.map((type: string) => ({ type: type.trim() }))
-				.filter((s: { type: string }) => s.type);
-			const body: Record<string, unknown> = {
-				query: params.query,
-				limit: Math.min(100, Math.max(1, (params.limit as number) ?? 5)),
-				sources,
-				...(params.country ? { country: String(params.country).toUpperCase() } : {}),
-				...(params.location ? { location: params.location } : {}),
-				...(params.tbs ? { tbs: params.tbs } : {}),
-				...(params.scrape ? { scrapeOptions: { formats: ["markdown"], onlyMainContent: true } } : {}),
-			};
-			const result = await firecrawlRequest(config, "POST", "/search", body, true, signal);
-			return { content: [{ type: "text" as const, text: truncateText(formatFirecrawlSearch((result as FirecrawlResult) as Record<string, unknown>, (params.content_chars as number) ?? 5000)) }], details: result };
-		},
-	});
-
-	// ── Firecrawl Scrape ─────────────────────────────────────────────────
-	pi.registerTool({
-		name: "firecrawl_scrape",
-		label: "Firecrawl Scrape",
-		description:
-			"Scrape a URL through Firecrawl as markdown/html/links/summary/json. Best for dynamic pages, higher-quality extraction, links, or structured JSON extraction.",
-		promptSnippet: "Scrape a URL with Firecrawl",
-		promptGuidelines: [
-			"Use for known URLs when extraction quality matters, pages are dynamic, or markdown/links/JSON extraction is needed.",
-			"Use prompt/schema only for explicit structured extraction tasks.",
-			"Prefer web_content for simple fast known-URL extraction when Firecrawl is unnecessary.",
-		],
-		parameters: Type.Object({
-			...firecrawlControlSchema,
-			url: Type.String(),
-			formats: Type.Optional(Type.String({ default: "markdown" })),
-			prompt: Type.Optional(Type.String()),
-			schema: Type.Optional(Type.Any()),
-			wait_for: Type.Optional(Type.Number()),
-			timeout: Type.Optional(Type.Number()),
-			mobile: Type.Optional(Type.Boolean()),
-			country: Type.Optional(Type.String()),
-			actions: Type.Optional(Type.Any()),
-			only_main_content: Type.Optional(Type.Boolean({ default: true })),
+			mode: Type.Optional(Type.Union(
+				[Type.Literal("auto"), Type.Literal("static"), Type.Literal("dynamic"), Type.Literal("full")],
+				{ default: "auto", description: "Extraction mode: auto (default), static, dynamic, or full." },
+			)),
+			prompt: Type.Optional(Type.String({ description: "Prompt for structured JSON extraction (dynamic mode only)." })),
+			schema: Type.Optional(Type.Any({ description: "JSON schema for structured extraction (dynamic mode only)." })),
 			content_chars: Type.Optional(Type.Number({ default: 20000 })),
+			wait_for: Type.Optional(Type.Number({ description: "Milliseconds to wait for Firecrawl dynamic rendering before extraction. Full mode uses Crawl4AI /md and may ignore this." })),
+			mobile: Type.Optional(Type.Boolean({ default: false, description: "Emulate mobile viewport (dynamic mode only)." })),
+			...sharedControlSchema,
 		}),
 		async execute(_id: string, params: Record<string, unknown>, signal: AbortSignal, _onUpdate: unknown, ctx: any) {
-			const formats = String(params.formats || "markdown")
-				.split(",")
-				.map((f: string) => f.trim())
-				.filter(Boolean)
-				.map((f: string) =>
-					f === "json" && (params.prompt || params.schema)
-						? { type: "json", ...(params.prompt ? { prompt: params.prompt } : {}), ...(params.schema ? { schema: params.schema } : {}) }
-						: f,
-				);
-			const body: Record<string, unknown> = {
-				url: params.url,
-				formats,
-				onlyMainContent: params.only_main_content !== false,
-				...(params.wait_for ? { waitFor: params.wait_for } : {}),
-				...(params.timeout ? { timeout: params.timeout } : {}),
-				...(params.mobile ? { mobile: true } : {}),
-				...(params.country ? { location: { country: String(params.country).toUpperCase() } } : {}),
-				...(params.actions ? { actions: params.actions } : {}),
-			};
-			const result = await firecrawlRequest(loadFirecrawlConfig(params, cwdFromContext(ctx), includeProjectEnv(ctx)), "POST", "/scrape", body, true, signal);
-			return { content: [{ type: "text" as const, text: truncateText(formatFirecrawlScrape((result as FirecrawlResult) as Record<string, unknown>, (params.content_chars as number) ?? 20000)) }], details: result };
+			const diagnostics = await extractWithDiagnostics({
+				url: params.url as string,
+				mode: params.mode as ExtractMode | undefined,
+				prompt: params.prompt as string | undefined,
+				schema: params.schema,
+				content_chars: params.content_chars as number | undefined,
+				timeout_ms: params.timeout_ms as number | undefined,
+				wait_for: params.wait_for as number | undefined,
+				mobile: params.mobile as boolean | undefined,
+				signal,
+				_ctx: ctx,
+			});
+			const result = diagnostics.result;
+			const attempts = diagnostics.attempts.map((a) => `${a.mode}: ${a.status}${a.message ? ` (${a.message})` : ""}`).join("\n");
+			const text = `${result.title ? `# ${result.title}\n\n` : ""}${result.markdown}\n\n--- Extraction diagnostics ---\nSelected mode: ${diagnostics.selectedMode}\nFallback used: ${diagnostics.fallbackUsed}\n${attempts}`;
+			return { content: [{ type: "text" as const, text: truncateText(text) }], details: { url: params.url, ...diagnostics } };
 		},
 	});
 
-	// ── Firecrawl Map ────────────────────────────────────────────────────
+	// ── web_map ──────────────────────────────────────────────────────────
 	pi.registerTool({
-		name: "firecrawl_map",
-		label: "Firecrawl Map",
+		name: "web_map",
+		label: "Site URL Discovery",
 		description:
-			"Discover URLs from a site with Firecrawl map. Best before crawling or when you need candidate pages from a site/docs section.",
+			"Discover URLs from a site using Firecrawl Map. Best for finding candidate pages from a site or docs section before targeted extraction. Works best on base domains; may return empty results on sub-paths.",
 		promptSnippet: "Map site URLs",
 		promptGuidelines: [
-			"Use before crawling to identify relevant URLs and avoid unnecessary broad crawls.",
-			"Keep limits small unless the user explicitly asks for broad site discovery.",
-			"Use mapped URLs with firecrawl_scrape for targeted extraction.",
+			"Use web_map to discover URLs from a site before crawling or targeted extraction.",
+			"Works best on base domains (e.g., https://example.com). Returns fewer results on sub-paths; use sitemap: 'only' for sitemap-only discovery on sub-paths.",
+			"Use sitemap: 'only' for sitemap-only discovery.",
+			"Use mapped URLs with web_extract for targeted extraction instead of web_crawl when only a few pages are needed.",
+			"Keep limits small unless broad site discovery is explicitly requested.",
 		],
 		parameters: Type.Object({
-			...firecrawlControlSchema,
 			url: Type.String(),
 			limit: Type.Optional(Type.Number({ default: 100 })),
 			include_subdomains: Type.Optional(Type.Boolean({ default: false })),
@@ -265,6 +200,7 @@ export default function piWebExtension(pi: ExtensionAPI) {
 			sitemap: Type.Optional(Type.Union([Type.Literal("only"), Type.Literal("include"), Type.Literal("skip")], { description: "Sitemap mode: include (default), only (sitemap-only), or skip." })),
 			use_index: Type.Optional(Type.Boolean({ default: true, description: "Whether to use the Firecrawl search index for URL discovery." })),
 			ignore_cache: Type.Optional(Type.Boolean({ default: false, description: "Ignore cached map results." })),
+			...firecrawlControlSchema,
 		}),
 		async execute(_id: string, params: Record<string, unknown>, signal: AbortSignal, _onUpdate: unknown, ctx: any) {
 			const body: Record<string, unknown> = {
@@ -276,14 +212,8 @@ export default function piWebExtension(pi: ExtensionAPI) {
 			if (params.sitemap !== undefined) body.sitemap = params.sitemap;
 			if (params.use_index !== undefined) body.useIndex = params.use_index;
 			if (params.ignore_cache !== undefined) body.ignoreCache = params.ignore_cache;
-			const result = await firecrawlRequest(
-				loadFirecrawlConfig(params, cwdFromContext(ctx), includeProjectEnv(ctx)),
-				"POST",
-				"/map",
-				body,
-				true,
-				signal,
-			);
+			const config = loadFirecrawlConfig(params as Record<string, unknown>, cwdFromContext(ctx), includeProjectEnv(ctx));
+			const result = await firecrawlRequest(config, "POST", "/map", body, true, signal);
 			const urls = result.data || result.links || result.urls || [];
 			const text = Array.isArray(urls) && urls.length > 0
 				? (urls as Array<Record<string, unknown> | string>).map((u: any) => u.url || u).join("\n")
@@ -292,46 +222,69 @@ export default function piWebExtension(pi: ExtensionAPI) {
 		},
 	});
 
-	// ── Firecrawl Crawl ──────────────────────────────────────────────────
+	// ── web_crawl ────────────────────────────────────────────────────────
 	pi.registerTool({
-		name: "firecrawl_crawl",
-		label: "Firecrawl Crawl",
+		name: "web_crawl",
+		label: "Site Crawl",
 		description:
-			"Start a conservative Firecrawl site crawl and optionally poll for results. Best for small docs/site sections after mapping or when multiple pages are required.",
+			"Crawl one or more pages from a site. Uses Firecrawl Crawl in 'light' mode (conservative, doc-focused) or Crawl4AI in 'full' mode (headless browser, rendered data, media, links). Accepts a single URL for Firecrawl-style crawling or multiple URLs for Crawl4AI-style crawling.",
 		promptSnippet: "Crawl a small site section",
 		promptGuidelines: [
-			"Use only when multiple pages from a site are needed; prefer firecrawl_map plus targeted scrape when possible.",
-			"Keep limits conservative and use include/exclude paths to narrow scope.",
-			"Avoid large crawls unless the user explicitly requests them and accepts runtime/load.",
+			"Use web_crawl only when multiple pages from a site are truly needed; prefer web_map + web_extract for small numbers of pages.",
+			"Use mode: 'light' (default) for conservative Firecrawl crawling of docs/sites.",
+			"Use mode: 'full' for Crawl4AI crawling with full rendered data (JS, media, links). More resource-intensive.",
+			"For Firecrawl mode ('light'), use the url parameter with optional include_paths/exclude_paths.",
+			"For Crawl4AI mode ('full'), use the urls (array) parameter for batch crawling up to 100 URLs.",
+			"Keep limits conservative (default 10) unless the user explicitly requests large crawls.",
+			"Try web_map first to identify candidate URLs and pass them to web_extract before resorting to a full crawl.",
 		],
 		parameters: Type.Object({
-			...firecrawlControlSchema,
-			url: Type.String(),
+			url: Type.Optional(Type.String({ description: "URL for Firecrawl-style single-page crawl (mode: 'light')." })),
+			urls: Type.Optional(Type.Array(Type.String(), { description: "URLs for Crawl4AI-style multi-URL crawl (mode: 'full'), up to 100." })),
+			mode: Type.Optional(Type.Union([Type.Literal("light"), Type.Literal("full")], { default: "light", description: "Crawl mode: 'light' (Firecrawl, conservative) or 'full' (Crawl4AI, rendered data)." })),
 			limit: Type.Optional(Type.Number({ default: 10 })),
-			include_paths: Type.Optional(Type.String()),
-			exclude_paths: Type.Optional(Type.String()),
-			poll: Type.Optional(Type.Boolean({ default: false })),
+			include_paths: Type.Optional(Type.String({ description: "Comma-separated path patterns to include (Firecrawl mode only)." })),
+			exclude_paths: Type.Optional(Type.String({ description: "Comma-separated path patterns to exclude (Firecrawl mode only)." })),
+			poll: Type.Optional(Type.Boolean({ default: false, description: "Poll for crawl completion (Firecrawl mode only)." })),
+			browser_config: Type.Optional(Type.Any({ description: "Optional BrowserConfig JSON object (Crawl4AI mode only)." })),
+			crawler_config: Type.Optional(Type.Any({ description: "Optional CrawlerRunConfig JSON object (Crawl4AI mode only)." })),
+			content_chars: Type.Optional(Type.Number({ default: 20000 })),
+			...firecrawlControlSchema,
+			...crawl4aiControlSchema,
 		}),
 		async execute(_id: string, params: Record<string, unknown>, signal: AbortSignal, _onUpdate: unknown, ctx: any) {
-			const config = loadFirecrawlConfig(params, cwdFromContext(ctx), includeProjectEnv(ctx));
+			const mode = (params.mode as string) || "light";
+			const cwd = cwdFromContext(ctx);
+			const trusted = includeProjectEnv(ctx);
+			const maxChars = (params.content_chars as number) ?? 20000;
+
+			if (mode === "full") {
+				// Crawl4AI mode
+				const urls = (params.urls as string[]) || (params.url ? [params.url as string] : []);
+				if (!urls.length) throw new Error("Either url or urls parameter is required for crawl.");
+				const config = loadCrawl4aiConfig(params as Record<string, unknown>, cwd, trusted);
+				const browserConfig = params.browser_config as Record<string, unknown> | undefined;
+				const crawlerConfig = params.crawler_config as Record<string, unknown> | undefined;
+				const result = await fetchCrawl4aiCrawl(config, urls, browserConfig, crawlerConfig, signal);
+				const text = formatCrawl4aiResult(result as unknown as Record<string, unknown>, maxChars);
+				return { content: [{ type: "text" as const, text: truncateText(text) }], details: result };
+			}
+
+			// Firecrawl mode ("light")
+			if (!params.url) throw new Error("The url parameter is required for Firecrawl mode ('light').");
+			const fcConfig = loadFirecrawlConfig(params as Record<string, unknown>, cwd, trusted);
 			let result = await firecrawlRequest(
-				config,
+				fcConfig,
 				"POST",
 				"/crawl",
 				{
-					url: params.url,
+					url: params.url as string,
 					limit: Math.min(10000, Math.max(1, (params.limit as number) ?? 10)),
 					includePaths: params.include_paths
-						? String(params.include_paths)
-								.split(",")
-								.map((s: string) => s.trim())
-								.filter(Boolean)
+						? String(params.include_paths).split(",").map((s: string) => s.trim()).filter(Boolean)
 						: [],
 					excludePaths: params.exclude_paths
-						? String(params.exclude_paths)
-								.split(",")
-								.map((s: string) => s.trim())
-								.filter(Boolean)
+						? String(params.exclude_paths).split(",").map((s: string) => s.trim()).filter(Boolean)
 						: [],
 					scrapeOptions: { formats: ["markdown"], onlyMainContent: true },
 				},
@@ -342,8 +295,10 @@ export default function piWebExtension(pi: ExtensionAPI) {
 			if (params.poll && id && !Array.isArray(result.data)) {
 				const { abortableSleep } = await import("./lib/retry");
 				for (let i = 0; i < 60; i++) {
-					result = await firecrawlRequest(config, "GET", `/crawl/${id}`, undefined, true, signal);
-					if (["completed", "failed", "cancelled"].includes(String(result.status || (result.data as Record<string, unknown> | undefined)?.status || ""))) break;
+					result = await firecrawlRequest(fcConfig, "GET", `/crawl/${id}`, undefined, true, signal);
+					if (["completed", "failed", "cancelled"].includes(
+						String(result.status || (result.data as Record<string, unknown> | undefined)?.status || ""),
+					)) break;
 					await abortableSleep(2000, signal);
 				}
 			}
@@ -351,9 +306,7 @@ export default function piWebExtension(pi: ExtensionAPI) {
 				? (result.data as Record<string, unknown>[])
 				: ((result.data as Record<string, unknown>)?.data as Record<string, unknown>[]) || [];
 			const text = pages.length
-				? pages
-						.map((p: Record<string, unknown>) => formatFirecrawlScrape({ data: p } as Record<string, unknown>, 12000))
-						.join("\n\n---\n\n")
+				? pages.map((p: Record<string, unknown>) => formatFirecrawlScrape({ data: p } as Record<string, unknown>, maxChars)).join("\n\n---\n\n")
 				: id
 					? `Crawl started: ${id}\nUse poll=true or check Firecrawl status/dashboard.`
 					: JSON.stringify(result, null, 2);
@@ -361,31 +314,105 @@ export default function piWebExtension(pi: ExtensionAPI) {
 		},
 	});
 
-	// ── Web Status ───────────────────────────────────────────────────────
+	// ── web_screenshot ───────────────────────────────────────────────────
+	pi.registerTool({
+		name: "web_screenshot",
+		label: "Web Page Screenshot",
+		description:
+			"Capture a full-page PNG screenshot of a URL using Crawl4AI's headless browser. Returns a base64-encoded PNG image suitable for visual inspection.",
+		promptSnippet: "Screenshot a webpage",
+		promptGuidelines: [
+			"Use web_screenshot when a visual snapshot of a rendered page is needed, or when web_extract fails on a heavily JS-dependent or bot-protected page.",
+			"The screenshot is returned as a base64-encoded PNG string.",
+			"Use wait_for to delay capture for dynamic content to render (default 2 seconds).",
+			"Use wait_for_images to ensure images are loaded before capture.",
+		],
+		parameters: Type.Object({
+			url: Type.String(),
+			wait_for: Type.Optional(Type.Number({ default: 2, description: "Seconds to wait before capturing screenshot." })),
+			wait_for_images: Type.Optional(Type.Boolean({ default: false, description: "Wait for images to load before capture." })),
+			...crawl4aiControlSchema,
+		}),
+		async execute(_id: string, params: Record<string, unknown>, signal: AbortSignal, _onUpdate: unknown, ctx: any) {
+			const config = loadCrawl4aiConfig(params as Record<string, unknown>, cwdFromContext(ctx), includeProjectEnv(ctx));
+			const result = await fetchCrawl4aiScreenshot(
+				config,
+				params.url as string,
+				params.wait_for as number | undefined,
+				params.wait_for_images as boolean | undefined,
+				signal,
+			);
+			const screenshot = result.screenshot as string | undefined;
+			const artifactUrl = result.url as string | undefined;
+			const mime = result.mime as string | undefined;
+			const size = result.size as number | undefined;
+			let text = `Screenshot: ${params.url}\n`;
+			if (screenshot) text += `Data: base64 PNG (${screenshot.length} chars)\n`;
+			if (artifactUrl) text += `Artifact: ${artifactUrl}\n`;
+			if (mime) text += `MIME: ${mime}\n`;
+			if (size) text += `Size: ${size} bytes\n`;
+			return { content: [{ type: "text" as const, text: truncateText(text) }], details: { ...result, url: params.url } };
+		},
+	});
+
+	// ── web_pdf ──────────────────────────────────────────────────────────
+	pi.registerTool({
+		name: "web_pdf",
+		label: "Web Page PDF",
+		description:
+			"Generate a PDF document of a URL using Crawl4AI's headless browser. Returns a base64-encoded PDF suitable for saving or archiving.",
+		promptSnippet: "PDF a webpage",
+		promptGuidelines: [
+			"Use web_pdf when a printable or archivable snapshot of a page is needed.",
+			"The PDF is returned as a base64-encoded string.",
+		],
+		parameters: Type.Object({
+			url: Type.String(),
+			...crawl4aiControlSchema,
+		}),
+		async execute(_id: string, params: Record<string, unknown>, signal: AbortSignal, _onUpdate: unknown, ctx: any) {
+			const config = loadCrawl4aiConfig(params as Record<string, unknown>, cwdFromContext(ctx), includeProjectEnv(ctx));
+			const result = await fetchCrawl4aiPdf(config, params.url as string, signal);
+			const pdf = result.pdf as string | undefined;
+			const artifactUrl = result.url as string | undefined;
+			const size = result.size as number | undefined;
+			let text = `PDF: ${params.url}\n`;
+			if (pdf) text += `Data: base64 PDF (${pdf.length} chars)\n`;
+			if (artifactUrl) text += `Artifact: ${artifactUrl}\n`;
+			if (size) text += `Size: ${size} bytes\n`;
+			return { content: [{ type: "text" as const, text: truncateText(text) }], details: { ...result, url: params.url } };
+		},
+	});
+
+	// ── web_status ───────────────────────────────────────────────────────
 	pi.registerTool({
 		name: "web_status",
 		label: "Web Provider Status",
 		description:
-			"Show Brave, SearXNG, and Firecrawl configuration status without printing secrets.",
-		promptSnippet: "Check web provider configuration",
+			"Show all web provider configuration status (Brave, SearXNG, Firecrawl, Crawl4AI) without printing secrets. Includes Crawl4AI server health check. Use when web tools fail due to missing credentials or configuration issues.",
+		promptSnippet: "Check web provider configuration and server status",
 		promptGuidelines: [
-			"Use when web tools fail due to missing credentials/config or before comparing Brave, SearXNG, and Firecrawl availability.",
+			"Use web_status when web tools fail due to missing credentials/config or to verify which backends are available.",
+			"The output shows which backends are configured; if a backend is missing, configure its env vars.",
 			"Never print API key values; this tool reports only presence and source.",
+			"Shows Crawl4AI server health, version, and auth status when the server is reachable.",
 		],
 		parameters: Type.Object({}),
-		async execute(_id: string, _params: Record<string, unknown>, _signal: AbortSignal, _onUpdate: unknown, ctx: any) {
+		async execute(_id: string, _params: Record<string, unknown>, signal: AbortSignal, _onUpdate: unknown, ctx: any) {
 			const cwd = cwdFromContext(ctx);
 			const trusted = includeProjectEnv(ctx);
+
+			// Provider config status
 			const braveKey = findEnvValue("BRAVE_API_KEY", cwd, trusted);
 			const searxngUrl = findEnvValue("SEARXNG_BASE_URL", cwd, trusted);
 			const fireKey = findEnvValue("FIRECRAWL_API_KEY", cwd, trusted);
 			const fireUrl = findEnvValue("FIRECRAWL_API_URL", cwd, trusted);
-			const status = {
+			const c4aiUrl = findEnvValue("CRAWL4AI_API_URL", cwd, trusted);
+			const c4aiToken = findEnvValue("CRAWL4AI_API_TOKEN", cwd, trusted);
+
+			const status: Record<string, unknown> = {
 				brave: { apiKeyFound: Boolean(braveKey.value), apiKeySource: braveKey.value ? braveKey.source : "not set" },
-				searxng: {
-					baseUrl: normalizeSearxngBaseUrl(searxngUrl.value),
-					baseUrlSource: searxngUrl.source || "default local",
-				},
+				searxng: { baseUrl: normalizeSearxngBaseUrl(searxngUrl.value), baseUrlSource: searxngUrl.source || "default local" },
 				firecrawl: {
 					baseUrl: normalizeFirecrawlBaseUrl(fireUrl.value),
 					apiUrlSource: fireUrl.source || "default hosted",
@@ -393,7 +420,24 @@ export default function piWebExtension(pi: ExtensionAPI) {
 					apiKeySource: fireKey.value ? fireKey.source : "not set",
 					hostedMode: !fireUrl.value || normalizeFirecrawlBaseUrl(fireUrl.value).startsWith(HOSTED_FIRECRAWL_BASE_URL),
 				},
+				crawl4ai: {
+					baseUrl: normalizeCrawl4aiApiUrl(c4aiUrl.value),
+					baseUrlSource: c4aiUrl.source || "default",
+					apiTokenFound: Boolean(c4aiToken.value),
+					apiTokenSource: c4aiToken.value ? c4aiToken.source : "not set",
+				},
 			};
+
+			// Crawl4AI health check
+			let c4aiHealth: Record<string, unknown> | undefined;
+			try {
+				const c4aiCfg = loadCrawl4aiConfig({}, cwd, trusted);
+				c4aiHealth = await fetchCrawl4aiHealth(c4aiCfg, signal);
+			} catch (e: any) {
+				c4aiHealth = { status: "unreachable", error: e?.message ?? String(e) };
+			}
+			status.crawl4ai = { ...(status.crawl4ai as Record<string, unknown>), health: c4aiHealth };
+
 			return { content: [{ type: "text" as const, text: JSON.stringify(status, null, 2) }], details: status };
 		},
 	});
@@ -408,8 +452,17 @@ export default function piWebExtension(pi: ExtensionAPI) {
 			const searxngUrl = findEnvValue("SEARXNG_BASE_URL", cwd, trusted);
 			const fireKey = findEnvValue("FIRECRAWL_API_KEY", cwd, trusted);
 			const fireUrl = findEnvValue("FIRECRAWL_API_URL", cwd, trusted);
+			const c4aiUrl = findEnvValue("CRAWL4AI_API_URL", cwd, trusted);
+			const c4aiToken = findEnvValue("CRAWL4AI_API_TOKEN", cwd, trusted);
 			ctx.ui.notify(
-				`Brave API key: ${braveKey.value ? `found (${braveKey.source})` : "not set"}\nSearXNG base URL: ${normalizeSearxngBaseUrl(searxngUrl.value)} (${searxngUrl.source || "default local"})\nFirecrawl base URL: ${normalizeFirecrawlBaseUrl(fireUrl.value)}\nFirecrawl API key: ${fireKey.value ? `found (${fireKey.source})` : "not set"}`,
+				[
+					`Brave API key: ${braveKey.value ? `found (${braveKey.source})` : "not set"}`,
+					`SearXNG base URL: ${normalizeSearxngBaseUrl(searxngUrl.value)} (${searxngUrl.source || "default local"})`,
+					`Firecrawl base URL: ${normalizeFirecrawlBaseUrl(fireUrl.value)}`,
+					`Firecrawl API key: ${fireKey.value ? `found (${fireKey.source})` : "not set"}`,
+					`Crawl4AI API URL: ${normalizeCrawl4aiApiUrl(c4aiUrl.value)} (${c4aiUrl.source || "default"})`,
+					`Crawl4AI API token: ${c4aiToken.value ? `found (${c4aiToken.source})` : "not set"}`,
+				].join("\n"),
 				"info",
 			);
 		},