@bacnh85/pi-web 0.2.1 → 0.3.0

package/README.md

@@ -1,6 +1,6 @@
 # pi-web
 
-Pi extension for web search, readable page extraction, and Firecrawl scraping/crawling.
+Pi extension for web search, readable page extraction, Firecrawl scraping/crawling, and Crawl4AI headless browser crawling.
 
 ## Install
 
@@ -43,6 +43,9 @@ Variables:
 - `FIRECRAWL_API_URL` — optional; defaults to `https://api.firecrawl.dev/v2`.
 - `FIRECRAWL_API_KEY` — required for hosted Firecrawl, optional for self-hosted instances without auth.
 - `FIRECRAWL_TIMEOUT_MS` — optional request timeout, default `60000`.
+- `CRAWL4AI_BASE_URL` — optional; defaults to `http://172.30.55.22:11235`.
+- `CRAWL4AI_API_TOKEN` — required if Crawl4AI server has auth enabled (v0.9+ default).
+- `CRAWL4AI_TIMEOUT_MS` — optional request timeout, default `60000`.
 
 Secrets are never printed; status reports only show presence/source.
 
@@ -67,9 +70,18 @@ Firecrawl:
 - `firecrawl_map` — discover URLs for a site.
 - `firecrawl_crawl` — start a conservative crawl, optionally polling for results.
 
+Crawl4AI (requires running Docker server at `CRAWL4AI_BASE_URL`):
+
+- `crawl4ai_scrape` — extract clean LLM-ready markdown with content filtering (fit/raw/bm25/llm).
+- `crawl4ai_crawl` — batch crawl 1-100 URLs with full result data (HTML, markdown, links, media).
+- `crawl4ai_stream` — streaming crawl via SSE-style newline-delimited JSON.
+- `crawl4ai_screenshot` — full-page PNG screenshot capture.
+- `crawl4ai_pdf` — PDF document generation.
+- `crawl4ai_status` — check Crawl4AI server health, version, and auth status.
+
 Utility:
 
-- `web_status` — show configured provider status without secrets.
+- `web_status` — show all provider configuration status without secrets.
 - `/web-status` — command version of the status check.
 
 ## Library structure
@@ -85,10 +97,11 @@ The extension is split into `lib/` modules:
 | `lib/brave.ts` | Brave Search API fetch client |
 | `lib/searxng.ts` | SearXNG metasearch fetch client |
 | `lib/firecrawl.ts` | Firecrawl API fetch client with v2→v1 fallback |
+| `lib/crawl4ai.ts` | Crawl4AI Docker API fetch client (scrape, crawl, stream, screenshot, PDF, health) |
 
 ## Retry behavior
 
-All HTTP-backed tools (`brave_search`, `searxng_search`, `firecrawl_*`) use a `withRetry` wrapper with exponential backoff (up to 2 retries, ~1s initial delay). Non-transient errors (auth, validation, not found) are never retried.
+All HTTP-backed tools (`brave_search`, `searxng_search`, `firecrawl_*`, `crawl4ai_*`) use a `withRetry` wrapper with exponential backoff (up to 2 retries, ~1s initial delay). Non-transient errors (auth, validation, not found) are never retried.
 
 ## Migration from 0.1.x
 
@@ -113,5 +126,10 @@ npm run test:unit
 - Use `firecrawl_scrape` for known URLs when extraction quality matters, pages are dynamic, links are needed, or structured JSON extraction is requested.
 - Use `firecrawl_map` before crawling to discover candidate URLs and keep crawls targeted.
 - Use `firecrawl_crawl` only when multiple pages are required; keep limits conservative and use include/exclude paths.
+- Use `crawl4ai_scrape` for JavaScript-rendered pages and when you need high-quality LLM-ready markdown.
+- Use `crawl4ai_crawl` when you need full crawl data (HTML, links, media) from multiple URLs.
+- Use `crawl4ai_screenshot` or `crawl4ai_pdf` for visual/printable snapshots of rendered pages.
+- Prefer `web_content` or `firecrawl_scrape` over Crawl4AI for simple static pages — Crawl4AI launches a full headless browser and is more resource-intensive.
 - When answering from web content, cite source URLs from result links or metadata.
 - Use `web_status` when provider configuration is uncertain or a web tool fails due to credentials/config.
+- Use `crawl4ai_status` when Crawl4AI-specific connection issues occur.

package/index.ts

@@ -9,15 +9,33 @@ import {
 	includeProjectEnv,
 	normalizeSearxngBaseUrl,
 	normalizeFirecrawlBaseUrl,
+	normalizeCrawl4aiBaseUrl,
 	loadSearxngConfig,
 	loadFirecrawlConfig,
+	loadCrawl4aiConfig,
 	HOSTED_FIRECRAWL_BASE_URL,
+	DEFAULT_CRAWL4AI_BASE_URL,
 } from "./lib/config";
-import { truncateText, formatSearchResults, formatFirecrawlScrape, formatFirecrawlSearch } from "./lib/format";
+import {
+	truncateText,
+	formatSearchResults,
+	formatFirecrawlScrape,
+	formatFirecrawlSearch,
+	formatCrawl4aiResult,
+	formatCrawl4aiStream,
+} from "./lib/format";
 import { fetchReadableContent } from "./lib/content";
 import { fetchBraveResults } from "./lib/brave";
 import { fetchSearxngResults } from "./lib/searxng";
 import { firecrawlRequest, type FirecrawlResult } from "./lib/firecrawl";
+import {
+	fetchCrawl4aiMarkdown,
+	fetchCrawl4aiCrawl,
+	fetchCrawl4aiStream,
+	fetchCrawl4aiScreenshot,
+	fetchCrawl4aiPdf,
+	fetchCrawl4aiHealth,
+} from "./lib/crawl4ai";
 
 // ---------------------------------------------------------------------------
 // Schema fragments shared across tool registrations
@@ -39,6 +57,12 @@ const searxngControlSchema = {
 	timeout_ms: Type.Optional(Type.Number({ description: "Request timeout in milliseconds." })),
 };
 
+const crawl4aiControlSchema = {
+	crawl4ai_base_url: Type.Optional(Type.String({ description: "Crawl4AI base URL override. Defaults to CRAWL4AI_BASE_URL or http://172.30.55.22:11235." })),
+	crawl4ai_api_token: Type.Optional(Type.String({ description: "Crawl4AI API token override. Prefer CRAWL4AI_API_TOKEN." })),
+	timeout_ms: Type.Optional(Type.Number({ description: "Request timeout in milliseconds." })),
+};
+
 // ---------------------------------------------------------------------------
 // Extension entry point
 // ---------------------------------------------------------------------------
@@ -361,15 +385,166 @@ export default function piWebExtension(pi: ExtensionAPI) {
 		},
 	});
 
+	// ── Crawl4AI Scrape ──────────────────────────────────────────────────
+	pi.registerTool({
+		name: "crawl4ai_scrape",
+		label: "Crawl4AI Scrape",
+		description:
+			"Extract clean LLM-ready markdown from a URL using Crawl4AI's headless browser engine, with optional content filtering (fit, raw, bm25, llm). Best for JavaScript-rendered pages and when you need high-quality markdown for AI consumption.",
+		promptSnippet: "Scrape with Crawl4AI",
+		promptGuidelines: [
+			"Prefer crawl4ai_scrape over web_content for dynamic/JS-rendered pages.",
+			"Use filter=raw for full DOM-to-markdown, filter=fit (default) for Readability-based clean content.",
+			"Use filter=bm25 with a query for relevance-ranked content extraction.",
+		],
+		parameters: Type.Object({
+			...crawl4aiControlSchema,
+			url: Type.String(),
+			filter: Type.Optional(Type.Union([Type.Literal("fit"), Type.Literal("raw"), Type.Literal("bm25"), Type.Literal("llm")], { default: "fit", description: "Content-filter strategy: fit (default), raw, bm25, or llm." })),
+			query: Type.Optional(Type.String({ description: "Query string used by BM25 or LLM filters." })),
+			content_chars: Type.Optional(Type.Number({ default: 20000 })),
+		}),
+		async execute(_id: string, params: Record<string, unknown>, signal: AbortSignal, _onUpdate: unknown, ctx: any) {
+			const config = loadCrawl4aiConfig(params, cwdFromContext(ctx), includeProjectEnv(ctx));
+			const filter = (params.filter as string) || "fit";
+			const query = params.query as string | undefined;
+			const result = await fetchCrawl4aiMarkdown(config, params.url as string, filter, query, signal);
+			const text = `# Crawl4AI Markdown (filter: ${filter})\nURL: ${params.url}\n\n${result.markdown.slice(0, (params.content_chars as number) ?? 20000)}`;
+			return { content: [{ type: "text" as const, text: truncateText(text) }], details: result };
+		},
+	});
+
+	// ── Crawl4AI Crawl ──────────────────────────────────────────────────
+	pi.registerTool({
+		name: "crawl4ai_crawl",
+		label: "Crawl4AI Crawl",
+		description:
+			"Crawl one or more URLs with Crawl4AI's headless browser, returning full crawl data including markdown, HTML, links, and media. Supports up to 100 URLs per request.",
+		promptSnippet: "Crawl with Crawl4AI",
+		promptGuidelines: [
+			"Use crawl4ai_crawl when you need full crawl data (HTML, links, media) from multiple URLs.",
+			"For single-URL markdown-only extraction, prefer crawl4ai_scrape.",
+			"Keep to 10 or fewer URLs per request for reasonable response times.",
+		],
+		parameters: Type.Object({
+			...crawl4aiControlSchema,
+			urls: Type.Array(Type.String(), { minItems: 1, maxItems: 100, description: "URLs to crawl (1-100)." }),
+			browser_config: Type.Optional(Type.Any({ description: "Optional BrowserConfig JSON object." })),
+			crawler_config: Type.Optional(Type.Any({ description: "Optional CrawlerRunConfig JSON object." })),
+			content_chars: Type.Optional(Type.Number({ default: 20000 })),
+		}),
+		async execute(_id: string, params: Record<string, unknown>, signal: AbortSignal, _onUpdate: unknown, ctx: any) {
+			const config = loadCrawl4aiConfig(params, cwdFromContext(ctx), includeProjectEnv(ctx));
+			const urls = params.urls as string[];
+			const browserConfig = params.browser_config as Record<string, unknown> | undefined;
+			const crawlerConfig = params.crawler_config as Record<string, unknown> | undefined;
+			const result = await fetchCrawl4aiCrawl(config, urls, browserConfig, crawlerConfig, signal);
+			const text = formatCrawl4aiResult(result as unknown as Record<string, unknown>, (params.content_chars as number) ?? 20000);
+			return { content: [{ type: "text" as const, text: truncateText(text) }], details: result };
+		},
+	});
+
+	// ── Crawl4AI Stream ─────────────────────────────────────────────────
+	pi.registerTool({
+		name: "crawl4ai_stream",
+		label: "Crawl4AI Stream",
+		description:
+			"Crawl one or more URLs with Crawl4AI using streaming response (SSE-style JSON lines). Results arrive as each URL is processed, useful for long-running multi-URL crawls.",
+		promptSnippet: "Stream crawl with Crawl4AI",
+		promptGuidelines: [
+			"Use crawl4ai_stream for large batch crawls where you want results incrementally.",
+			"The response is newline-delimited JSON; each line is a complete CrawlResult.",
+			"The stream ends with a line containing {\"status\": \"completed\"}.",
+		],
+		parameters: Type.Object({
+			...crawl4aiControlSchema,
+			urls: Type.Array(Type.String(), { minItems: 1, maxItems: 100, description: "URLs to crawl (1-100)." }),
+			browser_config: Type.Optional(Type.Any({ description: "Optional BrowserConfig JSON object." })),
+			crawler_config: Type.Optional(Type.Any({ description: "Optional CrawlerRunConfig JSON object." })),
+		}),
+		async execute(_id: string, params: Record<string, unknown>, signal: AbortSignal, _onUpdate: unknown, ctx: any) {
+			const config = loadCrawl4aiConfig(params, cwdFromContext(ctx), includeProjectEnv(ctx));
+			const urls = params.urls as string[];
+			const browserConfig = params.browser_config as Record<string, unknown> | undefined;
+			const crawlerConfig = params.crawler_config as Record<string, unknown> | undefined;
+			const raw = await fetchCrawl4aiStream(config, urls, browserConfig, crawlerConfig, signal);
+			const text = formatCrawl4aiStream(raw);
+			return { content: [{ type: "text" as const, text: truncateText(text) }], details: { raw_length: raw.length } };
+		},
+	});
+
+	// ── Crawl4AI Screenshot ─────────────────────────────────────────────
+	pi.registerTool({
+		name: "crawl4ai_screenshot",
+		label: "Crawl4AI Screenshot",
+		description:
+			"Capture a full-page PNG screenshot of a URL using Crawl4AI's headless browser. Returns a base64-encoded screenshot image.",
+		promptSnippet: "Screenshot with Crawl4AI",
+		promptGuidelines: [
+			"Use crawl4ai_screenshot when you need a visual snapshot of a rendered page.",
+			"The screenshot is returned as a base64-encoded PNG string.",
+			"Use the wait_for parameter to delay capture for dynamic content to render.",
+		],
+		parameters: Type.Object({
+			...crawl4aiControlSchema,
+			url: Type.String(),
+			wait_for: Type.Optional(Type.Number({ default: 2, description: "Seconds to wait before capturing screenshot." })),
+			wait_for_images: Type.Optional(Type.Boolean({ default: false, description: "Wait for images to load before capture." })),
+		}),
+		async execute(_id: string, params: Record<string, unknown>, signal: AbortSignal, _onUpdate: unknown, ctx: any) {
+			const config = loadCrawl4aiConfig(params, cwdFromContext(ctx), includeProjectEnv(ctx));
+			const result = await fetchCrawl4aiScreenshot(config, params.url as string, params.wait_for as number | undefined, params.wait_for_images as boolean | undefined, signal);
+			const screenshot = result.screenshot as string | undefined;
+			const artifactUrl = result.url as string | undefined;
+			const mime = result.mime as string | undefined;
+			const size = result.size as number | undefined;
+			let text = `Screenshot: ${params.url}\n`;
+			if (screenshot) text += `Data: base64 PNG (${screenshot.length} chars)\n`;
+			if (artifactUrl) text += `Artifact: ${artifactUrl}\n`;
+			if (mime) text += `MIME: ${mime}\n`;
+			if (size) text += `Size: ${size} bytes\n`;
+			return { content: [{ type: "text" as const, text: truncateText(text) }], details: { ...result, url: params.url } };
+		},
+	});
+
+	// ── Crawl4AI PDF ────────────────────────────────────────────────────
+	pi.registerTool({
+		name: "crawl4ai_pdf",
+		label: "Crawl4AI PDF",
+		description:
+			"Generate a PDF document of a URL using Crawl4AI's headless browser. Returns a base64-encoded PDF suitable for saving or further processing.",
+		promptSnippet: "PDF with Crawl4AI",
+		promptGuidelines: [
+			"Use crawl4ai_pdf when you need a printable or archivable snapshot of a page.",
+			"The PDF is returned as a base64-encoded string.",
+		],
+		parameters: Type.Object({
+			...crawl4aiControlSchema,
+			url: Type.String(),
+		}),
+		async execute(_id: string, params: Record<string, unknown>, signal: AbortSignal, _onUpdate: unknown, ctx: any) {
+			const config = loadCrawl4aiConfig(params, cwdFromContext(ctx), includeProjectEnv(ctx));
+			const result = await fetchCrawl4aiPdf(config, params.url as string, signal);
+			const pdf = result.pdf as string | undefined;
+			const artifactUrl = result.url as string | undefined;
+			const size = result.size as number | undefined;
+			let text = `PDF: ${params.url}\n`;
+			if (pdf) text += `Data: base64 PDF (${pdf.length} chars)\n`;
+			if (artifactUrl) text += `Artifact: ${artifactUrl}\n`;
+			if (size) text += `Size: ${size} bytes\n`;
+			return { content: [{ type: "text" as const, text: truncateText(text) }], details: { ...result, url: params.url } };
+		},
+	});
+
 	// ── Web Status ───────────────────────────────────────────────────────
 	pi.registerTool({
 		name: "web_status",
 		label: "Web Provider Status",
 		description:
-			"Show Brave, SearXNG, and Firecrawl configuration status without printing secrets.",
+			"Show Brave, SearXNG, Firecrawl, and Crawl4AI configuration status without printing secrets.",
 		promptSnippet: "Check web provider configuration",
 		promptGuidelines: [
-			"Use when web tools fail due to missing credentials/config or before comparing Brave, SearXNG, and Firecrawl availability.",
+			"Use when web tools fail due to missing credentials/config or before comparing Brave, SearXNG, Firecrawl, and Crawl4AI availability.",
 			"Never print API key values; this tool reports only presence and source.",
 		],
 		parameters: Type.Object({}),
@@ -380,6 +555,8 @@ export default function piWebExtension(pi: ExtensionAPI) {
 			const searxngUrl = findEnvValue("SEARXNG_BASE_URL", cwd, trusted);
 			const fireKey = findEnvValue("FIRECRAWL_API_KEY", cwd, trusted);
 			const fireUrl = findEnvValue("FIRECRAWL_API_URL", cwd, trusted);
+			const c4aiUrl = findEnvValue("CRAWL4AI_BASE_URL", cwd, trusted);
+			const c4aiToken = findEnvValue("CRAWL4AI_API_TOKEN", cwd, trusted);
 			const status = {
 				brave: { apiKeyFound: Boolean(braveKey.value), apiKeySource: braveKey.value ? braveKey.source : "not set" },
 				searxng: {
@@ -393,11 +570,46 @@ export default function piWebExtension(pi: ExtensionAPI) {
 					apiKeySource: fireKey.value ? fireKey.source : "not set",
 					hostedMode: !fireUrl.value || normalizeFirecrawlBaseUrl(fireUrl.value).startsWith(HOSTED_FIRECRAWL_BASE_URL),
 				},
+				crawl4ai: {
+					baseUrl: normalizeCrawl4aiBaseUrl(c4aiUrl.value),
+					baseUrlSource: c4aiUrl.source || "default",
+					apiTokenFound: Boolean(c4aiToken.value),
+					apiTokenSource: c4aiToken.value ? c4aiToken.source : "not set",
+				},
 			};
 			return { content: [{ type: "text" as const, text: JSON.stringify(status, null, 2) }], details: status };
 		},
 	});
 
+	// ── Crawl4AI Status ─────────────────────────────────────────────────
+	pi.registerTool({
+		name: "crawl4ai_status",
+		label: "Crawl4AI Server Status",
+		description:
+			"Check Crawl4AI server health and version. Connects to the configured Crawl4AI endpoint to verify availability.",
+		promptSnippet: "Check Crawl4AI server status",
+		promptGuidelines: [
+			"Use crawl4ai_status when Crawl4AI tools return connection errors.",
+			"Shows server version, response timestamp, and whether auth is configured.",
+		],
+		parameters: Type.Object({
+			...crawl4aiControlSchema,
+		}),
+		async execute(_id: string, params: Record<string, unknown>, signal: AbortSignal, _onUpdate: unknown, ctx: any) {
+			const config = loadCrawl4aiConfig(params, cwdFromContext(ctx), includeProjectEnv(ctx));
+			const health = await fetchCrawl4aiHealth(config, signal);
+			const text = [
+				`Crawl4AI Server Status`,
+				`Endpoint: ${config.baseUrl}`,
+				`Status: ${health.status || "unknown"}`,
+				`Version: ${health.version || "unknown"}`,
+				`Timestamp: ${health.timestamp ? new Date(health.timestamp * 1000).toISOString() : "unknown"}`,
+				`Auth token: ${config.apiToken ? "configured" : "not set (loopback only)"}`,
+			].join("\n");
+			return { content: [{ type: "text" as const, text: text }], details: { config: { baseUrl: config.baseUrl, apiTokenConfigured: Boolean(config.apiToken) }, health } };
+		},
+	});
+
 	// ── /web-status command ──────────────────────────────────────────────
 	pi.registerCommand("web-status", {
 		description: "Show web provider configuration status",
@@ -408,8 +620,17 @@ export default function piWebExtension(pi: ExtensionAPI) {
 			const searxngUrl = findEnvValue("SEARXNG_BASE_URL", cwd, trusted);
 			const fireKey = findEnvValue("FIRECRAWL_API_KEY", cwd, trusted);
 			const fireUrl = findEnvValue("FIRECRAWL_API_URL", cwd, trusted);
+			const c4aiUrl = findEnvValue("CRAWL4AI_BASE_URL", cwd, trusted);
+			const c4aiToken = findEnvValue("CRAWL4AI_API_TOKEN", cwd, trusted);
 			ctx.ui.notify(
-				`Brave API key: ${braveKey.value ? `found (${braveKey.source})` : "not set"}\nSearXNG base URL: ${normalizeSearxngBaseUrl(searxngUrl.value)} (${searxngUrl.source || "default local"})\nFirecrawl base URL: ${normalizeFirecrawlBaseUrl(fireUrl.value)}\nFirecrawl API key: ${fireKey.value ? `found (${fireKey.source})` : "not set"}`,
+				[
+					`Brave API key: ${braveKey.value ? `found (${braveKey.source})` : "not set"}`,
+					`SearXNG base URL: ${normalizeSearxngBaseUrl(searxngUrl.value)} (${searxngUrl.source || "default local"})`,
+					`Firecrawl base URL: ${normalizeFirecrawlBaseUrl(fireUrl.value)}`,
+					`Firecrawl API key: ${fireKey.value ? `found (${fireKey.source})` : "not set"}`,
+					`Crawl4AI base URL: ${normalizeCrawl4aiBaseUrl(c4aiUrl.value)} (${c4aiUrl.source || "default"})`,
+					`Crawl4AI API token: ${c4aiToken.value ? `found (${c4aiToken.source})` : "not set"}`,
+				].join("\n"),
 				"info",
 			);
 		},

package/lib/config.ts

@@ -12,6 +12,7 @@ export const OUTPUT_MAX_BYTES = 50 * 1024;
 export const OUTPUT_MAX_LINES = 2_000;
 export const HOSTED_FIRECRAWL_BASE_URL = "https://api.firecrawl.dev/v2";
 export const DEFAULT_SEARXNG_BASE_URL = "http://172.30.55.22:8888";
+export const DEFAULT_CRAWL4AI_BASE_URL = "http://172.30.55.22:11235";
 
 // ---------------------------------------------------------------------------
 // Environment loading
@@ -115,6 +116,39 @@ export function loadSearxngConfig(params: Record<string, unknown> = {}, cwd = pr
 	return { baseUrl, source };
 }
 
+// ---------------------------------------------------------------------------
+// Crawl4AI config
+// ---------------------------------------------------------------------------
+
+export interface Crawl4aiConfig {
+	baseUrl: string;
+	apiToken: string;
+	timeoutMs: number;
+}
+
+export function normalizeCrawl4aiBaseUrl(raw?: string): string {
+	let value = (raw || DEFAULT_CRAWL4AI_BASE_URL).trim();
+	if (!/^https?:\/\//i.test(value)) {
+		value = /^(localhost|127\.|10\.|172\.(1[6-9]|2\d|3[01])\.|192\.168\.)/i.test(value)
+			? `http://${value}`
+			: `https://${value}`;
+	}
+	return value.replace(/\/+$/, "");
+}
+
+export function loadCrawl4aiConfig(params: Record<string, unknown> = {}, cwd = process.cwd(), includeCwdEnv = false): Crawl4aiConfig {
+	const baseUrlLookup = findEnvValue("CRAWL4AI_BASE_URL", cwd, includeCwdEnv);
+	const apiTokenLookup = findEnvValue("CRAWL4AI_API_TOKEN", cwd, includeCwdEnv);
+	const explicitBaseUrl = params.crawl4ai_base_url as string | undefined;
+	const explicitApiToken = params.crawl4ai_api_token as string | undefined;
+	const baseUrl = normalizeCrawl4aiBaseUrl(explicitBaseUrl || baseUrlLookup.value);
+	const apiToken = explicitApiToken || apiTokenLookup.value || "";
+	const timeoutValue = (params.timeout_ms as number) || findEnvValue("CRAWL4AI_TIMEOUT_MS", cwd, includeCwdEnv).value;
+	const timeoutMs = timeoutValue ? Number.parseInt(String(timeoutValue), 10) : 60000;
+	if (!Number.isInteger(timeoutMs) || timeoutMs < 1000) throw new Error("CRAWL4AI_TIMEOUT_MS/timeout_ms must be an integer >= 1000.");
+	return { baseUrl, apiToken, timeoutMs };
+}
+
 // ---------------------------------------------------------------------------
 // Firecrawl config
 // ---------------------------------------------------------------------------

package/lib/crawl4ai.ts

@@ -0,0 +1,225 @@
+// Crawl4AI API client.
+// Talks to a self-hosted Crawl4AI Docker server (unclecode/crawl4ai).
+// Endpoints: /crawl, /crawl/stream, /md, /screenshot, /pdf, /health
+
+import type { Crawl4aiConfig } from "./config";
+import { signalWithTimeout, withRetry } from "./retry";
+
+// ---------------------------------------------------------------------------
+// Error
+// ---------------------------------------------------------------------------
+
+export class Crawl4aiHttpError extends Error {
+	status: number;
+	constructor(status: number, statusText: string, text: string) {
+		super(`HTTP ${status}: ${statusText}${text ? `\n${text}` : ""}`);
+		this.status = status;
+	}
+}
+
+// ---------------------------------------------------------------------------
+// Low-level HTTP helpers
+// ---------------------------------------------------------------------------
+
+async function crawl4aiRequestJson(
+	config: Crawl4aiConfig,
+	method: string,
+	endpoint: string,
+	body?: unknown,
+	signal?: AbortSignal,
+): Promise<Record<string, unknown>> {
+	const headers: Record<string, string> = { Accept: "application/json" };
+	if (body !== undefined) headers["Content-Type"] = "application/json";
+	if (config.apiToken) headers.Authorization = `Bearer ${config.apiToken}`;
+	const response = await fetch(`${config.baseUrl}${endpoint}`, {
+		method,
+		headers,
+		body: body === undefined ? undefined : JSON.stringify(body),
+		signal: signalWithTimeout(config.timeoutMs, signal),
+	});
+	const text = await response.text();
+	if (!response.ok) throw new Crawl4aiHttpError(response.status, response.statusText, text);
+	return text ? JSON.parse(text) : { success: true };
+}
+
+/**
+ * Crawl4AI API request with retry.
+ */
+export async function crawl4aiRequest(
+	config: Crawl4aiConfig,
+	method: string,
+	endpoint: string,
+	body?: unknown,
+	signal?: AbortSignal,
+): Promise<Record<string, unknown>> {
+	return withRetry(async () => {
+		return await crawl4aiRequestJson(config, method, endpoint, body, signal);
+	}, { maxRetries: 2, retryableErrors: ["timeout", "econn", "etimedout", "network", "socket"] });
+}
+
+// ---------------------------------------------------------------------------
+// Convenience wrappers
+// ---------------------------------------------------------------------------
+
+export interface Crawl4aiMediaItem {
+	url?: string;
+	src?: string;
+	alt?: string;
+	type?: string;
+	score?: number;
+}
+
+export interface Crawl4aiLinkItem {
+	href?: string;
+	text?: string;
+	domain?: string;
+}
+
+export interface Crawl4aiMarkdownResult {
+	raw_markdown?: string;
+	markdown_with_citations?: string;
+	references_markdown?: string;
+	fit_markdown?: string;
+	fit_html?: string;
+}
+
+export interface Crawl4aiResult {
+	url?: string;
+	html?: string;
+	success?: boolean;
+	cleaned_html?: string;
+	markdown?: string | Crawl4aiMarkdownResult;
+	media?: { images?: Crawl4aiMediaItem[]; videos?: Crawl4aiMediaItem[] };
+	links?: { internal?: Crawl4aiLinkItem[]; external?: Crawl4aiLinkItem[] };
+	extracted_content?: string;
+	screenshot?: string;
+	pdf?: string;
+	metadata?: Record<string, unknown>;
+	error_message?: string;
+	status_code?: number;
+	redirected_url?: string;
+	[key: string]: unknown;
+}
+
+/**
+ * Fetch clean markdown via POST /md.
+ * filter_ — one of "fit", "raw", "bm25", "llm"
+ * query — optional query for bm25/llm filters
+ */
+export async function fetchCrawl4aiMarkdown(
+	config: Crawl4aiConfig,
+	url: string,
+	filter_ = "fit",
+	query?: string,
+	signal?: AbortSignal,
+): Promise<{ markdown: string; success: boolean }> {
+	const body: Record<string, unknown> = { url, f: filter_ };
+	if (query) body.q = query;
+	const result = await crawl4aiRequest(config, "POST", "/md", body, signal);
+	return {
+		markdown: (result.markdown as string) || "",
+		success: result.success as boolean,
+	};
+}
+
+/**
+ * Crawl one or more URLs via POST /crawl.
+ */
+export async function fetchCrawl4aiCrawl(
+	config: Crawl4aiConfig,
+	urls: string[],
+	browserConfig?: Record<string, unknown>,
+	crawlerConfig?: Record<string, unknown>,
+	signal?: AbortSignal,
+): Promise<{ success: boolean; results?: Crawl4aiResult[] }> {
+	const body: Record<string, unknown> = { urls };
+	if (browserConfig) body.browser_config = browserConfig;
+	if (crawlerConfig) body.crawler_config = crawlerConfig;
+	const result = await crawl4aiRequest(config, "POST", "/crawl", body, signal);
+	return {
+		success: result.success as boolean,
+		results: (result.results as Crawl4aiResult[]) || [],
+	};
+}
+
+/**
+ * Crawl one or more URLs via POST /crawl/stream.
+ * Returns newline-delimited JSON lines as a string.
+ */
+export async function fetchCrawl4aiStream(
+	config: Crawl4aiConfig,
+	urls: string[],
+	browserConfig?: Record<string, unknown>,
+	crawlerConfig?: Record<string, unknown>,
+	signal?: AbortSignal,
+): Promise<string> {
+	const body: Record<string, unknown> = { urls };
+	if (browserConfig) body.browser_config = browserConfig;
+	if (crawlerConfig) body.crawler_config = crawlerConfig;
+	const headers: Record<string, string> = { Accept: "application/json" };
+	if (config.apiToken) headers.Authorization = `Bearer ${config.apiToken}`;
+	headers["Content-Type"] = "application/json";
+	const response = await fetch(`${config.baseUrl}/crawl/stream`, {
+		method: "POST",
+		headers,
+		body: JSON.stringify(body),
+		signal: signalWithTimeout(config.timeoutMs, signal),
+	});
+	if (!response.ok) {
+		const text = await response.text();
+		throw new Crawl4aiHttpError(response.status, response.statusText, text);
+	}
+	return await response.text();
+}
+
+/**
+ * Capture a screenshot via POST /screenshot.
+ */
+export async function fetchCrawl4aiScreenshot(
+	config: Crawl4aiConfig,
+	url: string,
+	waitFor?: number,
+	waitForImages?: boolean,
+	signal?: AbortSignal,
+): Promise<Record<string, unknown>> {
+	const body: Record<string, unknown> = { url };
+	if (waitFor !== undefined) body.screenshot_wait_for = waitFor;
+	if (waitForImages !== undefined) body.wait_for_images = waitForImages;
+	return await crawl4aiRequest(config, "POST", "/screenshot", body, signal);
+}
+
+/**
+ * Generate a PDF via POST /pdf.
+ */
+export async function fetchCrawl4aiPdf(
+	config: Crawl4aiConfig,
+	url: string,
+	signal?: AbortSignal,
+): Promise<Record<string, unknown>> {
+	return await crawl4aiRequest(config, "POST", "/pdf", { url }, signal);
+}
+
+/**
+ * Health check via GET /health.
+ */
+export async function fetchCrawl4aiHealth(
+	config: Crawl4aiConfig,
+	signal?: AbortSignal,
+): Promise<{ status?: string; version?: string; timestamp?: number }> {
+	const headers: Record<string, string> = { Accept: "application/json" };
+	if (config.apiToken) headers.Authorization = `Bearer ${config.apiToken}`;
+	const timeoutMs = Math.min(config.timeoutMs, 10000);
+	const response = await fetch(`${config.baseUrl}/health`, {
+		method: "GET",
+		headers,
+		signal: signalWithTimeout(timeoutMs, signal),
+	});
+	const text = await response.text();
+	if (!response.ok) throw new Crawl4aiHttpError(response.status, response.statusText, text);
+	const data = text ? JSON.parse(text) : {};
+	return {
+		status: data.status as string | undefined,
+		version: data.version as string | undefined,
+		timestamp: data.timestamp as number | undefined,
+	};
+}

package/lib/format.ts

@@ -90,6 +90,110 @@ export function formatFirecrawlScrape(data: Record<string, unknown>, maxChars =
 	return parts.join("\n\n") || JSON.stringify(data, null, 2);
 }
 
+// ---------------------------------------------------------------------------
+// Crawl4AI result formatting
+// ---------------------------------------------------------------------------
+
+function extractMarkdown(md: unknown): string {
+	if (!md) return "";
+	if (typeof md === "string") return md;
+	if (typeof md === "object") {
+		const obj = md as Record<string, unknown>;
+		return (obj.fit_markdown as string) || (obj.raw_markdown as string) || "";
+	}
+	return "";
+}
+
+function extractLinks(links: unknown): string[] {
+	if (!links) return [];
+	const arr = Array.isArray(links) ? links : typeof links === "object" ? Object.values(links) : [];
+	return arr.flatMap((item: any) => {
+		if (typeof item === "string") return [item];
+		if (item?.href) return [item.href];
+		if (item?.url) return [item.url];
+		return [];
+	});
+}
+
+export function formatCrawl4aiResult(data: Record<string, unknown>, maxChars = 20000): string {
+	// Handle wrapped response: { results: [...] }
+	const rawResults = (data.results as unknown[]) || [data];
+	return rawResults
+		.map((raw: any, i: number) => {
+			const parts: string[] = [];
+			const url = raw.url || raw.redirected_url || "";
+			const success = raw.success !== false;
+			const statusCode = raw.status_code || raw.metadata?.statusCode;
+
+			if (rawResults.length > 1) parts.push(`=== Result ${i + 1} ===`);
+			if (url) parts.push(`URL: ${url}`);
+			if (statusCode) parts.push(`Status: ${statusCode}`);
+			if (!success) {
+				parts.push(`Error: ${raw.error_message || "Crawl failed"}`);
+				return parts.join("\n");
+			}
+
+			// Markdown content
+			const md = extractMarkdown(raw.markdown);
+			if (md) {
+				const truncated = md.slice(0, maxChars);
+				parts.push(truncated);
+				if (truncated.length < md.length) parts.push("[Markdown truncated...]");
+			} else if (raw.extracted_content) {
+				const ec =
+					typeof raw.extracted_content === "string"
+						? raw.extracted_content
+						: JSON.stringify(raw.extracted_content, null, 2);
+				parts.push(ec.slice(0, maxChars));
+			} else if (raw.cleaned_html) {
+				parts.push(`(Cleaned HTML: ${raw.cleaned_html.length} chars)`);
+			}
+
+			// Links
+			const links = raw.links as Record<string, unknown> | undefined;
+			if (links) {
+				const internalLinks = extractLinks(links.internal);
+				const externalLinks = extractLinks(links.external);
+				const allLinks = [...internalLinks, ...externalLinks];
+				if (allLinks.length) {
+					parts.push(`Links (${allLinks.length}): ${allLinks.slice(0, 50).join(", ")}`);
+				}
+			}
+
+			// Media
+			const media = raw.media as Record<string, unknown> | undefined;
+			if (media) {
+				const imageCount = (media.images as unknown[])?.length || 0;
+				const videoCount = (media.videos as unknown[])?.length || 0;
+				if (imageCount || videoCount) {
+					parts.push(`Media: ${imageCount} images, ${videoCount} videos`);
+				}
+			}
+
+			return parts.join("\n\n");
+		})
+		.join("\n\n---\n\n") || "(No data)";
+}
+
+export function formatCrawl4aiStream(lines: string, _maxChars = 20000): string {
+	const results: string[] = [];
+	for (const line of lines.split("\n")) {
+		const trimmed = line.trim();
+		if (!trimmed) continue;
+		try {
+			const parsed = JSON.parse(trimmed);
+			if (parsed.status === "completed") {
+				results.push("[Stream complete]");
+				break;
+			}
+			results.push(formatCrawl4aiResult(parsed, _maxChars));
+		} catch {
+			results.push(trimmed.slice(0, 1000));
+		}
+	}
+	return results.join("\n\n---\n\n");
+}
+
 export function formatFirecrawlSearch(data: Record<string, unknown>, maxChars = 5000): string {
 	const rootData = data.data as Record<string, unknown> | undefined;
 	const web = (rootData?.web as unknown[]) || [];

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@bacnh85/pi-web",
-  "version": "0.2.1",
-  "description": "Pi extension for web search, page extraction, and Firecrawl scraping/crawling.",
+  "version": "0.3.0",
+  "description": "Pi extension for web search, page extraction, Firecrawl scraping/crawling, and Crawl4AI headless browser crawling.",
   "type": "module",
   "license": "MIT",
   "publishConfig": {
@@ -19,7 +19,9 @@
     "web-search",
     "brave-search",
     "firecrawl",
-    "scraping"
+    "crawl4ai",
+    "scraping",
+    "crawling"
   ],
   "scripts": {
     "test": "npx mocha",