@bacnh85/pi-web 0.2.1 → 0.4.1

package/lib/config.ts

@@ -12,6 +12,7 @@ export const OUTPUT_MAX_BYTES = 50 * 1024;
 export const OUTPUT_MAX_LINES = 2_000;
 export const HOSTED_FIRECRAWL_BASE_URL = "https://api.firecrawl.dev/v2";
 export const DEFAULT_SEARXNG_BASE_URL = "http://172.30.55.22:8888";
+export const DEFAULT_CRAWL4AI_API_URL = "http://172.30.55.22:11235";
 
 // ---------------------------------------------------------------------------
 // Environment loading
@@ -115,6 +116,39 @@ export function loadSearxngConfig(params: Record<string, unknown> = {}, cwd = pr
 	return { baseUrl, source };
 }
 
+// ---------------------------------------------------------------------------
+// Crawl4AI config
+// ---------------------------------------------------------------------------
+
+export interface Crawl4aiConfig {
+	baseUrl: string;
+	apiToken: string;
+	timeoutMs: number;
+}
+
+export function normalizeCrawl4aiApiUrl(raw?: string): string {
+	let value = (raw || DEFAULT_CRAWL4AI_API_URL).trim();
+	if (!/^https?:\/\//i.test(value)) {
+		value = /^(localhost|127\.|10\.|172\.(1[6-9]|2\d|3[01])\.|192\.168\.)/i.test(value)
+			? `http://${value}`
+			: `https://${value}`;
+	}
+	return value.replace(/\/+$/, "");
+}
+
+export function loadCrawl4aiConfig(params: Record<string, unknown> = {}, cwd = process.cwd(), includeCwdEnv = false): Crawl4aiConfig {
+	const apiUrlLookup = findEnvValue("CRAWL4AI_API_URL", cwd, includeCwdEnv);
+	const apiTokenLookup = findEnvValue("CRAWL4AI_API_TOKEN", cwd, includeCwdEnv);
+	const explicitApiUrl = params.crawl4ai_api_url as string | undefined;
+	const explicitApiToken = params.crawl4ai_api_token as string | undefined;
+	const baseUrl = normalizeCrawl4aiApiUrl(explicitApiUrl || apiUrlLookup.value);
+	const apiToken = explicitApiToken || apiTokenLookup.value || "";
+	const timeoutValue = (params.timeout_ms as number) || findEnvValue("CRAWL4AI_API_TIMEOUT_MS", cwd, includeCwdEnv).value;
+	const timeoutMs = timeoutValue ? Number.parseInt(String(timeoutValue), 10) : 60000;
+	if (!Number.isInteger(timeoutMs) || timeoutMs < 1000) throw new Error("CRAWL4AI_API_TIMEOUT_MS/timeout_ms must be an integer >= 1000.");
+	return { baseUrl, apiToken, timeoutMs };
+}
+
 // ---------------------------------------------------------------------------
 // Firecrawl config
 // ---------------------------------------------------------------------------

package/lib/crawl4ai.ts

@@ -0,0 +1,225 @@
+// Crawl4AI API client.
+// Talks to a self-hosted Crawl4AI Docker server (unclecode/crawl4ai).
+// Endpoints: /crawl, /crawl/stream, /md, /screenshot, /pdf, /health
+
+import type { Crawl4aiConfig } from "./config";
+import { signalWithTimeout, withRetry } from "./retry";
+
+// ---------------------------------------------------------------------------
+// Error
+// ---------------------------------------------------------------------------
+
+export class Crawl4aiHttpError extends Error {
+	status: number;
+	constructor(status: number, statusText: string, text: string) {
+		super(`HTTP ${status}: ${statusText}${text ? `\n${text}` : ""}`);
+		this.status = status;
+	}
+}
+
+// ---------------------------------------------------------------------------
+// Low-level HTTP helpers
+// ---------------------------------------------------------------------------
+
+async function crawl4aiRequestJson(
+	config: Crawl4aiConfig,
+	method: string,
+	endpoint: string,
+	body?: unknown,
+	signal?: AbortSignal,
+): Promise<Record<string, unknown>> {
+	const headers: Record<string, string> = { Accept: "application/json" };
+	if (body !== undefined) headers["Content-Type"] = "application/json";
+	if (config.apiToken) headers.Authorization = `Bearer ${config.apiToken}`;
+	const response = await fetch(`${config.baseUrl}${endpoint}`, {
+		method,
+		headers,
+		body: body === undefined ? undefined : JSON.stringify(body),
+		signal: signalWithTimeout(config.timeoutMs, signal),
+	});
+	const text = await response.text();
+	if (!response.ok) throw new Crawl4aiHttpError(response.status, response.statusText, text);
+	return text ? JSON.parse(text) : { success: true };
+}
+
+/**
+ * Crawl4AI API request with retry.
+ */
+export async function crawl4aiRequest(
+	config: Crawl4aiConfig,
+	method: string,
+	endpoint: string,
+	body?: unknown,
+	signal?: AbortSignal,
+): Promise<Record<string, unknown>> {
+	return withRetry(async () => {
+		return await crawl4aiRequestJson(config, method, endpoint, body, signal);
+	}, { maxRetries: 2, retryableErrors: ["timeout", "econn", "etimedout", "network", "socket"] });
+}
+
+// ---------------------------------------------------------------------------
+// Convenience wrappers
+// ---------------------------------------------------------------------------
+
+export interface Crawl4aiMediaItem {
+	url?: string;
+	src?: string;
+	alt?: string;
+	type?: string;
+	score?: number;
+}
+
+export interface Crawl4aiLinkItem {
+	href?: string;
+	text?: string;
+	domain?: string;
+}
+
+export interface Crawl4aiMarkdownResult {
+	raw_markdown?: string;
+	markdown_with_citations?: string;
+	references_markdown?: string;
+	fit_markdown?: string;
+	fit_html?: string;
+}
+
+export interface Crawl4aiResult {
+	url?: string;
+	html?: string;
+	success?: boolean;
+	cleaned_html?: string;
+	markdown?: string | Crawl4aiMarkdownResult;
+	media?: { images?: Crawl4aiMediaItem[]; videos?: Crawl4aiMediaItem[] };
+	links?: { internal?: Crawl4aiLinkItem[]; external?: Crawl4aiLinkItem[] };
+	extracted_content?: string;
+	screenshot?: string;
+	pdf?: string;
+	metadata?: Record<string, unknown>;
+	error_message?: string;
+	status_code?: number;
+	redirected_url?: string;
+	[key: string]: unknown;
+}
+
+/**
+ * Fetch clean markdown via POST /md.
+ * filter_ — one of "fit", "raw", "bm25", "llm"
+ * query — optional query for bm25/llm filters
+ */
+export async function fetchCrawl4aiMarkdown(
+	config: Crawl4aiConfig,
+	url: string,
+	filter_ = "fit",
+	query?: string,
+	signal?: AbortSignal,
+): Promise<{ markdown: string; success: boolean }> {
+	const body: Record<string, unknown> = { url, f: filter_ };
+	if (query) body.q = query;
+	const result = await crawl4aiRequest(config, "POST", "/md", body, signal);
+	return {
+		markdown: (result.markdown as string) || "",
+		success: result.success as boolean,
+	};
+}
+
+/**
+ * Crawl one or more URLs via POST /crawl.
+ */
+export async function fetchCrawl4aiCrawl(
+	config: Crawl4aiConfig,
+	urls: string[],
+	browserConfig?: Record<string, unknown>,
+	crawlerConfig?: Record<string, unknown>,
+	signal?: AbortSignal,
+): Promise<{ success: boolean; results?: Crawl4aiResult[] }> {
+	const body: Record<string, unknown> = { urls };
+	if (browserConfig) body.browser_config = browserConfig;
+	if (crawlerConfig) body.crawler_config = crawlerConfig;
+	const result = await crawl4aiRequest(config, "POST", "/crawl", body, signal);
+	return {
+		success: result.success as boolean,
+		results: (result.results as Crawl4aiResult[]) || [],
+	};
+}
+
+/**
+ * Crawl one or more URLs via POST /crawl/stream.
+ * Returns newline-delimited JSON lines as a string.
+ */
+export async function fetchCrawl4aiStream(
+	config: Crawl4aiConfig,
+	urls: string[],
+	browserConfig?: Record<string, unknown>,
+	crawlerConfig?: Record<string, unknown>,
+	signal?: AbortSignal,
+): Promise<string> {
+	const body: Record<string, unknown> = { urls };
+	if (browserConfig) body.browser_config = browserConfig;
+	if (crawlerConfig) body.crawler_config = crawlerConfig;
+	const headers: Record<string, string> = { Accept: "application/json" };
+	if (config.apiToken) headers.Authorization = `Bearer ${config.apiToken}`;
+	headers["Content-Type"] = "application/json";
+	const response = await fetch(`${config.baseUrl}/crawl/stream`, {
+		method: "POST",
+		headers,
+		body: JSON.stringify(body),
+		signal: signalWithTimeout(config.timeoutMs, signal),
+	});
+	if (!response.ok) {
+		const text = await response.text();
+		throw new Crawl4aiHttpError(response.status, response.statusText, text);
+	}
+	return await response.text();
+}
+
+/**
+ * Capture a screenshot via POST /screenshot.
+ */
+export async function fetchCrawl4aiScreenshot(
+	config: Crawl4aiConfig,
+	url: string,
+	waitFor?: number,
+	waitForImages?: boolean,
+	signal?: AbortSignal,
+): Promise<Record<string, unknown>> {
+	const body: Record<string, unknown> = { url };
+	if (waitFor !== undefined) body.screenshot_wait_for = waitFor;
+	if (waitForImages !== undefined) body.wait_for_images = waitForImages;
+	return await crawl4aiRequest(config, "POST", "/screenshot", body, signal);
+}
+
+/**
+ * Generate a PDF via POST /pdf.
+ */
+export async function fetchCrawl4aiPdf(
+	config: Crawl4aiConfig,
+	url: string,
+	signal?: AbortSignal,
+): Promise<Record<string, unknown>> {
+	return await crawl4aiRequest(config, "POST", "/pdf", { url }, signal);
+}
+
+/**
+ * Health check via GET /health.
+ */
+export async function fetchCrawl4aiHealth(
+	config: Crawl4aiConfig,
+	signal?: AbortSignal,
+): Promise<{ status?: string; version?: string; timestamp?: number }> {
+	const headers: Record<string, string> = { Accept: "application/json" };
+	if (config.apiToken) headers.Authorization = `Bearer ${config.apiToken}`;
+	const timeoutMs = Math.min(config.timeoutMs, 10000);
+	const response = await fetch(`${config.baseUrl}/health`, {
+		method: "GET",
+		headers,
+		signal: signalWithTimeout(timeoutMs, signal),
+	});
+	const text = await response.text();
+	if (!response.ok) throw new Crawl4aiHttpError(response.status, response.statusText, text);
+	const data = text ? JSON.parse(text) : {};
+	return {
+		status: data.status as string | undefined,
+		version: data.version as string | undefined,
+		timestamp: data.timestamp as number | undefined,
+	};
+}

package/lib/extract.ts

@@ -0,0 +1,189 @@
+// Unified content extraction orchestrator.
+// Modes:
+//   "static"  → JSDOM+Readability (no external API)
+//   "dynamic" → Firecrawl Scrape (JS rendering/structured JSON)
+//   "full"    → Crawl4AI markdown endpoint
+//   "auto"    → static → dynamic → full
+
+import { cwdFromContext, includeProjectEnv } from "./config";
+import { loadFirecrawlConfig, loadCrawl4aiConfig, type FirecrawlConfig, type Crawl4aiConfig } from "./config";
+import { fetchReadableContent } from "./content";
+import { firecrawlRequest, type FirecrawlResult } from "./firecrawl";
+import { fetchCrawl4aiMarkdown } from "./crawl4ai";
+import { formatFirecrawlScrape } from "./format";
+
+export type ExtractMode = "auto" | "static" | "dynamic" | "full";
+export type ExtractAttemptStatus = "success" | "empty" | "error";
+
+export interface ExtractParams {
+	url: string;
+	mode?: ExtractMode;
+	prompt?: string;
+	schema?: unknown;
+	content_chars?: number;
+	timeout_ms?: number;
+	wait_for?: number;
+	mobile?: boolean;
+	signal?: AbortSignal;
+	/** Internal: caller-provided ctx for env lookup. */
+	_ctx?: Record<string, unknown>;
+}
+
+export interface ExtractResult {
+	title: string;
+	markdown: string;
+	backend: string;
+	links?: string[];
+	structured?: unknown;
+}
+
+export interface ExtractAttempt {
+	mode: Exclude<ExtractMode, "auto">;
+	backend: string;
+	status: ExtractAttemptStatus;
+	message: string;
+	contentLength: number;
+}
+
+export interface ExtractDiagnostics {
+	result: ExtractResult;
+	attempts: ExtractAttempt[];
+	selectedMode: Exclude<ExtractMode, "auto">;
+	fallbackUsed: boolean;
+}
+
+const MIN_USEFUL_MARKDOWN_CHARS = 120;
+
+function sanitizeError(error: unknown): string {
+	const raw = error instanceof Error ? error.message : String(error);
+	return raw
+		.replace(/Bearer\s+[A-Za-z0-9._~+/=-]+/gi, "Bearer [REDACTED]")
+		.replace(/api[_-]?key[=:][^\s&]+/gi, "api_key=[REDACTED]")
+		.slice(0, 500);
+}
+
+function isUseful(result: ExtractResult | null, mode: ExtractMode, explicit: boolean): result is ExtractResult {
+	if (!result) return false;
+	const length = result.markdown.trim().length;
+	if (length === 0) return false;
+	return explicit || mode !== "static" || length >= MIN_USEFUL_MARKDOWN_CHARS;
+}
+
+function structuredSection(value: unknown): string {
+	if (value === undefined) return "";
+	return `\n\n## Structured extraction\n\n\`\`\`json\n${JSON.stringify(value, null, 2).slice(0, 10000)}\n\`\`\``;
+}
+
+async function extractStatic(params: ExtractParams): Promise<ExtractResult | null> {
+	const article = await fetchReadableContent(params.url, params.timeout_ms ?? 15000, params.signal);
+	return {
+		title: article.title ?? "",
+		markdown: article.markdown.slice(0, params.content_chars ?? 20000),
+		backend: "static",
+	};
+}
+
+function findStructuredPayload(result: FirecrawlResult): unknown {
+	const data = result.data as Record<string, unknown> | undefined;
+	if (!data) return undefined;
+	return data.json ?? data.extract ?? data.structuredData ?? data.llm_extraction;
+}
+
+async function extractDynamic(params: ExtractParams, ctx?: Record<string, unknown>): Promise<ExtractResult | null> {
+	let fcConfig: FirecrawlConfig;
+	try {
+		fcConfig = loadFirecrawlConfig({}, cwdFromContext(ctx ?? {}), includeProjectEnv(ctx ?? {}));
+	} catch (e) {
+		throw new Error(`Firecrawl configuration unavailable: ${sanitizeError(e)}`);
+	}
+
+	const formats: unknown[] = ["markdown"];
+	if (params.prompt || params.schema) {
+		const jsonEntry: Record<string, unknown> = { type: "json" };
+		if (params.prompt) jsonEntry.prompt = params.prompt;
+		if (params.schema) jsonEntry.schema = params.schema;
+		formats.push(jsonEntry);
+	}
+	const body: Record<string, unknown> = {
+		url: params.url,
+		formats,
+		onlyMainContent: true,
+		...(params.wait_for ? { waitFor: params.wait_for } : {}),
+		...(params.mobile ? { mobile: true } : {}),
+	};
+	const result = await firecrawlRequest(fcConfig, "POST", "/scrape", body, true, params.signal) as FirecrawlResult;
+	const structured = findStructuredPayload(result);
+	const raw = `${formatFirecrawlScrape(result as Record<string, unknown>, params.content_chars ?? 20000)}${structuredSection(structured)}`;
+	const titleMatch = raw.match(/^# (.+)$/m);
+	return {
+		title: titleMatch ? titleMatch[1] : "",
+		markdown: raw,
+		backend: "dynamic",
+		structured,
+	};
+}
+
+async function extractFull(params: ExtractParams, ctx?: Record<string, unknown>): Promise<ExtractResult | null> {
+	let c4aiConfig: Crawl4aiConfig;
+	try {
+		c4aiConfig = loadCrawl4aiConfig({}, cwdFromContext(ctx ?? {}), includeProjectEnv(ctx ?? {}));
+	} catch (e) {
+		throw new Error(`Crawl4AI configuration unavailable: ${sanitizeError(e)}`);
+	}
+	const result = await fetchCrawl4aiMarkdown(c4aiConfig, params.url, "fit", undefined, params.signal);
+	return {
+		title: "",
+		markdown: result.markdown.slice(0, params.content_chars ?? 20000),
+		backend: "full",
+	};
+}
+
+function modesFor(params: ExtractParams): Array<Exclude<ExtractMode, "auto">> {
+	const mode = params.mode ?? "auto";
+	if (mode !== "auto") return [mode];
+	return ["static", "dynamic", "full"];
+}
+
+async function runExtractor(mode: Exclude<ExtractMode, "auto">, params: ExtractParams, ctx?: Record<string, unknown>): Promise<ExtractResult | null> {
+	if (mode === "static") return extractStatic(params);
+	if (mode === "dynamic") return extractDynamic(params, ctx);
+	return extractFull(params, ctx);
+}
+
+export async function extractWithDiagnostics(params: ExtractParams): Promise<ExtractDiagnostics> {
+	const ctx = params._ctx;
+	const explicit = (params.mode ?? "auto") !== "auto";
+	const modes = modesFor(params);
+	const attempts: ExtractAttempt[] = [];
+
+	for (const mode of modes) {
+		try {
+			const result = await runExtractor(mode, params, ctx);
+			if (isUseful(result, mode, explicit)) {
+				if (!explicit && attempts.length > 0) {
+					result.markdown = `[Extraction fell back to ${mode === "dynamic" ? "Firecrawl Scrape (dynamic mode)" : "Crawl4AI (full browser mode)"}]\n\n${result.markdown}`;
+				}
+				attempts.push({ mode, backend: result.backend, status: "success", message: `Selected ${mode}`, contentLength: result.markdown.length });
+				return { result, attempts, selectedMode: mode, fallbackUsed: attempts.length > 1 };
+			}
+			const emptyResult = result as ExtractResult | null;
+			const length = emptyResult?.markdown.trim().length ?? 0;
+			attempts.push({ mode, backend: emptyResult?.backend ?? mode, status: "empty", message: `${mode} returned insufficient content`, contentLength: length });
+		} catch (e) {
+			attempts.push({ mode, backend: mode, status: "error", message: sanitizeError(e), contentLength: 0 });
+			if (explicit) break;
+		}
+	}
+
+	const diagnostics = attempts.map((a) => `${a.mode}: ${a.status}${a.message ? ` (${a.message})` : ""}`).join("\n");
+	const mode = params.mode ?? "auto";
+	throw new Error(
+		`Extraction failed for ${params.url} with mode="${mode}".\n${diagnostics}\n` +
+		"Try a different mode or use web_screenshot for a visual snapshot.",
+	);
+}
+
+/** Compatibility wrapper returning only the selected extraction result. */
+export async function universalExtract(params: ExtractParams): Promise<ExtractResult> {
+	return (await extractWithDiagnostics(params)).result;
+}

package/lib/format.ts

@@ -90,6 +90,110 @@ export function formatFirecrawlScrape(data: Record<string, unknown>, maxChars =
 	return parts.join("\n\n") || JSON.stringify(data, null, 2);
 }
 
+// ---------------------------------------------------------------------------
+// Crawl4AI result formatting
+// ---------------------------------------------------------------------------
+
+function extractMarkdown(md: unknown): string {
+	if (!md) return "";
+	if (typeof md === "string") return md;
+	if (typeof md === "object") {
+		const obj = md as Record<string, unknown>;
+		return (obj.fit_markdown as string) || (obj.raw_markdown as string) || "";
+	}
+	return "";
+}
+
+function extractLinks(links: unknown): string[] {
+	if (!links) return [];
+	const arr = Array.isArray(links) ? links : typeof links === "object" ? Object.values(links) : [];
+	return arr.flatMap((item: any) => {
+		if (typeof item === "string") return [item];
+		if (item?.href) return [item.href];
+		if (item?.url) return [item.url];
+		return [];
+	});
+}
+
+export function formatCrawl4aiResult(data: Record<string, unknown>, maxChars = 20000): string {
+	// Handle wrapped response: { results: [...] }
+	const rawResults = (data.results as unknown[]) || [data];
+	return rawResults
+		.map((raw: any, i: number) => {
+			const parts: string[] = [];
+			const url = raw.url || raw.redirected_url || "";
+			const success = raw.success !== false;
+			const statusCode = raw.status_code || raw.metadata?.statusCode;
+
+			if (rawResults.length > 1) parts.push(`=== Result ${i + 1} ===`);
+			if (url) parts.push(`URL: ${url}`);
+			if (statusCode) parts.push(`Status: ${statusCode}`);
+			if (!success) {
+				parts.push(`Error: ${raw.error_message || "Crawl failed"}`);
+				return parts.join("\n");
+			}
+
+			// Markdown content
+			const md = extractMarkdown(raw.markdown);
+			if (md) {
+				const truncated = md.slice(0, maxChars);
+				parts.push(truncated);
+				if (truncated.length < md.length) parts.push("[Markdown truncated...]");
+			} else if (raw.extracted_content) {
+				const ec =
+					typeof raw.extracted_content === "string"
+						? raw.extracted_content
+						: JSON.stringify(raw.extracted_content, null, 2);
+				parts.push(ec.slice(0, maxChars));
+			} else if (raw.cleaned_html) {
+				parts.push(`(Cleaned HTML: ${raw.cleaned_html.length} chars)`);
+			}
+
+			// Links
+			const links = raw.links as Record<string, unknown> | undefined;
+			if (links) {
+				const internalLinks = extractLinks(links.internal);
+				const externalLinks = extractLinks(links.external);
+				const allLinks = [...internalLinks, ...externalLinks];
+				if (allLinks.length) {
+					parts.push(`Links (${allLinks.length}): ${allLinks.slice(0, 50).join(", ")}`);
+				}
+			}
+
+			// Media
+			const media = raw.media as Record<string, unknown> | undefined;
+			if (media) {
+				const imageCount = (media.images as unknown[])?.length || 0;
+				const videoCount = (media.videos as unknown[])?.length || 0;
+				if (imageCount || videoCount) {
+					parts.push(`Media: ${imageCount} images, ${videoCount} videos`);
+				}
+			}
+
+			return parts.join("\n\n");
+		})
+		.join("\n\n---\n\n") || "(No data)";
+}
+
+export function formatCrawl4aiStream(lines: string, _maxChars = 20000): string {
+	const results: string[] = [];
+	for (const line of lines.split("\n")) {
+		const trimmed = line.trim();
+		if (!trimmed) continue;
+		try {
+			const parsed = JSON.parse(trimmed);
+			if (parsed.status === "completed") {
+				results.push("[Stream complete]");
+				break;
+			}
+			results.push(formatCrawl4aiResult(parsed, _maxChars));
+		} catch {
+			results.push(trimmed.slice(0, 1000));
+		}
+	}
+	return results.join("\n\n---\n\n");
+}
+
 export function formatFirecrawlSearch(data: Record<string, unknown>, maxChars = 5000): string {
 	const rootData = data.data as Record<string, unknown> | undefined;
 	const web = (rootData?.web as unknown[]) || [];