@bacnh85/pi-web 0.4.6 → 0.5.1

package/README.md

@@ -1,4 +1,4 @@
-# pi-web v0.4
+# pi-web v0.5
 
 Pi extension for **unified web search, content extraction, site crawling, and page capture**.
 

package/extensions/index.ts

@@ -148,6 +148,7 @@ export default function piWebExtension(pi: ExtensionAPI) {
 			content_chars: Type.Optional(Type.Number({ default: 20000 })),
 			wait_for: Type.Optional(Type.Number({ description: "Ms to wait for Firecrawl render before extraction." })),
 			mobile: Type.Optional(Type.Boolean({ default: false, description: "Mobile viewport (dynamic mode only)." })),
+			...crawl4aiControlSchema,
 			...sharedControlSchema,
 		}),
 		async execute(_id: string, params: Record<string, unknown>, signal: AbortSignal, _onUpdate: unknown, ctx: any) {
@@ -160,6 +161,8 @@ export default function piWebExtension(pi: ExtensionAPI) {
 				timeout_ms: params.timeout_ms as number | undefined,
 				wait_for: params.wait_for as number | undefined,
 				mobile: params.mobile as boolean | undefined,
+				crawl4ai_api_token: params.crawl4ai_api_token as string | undefined,
+				crawl4ai_api_url: params.crawl4ai_api_url as string | undefined,
 				signal,
 				_ctx: ctx,
 			});

package/extensions/lib/extract.ts

@@ -24,6 +24,8 @@ export interface ExtractParams {
 	timeout_ms?: number;
 	wait_for?: number;
 	mobile?: boolean;
+	crawl4ai_api_token?: string;
+	crawl4ai_api_url?: string;
 	signal?: AbortSignal;
 	/** Internal: caller-provided ctx for env lookup. */
 	_ctx?: Record<string, unknown>;
@@ -126,7 +128,11 @@ async function extractDynamic(params: ExtractParams, ctx?: Record<string, unknow
 async function extractFull(params: ExtractParams, ctx?: Record<string, unknown>): Promise<ExtractResult | null> {
 	let c4aiConfig: Crawl4aiConfig;
 	try {
-		c4aiConfig = loadCrawl4aiConfig({}, cwdFromContext(ctx ?? {}), includeProjectEnv(ctx ?? {}));
+		c4aiConfig = loadCrawl4aiConfig(
+		{ crawl4ai_api_token: params.crawl4ai_api_token, crawl4ai_api_url: params.crawl4ai_api_url },
+		cwdFromContext(ctx ?? {}),
+		includeProjectEnv(ctx ?? {}),
+	);
 	} catch (e) {
 		throw new Error(`Crawl4AI configuration unavailable: ${sanitizeError(e)}`);
 	}
@@ -175,11 +181,20 @@ export async function extractWithDiagnostics(params: ExtractParams): Promise<Ext
 		}
 	}
 
-	const diagnostics = attempts.map((a) => `${a.mode}: ${a.status}${a.message ? ` (${a.message})` : ""}`).join("\n");
-	const mode = params.mode ?? "auto";
-	throw new Error(
-		`Extraction failed for ${params.url} with mode="${mode}".\n${diagnostics}\n` +
-		"Try a different mode or use web_screenshot for a visual snapshot.",
-	);
+	// All modes exhausted — return the best attempt with diagnostics instead of throwing
+	const lastAttempt = attempts[attempts.length - 1];
+	const bestResult: ExtractResult = {
+		title: "",
+		markdown: `[All extraction modes failed for ${params.url}]\n\n` +
+			attempts.map((a) => `  ${a.mode}: ${a.status}${a.message ? ` (${a.message})` : ""}`).join("\n") +
+			"\n\nTry web_screenshot for a visual snapshot, or verify the URL is accessible.",
+		backend: lastAttempt?.backend ?? "none",
+	};
+	return {
+		result: bestResult,
+		attempts,
+		selectedMode: lastAttempt?.mode ?? "static",
+		fallbackUsed: true,
+	};
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bacnh85/pi-web",
-  "version": "0.4.6",
+  "version": "0.5.1",
   "description": "Pi extension for web search, page extraction, Firecrawl scraping/crawling, and Crawl4AI headless browser crawling.",
   "type": "module",
   "license": "MIT",
@@ -24,11 +24,13 @@
     "crawling"
   ],
   "scripts": {
-    "test": "cd extensions && mocha"
+    "test": "cd extensions && npx mocha"
   },
   "files": [
     "README.md",
-    "extensions/",
+    "extensions/index.ts",
+    "extensions/lib/",
+    "extensions/types.d.ts",
     "skills/"
   ],
   "pi": {

package/extensions/.mocharc.yml

@@ -1,5 +0,0 @@
-require:
-  - tsx
-spec:
-  - test/**/*.test.ts
-timeout: 10000

package/extensions/package.json

@@ -1,3 +0,0 @@
-{
-  "type": "module"
-}

package/extensions/test/unit/config.test.ts

@@ -1,309 +0,0 @@
-/**
- * Unit tests for pi-web config module.
- */
-
-import { expect } from "chai";
-import {
-	stripInlineComment,
-	parseDotenvValue,
-	parseDotenvFile,
-	piConfigDirs,
-	envFileCandidates,
-	findEnvValue,
-	normalizeSearxngBaseUrl,
-	normalizeFirecrawlBaseUrl,
-	normalizeCrawl4aiApiUrl,
-	loadCrawl4aiConfig,
-	DEFAULT_SEARXNG_BASE_URL,
-	DEFAULT_CRAWL4AI_API_URL,
-	HOSTED_FIRECRAWL_BASE_URL,
-} from "../../lib/config";
-
-describe("stripInlineComment", () => {
-	it("returns full value when no comment", () => {
-		expect(stripInlineComment("hello")).to.equal("hello");
-		expect(stripInlineComment("")).to.equal("");
-	});
-
-	it("strips comment after space", () => {
-		expect(stripInlineComment("hello # world")).to.equal("hello ");
-		expect(stripInlineComment("value #comment")).to.equal("value ");
-	});
-
-	it("preserves # inside single quotes", () => {
-		expect(stripInlineComment("'hello # world'")).to.equal("'hello # world'");
-	});
-
-	it("preserves # inside double quotes", () => {
-		expect(stripInlineComment('"hello # world"')).to.equal('"hello # world"');
-	});
-
-	it("handles escaped characters inside quotes", () => {
-		expect(stripInlineComment('"hello \\" world" # comment')).to.equal(
-			'"hello \\" world" ',
-		);
-	});
-
-	it("strips comment when # is first character", () => {
-		expect(stripInlineComment("# comment")).to.equal("");
-	});
-
-	it("handles mixed quoted and unquoted content", () => {
-		expect(stripInlineComment("key='val' # comment")).to.equal("key='val' ");
-	});
-
-	it("handles escaped # with backslash", () => {
-		expect(stripInlineComment("hello \\# not a comment")).to.equal(
-			"hello \\# not a comment",
-		);
-	});
-});
-
-describe("parseDotenvValue", () => {
-	it("trims and returns unquoted value", () => {
-		expect(parseDotenvValue("  hello  ")).to.equal("hello");
-	});
-
-	it("strips double quotes", () => {
-		expect(parseDotenvValue('"hello"')).to.equal("hello");
-	});
-
-	it("strips single quotes", () => {
-		expect(parseDotenvValue("'hello'")).to.equal("hello");
-	});
-
-	it("handles escape sequences in double-quoted values", () => {
-		expect(parseDotenvValue('"hello\\nworld"')).to.equal("hello\nworld");
-		expect(parseDotenvValue('"path\\tto\\truby"')).to.equal("path\tto\truby");
-	});
-
-	it("trims whitespace in unquoted values", () => {
-		expect(parseDotenvValue("  hello world  ")).to.equal("hello world");
-	});
-
-	it("refuses to remove double quotes from single-quoted value", () => {
-		expect(parseDotenvValue("'hello\"'")).to.equal('hello"');
-	});
-
-	it("strips inline comment from unquoted value", () => {
-		expect(parseDotenvValue("hello # comment")).to.equal("hello");
-	});
-});
-
-describe("parseDotenvFile", () => {
-	it("returns null for non-existent file", () => {
-		const result = parseDotenvFile("/tmp/nonexistent-file-12345.env");
-		expect(result).to.be.null;
-	});
-
-	it("parses a simple env file content via temp testing", async () => {
-		const fs = await import("node:fs");
-		const path = await import("node:path");
-		const tmpDir = fs.mkdtempSync("/tmp/pi-web-test-");
-		const testFile = path.join(tmpDir, ".env");
-		fs.writeFileSync(
-			testFile,
-			"FOO=bar\nBAZ=qux # inline comment\n# FULL LINE COMMENT\nEMPTY=\n",
-			"utf8",
-		);
-		try {
-			const result = parseDotenvFile(testFile);
-			expect(result).to.deep.equal({ FOO: "bar", BAZ: "qux", EMPTY: "" });
-		} finally {
-			fs.rmSync(tmpDir, { recursive: true, force: true });
-		}
-	});
-});
-
-describe("piConfigDirs", () => {
-	const origEnv = process.env.PI_CODING_AGENT_DIR;
-
-	afterEach(() => {
-		if (origEnv === undefined) {
-			delete process.env.PI_CODING_AGENT_DIR;
-		} else {
-			process.env.PI_CODING_AGENT_DIR = origEnv;
-		}
-	});
-
-	it("returns custom dir when PI_CODING_AGENT_DIR is set", () => {
-		process.env.PI_CODING_AGENT_DIR = "/custom/pi";
-		const dirs = piConfigDirs();
-		expect(dirs).to.deep.equal(["/custom/pi"]);
-	});
-
-	it("returns default dir when PI_CODING_AGENT_DIR is not set", () => {
-		delete process.env.PI_CODING_AGENT_DIR;
-		const dirs = piConfigDirs();
-		expect(dirs).to.have.length(1);
-		expect(dirs[0]).to.include(".pi/agent");
-	});
-});
-
-describe("envFileCandidates", () => {
-	it("includes cwd files and pi config dirs", () => {
-		const candidates = envFileCandidates("/test/project", true);
-		expect(candidates[0]).to.equal("/test/project/.env.local");
-		expect(candidates[1]).to.equal("/test/project/.env");
-	});
-
-	it("omits cwd files when includeCwd is false", () => {
-		const candidates = envFileCandidates("/test/project", false);
-		expect(candidates[0]).not.to.include("/test/project");
-	});
-
-	it("returns pi config dir entries", () => {
-		const candidates = envFileCandidates("/test/project", true);
-		expect(candidates.length).to.be.at.least(2); // cwd .env.local, cwd .env
-	});
-});
-
-describe("normalizeSearxngBaseUrl", () => {
-	it("returns default when nothing provided", () => {
-		expect(normalizeSearxngBaseUrl()).to.equal(DEFAULT_SEARXNG_BASE_URL);
-	});
-
-	it("preserves full URL with scheme", () => {
-		expect(normalizeSearxngBaseUrl("https://search.example.com")).to.equal(
-			"https://search.example.com",
-		);
-	});
-
-	it("adds http for localhost", () => {
-		expect(normalizeSearxngBaseUrl("localhost:8888")).to.equal(
-			"http://localhost:8888",
-		);
-	});
-
-	it("adds http for private IP", () => {
-		expect(normalizeSearxngBaseUrl("192.168.1.1:8888")).to.equal(
-			"http://192.168.1.1:8888",
-		);
-	});
-
-	it("adds https for public hostname", () => {
-		expect(normalizeSearxngBaseUrl("search.example.com")).to.equal(
-			"https://search.example.com",
-		);
-	});
-
-	it("removes trailing slashes", () => {
-		expect(normalizeSearxngBaseUrl("http://localhost:8888/")).to.equal(
-			"http://localhost:8888",
-		);
-	});
-});
-
-describe("normalizeFirecrawlBaseUrl", () => {
-	it("returns hosted default when nothing provided", () => {
-		expect(normalizeFirecrawlBaseUrl()).to.equal(HOSTED_FIRECRAWL_BASE_URL);
-	});
-
-	it("appends /v2 if no version segment", () => {
-		expect(
-			normalizeFirecrawlBaseUrl("http://localhost:3002"),
-		).to.equal("http://localhost:3002/v2");
-	});
-
-	it("preserves existing version segment", () => {
-		expect(
-			normalizeFirecrawlBaseUrl("http://localhost:3002/v1"),
-		).to.equal("http://localhost:3002/v1");
-	});
-
-	it("removes trailing slash before appending version", () => {
-		expect(
-			normalizeFirecrawlBaseUrl("http://localhost:3002/"),
-		).to.equal("http://localhost:3002/v2");
-	});
-});
-
-describe("findEnvValue", () => {
-	it("reads from process.env first", () => {
-		process.env.TEST_PI_WEB_VAR = "from_process";
-		try {
-			const result = findEnvValue("TEST_PI_WEB_VAR", "/tmp", false);
-			expect(result.value).to.equal("from_process");
-			expect(result.source).to.equal("process.env");
-		} finally {
-			delete process.env.TEST_PI_WEB_VAR;
-		}
-	});
-
-	it("returns undefined when not found", () => {
-		const result = findEnvValue("THIS_VAR_DOES_NOT_EXIST_12345", "/tmp", false);
-		expect(result.value).to.be.undefined;
-		expect(result.source).to.equal("");
-	});
-});
-
-describe("normalizeCrawl4aiApiUrl", () => {
-	it("returns default when nothing provided", () => {
-		expect(normalizeCrawl4aiApiUrl()).to.equal(DEFAULT_CRAWL4AI_API_URL);
-	});
-
-	it("preserves full URL with scheme", () => {
-		expect(normalizeCrawl4aiApiUrl("https://crawl.example.com")).to.equal(
-			"https://crawl.example.com",
-		);
-	});
-
-	it("adds http for private IP", () => {
-		expect(normalizeCrawl4aiApiUrl("172.30.55.22:11235")).to.equal(
-			"http://172.30.55.22:11235",
-		);
-	});
-
-	it("adds https for public hostname", () => {
-		expect(normalizeCrawl4aiApiUrl("crawl.example.com")).to.equal(
-			"https://crawl.example.com",
-		);
-	});
-
-	it("removes trailing slashes", () => {
-		expect(normalizeCrawl4aiApiUrl("http://localhost:11235/")).to.equal(
-			"http://localhost:11235",
-		);
-	});
-
-	it("does not append /v2 like Firecrawl", () => {
-		expect(normalizeCrawl4aiApiUrl("http://localhost:11235")).to.equal(
-			"http://localhost:11235",
-		);
-	});
-});
-
-describe("loadCrawl4aiConfig", () => {
-	it("returns default config from defaults", () => {
-		const config = loadCrawl4aiConfig({}, "/tmp", false);
-		expect(config.baseUrl).to.equal(DEFAULT_CRAWL4AI_API_URL);
-		expect(config.timeoutMs).to.equal(60000);
-	});
-
-	it("accepts explicit API URL from params", () => {
-		const config = loadCrawl4aiConfig(
-			{ crawl4ai_api_url: "http://custom:12345" },
-			"/tmp",
-			false,
-		);
-		expect(config.baseUrl).to.equal("http://custom:12345");
-	});
-
-	it("accepts explicit API token from params", () => {
-		const config = loadCrawl4aiConfig(
-			{ crawl4ai_api_token: "my-token" },
-			"/tmp",
-			false,
-		);
-		expect(config.apiToken).to.equal("my-token");
-	});
-
-	it("reads timeout from params", () => {
-		const config = loadCrawl4aiConfig({ timeout_ms: 30000 }, "/tmp", false);
-		expect(config.timeoutMs).to.equal(30000);
-	});
-
-	it("throws on invalid timeout", () => {
-		expect(() => loadCrawl4aiConfig({ timeout_ms: 500 }, "/tmp", false)).to.throw();
-	});
-});

package/extensions/test/unit/crawl4ai.test.ts

@@ -1,69 +0,0 @@
-/**
- * Unit tests for pi-web Crawl4AI client.
- */
-
-import { expect } from "chai";
-import { Crawl4aiHttpError } from "../../lib/crawl4ai";
-
-describe("Crawl4aiHttpError", () => {
-	it("formats error message with status and text", () => {
-		const err = new Crawl4aiHttpError(401, "Unauthorized", '{"detail":"bad token"}');
-		expect(err.status).to.equal(401);
-		expect(err.message).to.include("HTTP 401: Unauthorized");
-		expect(err.message).to.include('{"detail":"bad token"}');
-	});
-
-	it("handles empty response text", () => {
-		const err = new Crawl4aiHttpError(500, "Internal Server Error", "");
-		expect(err.status).to.equal(500);
-		expect(err.message).to.equal("HTTP 500: Internal Server Error");
-	});
-
-	it("is instance of Error", () => {
-		const err = new Crawl4aiHttpError(403, "Forbidden", "rate limit");
-		expect(err).to.be.instanceOf(Error);
-	});
-});
-
-// ---------------------------------------------------------------------------
-// Config + format integration: verify imported functions exist with correct sigs
-// ---------------------------------------------------------------------------
-
-import type { Crawl4aiConfig } from "../../lib/config";
-
-describe("Crawl4aiConfig interface", () => {
-	it("has expected shape", () => {
-		const config: Crawl4aiConfig = {
-			baseUrl: "http://localhost:11235",
-			apiToken: "token123",
-			timeoutMs: 30000,
-		};
-		expect(config.baseUrl).to.equal("http://localhost:11235");
-		expect(config.apiToken).to.equal("token123");
-		expect(config.timeoutMs).to.equal(30000);
-	});
-});
-
-// ---------------------------------------------------------------------------
-// Verify client exports exist
-// ---------------------------------------------------------------------------
-
-import {
-	fetchCrawl4aiMarkdown,
-	fetchCrawl4aiCrawl,
-	fetchCrawl4aiScreenshot,
-	fetchCrawl4aiPdf,
-	fetchCrawl4aiHealth,
-	crawl4aiRequest,
-} from "../../lib/crawl4ai";
-
-describe("Crawl4AI client exports", () => {
-	it("exports all expected functions", () => {
-		expect(fetchCrawl4aiMarkdown).to.be.a("function");
-		expect(fetchCrawl4aiCrawl).to.be.a("function");
-		expect(fetchCrawl4aiScreenshot).to.be.a("function");
-		expect(fetchCrawl4aiPdf).to.be.a("function");
-		expect(fetchCrawl4aiHealth).to.be.a("function");
-		expect(crawl4aiRequest).to.be.a("function");
-	});
-});

package/extensions/test/unit/extract.test.ts

@@ -1,134 +0,0 @@
-import { expect } from "chai";
-
-import {
-	extractWithDiagnostics,
-	type ExtractMode,
-	type ExtractParams,
-	type ExtractResult,
-} from "../../lib/extract";
-
-const ORIGINAL_ENV = { ...process.env };
-const ORIGINAL_FETCH = globalThis.fetch;
-
-function jsonResponse(body: unknown, status = 200): Response {
-	return new Response(JSON.stringify(body), {
-		status,
-		headers: { "content-type": "application/json" },
-	});
-}
-
-function htmlResponse(body: string, status = 200): Response {
-	return new Response(body, {
-		status,
-		headers: { "content-type": "text/html" },
-	});
-}
-
-function installMockFetch(handler: (url: string, init?: RequestInit) => Response | Promise<Response>): string[] {
-	const calls: string[] = [];
-	globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => {
-		const url = String(input);
-		calls.push(url);
-		return handler(url, init);
-	}) as typeof fetch;
-	return calls;
-}
-
-function restoreEnv(): void {
-	process.env = { ...ORIGINAL_ENV };
-}
-
-describe("ExtractParams and ExtractResult types", () => {
-	it("accept expected fields", () => {
-		const modes: ExtractMode[] = ["auto", "static", "dynamic", "full"];
-		const params: ExtractParams = { url: "https://example.com", mode: modes[0], wait_for: 1000, mobile: true };
-		const result: ExtractResult = { title: "Title", markdown: "Content", backend: "static", structured: { ok: true } };
-		expect(params.mode).to.equal("auto");
-		expect(result.structured).to.deep.equal({ ok: true });
-	});
-});
-
-describe("extractWithDiagnostics", () => {
-	beforeEach(() => {
-		restoreEnv();
-		process.env.FIRECRAWL_API_URL = "http://firecrawl.test/v2";
-		process.env.CRAWL4AI_API_URL = "http://crawl4ai.test";
-	});
-
-	afterEach(() => {
-		globalThis.fetch = ORIGINAL_FETCH;
-		restoreEnv();
-	});
-
-	it("uses static extraction when it returns useful content", async () => {
-		installMockFetch((url) => {
-			if (url === "https://example.com/static") {
-				return htmlResponse("<html><head><title>Static</title></head><body><main><h1>Static</h1><p>This static article has enough readable text to pass the useful-content threshold in auto mode without falling back.</p><p>Additional words make it reliably longer than the minimum threshold.</p></main></body></html>");
-			}
-			throw new Error(`unexpected fetch ${url}`);
-		});
-
-		const diagnostics = await extractWithDiagnostics({ url: "https://example.com/static" });
-		expect(diagnostics.selectedMode).to.equal("static");
-		expect(diagnostics.fallbackUsed).to.be.false;
-		expect(diagnostics.result.markdown).to.include("static article");
-	});
-
-	it("falls through from short static content to dynamic extraction", async () => {
-		const calls = installMockFetch((url) => {
-			if (url === "https://example.com/short") return htmlResponse("<html><body><main>short</main></body></html>");
-			if (url === "http://firecrawl.test/v2/scrape") {
-				return jsonResponse({ data: { markdown: "# Dynamic\n\nDynamic content from Firecrawl after static extraction was too short.", metadata: { title: "Dynamic", sourceURL: "https://example.com/short" } } });
-			}
-			throw new Error(`unexpected fetch ${url}`);
-		});
-
-		const diagnostics = await extractWithDiagnostics({ url: "https://example.com/short" });
-		expect(diagnostics.selectedMode).to.equal("dynamic");
-		expect(diagnostics.fallbackUsed).to.be.true;
-		expect(diagnostics.attempts.map((a) => a.mode)).to.deep.equal(["static", "dynamic"]);
-		expect(diagnostics.result.markdown).to.include("fell back to Firecrawl");
-		expect(calls).to.deep.equal(["https://example.com/short", "http://firecrawl.test/v2/scrape"]);
-	});
-
-	it("explicit static mode does not fall through", async () => {
-		installMockFetch((url) => {
-			if (url === "https://example.com/short") return htmlResponse("<html><body><main>short</main></body></html>");
-			throw new Error(`unexpected fetch ${url}`);
-		});
-
-		const diagnostics = await extractWithDiagnostics({ url: "https://example.com/short", mode: "static" });
-		expect(diagnostics.selectedMode).to.equal("static");
-		expect(diagnostics.result.markdown).to.include("short");
-	});
-
-	it("dynamic mode includes structured JSON output when present", async () => {
-		installMockFetch((url, init) => {
-			if (url === "http://firecrawl.test/v2/scrape") {
-				const body = JSON.parse(String(init?.body));
-				expect(body.formats).to.deep.include({ type: "json", prompt: "Extract title" });
-				return jsonResponse({ data: { markdown: "# Dynamic\n\nBody", json: { title: "Structured" }, metadata: { title: "Dynamic" } } });
-			}
-			throw new Error(`unexpected fetch ${url}`);
-		});
-
-		const diagnostics = await extractWithDiagnostics({ url: "https://example.com/dynamic", mode: "dynamic", prompt: "Extract title" });
-		expect(diagnostics.result.structured).to.deep.equal({ title: "Structured" });
-		expect(diagnostics.result.markdown).to.include("## Structured extraction");
-	});
-
-	it("falls through to full mode when static and dynamic fail", async () => {
-		installMockFetch((url) => {
-			if (url === "https://example.com/full") return htmlResponse("<html><body><main>tiny</main></body></html>");
-			if (url === "http://firecrawl.test/v2/scrape") return jsonResponse({ error: "blocked" }, 500);
-			if (url === "http://crawl4ai.test/md") return jsonResponse({ success: true, markdown: "# Full\n\nCrawl4AI markdown content" });
-			throw new Error(`unexpected fetch ${url}`);
-		});
-
-		const diagnostics = await extractWithDiagnostics({ url: "https://example.com/full" });
-		expect(diagnostics.selectedMode).to.equal("full");
-		expect(diagnostics.attempts.map((a) => a.mode)).to.deep.equal(["static", "dynamic", "full"]);
-		expect(diagnostics.result.markdown).to.include("Crawl4AI");
-	});
-
-});

package/extensions/test/unit/format.test.ts

@@ -1,194 +0,0 @@
-/**
- * Unit tests for pi-web format module.
- */
-
-import { expect } from "chai";
-import {
-	sanitizeSnippet,
-	truncateText,
-	formatFirecrawlScrape,
-	formatCrawl4aiResult,
-	type SearchResultItem,
-} from "../../lib/format";
-
-const OUTPUT_MAX_BYTES = 50 * 1024;
-const OUTPUT_MAX_LINES = 2_000;
-
-describe("sanitizeSnippet", () => {
-	it("returns empty string for empty input", () => {
-		expect(sanitizeSnippet()).to.equal("");
-		expect(sanitizeSnippet("")).to.equal("");
-	});
-
-	it("strips HTML tags", () => {
-		expect(sanitizeSnippet("<b>hello</b> world")).to.equal("hello world");
-	});
-
-	it("decodes HTML entities", () => {
-		expect(sanitizeSnippet("foo &amp; bar")).to.equal("foo & bar");
-		expect(sanitizeSnippet("&lt;tag&gt;")).to.equal("<tag>");
-		expect(sanitizeSnippet("&quot;quoted&quot;")).to.equal('"quoted"');
-		expect(sanitizeSnippet("hello&nbsp;world")).to.equal("hello world");
-	});
-
-	it("decodes numeric HTML entities", () => {
-		expect(sanitizeSnippet("&#65;")).to.equal("A");
-		expect(sanitizeSnippet("&#x41;")).to.equal("A");
-	});
-
-	it("collapses whitespace", () => {
-		expect(sanitizeSnippet("hello   world")).to.equal("hello world");
-		expect(sanitizeSnippet("  hello  ")).to.equal("hello");
-	});
-});
-
-describe("truncateText", () => {
-	it("returns short text as-is", () => {
-		const text = "hello world";
-		expect(truncateText(text)).to.equal(text);
-	});
-
-	it("truncates by line count when exceeding OUTPUT_MAX_LINES", () => {
-		const lines = Array.from({ length: OUTPUT_MAX_LINES + 10 }, (_, i) => `line ${i}`);
-		const text = lines.join("\n");
-		const result = truncateText(text);
-		expect(result).to.include("[Web output truncated to");
-		expect(result.split("\n").length).to.be.lessThan(OUTPUT_MAX_LINES + 15);
-	});
-
-	it("truncates by byte count when exceeding OUTPUT_MAX_BYTES", () => {
-		const text = "x".repeat(OUTPUT_MAX_BYTES + 50000);
-		const result = truncateText(text);
-		expect(result).to.include("[Web output truncated to");
-	});
-
-	it("handles empty string", () => {
-		expect(truncateText("")).to.equal("");
-	});
-});
-
-describe("formatFirecrawlScrape", () => {
-	it("formats scrape data with metadata", () => {
-		const data = {
-			data: {
-				metadata: {
-					title: "Test Page",
-					sourceURL: "https://example.com",
-					statusCode: 200,
-				},
-				markdown: "# Hello\n\nThis is content.",
-			},
-		};
-		const output = formatFirecrawlScrape(data);
-		expect(output).to.include("# Test Page");
-		expect(output).to.include("Source: https://example.com");
-		expect(output).to.include("Status: 200");
-		expect(output).to.include("# Hello");
-		expect(output).to.include("This is content.");
-	});
-
-	it("resolves data at top level when no nested data", () => {
-		const data = {
-			metadata: { title: "Direct" },
-			markdown: "Content",
-		};
-		const output = formatFirecrawlScrape(data);
-		expect(output).to.include("# Direct");
-		expect(output).to.include("Content");
-	});
-
-	it("includes warning when present", () => {
-		const data = { data: { markdown: "x" }, warning: "Rate limited" };
-		const output = formatFirecrawlScrape(data);
-		expect(output).to.include("Warning: Rate limited");
-	});
-
-	it("includes links when present", () => {
-		const data = {
-			data: {
-				markdown: "body",
-				links: ["https://a.com", "https://b.com"],
-			},
-		};
-		const output = formatFirecrawlScrape(data);
-		expect(output).to.include("## Links");
-		expect(output).to.include("- https://a.com");
-	});
-
-	it("returns JSON stringify fallback for empty data", () => {
-		const data = { foo: "bar" };
-		const output = formatFirecrawlScrape(data);
-		expect(output).to.include("foo");
-	});
-});
-
-describe("formatCrawl4aiResult", () => {
-	it("formats a single successful crawl result with markdown", () => {
-		const data = {
-			url: "https://example.com",
-			success: true,
-			status_code: 200,
-			markdown: { fit_markdown: "# Hello\n\nWorld content." },
-		};
-		const output = formatCrawl4aiResult(data);
-		expect(output).to.include("URL: https://example.com");
-		expect(output).to.include("Status: 200");
-		expect(output).to.include("# Hello");
-	});
-
-	it("shows error for failed crawl", () => {
-		const data = {
-			url: "https://example.com/404",
-			success: false,
-			error_message: "Not Found",
-		};
-		const output = formatCrawl4aiResult(data);
-		expect(output).to.include("Error: Not Found");
-		expect(output).to.include("URL: https://example.com/404");
-	});
-
-	it("formats multiple results from wrapped response", () => {
-		const data = {
-			results: [
-				{ url: "https://a.com", success: true, markdown: { raw_markdown: "Page A" } },
-				{ url: "https://b.com", success: true, markdown: { raw_markdown: "Page B" } },
-			],
-		};
-		const output = formatCrawl4aiResult(data);
-		expect(output).to.include("=== Result 1 ===");
-		expect(output).to.include("=== Result 2 ===");
-		expect(output).to.include("Page A");
-		expect(output).to.include("Page B");
-	});
-
-	it("falls back to raw_markdown when fit_markdown is absent", () => {
-		const data = {
-			url: "https://example.com",
-			success: true,
-			markdown: { raw_markdown: "Raw content only" },
-		};
-		const output = formatCrawl4aiResult(data);
-		expect(output).to.include("Raw content only");
-	});
-
-	it("includes links when present", () => {
-		const data = {
-			url: "https://example.com",
-			success: true,
-			markdown: { fit_markdown: "body" },
-			links: {
-				internal: [{ href: "https://example.com/about" }],
-				external: [{ href: "https://other.com" }],
-			},
-		};
-		const output = formatCrawl4aiResult(data);
-		expect(output).to.include("Links");
-		expect(output).to.include("https://example.com/about");
-	});
-
-	it("returns fallback for empty data", () => {
-		expect(formatCrawl4aiResult({})).to.equal("(No data)");
-	});
-});
-
-

package/extensions/test/unit/search.test.ts

@@ -1,147 +0,0 @@
-import { expect } from "chai";
-
-import {
-	searchWithDiagnostics,
-	selectSearchBackendOrder,
-	type SearchParams,
-	type SearchResult,
-} from "../../lib/search";
-
-const ORIGINAL_ENV = { ...process.env };
-const ORIGINAL_FETCH = globalThis.fetch;
-
-function jsonResponse(body: unknown, status = 200): Response {
-	return new Response(JSON.stringify(body), {
-		status,
-		headers: { "content-type": "application/json" },
-	});
-}
-
-function htmlResponse(body: string, status = 200): Response {
-	return new Response(body, {
-		status,
-		headers: { "content-type": "text/html" },
-	});
-}
-
-function installMockFetch(handler: (url: string, init?: RequestInit) => Response | Promise<Response>): string[] {
-	const calls: string[] = [];
-	globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => {
-		const url = String(input);
-		calls.push(url);
-		return handler(url, init);
-	}) as typeof fetch;
-	return calls;
-}
-
-function restoreEnv(): void {
-	process.env = { ...ORIGINAL_ENV };
-}
-
-describe("SearchParams and SearchResult types", () => {
-	it("accept expected fields", () => {
-		const params: SearchParams = { query: "test", backend: "auto", signal: new AbortController().signal };
-		const result: SearchResult = { title: "T", url: "https://example.com", snippet: "S", age: "", content: "", backend: "brave" };
-		expect(params.backend).to.equal("auto");
-		expect(result.backend).to.equal("brave");
-	});
-});
-
-describe("selectSearchBackendOrder", () => {
-	it("selects Brave first for include_content", () => {
-		expect(selectSearchBackendOrder({ query: "homelab ansible", include_content: true })).to.deep.equal(["brave", "searxng", "firecrawl"]);
-	});
-
-	it("selects Brave first for site: and precision queries", () => {
-		expect(selectSearchBackendOrder({ query: "site:docs.ansible.com podman quadlet" })).to.deep.equal(["brave", "searxng", "firecrawl"]);
-		expect(selectSearchBackendOrder({ query: '"exact phrase" release notes' })).to.deep.equal(["brave", "searxng", "firecrawl"]);
-	});
-
-	it("keeps SearXNG first for broad discovery and explicit engines", () => {
-		expect(selectSearchBackendOrder({ query: "homelab ansible ideas" })).to.deep.equal(["searxng", "brave", "firecrawl"]);
-		expect(selectSearchBackendOrder({ query: "docs api", engines: "google,github" })).to.deep.equal(["searxng", "brave", "firecrawl"]);
-	});
-
-	it("honors explicit backend", () => {
-		expect(selectSearchBackendOrder({ query: "test", backend: "firecrawl" })).to.deep.equal(["firecrawl"]);
-	});
-});
-
-describe("searchWithDiagnostics", () => {
-	beforeEach(() => {
-		restoreEnv();
-		process.env.BRAVE_API_KEY = "test-brave-key";
-		process.env.SEARXNG_BASE_URL = "http://searxng.test";
-		process.env.FIRECRAWL_API_URL = "http://firecrawl.test/v2";
-	});
-
-	afterEach(() => {
-		globalThis.fetch = ORIGINAL_FETCH;
-		restoreEnv();
-	});
-
-	it("uses Brave first for include_content and fetches inline content", async () => {
-		const calls = installMockFetch((url) => {
-			if (url.startsWith("https://api.search.brave.com/")) {
-				return jsonResponse({ web: { results: [{ title: "Brave", url: "https://example.com/page", description: "Snippet" }] } });
-			}
-			if (url === "https://example.com/page") {
-				return htmlResponse("<html><head><title>Page</title></head><body><main><h1>Page</h1><p>This is long enough readable content for the inline content fetch test.</p></main></body></html>");
-			}
-			throw new Error(`unexpected fetch ${url}`);
-		});
-
-		const result = await searchWithDiagnostics({ query: "homelab ansible", include_content: true });
-		expect(result.selectedBackend).to.equal("brave");
-		expect(result.backendOrder[0]).to.equal("brave");
-		expect(result.results[0].content).to.include("readable content");
-		expect(calls[0]).to.include("api.search.brave.com");
-	});
-
-	it("uses SearXNG first for broad discovery", async () => {
-		const calls = installMockFetch((url) => {
-			if (url.startsWith("http://searxng.test/search")) {
-				return jsonResponse({ results: [{ title: "SearXNG", url: "https://example.com", content: "Snippet" }] });
-			}
-			throw new Error(`unexpected fetch ${url}`);
-		});
-
-		const result = await searchWithDiagnostics({ query: "homelab ansible ideas" });
-		expect(result.selectedBackend).to.equal("searxng");
-		expect(result.attempts).to.deep.include({ backend: "searxng", status: "success", message: "Selected searxng", resultCount: 1 });
-		expect(calls[0]).to.include("searxng.test");
-	});
-
-	it("captures backend errors and falls through to Firecrawl last", async () => {
-		installMockFetch((url) => {
-			if (url.startsWith("http://searxng.test/search")) return jsonResponse({ results: [] });
-			if (url.startsWith("https://api.search.brave.com/")) return jsonResponse({ error: "rate limited" }, 429);
-			if (url.startsWith("http://firecrawl.test/v2/search")) {
-				return jsonResponse({ data: { web: [{ title: "Fire", url: "https://fire.example", description: "Fallback" }] } });
-			}
-			throw new Error(`unexpected fetch ${url}`);
-		});
-
-		const result = await searchWithDiagnostics({ query: "homelab ansible ideas" });
-		expect(result.selectedBackend).to.equal("firecrawl");
-		expect(result.attempts.map((a) => a.backend)).to.deep.equal(["searxng", "brave", "firecrawl"]);
-		expect(result.attempts[1].status).to.equal("error");
-		expect(result.attempts[1].message).not.to.include("test-brave-key");
-	});
-
-	it("explicit backend tries only that backend", async () => {
-		const calls = installMockFetch((url) => {
-			if (url.startsWith("http://searxng.test/search")) return jsonResponse({ results: [] });
-			throw new Error(`unexpected fetch ${url}`);
-		});
-
-		try {
-			await searchWithDiagnostics({ query: "nothing", backend: "searxng" });
-			expect.fail("expected search to fail");
-		} catch (e: any) {
-			expect(e.message).to.include("searxng: empty");
-			expect(calls).to.have.length(1);
-		}
-	});
-
-});