@bacnh85/pi-web 0.1.0

package/README.md

@@ -0,0 +1,72 @@
+# pi-web
+
+Pi extension for web search, readable page extraction, and Firecrawl scraping/crawling.
+
+## Install
+
+```bash
+cd extensions/pi-web
+npm install
+cd ../..
+
+pi install ./extensions/pi-web
+# or test directly
+pi -e ./extensions/pi-web
+```
+
+If you manually copy this directory instead of using `pi install`, run `npm install --omit=dev` in the copied `pi-web` directory so readable-content dependencies such as `@mozilla/readability` are present.
+
+## Configuration
+
+Environment lookup order:
+
+1. Process environment
+2. Current working directory `.env.local`
+3. Current working directory `.env`
+4. Pi global config `.env.local` (`$PI_CODING_AGENT_DIR/.env.local` when set; otherwise `~/.pi/agent/.env.local` then `~/.pi/agents/.env.local`)
+5. Pi global config `.env` (`$PI_CODING_AGENT_DIR/.env` when set; otherwise `~/.pi/agent/.env` then `~/.pi/agents/.env`)
+
+Variables:
+
+- `BRAVE_API_KEY` — required for Brave Search.
+- `SEARXNG_BASE_URL` — optional; defaults to `http://172.30.55.22:8888`.
+- `FIRECRAWL_API_URL` — optional; defaults to `https://api.firecrawl.dev/v2`.
+- `FIRECRAWL_API_KEY` — required for hosted Firecrawl, optional for self-hosted instances without auth.
+- `FIRECRAWL_TIMEOUT_MS` — optional request timeout, default `60000`.
+
+Secrets are never printed; status reports only show presence/source.
+
+## Tools
+
+Brave:
+
+- `brave_search` — search web results, optionally fetch readable result content.
+- `brave_content` — fetch a URL and extract readable markdown.
+
+SearXNG:
+
+- `searxng_search` — search web results through a configured self-hosted SearXNG metasearch instance.
+
+Firecrawl:
+
+- `firecrawl_search` — search web/news/images, optionally scrape result markdown.
+- `firecrawl_scrape` — scrape a URL as markdown/html/links/summary/json.
+- `firecrawl_map` — discover URLs for a site.
+- `firecrawl_crawl` — start a conservative crawl, optionally polling for results.
+
+Utility:
+
+- `web_status` — show configured provider status without secrets.
+- `/web-status` — command version of the status check.
+
+## Practical Guidance
+
+- Use `searxng_search` first for general web search, docs lookup, current facts, and source discovery; it is fast, self-hosted, and avoids hosted API costs/rate limits.
+- Use `brave_search` as a fast hosted fallback when SearXNG results are weak/unavailable, or when an independent search index is useful. Use `include_content` sparingly.
+- Use `brave_content` for fast known-URL article/docs extraction when simple readable markdown is enough.
+- Use `firecrawl_search` when SearXNG/Brave are unavailable, when Firecrawl search is preferred, or when search results should include scraped markdown.
+- Use `firecrawl_scrape` for known URLs when extraction quality matters, pages are dynamic, links are needed, or structured JSON extraction is requested.
+- Use `firecrawl_map` before crawling to discover candidate URLs and keep crawls targeted.
+- Use `firecrawl_crawl` only when multiple pages are required; keep limits conservative and use include/exclude paths.
+- When answering from web content, cite source URLs from result links or metadata.
+- Use `web_status` when provider configuration is uncertain or a web tool fails due to credentials/config.

package/index.ts

@@ -0,0 +1,329 @@
+/// <reference path="./types.d.ts" />
+
+import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+import { Type } from "typebox";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { createRequire } from "node:module";
+
+const require = createRequire(import.meta.url);
+
+const OUTPUT_MAX_BYTES = 50 * 1024;
+const OUTPUT_MAX_LINES = 2_000;
+const HOSTED_FIRECRAWL_BASE_URL = "https://api.firecrawl.dev/v2";
+const DEFAULT_SEARXNG_BASE_URL = "http://172.30.55.22:8888";
+
+function piConfigDirs(): string[] {
+	return process.env.PI_CODING_AGENT_DIR ? [process.env.PI_CODING_AGENT_DIR] : [path.join(os.homedir(), ".pi", "agent"), path.join(os.homedir(), ".pi", "agents")];
+}
+
+function envFileCandidates(cwd = process.cwd()): string[] {
+	return [path.resolve(cwd, ".env.local"), path.resolve(cwd, ".env"), ...piConfigDirs().flatMap((dir) => [path.join(dir, ".env.local"), path.join(dir, ".env")])];
+}
+
+function stripInlineComment(value: string): string {
+	let quote = "";
+	let escaped = false;
+	for (let i = 0; i < value.length; i++) {
+		const char = value[i];
+		if (escaped) { escaped = false; continue; }
+		if (char === "\\") { escaped = true; continue; }
+		if (quote) { if (char === quote) quote = ""; continue; }
+		if (char === '"' || char === "'") { quote = char; continue; }
+		if (char === "#" && (i === 0 || /\s/.test(value[i - 1]))) return value.slice(0, i);
+	}
+	return value;
+}
+
+function parseDotenvValue(rawValue: string): string {
+	let value = stripInlineComment(rawValue).trim();
+	if ((value.startsWith('"') && value.endsWith('"')) || (value.startsWith("'") && value.endsWith("'"))) {
+		value = value.slice(1, -1);
+		return value.replace(/\\n/g, "\n").replace(/\\r/g, "\r").replace(/\\t/g, "\t");
+	}
+	return value.trim();
+}
+
+function parseDotenvFile(file: string): Record<string, string> | null {
+	let text: string;
+	try { text = fs.readFileSync(file, "utf8"); } catch (e: any) { if (e?.code === "ENOENT") return null; throw e; }
+	const values: Record<string, string> = {};
+	for (const rawLine of text.split(/\r?\n/)) {
+		const line = rawLine.trim();
+		if (!line || line.startsWith("#")) continue;
+		const match = line.match(/^(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)$/);
+		if (match) values[match[1]] = parseDotenvValue(match[2]);
+	}
+	return values;
+}
+
+function findEnvValue(name: string, cwd = process.cwd()): { value?: string; source: string; checkedFiles: string[] } {
+	if (process.env[name]) return { value: process.env[name], source: "process.env", checkedFiles: [] };
+	const checkedFiles = envFileCandidates(cwd);
+	for (const file of checkedFiles) {
+		const parsed = parseDotenvFile(file);
+		if (parsed?.[name]) return { value: parsed[name], source: file, checkedFiles };
+	}
+	return { value: undefined, source: "", checkedFiles };
+}
+
+function truncateText(text: string): string {
+	const lines = text.split("\n");
+	let output = lines.slice(0, OUTPUT_MAX_LINES).join("\n");
+	while (Buffer.byteLength(output, "utf8") > OUTPUT_MAX_BYTES) output = output.slice(0, Math.max(0, output.length - 1024));
+	if (lines.length > OUTPUT_MAX_LINES || output.length < text.length) output += `\n\n[Web output truncated to ${OUTPUT_MAX_LINES} lines / ${OUTPUT_MAX_BYTES} bytes.]`;
+	return output;
+}
+
+function cwdFromContext(ctx: any): string {
+	return typeof ctx?.cwd === "string" && ctx.cwd ? ctx.cwd : process.cwd();
+}
+
+function signalWithTimeout(timeoutMs: number, signal?: AbortSignal): AbortSignal {
+	const timeoutSignal = AbortSignal.timeout(timeoutMs);
+	if (!signal) return timeoutSignal;
+	return AbortSignal.any([signal, timeoutSignal]);
+}
+
+function abortableSleep(ms: number, signal?: AbortSignal): Promise<void> {
+	if (signal?.aborted) return Promise.reject(signal.reason ?? new Error("Operation aborted."));
+	return new Promise((resolve, reject) => {
+		const timer = setTimeout(cleanupResolve, ms);
+		function cleanupResolve() {
+			signal?.removeEventListener("abort", cleanupReject);
+			resolve();
+		}
+		function cleanupReject() {
+			clearTimeout(timer);
+			reject(signal?.reason ?? new Error("Operation aborted."));
+		}
+		signal?.addEventListener("abort", cleanupReject, { once: true });
+	});
+}
+
+function sanitizeSnippet(text = ""): string {
+	return text.replace(/<[^>]*>/g, "").replace(/&nbsp;/gi, " ").replace(/&amp;/gi, "&").replace(/&lt;/gi, "<").replace(/&gt;/gi, ">").replace(/&quot;/gi, '"').replace(/&#39;|&#x27;/gi, "'").replace(/&#(\d+);/g, (_, code) => String.fromCodePoint(Number(code))).replace(/&#x([0-9a-f]+);/gi, (_, code) => String.fromCodePoint(Number.parseInt(code, 16))).replace(/\s+/g, " ").trim();
+}
+
+function loadReadableContentDependencies(): { Readability: any; JSDOM: any; TurndownService: any; gfm: any } {
+	try {
+		return {
+			Readability: require("@mozilla/readability").Readability,
+			JSDOM: require("jsdom").JSDOM,
+			TurndownService: require("turndown"),
+			gfm: require("turndown-plugin-gfm").gfm,
+		};
+	} catch (error: any) {
+		throw new Error(`Readable content extraction dependencies are missing. Install pi-web dependencies with \`npm install\` in the pi-web extension directory, or install the package with Pi so runtime dependencies are installed. Original error: ${error?.message || error}`);
+	}
+}
+
+function htmlToMarkdown(html: string, deps = loadReadableContentDependencies()): string {
+	const turndown = new deps.TurndownService({ headingStyle: "atx", codeBlockStyle: "fenced" });
+	turndown.use(deps.gfm as any);
+	turndown.addRule("removeEmptyLinks", { filter: (node: any) => node.nodeName === "A" && !node.textContent?.trim(), replacement: () => "" });
+	return turndown.turndown(html).replace(/\[\\?\[\s*\\?\]\]\([^)]*\)/g, "").replace(/ +/g, " ").replace(/\s+,/g, ",").replace(/\s+\./g, ".").replace(/\n{3,}/g, "\n\n").trim();
+}
+
+async function fetchReadableContent(url: string, timeoutMs = 15000, signal?: AbortSignal): Promise<{ title: string; markdown: string }> {
+	const response = await fetch(url, { headers: { "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/120 Safari/537.36", Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "en-US,en;q=0.9" }, signal: signalWithTimeout(timeoutMs, signal) });
+	if (!response.ok) throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+	const html = await response.text();
+	const deps = loadReadableContentDependencies();
+	const dom = new deps.JSDOM(html, { url });
+	const article = new deps.Readability(dom.window.document).parse();
+	if (article?.content) return { title: article.title || "", markdown: htmlToMarkdown(article.content, deps) };
+	const fallbackDoc = new deps.JSDOM(html, { url });
+	const doc = fallbackDoc.window.document;
+	doc.querySelectorAll("script, style, noscript, nav, header, footer, aside").forEach((el: Element) => el.remove());
+	const title = doc.querySelector("title")?.textContent?.trim() || "";
+	const main = doc.querySelector("main, article, [role='main'], .content, #content") || doc.body;
+	const fallbackHtml = main?.innerHTML || "";
+	if (fallbackHtml.trim().length <= 100) throw new Error("Could not extract readable content from this page.");
+	return { title, markdown: htmlToMarkdown(fallbackHtml, deps) };
+}
+
+async function fetchBraveResults(query: string, count: number, country: string, freshness: string, apiKey: string, signal?: AbortSignal) {
+	const params = new URLSearchParams({ q: query, count: String(count), country });
+	if (freshness) params.append("freshness", freshness);
+	const response = await fetch(`https://api.search.brave.com/res/v1/web/search?${params}`, { headers: { Accept: "application/json", "Accept-Encoding": "gzip", "X-Subscription-Token": apiKey }, signal });
+	if (!response.ok) throw new Error(`HTTP ${response.status}: ${response.statusText}\n${await response.text()}`);
+	const data: any = await response.json();
+	return (data.web?.results || []).slice(0, count).map((r: any) => ({ title: sanitizeSnippet(r.title || ""), url: r.url || "", snippet: sanitizeSnippet(r.description || ""), age: sanitizeSnippet(r.age || r.page_age || "") }));
+}
+
+function normalizeSearxngBaseUrl(raw?: string): string {
+	let value = (raw || DEFAULT_SEARXNG_BASE_URL).trim();
+	if (!/^https?:\/\//i.test(value)) value = /^(localhost|127\.|10\.|172\.(1[6-9]|2\d|3[01])\.|192\.168\.)/i.test(value) ? `http://${value}` : `https://${value}`;
+	return value.replace(/\/+$/, "");
+}
+
+function loadSearxngConfig(params: any = {}, cwd = process.cwd()) {
+	const baseUrlEnv = findEnvValue("SEARXNG_BASE_URL", cwd);
+	return { baseUrl: normalizeSearxngBaseUrl(params.searxng_base_url || params.base_url || baseUrlEnv.value), source: params.searxng_base_url || params.base_url ? "tool parameter" : baseUrlEnv.source || "default local" };
+}
+
+async function fetchSearxngResults(params: any, baseUrl: string, signal?: AbortSignal) {
+	const limit = Math.min(50, Math.max(1, params.count ?? 5));
+	const url = new URL(`${baseUrl}/search`);
+	url.searchParams.set("q", params.query);
+	url.searchParams.set("format", "json");
+	url.searchParams.set("pageno", String(Math.max(1, params.pageno ?? 1)));
+	for (const key of ["categories", "engines", "language", "time_range", "safesearch"] as const) if (params[key] !== undefined && params[key] !== "") url.searchParams.set(key, String(params[key]));
+	const response = await fetch(url, { headers: { Accept: "application/json", "User-Agent": "pi-web/0.1 SearXNG" }, signal: signalWithTimeout(params.timeout_ms || 15000, signal) });
+	const text = await response.text();
+	if (!response.ok) {
+		const hint = response.status === 403 ? "\nHint: enable JSON output in SearXNG settings.yml with search.formats including json." : "";
+		throw new Error(`HTTP ${response.status}: ${response.statusText}${hint}${text ? `\n${text}` : ""}`);
+	}
+	const data: any = text ? JSON.parse(text) : {};
+	const results = (data.results || []).slice(0, limit).map((r: any) => ({ title: sanitizeSnippet(r.title || ""), url: r.url || "", snippet: sanitizeSnippet(r.content || r.snippet || ""), engine: r.engine || "", category: r.category || "", score: r.score, publishedDate: r.publishedDate || r.published_date || "" }));
+	return { ...data, results };
+}
+
+function formatSearchResults(results: any[]): string {
+	if (!results.length) return "No results found.";
+	return results.map((r, i) => [`--- Result ${i + 1} ---`, `Title: ${r.title || ""}`, `Link: ${r.url || r.link || ""}`, r.age ? `Age: ${r.age}` : "", r.snippet || r.description ? `Snippet: ${r.snippet || r.description}` : "", r.content || r.markdown ? `Content:\n${r.content || r.markdown}` : ""].filter(Boolean).join("\n")).join("\n\n");
+}
+
+function normalizeFirecrawlBaseUrl(raw?: string): string {
+	let value = (raw || HOSTED_FIRECRAWL_BASE_URL).trim();
+	if (!/^https?:\/\//i.test(value)) value = /^(localhost|127\.|10\.|172\.(1[6-9]|2\d|3[01])\.|192\.168\.)/i.test(value) ? `http://${value}` : `https://${value}`;
+	value = value.replace(/\/+$/, "");
+	if (!/\/v\d+$/i.test(value)) value += "/v2";
+	return value;
+}
+
+function loadFirecrawlConfig(params: any = {}, cwd = process.cwd()) {
+	const apiUrl = params.firecrawl_api_url || params.api_url || findEnvValue("FIRECRAWL_API_URL", cwd).value;
+	const apiKey = params.firecrawl_api_key || params.api_key || findEnvValue("FIRECRAWL_API_KEY", cwd).value;
+	const timeoutValue = params.timeout_ms || findEnvValue("FIRECRAWL_TIMEOUT_MS", cwd).value;
+	const baseUrl = normalizeFirecrawlBaseUrl(apiUrl);
+	const isHosted = !apiUrl || baseUrl.startsWith(HOSTED_FIRECRAWL_BASE_URL);
+	const timeoutMs = timeoutValue ? Number.parseInt(String(timeoutValue), 10) : 60000;
+	if (!Number.isInteger(timeoutMs) || timeoutMs < 1000) throw new Error("FIRECRAWL_TIMEOUT_MS/timeout_ms must be an integer >= 1000.");
+	if (isHosted && !apiKey) throw new Error("FIRECRAWL_API_KEY is required for hosted Firecrawl.");
+	return { baseUrl, apiKey, isHosted, timeoutMs };
+}
+
+class FirecrawlHttpError extends Error { status: number; constructor(status: number, statusText: string, text: string) { super(`HTTP ${status}: ${statusText}${text ? `\n${text}` : ""}`); this.status = status; } }
+
+async function firecrawlRequest(config: any, method: string, endpoint: string, body?: unknown, allowFallback = true, signal?: AbortSignal): Promise<any> {
+	try { return await firecrawlRequestJson(config, method, endpoint, body, signal); }
+	catch (e) {
+		if (!allowFallback || config.isHosted || !config.baseUrl.endsWith("/v2") || !(e instanceof FirecrawlHttpError) || e.status !== 404) throw e;
+		return firecrawlRequestJson({ ...config, baseUrl: config.baseUrl.replace(/\/v2$/, "/v1") }, method, endpoint, body, signal);
+	}
+}
+
+async function firecrawlRequestJson(config: any, method: string, endpoint: string, body?: unknown, signal?: AbortSignal): Promise<any> {
+	const headers: Record<string, string> = { Accept: "application/json" };
+	if (body !== undefined) headers["Content-Type"] = "application/json";
+	if (config.apiKey) headers.Authorization = `Bearer ${config.apiKey}`;
+	const response = await fetch(`${config.baseUrl}${endpoint}`, { method, headers, body: body === undefined ? undefined : JSON.stringify(body), signal: signalWithTimeout(config.timeoutMs, signal) });
+	const text = await response.text();
+	if (!response.ok) throw new FirecrawlHttpError(response.status, response.statusText, text);
+	return text ? JSON.parse(text) : { success: true };
+}
+
+function formatFirecrawlScrape(data: any, maxChars = 20000): string {
+	const page = data.data || data;
+	const meta = page.metadata || {};
+	const parts: string[] = [];
+	if (meta.title) parts.push(`# ${meta.title}`);
+	if (meta.sourceURL || meta.url) parts.push(`Source: ${meta.sourceURL || meta.url}`);
+	if (meta.statusCode) parts.push(`Status: ${meta.statusCode}`);
+	if (data.warning || page.warning) parts.push(`Warning: ${data.warning || page.warning}`);
+	const body = page.markdown || page.summary || page.answer || page.html || page.rawHtml || "";
+	if (body) parts.push(String(body).slice(0, maxChars));
+	if (Array.isArray(page.links)) parts.push(["## Links", ...page.links.slice(0, 100).map((l: string) => `- ${l}`)].join("\n"));
+	return parts.join("\n\n") || JSON.stringify(data, null, 2);
+}
+
+function formatFirecrawlSearch(data: any, maxChars = 5000): string {
+	const web = data.data?.web || [];
+	const news = data.data?.news || [];
+	const images = data.data?.images || [];
+	const results = [...web, ...news].map((r: any) => ({ title: r.title || r.metadata?.title || "", url: r.url || r.metadata?.sourceURL || "", snippet: r.description || r.snippet || "", markdown: r.markdown ? String(r.markdown).slice(0, maxChars) : "" }));
+	let text = formatSearchResults(results);
+	if (images.length) text += "\n\n" + images.map((img: any, i: number) => [`--- Image ${i + 1} ---`, `Title: ${img.title || ""}`, `Image: ${img.imageUrl || ""}`, `Page: ${img.url || ""}`].join("\n")).join("\n\n");
+	if (data.warning) text += `\n\nWarning: ${data.warning}`;
+	return text;
+}
+
+const controlSchema = { api_key: Type.Optional(Type.String({ description: "Provider API key override. Prefer env vars." })), timeout_ms: Type.Optional(Type.Number({ description: "Request timeout in milliseconds." })) };
+const firecrawlControlSchema = { firecrawl_api_key: Type.Optional(Type.String({ description: "Firecrawl API key override. Prefer FIRECRAWL_API_KEY." })), firecrawl_api_url: Type.Optional(Type.String({ description: "Firecrawl API URL override. Defaults to FIRECRAWL_API_URL or hosted." })), timeout_ms: Type.Optional(Type.Number({ description: "Request timeout in milliseconds." })) };
+const searxngControlSchema = { searxng_base_url: Type.Optional(Type.String({ description: "SearXNG base URL override. Defaults to SEARXNG_BASE_URL or http://172.30.55.22:8888." })), timeout_ms: Type.Optional(Type.Number({ description: "Request timeout in milliseconds." })) };
+
+export default function piWebExtension(pi: ExtensionAPI) {
+	pi.registerTool({ name: "brave_search", label: "Brave Search", description: "Search the web with Brave Search API, optionally fetching readable page content. Best as a fast hosted fallback or independent index for source discovery, current facts, docs lookup, and general web search.", promptSnippet: "Search current web results with Brave", promptGuidelines: ["Use SearXNG first for general web search when the self-hosted instance is available; use Brave as a fast hosted fallback or independent second source.", "Use include_content only for a small number of results to avoid rate limits and extra page fetches.", "Cite result URLs when web results materially support the answer; if Brave is rate-limited, try SearXNG or Firecrawl search."], parameters: Type.Object({ ...controlSchema, query: Type.String(), count: Type.Optional(Type.Number({ default: 5 })), country: Type.Optional(Type.String({ default: "US" })), freshness: Type.Optional(Type.String()), include_content: Type.Optional(Type.Boolean({ default: false })), content_chars: Type.Optional(Type.Number({ default: 5000 })) }), async execute(_id, params: any, signal: AbortSignal, _onUpdate: unknown, ctx: any) {
+		const cwd = cwdFromContext(ctx);
+		const apiKey = params.api_key || findEnvValue("BRAVE_API_KEY", cwd).value;
+		if (!apiKey) throw new Error("BRAVE_API_KEY is required.");
+		const count = Math.min(20, Math.max(1, params.count ?? 5));
+		const results = await fetchBraveResults(params.query, count, (params.country || "US").toUpperCase(), params.freshness || "", apiKey, signal);
+		if (params.include_content) await Promise.all(results.map(async (r: any) => { try { r.content = (await fetchReadableContent(r.url, params.timeout_ms || 10000, signal)).markdown.slice(0, params.content_chars ?? 5000); } catch (e: any) { r.content = `(Fetch error for ${r.url}: ${e?.message || String(e)})`; } }));
+		return { content: [{ type: "text" as const, text: truncateText(formatSearchResults(results)) }], details: results };
+	} });
+
+	pi.registerTool({ name: "brave_content", label: "Brave Content", description: "Fetch a URL and extract readable markdown content without using the Brave Search API. Best for fast known-URL article/docs extraction.", promptSnippet: "Extract readable webpage content as markdown", promptGuidelines: ["Use for known URLs when simple readable markdown extraction is enough.", "Prefer Firecrawl scrape for dynamic pages, pages that block direct fetching, or when links/structured extraction are needed.", "Cite the source URL when using extracted content in an answer."], parameters: Type.Object({ url: Type.String(), timeout_ms: Type.Optional(Type.Number({ default: 15000 })), content_chars: Type.Optional(Type.Number({ default: 20000 })) }), async execute(_id, params: any, signal: AbortSignal) {
+		const parsed = new URL(params.url); if (!["http:", "https:"].includes(parsed.protocol)) throw new Error("URL must use http or https.");
+		const article = await fetchReadableContent(parsed.href, params.timeout_ms || 15000, signal);
+		const text = `${article.title ? `# ${article.title}\n\n` : ""}${article.markdown.slice(0, params.content_chars ?? 20000)}`;
+		return { content: [{ type: "text" as const, text: truncateText(text) }], details: { url: parsed.href, ...article } };
+	} });
+
+	pi.registerTool({ name: "searxng_search", label: "SearXNG Search", description: "Search the web through the configured self-hosted SearXNG metasearch instance. Best for free, self-hosted source discovery without hosted API keys.", promptSnippet: "Search web results with self-hosted SearXNG", promptGuidelines: ["Use SearXNG for self-hosted metasearch and source discovery, especially when Brave is unavailable or avoiding hosted search APIs.", "Use SearXNG before browser/scrape tools when the task starts with finding URLs or sources.", "Cite result URLs when SearXNG results materially support the answer."], parameters: Type.Object({ ...searxngControlSchema, query: Type.String(), count: Type.Optional(Type.Number({ default: 5 })), pageno: Type.Optional(Type.Number({ default: 1 })), categories: Type.Optional(Type.String({ description: "Comma-separated SearXNG categories, e.g. general,science." })), engines: Type.Optional(Type.String({ description: "Comma-separated SearXNG engines, e.g. brave,wikipedia." })), language: Type.Optional(Type.String()), time_range: Type.Optional(Type.Union([Type.Literal("day"), Type.Literal("month"), Type.Literal("year")])), safesearch: Type.Optional(Type.Number({ description: "SearXNG safesearch level: 0, 1, or 2." })) }), async execute(_id, params: any, signal: AbortSignal, _onUpdate: unknown, ctx: any) {
+		const config = loadSearxngConfig(params, cwdFromContext(ctx));
+		const result = await fetchSearxngResults(params, config.baseUrl, signal);
+		return { content: [{ type: "text" as const, text: truncateText(formatSearchResults(result.results || [])) }], details: { ...result, baseUrl: config.baseUrl } };
+	} });
+
+	pi.registerTool({ name: "firecrawl_search", label: "Firecrawl Search", description: "Search web/news/images through Firecrawl, optionally scraping result markdown. Best when SearXNG/Brave are unavailable or when Firecrawl-backed search/scrape is preferred.", promptSnippet: "Search with Firecrawl", promptGuidelines: ["Use when SearXNG and Brave are missing/rate-limited/insufficient, when Firecrawl search is preferred, or when you need scraped markdown from results.", "Keep limits conservative; scraped search is slower and heavier than plain search.", "Cite result URLs and verify scraped content before relying on it."], parameters: Type.Object({ ...firecrawlControlSchema, query: Type.String(), limit: Type.Optional(Type.Number({ default: 5 })), sources: Type.Optional(Type.String({ description: "Comma-separated sources: web,news,images", default: "web" })), country: Type.Optional(Type.String()), location: Type.Optional(Type.String()), tbs: Type.Optional(Type.String()), scrape: Type.Optional(Type.Boolean({ default: false })), content_chars: Type.Optional(Type.Number({ default: 5000 })) }), async execute(_id, params: any, signal: AbortSignal, _onUpdate: unknown, ctx: any) {
+		const config = loadFirecrawlConfig(params, cwdFromContext(ctx));
+		const sources = String(params.sources || "web").split(",").map((type) => ({ type: type.trim() })).filter((s) => s.type);
+		const body = { query: params.query, limit: Math.min(100, Math.max(1, params.limit ?? 5)), sources, ...(params.country ? { country: String(params.country).toUpperCase() } : {}), ...(params.location ? { location: params.location } : {}), ...(params.tbs ? { tbs: params.tbs } : {}), ...(params.scrape ? { scrapeOptions: { formats: ["markdown"], onlyMainContent: true } } : {}) };
+		const result = await firecrawlRequest(config, "POST", "/search", body, true, signal);
+		return { content: [{ type: "text" as const, text: truncateText(formatFirecrawlSearch(result, params.content_chars ?? 5000)) }], details: result };
+	} });
+
+	pi.registerTool({ name: "firecrawl_scrape", label: "Firecrawl Scrape", description: "Scrape a URL through Firecrawl as markdown/html/links/summary/json. Best for dynamic pages, higher-quality extraction, links, or structured JSON extraction.", promptSnippet: "Scrape a URL with Firecrawl", promptGuidelines: ["Use for known URLs when extraction quality matters, pages are dynamic, or markdown/links/JSON extraction is needed.", "Use prompt/schema only for explicit structured extraction tasks.", "Prefer brave_content for simple fast known-URL extraction when Firecrawl is unnecessary."], parameters: Type.Object({ ...firecrawlControlSchema, url: Type.String(), formats: Type.Optional(Type.String({ default: "markdown" })), prompt: Type.Optional(Type.String()), schema: Type.Optional(Type.Any()), wait_for: Type.Optional(Type.Number()), timeout: Type.Optional(Type.Number()), mobile: Type.Optional(Type.Boolean()), country: Type.Optional(Type.String()), actions: Type.Optional(Type.Any()), only_main_content: Type.Optional(Type.Boolean({ default: true })), content_chars: Type.Optional(Type.Number({ default: 20000 })) }), async execute(_id, params: any, signal: AbortSignal, _onUpdate: unknown, ctx: any) {
+		const formats = String(params.formats || "markdown").split(",").map((f) => f.trim()).filter(Boolean).map((f) => f === "json" && (params.prompt || params.schema) ? { type: "json", ...(params.prompt ? { prompt: params.prompt } : {}), ...(params.schema ? { schema: params.schema } : {}) } : f);
+		const body = { url: params.url, formats, onlyMainContent: params.only_main_content !== false, ...(params.wait_for ? { waitFor: params.wait_for } : {}), ...(params.timeout ? { timeout: params.timeout } : {}), ...(params.mobile ? { mobile: true } : {}), ...(params.country ? { location: { country: String(params.country).toUpperCase() } } : {}), ...(params.actions ? { actions: params.actions } : {}) };
+		const result = await firecrawlRequest(loadFirecrawlConfig(params, cwdFromContext(ctx)), "POST", "/scrape", body, true, signal);
+		return { content: [{ type: "text" as const, text: truncateText(formatFirecrawlScrape(result, params.content_chars ?? 20000)) }], details: result };
+	} });
+
+	pi.registerTool({ name: "firecrawl_map", label: "Firecrawl Map", description: "Discover URLs from a site with Firecrawl map. Best before crawling or when you need candidate pages from a site/docs section.", promptSnippet: "Map site URLs", promptGuidelines: ["Use before crawling to identify relevant URLs and avoid unnecessary broad crawls.", "Keep limits small unless the user explicitly asks for broad site discovery.", "Use mapped URLs with firecrawl_scrape for targeted extraction."], parameters: Type.Object({ ...firecrawlControlSchema, url: Type.String(), limit: Type.Optional(Type.Number({ default: 100 })), include_subdomains: Type.Optional(Type.Boolean({ default: false })) }), async execute(_id, params: any, signal: AbortSignal, _onUpdate: unknown, ctx: any) {
+		const result = await firecrawlRequest(loadFirecrawlConfig(params, cwdFromContext(ctx)), "POST", "/map", { url: params.url, limit: params.limit ?? 100, includeSubdomains: Boolean(params.include_subdomains) }, true, signal);
+		const urls = result.data || result.links || result.urls || [];
+		const text = Array.isArray(urls) ? urls.map((u: any) => u.url || u).join("\n") : JSON.stringify(result, null, 2);
+		return { content: [{ type: "text" as const, text: truncateText(text) }], details: result };
+	} });
+
+	pi.registerTool({ name: "firecrawl_crawl", label: "Firecrawl Crawl", description: "Start a conservative Firecrawl site crawl and optionally poll for results. Best for small docs/site sections after mapping or when multiple pages are required.", promptSnippet: "Crawl a small site section", promptGuidelines: ["Use only when multiple pages from a site are needed; prefer firecrawl_map plus targeted scrape when possible.", "Keep limits conservative and use include/exclude paths to narrow scope.", "Avoid large crawls unless the user explicitly requests them and accepts runtime/load."], parameters: Type.Object({ ...firecrawlControlSchema, url: Type.String(), limit: Type.Optional(Type.Number({ default: 10 })), include_paths: Type.Optional(Type.String()), exclude_paths: Type.Optional(Type.String()), poll: Type.Optional(Type.Boolean({ default: false })) }), async execute(_id, params: any, signal: AbortSignal, _onUpdate: unknown, ctx: any) {
+		const config = loadFirecrawlConfig(params, cwdFromContext(ctx));
+		let result = await firecrawlRequest(config, "POST", "/crawl", { url: params.url, limit: Math.min(10000, Math.max(1, params.limit ?? 10)), includePaths: params.include_paths ? String(params.include_paths).split(",").map((s) => s.trim()).filter(Boolean) : [], excludePaths: params.exclude_paths ? String(params.exclude_paths).split(",").map((s) => s.trim()).filter(Boolean) : [], scrapeOptions: { formats: ["markdown"], onlyMainContent: true } }, true, signal);
+		const id = result.id || result.data?.id;
+		if (params.poll && id && !Array.isArray(result.data)) {
+			for (let i = 0; i < 60; i++) { result = await firecrawlRequest(config, "GET", `/crawl/${id}`, undefined, true, signal); if (["completed", "failed", "cancelled"].includes(result.status || result.data?.status)) break; await abortableSleep(2000, signal); }
+		}
+		const pages = Array.isArray(result.data) ? result.data : result.data?.data || [];
+		const text = pages.length ? pages.map((p: any) => formatFirecrawlScrape({ data: p }, 12000)).join("\n\n---\n\n") : id ? `Crawl started: ${id}\nUse poll=true or check Firecrawl status/dashboard.` : JSON.stringify(result, null, 2);
+		return { content: [{ type: "text" as const, text: truncateText(text) }], details: result };
+	} });
+
+	pi.registerTool({ name: "web_status", label: "Web Provider Status", description: "Show Brave, SearXNG, and Firecrawl configuration status without printing secrets.", promptSnippet: "Check web provider configuration", promptGuidelines: ["Use when web tools fail due to missing credentials/config or before comparing Brave, SearXNG, and Firecrawl availability.", "Never print API key values; this tool reports only presence and source."], parameters: Type.Object({}), async execute(_id, _params, _signal, _onUpdate, ctx: any) {
+		const cwd = cwdFromContext(ctx);
+		const braveKey = findEnvValue("BRAVE_API_KEY", cwd); const searxngUrl = findEnvValue("SEARXNG_BASE_URL", cwd); const fireKey = findEnvValue("FIRECRAWL_API_KEY", cwd); const fireUrl = findEnvValue("FIRECRAWL_API_URL", cwd);
+		const status = { brave: { apiKeyFound: Boolean(braveKey.value), apiKeySource: braveKey.value ? braveKey.source : "not set" }, searxng: { baseUrl: normalizeSearxngBaseUrl(searxngUrl.value), baseUrlSource: searxngUrl.source || "default local" }, firecrawl: { baseUrl: normalizeFirecrawlBaseUrl(fireUrl.value), apiUrlSource: fireUrl.source || "default hosted", apiKeyFound: Boolean(fireKey.value), apiKeySource: fireKey.value ? fireKey.source : "not set", hostedMode: !fireUrl.value || normalizeFirecrawlBaseUrl(fireUrl.value).startsWith(HOSTED_FIRECRAWL_BASE_URL) } };
+		return { content: [{ type: "text" as const, text: JSON.stringify(status, null, 2) }], details: status };
+	} });
+
+	pi.registerCommand("web-status", { description: "Show web provider configuration status", handler: async (_args, ctx) => {
+		const cwd = cwdFromContext(ctx);
+		const braveKey = findEnvValue("BRAVE_API_KEY", cwd); const searxngUrl = findEnvValue("SEARXNG_BASE_URL", cwd); const fireKey = findEnvValue("FIRECRAWL_API_KEY", cwd); const fireUrl = findEnvValue("FIRECRAWL_API_URL", cwd);
+		ctx.ui.notify(`Brave API key: ${braveKey.value ? `found (${braveKey.source})` : "not set"}\nSearXNG base URL: ${normalizeSearxngBaseUrl(searxngUrl.value)} (${searxngUrl.source || "default local"})\nFirecrawl base URL: ${normalizeFirecrawlBaseUrl(fireUrl.value)}\nFirecrawl API key: ${fireKey.value ? `found (${fireKey.source})` : "not set"}`, "info");
+	} });
+}

package/package.json

@@ -0,0 +1,48 @@
+{
+  "name": "@bacnh85/pi-web",
+  "version": "0.1.0",
+  "description": "Pi extension for web search, page extraction, and Firecrawl scraping/crawling.",
+  "type": "module",
+  "license": "MIT",
+  "publishConfig": {
+    "access": "public"
+  },
+  "homepage": "https://github.com/bacnh85/skills#readme",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/bacnh85/skills.git",
+    "directory": "extensions/pi-web"
+  },
+  "keywords": [
+    "pi-package",
+    "pi-extension",
+    "web-search",
+    "brave-search",
+    "firecrawl",
+    "scraping"
+  ],
+  "files": [
+    "README.md",
+    "index.ts",
+    "types.d.ts"
+  ],
+  "pi": {
+    "extensions": [
+      "./index.ts"
+    ]
+  },
+  "dependencies": {
+    "@mozilla/readability": "^0.6.0",
+    "jsdom": "^27.0.1",
+    "turndown": "^7.2.2",
+    "turndown-plugin-gfm": "^1.0.2"
+  },
+  "peerDependencies": {
+    "@earendil-works/pi-coding-agent": "*",
+    "typebox": "*"
+  },
+  "devDependencies": {
+    "@types/jsdom": "^28.0.3",
+    "@types/turndown": "^5.0.6"
+  }
+}

package/types.d.ts

@@ -0,0 +1,3 @@
+declare module "turndown-plugin-gfm" {
+	export const gfm: unknown;
+}