@bacnh85/pi-web 0.1.0 → 0.1.2

package/README.md

@@ -4,6 +4,14 @@ Pi extension for web search, readable page extraction, and Firecrawl scraping/cr
 
 ## Install
 
+Install the published package from npm:
+
+```bash
+pi install npm:@bacnh85/pi-web
+```
+
+From this repository checkout, install only this extension package:
+
 ```bash
 cd extensions/pi-web
 npm install
@@ -14,6 +22,8 @@ pi install ./extensions/pi-web
 pi -e ./extensions/pi-web
 ```
 
+The package manifest points Pi directly at `./index.ts`, so published npm installs and local installs load the same extension entrypoint.
+
 If you manually copy this directory instead of using `pi install`, run `npm install --omit=dev` in the copied `pi-web` directory so readable-content dependencies such as `@mozilla/readability` are present.
 
 ## Configuration

package/index.ts

@@ -18,8 +18,8 @@ function piConfigDirs(): string[] {
 	return process.env.PI_CODING_AGENT_DIR ? [process.env.PI_CODING_AGENT_DIR] : [path.join(os.homedir(), ".pi", "agent"), path.join(os.homedir(), ".pi", "agents")];
 }
 
-function envFileCandidates(cwd = process.cwd()): string[] {
-	return [path.resolve(cwd, ".env.local"), path.resolve(cwd, ".env"), ...piConfigDirs().flatMap((dir) => [path.join(dir, ".env.local"), path.join(dir, ".env")])];
+function envFileCandidates(cwd = process.cwd(), includeCwd = true): string[] {
+	return [...(includeCwd ? [path.resolve(cwd, ".env.local"), path.resolve(cwd, ".env")] : []), ...piConfigDirs().flatMap((dir) => [path.join(dir, ".env.local"), path.join(dir, ".env")])];
 }
 
 function stripInlineComment(value: string): string {
@@ -58,9 +58,9 @@ function parseDotenvFile(file: string): Record<string, string> | null {
 	return values;
 }
 
-function findEnvValue(name: string, cwd = process.cwd()): { value?: string; source: string; checkedFiles: string[] } {
+function findEnvValue(name: string, cwd = process.cwd(), includeCwd = true): { value?: string; source: string; checkedFiles: string[] } {
 	if (process.env[name]) return { value: process.env[name], source: "process.env", checkedFiles: [] };
-	const checkedFiles = envFileCandidates(cwd);
+	const checkedFiles = envFileCandidates(cwd, includeCwd);
 	for (const file of checkedFiles) {
 		const parsed = parseDotenvFile(file);
 		if (parsed?.[name]) return { value: parsed[name], source: file, checkedFiles };
@@ -80,6 +80,10 @@ function cwdFromContext(ctx: any): string {
 	return typeof ctx?.cwd === "string" && ctx.cwd ? ctx.cwd : process.cwd();
 }
 
+function includeProjectEnv(ctx: any): boolean {
+	return typeof ctx?.isProjectTrusted === "function" ? ctx.isProjectTrusted() : false;
+}
+
 function signalWithTimeout(timeoutMs: number, signal?: AbortSignal): AbortSignal {
 	const timeoutSignal = AbortSignal.timeout(timeoutMs);
 	if (!signal) return timeoutSignal;
@@ -159,8 +163,8 @@ function normalizeSearxngBaseUrl(raw?: string): string {
 	return value.replace(/\/+$/, "");
 }
 
-function loadSearxngConfig(params: any = {}, cwd = process.cwd()) {
-	const baseUrlEnv = findEnvValue("SEARXNG_BASE_URL", cwd);
+function loadSearxngConfig(params: any = {}, cwd = process.cwd(), includeCwdEnv = false) {
+	const baseUrlEnv = findEnvValue("SEARXNG_BASE_URL", cwd, includeCwdEnv);
 	return { baseUrl: normalizeSearxngBaseUrl(params.searxng_base_url || params.base_url || baseUrlEnv.value), source: params.searxng_base_url || params.base_url ? "tool parameter" : baseUrlEnv.source || "default local" };
 }
 
@@ -195,14 +199,18 @@ function normalizeFirecrawlBaseUrl(raw?: string): string {
 	return value;
 }
 
-function loadFirecrawlConfig(params: any = {}, cwd = process.cwd()) {
-	const apiUrl = params.firecrawl_api_url || params.api_url || findEnvValue("FIRECRAWL_API_URL", cwd).value;
-	const apiKey = params.firecrawl_api_key || params.api_key || findEnvValue("FIRECRAWL_API_KEY", cwd).value;
+function loadFirecrawlConfig(params: any = {}, cwd = process.cwd(), includeCwdEnv = false) {
+	const apiUrlLookup = findEnvValue("FIRECRAWL_API_URL", cwd, includeCwdEnv);
+	const apiKeyLookup = findEnvValue("FIRECRAWL_API_KEY", cwd, includeCwdEnv);
+	const explicitApiKey = params.firecrawl_api_key || params.api_key;
+	const apiUrl = params.firecrawl_api_url || params.api_url || apiUrlLookup.value;
+	const apiKey = explicitApiKey || apiKeyLookup.value;
 	const timeoutValue = params.timeout_ms || findEnvValue("FIRECRAWL_TIMEOUT_MS", cwd).value;
 	const baseUrl = normalizeFirecrawlBaseUrl(apiUrl);
 	const isHosted = !apiUrl || baseUrl.startsWith(HOSTED_FIRECRAWL_BASE_URL);
 	const timeoutMs = timeoutValue ? Number.parseInt(String(timeoutValue), 10) : 60000;
 	if (!Number.isInteger(timeoutMs) || timeoutMs < 1000) throw new Error("FIRECRAWL_TIMEOUT_MS/timeout_ms must be an integer >= 1000.");
+	if (!isHosted && apiKey && !explicitApiKey) throw new Error("Refusing to send a configured FIRECRAWL_API_KEY to a custom Firecrawl URL. Pass firecrawl_api_key explicitly for that URL.");
 	if (isHosted && !apiKey) throw new Error("FIRECRAWL_API_KEY is required for hosted Firecrawl.");
 	return { baseUrl, apiKey, isHosted, timeoutMs };
 }
@@ -259,7 +267,7 @@ const searxngControlSchema = { searxng_base_url: Type.Optional(Type.String({ des
 export default function piWebExtension(pi: ExtensionAPI) {
 	pi.registerTool({ name: "brave_search", label: "Brave Search", description: "Search the web with Brave Search API, optionally fetching readable page content. Best as a fast hosted fallback or independent index for source discovery, current facts, docs lookup, and general web search.", promptSnippet: "Search current web results with Brave", promptGuidelines: ["Use SearXNG first for general web search when the self-hosted instance is available; use Brave as a fast hosted fallback or independent second source.", "Use include_content only for a small number of results to avoid rate limits and extra page fetches.", "Cite result URLs when web results materially support the answer; if Brave is rate-limited, try SearXNG or Firecrawl search."], parameters: Type.Object({ ...controlSchema, query: Type.String(), count: Type.Optional(Type.Number({ default: 5 })), country: Type.Optional(Type.String({ default: "US" })), freshness: Type.Optional(Type.String()), include_content: Type.Optional(Type.Boolean({ default: false })), content_chars: Type.Optional(Type.Number({ default: 5000 })) }), async execute(_id, params: any, signal: AbortSignal, _onUpdate: unknown, ctx: any) {
 		const cwd = cwdFromContext(ctx);
-		const apiKey = params.api_key || findEnvValue("BRAVE_API_KEY", cwd).value;
+		const apiKey = params.api_key || findEnvValue("BRAVE_API_KEY", cwd, includeProjectEnv(ctx)).value;
 		if (!apiKey) throw new Error("BRAVE_API_KEY is required.");
 		const count = Math.min(20, Math.max(1, params.count ?? 5));
 		const results = await fetchBraveResults(params.query, count, (params.country || "US").toUpperCase(), params.freshness || "", apiKey, signal);
@@ -275,13 +283,13 @@ export default function piWebExtension(pi: ExtensionAPI) {
 	} });
 
 	pi.registerTool({ name: "searxng_search", label: "SearXNG Search", description: "Search the web through the configured self-hosted SearXNG metasearch instance. Best for free, self-hosted source discovery without hosted API keys.", promptSnippet: "Search web results with self-hosted SearXNG", promptGuidelines: ["Use SearXNG for self-hosted metasearch and source discovery, especially when Brave is unavailable or avoiding hosted search APIs.", "Use SearXNG before browser/scrape tools when the task starts with finding URLs or sources.", "Cite result URLs when SearXNG results materially support the answer."], parameters: Type.Object({ ...searxngControlSchema, query: Type.String(), count: Type.Optional(Type.Number({ default: 5 })), pageno: Type.Optional(Type.Number({ default: 1 })), categories: Type.Optional(Type.String({ description: "Comma-separated SearXNG categories, e.g. general,science." })), engines: Type.Optional(Type.String({ description: "Comma-separated SearXNG engines, e.g. brave,wikipedia." })), language: Type.Optional(Type.String()), time_range: Type.Optional(Type.Union([Type.Literal("day"), Type.Literal("month"), Type.Literal("year")])), safesearch: Type.Optional(Type.Number({ description: "SearXNG safesearch level: 0, 1, or 2." })) }), async execute(_id, params: any, signal: AbortSignal, _onUpdate: unknown, ctx: any) {
-		const config = loadSearxngConfig(params, cwdFromContext(ctx));
+		const config = loadSearxngConfig(params, cwdFromContext(ctx), includeProjectEnv(ctx));
 		const result = await fetchSearxngResults(params, config.baseUrl, signal);
 		return { content: [{ type: "text" as const, text: truncateText(formatSearchResults(result.results || [])) }], details: { ...result, baseUrl: config.baseUrl } };
 	} });
 
 	pi.registerTool({ name: "firecrawl_search", label: "Firecrawl Search", description: "Search web/news/images through Firecrawl, optionally scraping result markdown. Best when SearXNG/Brave are unavailable or when Firecrawl-backed search/scrape is preferred.", promptSnippet: "Search with Firecrawl", promptGuidelines: ["Use when SearXNG and Brave are missing/rate-limited/insufficient, when Firecrawl search is preferred, or when you need scraped markdown from results.", "Keep limits conservative; scraped search is slower and heavier than plain search.", "Cite result URLs and verify scraped content before relying on it."], parameters: Type.Object({ ...firecrawlControlSchema, query: Type.String(), limit: Type.Optional(Type.Number({ default: 5 })), sources: Type.Optional(Type.String({ description: "Comma-separated sources: web,news,images", default: "web" })), country: Type.Optional(Type.String()), location: Type.Optional(Type.String()), tbs: Type.Optional(Type.String()), scrape: Type.Optional(Type.Boolean({ default: false })), content_chars: Type.Optional(Type.Number({ default: 5000 })) }), async execute(_id, params: any, signal: AbortSignal, _onUpdate: unknown, ctx: any) {
-		const config = loadFirecrawlConfig(params, cwdFromContext(ctx));
+		const config = loadFirecrawlConfig(params, cwdFromContext(ctx), includeProjectEnv(ctx));
 		const sources = String(params.sources || "web").split(",").map((type) => ({ type: type.trim() })).filter((s) => s.type);
 		const body = { query: params.query, limit: Math.min(100, Math.max(1, params.limit ?? 5)), sources, ...(params.country ? { country: String(params.country).toUpperCase() } : {}), ...(params.location ? { location: params.location } : {}), ...(params.tbs ? { tbs: params.tbs } : {}), ...(params.scrape ? { scrapeOptions: { formats: ["markdown"], onlyMainContent: true } } : {}) };
 		const result = await firecrawlRequest(config, "POST", "/search", body, true, signal);
@@ -291,19 +299,19 @@ export default function piWebExtension(pi: ExtensionAPI) {
 	pi.registerTool({ name: "firecrawl_scrape", label: "Firecrawl Scrape", description: "Scrape a URL through Firecrawl as markdown/html/links/summary/json. Best for dynamic pages, higher-quality extraction, links, or structured JSON extraction.", promptSnippet: "Scrape a URL with Firecrawl", promptGuidelines: ["Use for known URLs when extraction quality matters, pages are dynamic, or markdown/links/JSON extraction is needed.", "Use prompt/schema only for explicit structured extraction tasks.", "Prefer brave_content for simple fast known-URL extraction when Firecrawl is unnecessary."], parameters: Type.Object({ ...firecrawlControlSchema, url: Type.String(), formats: Type.Optional(Type.String({ default: "markdown" })), prompt: Type.Optional(Type.String()), schema: Type.Optional(Type.Any()), wait_for: Type.Optional(Type.Number()), timeout: Type.Optional(Type.Number()), mobile: Type.Optional(Type.Boolean()), country: Type.Optional(Type.String()), actions: Type.Optional(Type.Any()), only_main_content: Type.Optional(Type.Boolean({ default: true })), content_chars: Type.Optional(Type.Number({ default: 20000 })) }), async execute(_id, params: any, signal: AbortSignal, _onUpdate: unknown, ctx: any) {
 		const formats = String(params.formats || "markdown").split(",").map((f) => f.trim()).filter(Boolean).map((f) => f === "json" && (params.prompt || params.schema) ? { type: "json", ...(params.prompt ? { prompt: params.prompt } : {}), ...(params.schema ? { schema: params.schema } : {}) } : f);
 		const body = { url: params.url, formats, onlyMainContent: params.only_main_content !== false, ...(params.wait_for ? { waitFor: params.wait_for } : {}), ...(params.timeout ? { timeout: params.timeout } : {}), ...(params.mobile ? { mobile: true } : {}), ...(params.country ? { location: { country: String(params.country).toUpperCase() } } : {}), ...(params.actions ? { actions: params.actions } : {}) };
-		const result = await firecrawlRequest(loadFirecrawlConfig(params, cwdFromContext(ctx)), "POST", "/scrape", body, true, signal);
+		const result = await firecrawlRequest(loadFirecrawlConfig(params, cwdFromContext(ctx), includeProjectEnv(ctx)), "POST", "/scrape", body, true, signal);
 		return { content: [{ type: "text" as const, text: truncateText(formatFirecrawlScrape(result, params.content_chars ?? 20000)) }], details: result };
 	} });
 
 	pi.registerTool({ name: "firecrawl_map", label: "Firecrawl Map", description: "Discover URLs from a site with Firecrawl map. Best before crawling or when you need candidate pages from a site/docs section.", promptSnippet: "Map site URLs", promptGuidelines: ["Use before crawling to identify relevant URLs and avoid unnecessary broad crawls.", "Keep limits small unless the user explicitly asks for broad site discovery.", "Use mapped URLs with firecrawl_scrape for targeted extraction."], parameters: Type.Object({ ...firecrawlControlSchema, url: Type.String(), limit: Type.Optional(Type.Number({ default: 100 })), include_subdomains: Type.Optional(Type.Boolean({ default: false })) }), async execute(_id, params: any, signal: AbortSignal, _onUpdate: unknown, ctx: any) {
-		const result = await firecrawlRequest(loadFirecrawlConfig(params, cwdFromContext(ctx)), "POST", "/map", { url: params.url, limit: params.limit ?? 100, includeSubdomains: Boolean(params.include_subdomains) }, true, signal);
+		const result = await firecrawlRequest(loadFirecrawlConfig(params, cwdFromContext(ctx), includeProjectEnv(ctx)), "POST", "/map", { url: params.url, limit: params.limit ?? 100, includeSubdomains: Boolean(params.include_subdomains) }, true, signal);
 		const urls = result.data || result.links || result.urls || [];
 		const text = Array.isArray(urls) ? urls.map((u: any) => u.url || u).join("\n") : JSON.stringify(result, null, 2);
 		return { content: [{ type: "text" as const, text: truncateText(text) }], details: result };
 	} });
 
 	pi.registerTool({ name: "firecrawl_crawl", label: "Firecrawl Crawl", description: "Start a conservative Firecrawl site crawl and optionally poll for results. Best for small docs/site sections after mapping or when multiple pages are required.", promptSnippet: "Crawl a small site section", promptGuidelines: ["Use only when multiple pages from a site are needed; prefer firecrawl_map plus targeted scrape when possible.", "Keep limits conservative and use include/exclude paths to narrow scope.", "Avoid large crawls unless the user explicitly requests them and accepts runtime/load."], parameters: Type.Object({ ...firecrawlControlSchema, url: Type.String(), limit: Type.Optional(Type.Number({ default: 10 })), include_paths: Type.Optional(Type.String()), exclude_paths: Type.Optional(Type.String()), poll: Type.Optional(Type.Boolean({ default: false })) }), async execute(_id, params: any, signal: AbortSignal, _onUpdate: unknown, ctx: any) {
-		const config = loadFirecrawlConfig(params, cwdFromContext(ctx));
+		const config = loadFirecrawlConfig(params, cwdFromContext(ctx), includeProjectEnv(ctx));
 		let result = await firecrawlRequest(config, "POST", "/crawl", { url: params.url, limit: Math.min(10000, Math.max(1, params.limit ?? 10)), includePaths: params.include_paths ? String(params.include_paths).split(",").map((s) => s.trim()).filter(Boolean) : [], excludePaths: params.exclude_paths ? String(params.exclude_paths).split(",").map((s) => s.trim()).filter(Boolean) : [], scrapeOptions: { formats: ["markdown"], onlyMainContent: true } }, true, signal);
 		const id = result.id || result.data?.id;
 		if (params.poll && id && !Array.isArray(result.data)) {
@@ -316,14 +324,16 @@ export default function piWebExtension(pi: ExtensionAPI) {
 
 	pi.registerTool({ name: "web_status", label: "Web Provider Status", description: "Show Brave, SearXNG, and Firecrawl configuration status without printing secrets.", promptSnippet: "Check web provider configuration", promptGuidelines: ["Use when web tools fail due to missing credentials/config or before comparing Brave, SearXNG, and Firecrawl availability.", "Never print API key values; this tool reports only presence and source."], parameters: Type.Object({}), async execute(_id, _params, _signal, _onUpdate, ctx: any) {
 		const cwd = cwdFromContext(ctx);
-		const braveKey = findEnvValue("BRAVE_API_KEY", cwd); const searxngUrl = findEnvValue("SEARXNG_BASE_URL", cwd); const fireKey = findEnvValue("FIRECRAWL_API_KEY", cwd); const fireUrl = findEnvValue("FIRECRAWL_API_URL", cwd);
+		const trusted = includeProjectEnv(ctx);
+		const braveKey = findEnvValue("BRAVE_API_KEY", cwd, trusted); const searxngUrl = findEnvValue("SEARXNG_BASE_URL", cwd, trusted); const fireKey = findEnvValue("FIRECRAWL_API_KEY", cwd, trusted); const fireUrl = findEnvValue("FIRECRAWL_API_URL", cwd, trusted);
 		const status = { brave: { apiKeyFound: Boolean(braveKey.value), apiKeySource: braveKey.value ? braveKey.source : "not set" }, searxng: { baseUrl: normalizeSearxngBaseUrl(searxngUrl.value), baseUrlSource: searxngUrl.source || "default local" }, firecrawl: { baseUrl: normalizeFirecrawlBaseUrl(fireUrl.value), apiUrlSource: fireUrl.source || "default hosted", apiKeyFound: Boolean(fireKey.value), apiKeySource: fireKey.value ? fireKey.source : "not set", hostedMode: !fireUrl.value || normalizeFirecrawlBaseUrl(fireUrl.value).startsWith(HOSTED_FIRECRAWL_BASE_URL) } };
 		return { content: [{ type: "text" as const, text: JSON.stringify(status, null, 2) }], details: status };
 	} });
 
 	pi.registerCommand("web-status", { description: "Show web provider configuration status", handler: async (_args, ctx) => {
 		const cwd = cwdFromContext(ctx);
-		const braveKey = findEnvValue("BRAVE_API_KEY", cwd); const searxngUrl = findEnvValue("SEARXNG_BASE_URL", cwd); const fireKey = findEnvValue("FIRECRAWL_API_KEY", cwd); const fireUrl = findEnvValue("FIRECRAWL_API_URL", cwd);
+		const trusted = includeProjectEnv(ctx);
+		const braveKey = findEnvValue("BRAVE_API_KEY", cwd, trusted); const searxngUrl = findEnvValue("SEARXNG_BASE_URL", cwd, trusted); const fireKey = findEnvValue("FIRECRAWL_API_KEY", cwd, trusted); const fireUrl = findEnvValue("FIRECRAWL_API_URL", cwd, trusted);
 		ctx.ui.notify(`Brave API key: ${braveKey.value ? `found (${braveKey.source})` : "not set"}\nSearXNG base URL: ${normalizeSearxngBaseUrl(searxngUrl.value)} (${searxngUrl.source || "default local"})\nFirecrawl base URL: ${normalizeFirecrawlBaseUrl(fireUrl.value)}\nFirecrawl API key: ${fireKey.value ? `found (${fireKey.source})` : "not set"}`, "info");
 	} });
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bacnh85/pi-web",
-  "version": "0.1.0",
+  "version": "0.1.2",
   "description": "Pi extension for web search, page extraction, and Firecrawl scraping/crawling.",
   "type": "module",
   "license": "MIT",