@bacnh85/pi-web 0.3.0 → 0.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (6) hide show
  1. package/README.md +171 -77
  2. package/index.ts +216 -384
  3. package/lib/config.ts +8 -8
  4. package/lib/extract.ts +189 -0
  5. package/lib/search.ts +244 -0
  6. package/package.json +1 -1
package/index.ts CHANGED
@@ -9,40 +9,31 @@ import {
9
9
  includeProjectEnv,
10
10
  normalizeSearxngBaseUrl,
11
11
  normalizeFirecrawlBaseUrl,
12
- normalizeCrawl4aiBaseUrl,
13
- loadSearxngConfig,
12
+ normalizeCrawl4aiApiUrl,
14
13
  loadFirecrawlConfig,
15
14
  loadCrawl4aiConfig,
16
15
  HOSTED_FIRECRAWL_BASE_URL,
17
- DEFAULT_CRAWL4AI_BASE_URL,
18
16
  } from "./lib/config";
19
17
  import {
20
18
  truncateText,
21
- formatSearchResults,
22
19
  formatFirecrawlScrape,
23
- formatFirecrawlSearch,
24
20
  formatCrawl4aiResult,
25
- formatCrawl4aiStream,
26
21
  } from "./lib/format";
27
- import { fetchReadableContent } from "./lib/content";
28
- import { fetchBraveResults } from "./lib/brave";
29
- import { fetchSearxngResults } from "./lib/searxng";
22
+ import { searchWithDiagnostics, type SearchResult as UnifiedSearchResult } from "./lib/search";
23
+ import { extractWithDiagnostics, type ExtractMode } from "./lib/extract";
30
24
  import { firecrawlRequest, type FirecrawlResult } from "./lib/firecrawl";
31
25
  import {
32
- fetchCrawl4aiMarkdown,
33
26
  fetchCrawl4aiCrawl,
34
- fetchCrawl4aiStream,
35
27
  fetchCrawl4aiScreenshot,
36
28
  fetchCrawl4aiPdf,
37
29
  fetchCrawl4aiHealth,
38
30
  } from "./lib/crawl4ai";
39
31
 
40
32
  // ---------------------------------------------------------------------------
41
- // Schema fragments shared across tool registrations
33
+ // Shared schema fragment
42
34
  // ---------------------------------------------------------------------------
43
35
 
44
- const controlSchema = {
45
- api_key: Type.Optional(Type.String({ description: "Provider API key override. Prefer env vars." })),
36
+ const sharedControlSchema = {
46
37
  timeout_ms: Type.Optional(Type.Number({ description: "Request timeout in milliseconds." })),
47
38
  };
48
39
 
@@ -52,236 +43,156 @@ const firecrawlControlSchema = {
52
43
  timeout_ms: Type.Optional(Type.Number({ description: "Request timeout in milliseconds." })),
53
44
  };
54
45
 
55
- const searxngControlSchema = {
56
- searxng_base_url: Type.Optional(Type.String({ description: "SearXNG base URL override. Defaults to SEARXNG_BASE_URL or http://172.30.55.22:8888." })),
57
- timeout_ms: Type.Optional(Type.Number({ description: "Request timeout in milliseconds." })),
58
- };
59
-
60
46
  const crawl4aiControlSchema = {
61
- crawl4ai_base_url: Type.Optional(Type.String({ description: "Crawl4AI base URL override. Defaults to CRAWL4AI_BASE_URL or http://172.30.55.22:11235." })),
47
+ crawl4ai_api_url: Type.Optional(Type.String({ description: "Crawl4AI API URL override. Defaults to CRAWL4AI_API_URL or http://172.30.55.22:11235." })),
62
48
  crawl4ai_api_token: Type.Optional(Type.String({ description: "Crawl4AI API token override. Prefer CRAWL4AI_API_TOKEN." })),
63
49
  timeout_ms: Type.Optional(Type.Number({ description: "Request timeout in milliseconds." })),
64
50
  };
65
51
 
52
+ // ---------------------------------------------------------------------------
53
+ // Helper: format unified search results for text output
54
+ // ---------------------------------------------------------------------------
55
+
56
+ function formatUnifiedSearchResults(results: UnifiedSearchResult[]): string {
57
+ if (!results.length) return "No results found.";
58
+ return results
59
+ .map((r, i) =>
60
+ [
61
+ `--- Result ${i + 1} (backend: ${r.backend}) ---`,
62
+ `Title: ${r.title || ""}`,
63
+ `Link: ${r.url || ""}`,
64
+ r.age ? `Age: ${r.age}` : "",
65
+ r.snippet ? `Snippet: ${r.snippet}` : "",
66
+ r.content ? `Content:\n${r.content}` : "",
67
+ ]
68
+ .filter(Boolean)
69
+ .join("\n"),
70
+ )
71
+ .join("\n\n");
72
+ }
73
+
66
74
  // ---------------------------------------------------------------------------
67
75
  // Extension entry point
68
76
  // ---------------------------------------------------------------------------
69
77
 
70
78
  export default function piWebExtension(pi: ExtensionAPI) {
71
- // ── Brave Search ──────────────────────────────────────────────────────
79
+ // ── web_search ────────────────────────────────────────────────────────
72
80
  pi.registerTool({
73
- name: "brave_search",
74
- label: "Brave Search",
81
+ name: "web_search",
82
+ label: "Web Search",
75
83
  description:
76
- "Search the web with Brave Search API, optionally fetching readable page content. Best as a fast hosted fallback or independent index for source discovery, current facts, docs lookup, and general web search.",
77
- promptSnippet: "Search current web results with Brave",
84
+ "Search the web with adaptive backend selection. Uses SearXNG (self-hosted) for broad discovery, Brave (hosted API) for precision-sensitive queries and include_content, and Firecrawl as a last resort. Use backend to force a provider.",
85
+ promptSnippet: "Search current web results",
78
86
  promptGuidelines: [
79
- "Use SearXNG first for general web search when the self-hosted instance is available; use Brave as a fast hosted fallback or independent second source.",
80
- "Use include_content only for a small number of results to avoid rate limits and extra page fetches.",
81
- "Cite result URLs when web results materially support the answer; if Brave is rate-limited, try SearXNG or Firecrawl search.",
87
+ "Use web_search for source discovery, documentation lookup, current facts, and general web search.",
88
+ "Backend auto-selection is adaptive: broad discovery uses SearXNG first; precision queries, site: searches, docs/source lookups, and include_content use Brave first; Firecrawl is last resort.",
89
+ "If auto-selection returns poor-quality results, force backend: 'brave' or backend: 'searxng' for a different search index.",
90
+ "Use the backend parameter to force a specific search engine (e.g., backend: 'brave').",
91
+ "Use engines parameter for SearXNG tuning (e.g., engines: 'google,github' for technical queries).",
92
+ "include_content prefers Brave automatically because only Brave search fetches inline page content; for other result URLs, use web_extract.",
93
+ "Note: Firecrawl Search has poor semantic accuracy on domain-specific/ambiguous queries; prefer SearXNG or Brave for precision.",
94
+ "Cite result URLs when web results materially support the answer.",
82
95
  ],
83
96
  parameters: Type.Object({
84
- ...controlSchema,
85
97
  query: Type.String(),
86
98
  count: Type.Optional(Type.Number({ default: 5 })),
99
+ freshness: Type.Optional(Type.String({ description: "Time filter: e.g., 'pw' (past week), 'pm' (past month), 'py' (past year), or a date range like '2024-01-01to2024-06-30'." })),
87
100
  country: Type.Optional(Type.String({ default: "US" })),
88
- freshness: Type.Optional(Type.String()),
89
- include_content: Type.Optional(Type.Boolean({ default: false })),
101
+ backend: Type.Optional(Type.Union(
102
+ [Type.Literal("auto"), Type.Literal("searxng"), Type.Literal("brave"), Type.Literal("firecrawl")],
103
+ { default: "auto", description: "Search backend: auto (default), searxng, brave, or firecrawl." },
104
+ )),
105
+ engines: Type.Optional(Type.String({ description: "SearXNG engine override (e.g., 'google,github' for technical queries). Only used when backend is searxng or auto." })),
106
+ include_content: Type.Optional(Type.Boolean({ default: false, description: "Fetch page content alongside search results. Slower but provides inline content." })),
90
107
  content_chars: Type.Optional(Type.Number({ default: 5000 })),
108
+ ...sharedControlSchema,
91
109
  }),
92
110
  async execute(_id: string, params: Record<string, unknown>, signal: AbortSignal, _onUpdate: unknown, ctx: any) {
93
- const cwd = cwdFromContext(ctx);
94
- const apiKey = (params.api_key as string) || findEnvValue("BRAVE_API_KEY", cwd, includeProjectEnv(ctx)).value;
95
- if (!apiKey) throw new Error("BRAVE_API_KEY is required.");
96
- const count = Math.min(20, Math.max(1, (params.count as number) ?? 5));
97
- const results = await fetchBraveResults(
98
- params.query as string,
99
- count,
100
- ((params.country as string) || "US").toUpperCase(),
101
- (params.freshness as string) || "",
102
- apiKey,
111
+ const diagnostics = await searchWithDiagnostics({
112
+ query: params.query as string,
113
+ count: params.count as number | undefined,
114
+ freshness: params.freshness as string | undefined,
115
+ country: params.country as string | undefined,
116
+ backend: params.backend as "auto" | "searxng" | "brave" | "firecrawl" | undefined,
117
+ engines: params.engines as string | undefined,
118
+ include_content: params.include_content as boolean | undefined,
119
+ content_chars: params.content_chars as number | undefined,
120
+ timeout_ms: params.timeout_ms as number | undefined,
103
121
  signal,
104
- );
105
- if (params.include_content) {
106
- await Promise.all(
107
- results.map(async (r) => {
108
- try {
109
- const content = await fetchReadableContent(r.url, (params.timeout_ms as number) || 10000, signal);
110
- r.content = content.markdown.slice(0, (params.content_chars as number) ?? 5000);
111
- } catch (e: any) {
112
- r.content = `(Fetch error for ${r.url}: ${e?.message || String(e)})`;
113
- }
114
- }),
115
- );
116
- }
117
- return { content: [{ type: "text" as const, text: truncateText(formatSearchResults(results)) }], details: results };
122
+ _ctx: ctx,
123
+ });
124
+ const attempts = diagnostics.attempts.map((a) => `${a.backend}: ${a.status}${a.message ? ` (${a.message})` : ""}`).join("\n");
125
+ const text = `${formatUnifiedSearchResults(diagnostics.results)}\n\n--- Search diagnostics ---\nSelected backend: ${diagnostics.selectedBackend}\n${attempts}`;
126
+ return { content: [{ type: "text" as const, text: truncateText(text) }], details: diagnostics };
118
127
  },
119
128
  });
120
129
 
121
- // ── Web Content (readable extraction) ─────────────────────────────────
130
+ // ── web_extract ──────────────────────────────────────────────────────
122
131
  pi.registerTool({
123
- name: "web_content",
124
- label: "Web Content",
132
+ name: "web_extract",
133
+ label: "Web Content Extraction",
125
134
  description:
126
- "Fetch a URL and extract readable markdown content using JSDOM + Readability + Turndown (does not use the Brave API). Best for fast known-URL article/docs extraction.",
135
+ "Extract readable content from a URL with automatic backend selection. Tries static extraction (JSDOM+Readability, no API key) first, then dynamic extraction (Firecrawl Scrape, JS rendering), then full browser extraction (Crawl4AI, headless browser). Use the mode parameter for explicit control. If all modes fail, try web_screenshot for a visual snapshot.",
127
136
  promptSnippet: "Extract readable webpage content as markdown",
128
137
  promptGuidelines: [
129
- "Use for known URLs when simple readable markdown extraction is enough.",
130
- "Prefer Firecrawl scrape for dynamic pages, pages that block direct fetching, or when links/structured extraction are needed.",
138
+ "Use web_extract to get clean markdown content from a known URL.",
139
+ "Use mode: 'static' for simple static pages (fast, no API key needed, uses JSDOM+Readability).",
140
+ "Use mode: 'dynamic' for JavaScript-rendered pages (uses Firecrawl Scrape).",
141
+ "Use mode: 'full' for pages that need Crawl4AI rendering when other modes fail (resource-intensive).",
142
+ "Default mode 'auto' tries static → dynamic → full. Successive fallbacks are noted in output.",
143
+ "⚠️ Dynamic mode (Firecrawl Scrape) may fail on bot-protected sites (Ansible docs, CDN-backed doc sites). Full mode (Crawl4AI) handles more sites but is heavier.",
144
+ "If all extraction modes fail, try web_screenshot for a visual snapshot, or the page may require interactive login.",
145
+ "Use the prompt and schema parameters for structured JSON extraction (dynamic mode only); structured JSON is returned in details and text when available.",
131
146
  "Cite the source URL when using extracted content in an answer.",
132
147
  ],
133
148
  parameters: Type.Object({
134
149
  url: Type.String(),
135
- timeout_ms: Type.Optional(Type.Number({ default: 15000 })),
136
- content_chars: Type.Optional(Type.Number({ default: 20000 })),
137
- }),
138
- async execute(_id: string, params: Record<string, unknown>, signal: AbortSignal) {
139
- const parsed = new URL(params.url as string);
140
- if (!["http:", "https:"].includes(parsed.protocol)) throw new Error("URL must use http or https.");
141
- const article = await fetchReadableContent(parsed.href, (params.timeout_ms as number) || 15000, signal);
142
- const text = `${article.title ? `# ${article.title}\n\n` : ""}${article.markdown.slice(0, (params.content_chars as number) ?? 20000)}`;
143
- return { content: [{ type: "text" as const, text: truncateText(text) }], details: { url: parsed.href, ...article } };
144
- },
145
- });
146
-
147
- // ── SearXNG Search ───────────────────────────────────────────────────
148
- pi.registerTool({
149
- name: "searxng_search",
150
- label: "SearXNG Search",
151
- description:
152
- "Search the web through the configured self-hosted SearXNG metasearch instance. Best for free, self-hosted source discovery without hosted API keys.",
153
- promptSnippet: "Search web results with self-hosted SearXNG",
154
- promptGuidelines: [
155
- "Use SearXNG for self-hosted metasearch and source discovery, especially when Brave is unavailable or avoiding hosted search APIs.",
156
- "Use SearXNG before browser/scrape tools when the task starts with finding URLs or sources.",
157
- "Cite result URLs when SearXNG results materially support the answer.",
158
- ],
159
- parameters: Type.Object({
160
- ...searxngControlSchema,
161
- query: Type.String(),
162
- count: Type.Optional(Type.Number({ default: 5 })),
163
- pageno: Type.Optional(Type.Number({ default: 1 })),
164
- categories: Type.Optional(Type.String({ description: "Comma-separated SearXNG categories, e.g. general,science." })),
165
- engines: Type.Optional(Type.String({ description: "Comma-separated SearXNG engines, e.g. brave,wikipedia." })),
166
- language: Type.Optional(Type.String()),
167
- time_range: Type.Optional(Type.Union([Type.Literal("day"), Type.Literal("month"), Type.Literal("year")])),
168
- safesearch: Type.Optional(Type.Number({ description: "SearXNG safesearch level: 0, 1, or 2." })),
169
- }),
170
- async execute(_id: string, params: Record<string, unknown>, signal: AbortSignal, _onUpdate: unknown, ctx: any) {
171
- const config = loadSearxngConfig(params, cwdFromContext(ctx), includeProjectEnv(ctx));
172
- const result = await fetchSearxngResults(params, config.baseUrl, signal);
173
- return { content: [{ type: "text" as const, text: truncateText(formatSearchResults(result.results || [])) }], details: { ...result, baseUrl: config.baseUrl } };
174
- },
175
- });
176
-
177
- // ── Firecrawl Search ─────────────────────────────────────────────────
178
- pi.registerTool({
179
- name: "firecrawl_search",
180
- label: "Firecrawl Search",
181
- description:
182
- "Search web/news/images through Firecrawl, optionally scraping result markdown. Best when SearXNG/Brave are unavailable or when Firecrawl-backed search/scrape is preferred.",
183
- promptSnippet: "Search with Firecrawl",
184
- promptGuidelines: [
185
- "Use when SearXNG and Brave are missing/rate-limited/insufficient, when Firecrawl search is preferred, or when you need scraped markdown from results.",
186
- "Keep limits conservative; scraped search is slower and heavier than plain search.",
187
- "Cite result URLs and verify scraped content before relying on it.",
188
- ],
189
- parameters: Type.Object({
190
- ...firecrawlControlSchema,
191
- query: Type.String(),
192
- limit: Type.Optional(Type.Number({ default: 5 })),
193
- sources: Type.Optional(Type.String({ description: "Comma-separated sources: web,news,images", default: "web" })),
194
- country: Type.Optional(Type.String()),
195
- location: Type.Optional(Type.String()),
196
- tbs: Type.Optional(Type.String()),
197
- scrape: Type.Optional(Type.Boolean({ default: false })),
198
- content_chars: Type.Optional(Type.Number({ default: 5000 })),
199
- }),
200
- async execute(_id: string, params: Record<string, unknown>, signal: AbortSignal, _onUpdate: unknown, ctx: any) {
201
- const config = loadFirecrawlConfig(params, cwdFromContext(ctx), includeProjectEnv(ctx));
202
- const sources = String(params.sources || "web")
203
- .split(",")
204
- .map((type: string) => ({ type: type.trim() }))
205
- .filter((s: { type: string }) => s.type);
206
- const body: Record<string, unknown> = {
207
- query: params.query,
208
- limit: Math.min(100, Math.max(1, (params.limit as number) ?? 5)),
209
- sources,
210
- ...(params.country ? { country: String(params.country).toUpperCase() } : {}),
211
- ...(params.location ? { location: params.location } : {}),
212
- ...(params.tbs ? { tbs: params.tbs } : {}),
213
- ...(params.scrape ? { scrapeOptions: { formats: ["markdown"], onlyMainContent: true } } : {}),
214
- };
215
- const result = await firecrawlRequest(config, "POST", "/search", body, true, signal);
216
- return { content: [{ type: "text" as const, text: truncateText(formatFirecrawlSearch((result as FirecrawlResult) as Record<string, unknown>, (params.content_chars as number) ?? 5000)) }], details: result };
217
- },
218
- });
219
-
220
- // ── Firecrawl Scrape ─────────────────────────────────────────────────
221
- pi.registerTool({
222
- name: "firecrawl_scrape",
223
- label: "Firecrawl Scrape",
224
- description:
225
- "Scrape a URL through Firecrawl as markdown/html/links/summary/json. Best for dynamic pages, higher-quality extraction, links, or structured JSON extraction.",
226
- promptSnippet: "Scrape a URL with Firecrawl",
227
- promptGuidelines: [
228
- "Use for known URLs when extraction quality matters, pages are dynamic, or markdown/links/JSON extraction is needed.",
229
- "Use prompt/schema only for explicit structured extraction tasks.",
230
- "Prefer web_content for simple fast known-URL extraction when Firecrawl is unnecessary.",
231
- ],
232
- parameters: Type.Object({
233
- ...firecrawlControlSchema,
234
- url: Type.String(),
235
- formats: Type.Optional(Type.String({ default: "markdown" })),
236
- prompt: Type.Optional(Type.String()),
237
- schema: Type.Optional(Type.Any()),
238
- wait_for: Type.Optional(Type.Number()),
239
- timeout: Type.Optional(Type.Number()),
240
- mobile: Type.Optional(Type.Boolean()),
241
- country: Type.Optional(Type.String()),
242
- actions: Type.Optional(Type.Any()),
243
- only_main_content: Type.Optional(Type.Boolean({ default: true })),
150
+ mode: Type.Optional(Type.Union(
151
+ [Type.Literal("auto"), Type.Literal("static"), Type.Literal("dynamic"), Type.Literal("full")],
152
+ { default: "auto", description: "Extraction mode: auto (default), static, dynamic, or full." },
153
+ )),
154
+ prompt: Type.Optional(Type.String({ description: "Prompt for structured JSON extraction (dynamic mode only)." })),
155
+ schema: Type.Optional(Type.Any({ description: "JSON schema for structured extraction (dynamic mode only)." })),
244
156
  content_chars: Type.Optional(Type.Number({ default: 20000 })),
157
+ wait_for: Type.Optional(Type.Number({ description: "Milliseconds to wait for Firecrawl dynamic rendering before extraction. Full mode uses Crawl4AI /md and may ignore this." })),
158
+ mobile: Type.Optional(Type.Boolean({ default: false, description: "Emulate mobile viewport (dynamic mode only)." })),
159
+ ...sharedControlSchema,
245
160
  }),
246
161
  async execute(_id: string, params: Record<string, unknown>, signal: AbortSignal, _onUpdate: unknown, ctx: any) {
247
- const formats = String(params.formats || "markdown")
248
- .split(",")
249
- .map((f: string) => f.trim())
250
- .filter(Boolean)
251
- .map((f: string) =>
252
- f === "json" && (params.prompt || params.schema)
253
- ? { type: "json", ...(params.prompt ? { prompt: params.prompt } : {}), ...(params.schema ? { schema: params.schema } : {}) }
254
- : f,
255
- );
256
- const body: Record<string, unknown> = {
257
- url: params.url,
258
- formats,
259
- onlyMainContent: params.only_main_content !== false,
260
- ...(params.wait_for ? { waitFor: params.wait_for } : {}),
261
- ...(params.timeout ? { timeout: params.timeout } : {}),
262
- ...(params.mobile ? { mobile: true } : {}),
263
- ...(params.country ? { location: { country: String(params.country).toUpperCase() } } : {}),
264
- ...(params.actions ? { actions: params.actions } : {}),
265
- };
266
- const result = await firecrawlRequest(loadFirecrawlConfig(params, cwdFromContext(ctx), includeProjectEnv(ctx)), "POST", "/scrape", body, true, signal);
267
- return { content: [{ type: "text" as const, text: truncateText(formatFirecrawlScrape((result as FirecrawlResult) as Record<string, unknown>, (params.content_chars as number) ?? 20000)) }], details: result };
162
+ const diagnostics = await extractWithDiagnostics({
163
+ url: params.url as string,
164
+ mode: params.mode as ExtractMode | undefined,
165
+ prompt: params.prompt as string | undefined,
166
+ schema: params.schema,
167
+ content_chars: params.content_chars as number | undefined,
168
+ timeout_ms: params.timeout_ms as number | undefined,
169
+ wait_for: params.wait_for as number | undefined,
170
+ mobile: params.mobile as boolean | undefined,
171
+ signal,
172
+ _ctx: ctx,
173
+ });
174
+ const result = diagnostics.result;
175
+ const attempts = diagnostics.attempts.map((a) => `${a.mode}: ${a.status}${a.message ? ` (${a.message})` : ""}`).join("\n");
176
+ const text = `${result.title ? `# ${result.title}\n\n` : ""}${result.markdown}\n\n--- Extraction diagnostics ---\nSelected mode: ${diagnostics.selectedMode}\nFallback used: ${diagnostics.fallbackUsed}\n${attempts}`;
177
+ return { content: [{ type: "text" as const, text: truncateText(text) }], details: { url: params.url, ...diagnostics } };
268
178
  },
269
179
  });
270
180
 
271
- // ── Firecrawl Map ────────────────────────────────────────────────────
181
+ // ── web_map ──────────────────────────────────────────────────────────
272
182
  pi.registerTool({
273
- name: "firecrawl_map",
274
- label: "Firecrawl Map",
183
+ name: "web_map",
184
+ label: "Site URL Discovery",
275
185
  description:
276
- "Discover URLs from a site with Firecrawl map. Best before crawling or when you need candidate pages from a site/docs section.",
186
+ "Discover URLs from a site using Firecrawl Map. Best for finding candidate pages from a site or docs section before targeted extraction. Works best on base domains; may return empty results on sub-paths.",
277
187
  promptSnippet: "Map site URLs",
278
188
  promptGuidelines: [
279
- "Use before crawling to identify relevant URLs and avoid unnecessary broad crawls.",
280
- "Keep limits small unless the user explicitly asks for broad site discovery.",
281
- "Use mapped URLs with firecrawl_scrape for targeted extraction.",
189
+ "Use web_map to discover URLs from a site before crawling or targeted extraction.",
190
+ "Works best on base domains (e.g., https://example.com). Returns fewer results on sub-paths; use sitemap: 'only' for sitemap-only discovery on sub-paths.",
191
+ "Use sitemap: 'only' for sitemap-only discovery.",
192
+ "Use mapped URLs with web_extract for targeted extraction instead of web_crawl when only a few pages are needed.",
193
+ "Keep limits small unless broad site discovery is explicitly requested.",
282
194
  ],
283
195
  parameters: Type.Object({
284
- ...firecrawlControlSchema,
285
196
  url: Type.String(),
286
197
  limit: Type.Optional(Type.Number({ default: 100 })),
287
198
  include_subdomains: Type.Optional(Type.Boolean({ default: false })),
@@ -289,6 +200,7 @@ export default function piWebExtension(pi: ExtensionAPI) {
289
200
  sitemap: Type.Optional(Type.Union([Type.Literal("only"), Type.Literal("include"), Type.Literal("skip")], { description: "Sitemap mode: include (default), only (sitemap-only), or skip." })),
290
201
  use_index: Type.Optional(Type.Boolean({ default: true, description: "Whether to use the Firecrawl search index for URL discovery." })),
291
202
  ignore_cache: Type.Optional(Type.Boolean({ default: false, description: "Ignore cached map results." })),
203
+ ...firecrawlControlSchema,
292
204
  }),
293
205
  async execute(_id: string, params: Record<string, unknown>, signal: AbortSignal, _onUpdate: unknown, ctx: any) {
294
206
  const body: Record<string, unknown> = {
@@ -300,14 +212,8 @@ export default function piWebExtension(pi: ExtensionAPI) {
300
212
  if (params.sitemap !== undefined) body.sitemap = params.sitemap;
301
213
  if (params.use_index !== undefined) body.useIndex = params.use_index;
302
214
  if (params.ignore_cache !== undefined) body.ignoreCache = params.ignore_cache;
303
- const result = await firecrawlRequest(
304
- loadFirecrawlConfig(params, cwdFromContext(ctx), includeProjectEnv(ctx)),
305
- "POST",
306
- "/map",
307
- body,
308
- true,
309
- signal,
310
- );
215
+ const config = loadFirecrawlConfig(params as Record<string, unknown>, cwdFromContext(ctx), includeProjectEnv(ctx));
216
+ const result = await firecrawlRequest(config, "POST", "/map", body, true, signal);
311
217
  const urls = result.data || result.links || result.urls || [];
312
218
  const text = Array.isArray(urls) && urls.length > 0
313
219
  ? (urls as Array<Record<string, unknown> | string>).map((u: any) => u.url || u).join("\n")
@@ -316,46 +222,69 @@ export default function piWebExtension(pi: ExtensionAPI) {
316
222
  },
317
223
  });
318
224
 
319
- // ── Firecrawl Crawl ──────────────────────────────────────────────────
225
+ // ── web_crawl ────────────────────────────────────────────────────────
320
226
  pi.registerTool({
321
- name: "firecrawl_crawl",
322
- label: "Firecrawl Crawl",
227
+ name: "web_crawl",
228
+ label: "Site Crawl",
323
229
  description:
324
- "Start a conservative Firecrawl site crawl and optionally poll for results. Best for small docs/site sections after mapping or when multiple pages are required.",
230
+ "Crawl one or more pages from a site. Uses Firecrawl Crawl in 'light' mode (conservative, doc-focused) or Crawl4AI in 'full' mode (headless browser, rendered data, media, links). Accepts a single URL for Firecrawl-style crawling or multiple URLs for Crawl4AI-style crawling.",
325
231
  promptSnippet: "Crawl a small site section",
326
232
  promptGuidelines: [
327
- "Use only when multiple pages from a site are needed; prefer firecrawl_map plus targeted scrape when possible.",
328
- "Keep limits conservative and use include/exclude paths to narrow scope.",
329
- "Avoid large crawls unless the user explicitly requests them and accepts runtime/load.",
233
+ "Use web_crawl only when multiple pages from a site are truly needed; prefer web_map + web_extract for small numbers of pages.",
234
+ "Use mode: 'light' (default) for conservative Firecrawl crawling of docs/sites.",
235
+ "Use mode: 'full' for Crawl4AI crawling with full rendered data (JS, media, links). More resource-intensive.",
236
+ "For Firecrawl mode ('light'), use the url parameter with optional include_paths/exclude_paths.",
237
+ "For Crawl4AI mode ('full'), use the urls (array) parameter for batch crawling up to 100 URLs.",
238
+ "Keep limits conservative (default 10) unless the user explicitly requests large crawls.",
239
+ "Try web_map first to identify candidate URLs and pass them to web_extract before resorting to a full crawl.",
330
240
  ],
331
241
  parameters: Type.Object({
332
- ...firecrawlControlSchema,
333
- url: Type.String(),
242
+ url: Type.Optional(Type.String({ description: "URL for Firecrawl-style single-page crawl (mode: 'light')." })),
243
+ urls: Type.Optional(Type.Array(Type.String(), { description: "URLs for Crawl4AI-style multi-URL crawl (mode: 'full'), up to 100." })),
244
+ mode: Type.Optional(Type.Union([Type.Literal("light"), Type.Literal("full")], { default: "light", description: "Crawl mode: 'light' (Firecrawl, conservative) or 'full' (Crawl4AI, rendered data)." })),
334
245
  limit: Type.Optional(Type.Number({ default: 10 })),
335
- include_paths: Type.Optional(Type.String()),
336
- exclude_paths: Type.Optional(Type.String()),
337
- poll: Type.Optional(Type.Boolean({ default: false })),
246
+ include_paths: Type.Optional(Type.String({ description: "Comma-separated path patterns to include (Firecrawl mode only)." })),
247
+ exclude_paths: Type.Optional(Type.String({ description: "Comma-separated path patterns to exclude (Firecrawl mode only)." })),
248
+ poll: Type.Optional(Type.Boolean({ default: false, description: "Poll for crawl completion (Firecrawl mode only)." })),
249
+ browser_config: Type.Optional(Type.Any({ description: "Optional BrowserConfig JSON object (Crawl4AI mode only)." })),
250
+ crawler_config: Type.Optional(Type.Any({ description: "Optional CrawlerRunConfig JSON object (Crawl4AI mode only)." })),
251
+ content_chars: Type.Optional(Type.Number({ default: 20000 })),
252
+ ...firecrawlControlSchema,
253
+ ...crawl4aiControlSchema,
338
254
  }),
339
255
  async execute(_id: string, params: Record<string, unknown>, signal: AbortSignal, _onUpdate: unknown, ctx: any) {
340
- const config = loadFirecrawlConfig(params, cwdFromContext(ctx), includeProjectEnv(ctx));
256
+ const mode = (params.mode as string) || "light";
257
+ const cwd = cwdFromContext(ctx);
258
+ const trusted = includeProjectEnv(ctx);
259
+ const maxChars = (params.content_chars as number) ?? 20000;
260
+
261
+ if (mode === "full") {
262
+ // Crawl4AI mode
263
+ const urls = (params.urls as string[]) || (params.url ? [params.url as string] : []);
264
+ if (!urls.length) throw new Error("Either url or urls parameter is required for crawl.");
265
+ const config = loadCrawl4aiConfig(params as Record<string, unknown>, cwd, trusted);
266
+ const browserConfig = params.browser_config as Record<string, unknown> | undefined;
267
+ const crawlerConfig = params.crawler_config as Record<string, unknown> | undefined;
268
+ const result = await fetchCrawl4aiCrawl(config, urls, browserConfig, crawlerConfig, signal);
269
+ const text = formatCrawl4aiResult(result as unknown as Record<string, unknown>, maxChars);
270
+ return { content: [{ type: "text" as const, text: truncateText(text) }], details: result };
271
+ }
272
+
273
+ // Firecrawl mode ("light")
274
+ if (!params.url) throw new Error("The url parameter is required for Firecrawl mode ('light').");
275
+ const fcConfig = loadFirecrawlConfig(params as Record<string, unknown>, cwd, trusted);
341
276
  let result = await firecrawlRequest(
342
- config,
277
+ fcConfig,
343
278
  "POST",
344
279
  "/crawl",
345
280
  {
346
- url: params.url,
281
+ url: params.url as string,
347
282
  limit: Math.min(10000, Math.max(1, (params.limit as number) ?? 10)),
348
283
  includePaths: params.include_paths
349
- ? String(params.include_paths)
350
- .split(",")
351
- .map((s: string) => s.trim())
352
- .filter(Boolean)
284
+ ? String(params.include_paths).split(",").map((s: string) => s.trim()).filter(Boolean)
353
285
  : [],
354
286
  excludePaths: params.exclude_paths
355
- ? String(params.exclude_paths)
356
- .split(",")
357
- .map((s: string) => s.trim())
358
- .filter(Boolean)
287
+ ? String(params.exclude_paths).split(",").map((s: string) => s.trim()).filter(Boolean)
359
288
  : [],
360
289
  scrapeOptions: { formats: ["markdown"], onlyMainContent: true },
361
290
  },
@@ -366,8 +295,10 @@ export default function piWebExtension(pi: ExtensionAPI) {
366
295
  if (params.poll && id && !Array.isArray(result.data)) {
367
296
  const { abortableSleep } = await import("./lib/retry");
368
297
  for (let i = 0; i < 60; i++) {
369
- result = await firecrawlRequest(config, "GET", `/crawl/${id}`, undefined, true, signal);
370
- if (["completed", "failed", "cancelled"].includes(String(result.status || (result.data as Record<string, unknown> | undefined)?.status || ""))) break;
298
+ result = await firecrawlRequest(fcConfig, "GET", `/crawl/${id}`, undefined, true, signal);
299
+ if (["completed", "failed", "cancelled"].includes(
300
+ String(result.status || (result.data as Record<string, unknown> | undefined)?.status || ""),
301
+ )) break;
371
302
  await abortableSleep(2000, signal);
372
303
  }
373
304
  }
@@ -375,9 +306,7 @@ export default function piWebExtension(pi: ExtensionAPI) {
375
306
  ? (result.data as Record<string, unknown>[])
376
307
  : ((result.data as Record<string, unknown>)?.data as Record<string, unknown>[]) || [];
377
308
  const text = pages.length
378
- ? pages
379
- .map((p: Record<string, unknown>) => formatFirecrawlScrape({ data: p } as Record<string, unknown>, 12000))
380
- .join("\n\n---\n\n")
309
+ ? pages.map((p: Record<string, unknown>) => formatFirecrawlScrape({ data: p } as Record<string, unknown>, maxChars)).join("\n\n---\n\n")
381
310
  : id
382
311
  ? `Crawl started: ${id}\nUse poll=true or check Firecrawl status/dashboard.`
383
312
  : JSON.stringify(result, null, 2);
@@ -385,115 +314,34 @@ export default function piWebExtension(pi: ExtensionAPI) {
385
314
  },
386
315
  });
387
316
 
388
- // ── Crawl4AI Scrape ──────────────────────────────────────────────────
389
- pi.registerTool({
390
- name: "crawl4ai_scrape",
391
- label: "Crawl4AI Scrape",
392
- description:
393
- "Extract clean LLM-ready markdown from a URL using Crawl4AI's headless browser engine, with optional content filtering (fit, raw, bm25, llm). Best for JavaScript-rendered pages and when you need high-quality markdown for AI consumption.",
394
- promptSnippet: "Scrape with Crawl4AI",
395
- promptGuidelines: [
396
- "Prefer crawl4ai_scrape over web_content for dynamic/JS-rendered pages.",
397
- "Use filter=raw for full DOM-to-markdown, filter=fit (default) for Readability-based clean content.",
398
- "Use filter=bm25 with a query for relevance-ranked content extraction.",
399
- ],
400
- parameters: Type.Object({
401
- ...crawl4aiControlSchema,
402
- url: Type.String(),
403
- filter: Type.Optional(Type.Union([Type.Literal("fit"), Type.Literal("raw"), Type.Literal("bm25"), Type.Literal("llm")], { default: "fit", description: "Content-filter strategy: fit (default), raw, bm25, or llm." })),
404
- query: Type.Optional(Type.String({ description: "Query string used by BM25 or LLM filters." })),
405
- content_chars: Type.Optional(Type.Number({ default: 20000 })),
406
- }),
407
- async execute(_id: string, params: Record<string, unknown>, signal: AbortSignal, _onUpdate: unknown, ctx: any) {
408
- const config = loadCrawl4aiConfig(params, cwdFromContext(ctx), includeProjectEnv(ctx));
409
- const filter = (params.filter as string) || "fit";
410
- const query = params.query as string | undefined;
411
- const result = await fetchCrawl4aiMarkdown(config, params.url as string, filter, query, signal);
412
- const text = `# Crawl4AI Markdown (filter: ${filter})\nURL: ${params.url}\n\n${result.markdown.slice(0, (params.content_chars as number) ?? 20000)}`;
413
- return { content: [{ type: "text" as const, text: truncateText(text) }], details: result };
414
- },
415
- });
416
-
417
- // ── Crawl4AI Crawl ──────────────────────────────────────────────────
317
+ // ── web_screenshot ───────────────────────────────────────────────────
418
318
  pi.registerTool({
419
- name: "crawl4ai_crawl",
420
- label: "Crawl4AI Crawl",
319
+ name: "web_screenshot",
320
+ label: "Web Page Screenshot",
421
321
  description:
422
- "Crawl one or more URLs with Crawl4AI's headless browser, returning full crawl data including markdown, HTML, links, and media. Supports up to 100 URLs per request.",
423
- promptSnippet: "Crawl with Crawl4AI",
322
+ "Capture a full-page PNG screenshot of a URL using Crawl4AI's headless browser. Returns a base64-encoded PNG image suitable for visual inspection.",
323
+ promptSnippet: "Screenshot a webpage",
424
324
  promptGuidelines: [
425
- "Use crawl4ai_crawl when you need full crawl data (HTML, links, media) from multiple URLs.",
426
- "For single-URL markdown-only extraction, prefer crawl4ai_scrape.",
427
- "Keep to 10 or fewer URLs per request for reasonable response times.",
428
- ],
429
- parameters: Type.Object({
430
- ...crawl4aiControlSchema,
431
- urls: Type.Array(Type.String(), { minItems: 1, maxItems: 100, description: "URLs to crawl (1-100)." }),
432
- browser_config: Type.Optional(Type.Any({ description: "Optional BrowserConfig JSON object." })),
433
- crawler_config: Type.Optional(Type.Any({ description: "Optional CrawlerRunConfig JSON object." })),
434
- content_chars: Type.Optional(Type.Number({ default: 20000 })),
435
- }),
436
- async execute(_id: string, params: Record<string, unknown>, signal: AbortSignal, _onUpdate: unknown, ctx: any) {
437
- const config = loadCrawl4aiConfig(params, cwdFromContext(ctx), includeProjectEnv(ctx));
438
- const urls = params.urls as string[];
439
- const browserConfig = params.browser_config as Record<string, unknown> | undefined;
440
- const crawlerConfig = params.crawler_config as Record<string, unknown> | undefined;
441
- const result = await fetchCrawl4aiCrawl(config, urls, browserConfig, crawlerConfig, signal);
442
- const text = formatCrawl4aiResult(result as unknown as Record<string, unknown>, (params.content_chars as number) ?? 20000);
443
- return { content: [{ type: "text" as const, text: truncateText(text) }], details: result };
444
- },
445
- });
446
-
447
- // ── Crawl4AI Stream ─────────────────────────────────────────────────
448
- pi.registerTool({
449
- name: "crawl4ai_stream",
450
- label: "Crawl4AI Stream",
451
- description:
452
- "Crawl one or more URLs with Crawl4AI using streaming response (SSE-style JSON lines). Results arrive as each URL is processed, useful for long-running multi-URL crawls.",
453
- promptSnippet: "Stream crawl with Crawl4AI",
454
- promptGuidelines: [
455
- "Use crawl4ai_stream for large batch crawls where you want results incrementally.",
456
- "The response is newline-delimited JSON; each line is a complete CrawlResult.",
457
- "The stream ends with a line containing {\"status\": \"completed\"}.",
458
- ],
459
- parameters: Type.Object({
460
- ...crawl4aiControlSchema,
461
- urls: Type.Array(Type.String(), { minItems: 1, maxItems: 100, description: "URLs to crawl (1-100)." }),
462
- browser_config: Type.Optional(Type.Any({ description: "Optional BrowserConfig JSON object." })),
463
- crawler_config: Type.Optional(Type.Any({ description: "Optional CrawlerRunConfig JSON object." })),
464
- }),
465
- async execute(_id: string, params: Record<string, unknown>, signal: AbortSignal, _onUpdate: unknown, ctx: any) {
466
- const config = loadCrawl4aiConfig(params, cwdFromContext(ctx), includeProjectEnv(ctx));
467
- const urls = params.urls as string[];
468
- const browserConfig = params.browser_config as Record<string, unknown> | undefined;
469
- const crawlerConfig = params.crawler_config as Record<string, unknown> | undefined;
470
- const raw = await fetchCrawl4aiStream(config, urls, browserConfig, crawlerConfig, signal);
471
- const text = formatCrawl4aiStream(raw);
472
- return { content: [{ type: "text" as const, text: truncateText(text) }], details: { raw_length: raw.length } };
473
- },
474
- });
475
-
476
- // ── Crawl4AI Screenshot ─────────────────────────────────────────────
477
- pi.registerTool({
478
- name: "crawl4ai_screenshot",
479
- label: "Crawl4AI Screenshot",
480
- description:
481
- "Capture a full-page PNG screenshot of a URL using Crawl4AI's headless browser. Returns a base64-encoded screenshot image.",
482
- promptSnippet: "Screenshot with Crawl4AI",
483
- promptGuidelines: [
484
- "Use crawl4ai_screenshot when you need a visual snapshot of a rendered page.",
325
+ "Use web_screenshot when a visual snapshot of a rendered page is needed, or when web_extract fails on a heavily JS-dependent or bot-protected page.",
485
326
  "The screenshot is returned as a base64-encoded PNG string.",
486
- "Use the wait_for parameter to delay capture for dynamic content to render.",
327
+ "Use wait_for to delay capture for dynamic content to render (default 2 seconds).",
328
+ "Use wait_for_images to ensure images are loaded before capture.",
487
329
  ],
488
330
  parameters: Type.Object({
489
- ...crawl4aiControlSchema,
490
331
  url: Type.String(),
491
332
  wait_for: Type.Optional(Type.Number({ default: 2, description: "Seconds to wait before capturing screenshot." })),
492
333
  wait_for_images: Type.Optional(Type.Boolean({ default: false, description: "Wait for images to load before capture." })),
334
+ ...crawl4aiControlSchema,
493
335
  }),
494
336
  async execute(_id: string, params: Record<string, unknown>, signal: AbortSignal, _onUpdate: unknown, ctx: any) {
495
- const config = loadCrawl4aiConfig(params, cwdFromContext(ctx), includeProjectEnv(ctx));
496
- const result = await fetchCrawl4aiScreenshot(config, params.url as string, params.wait_for as number | undefined, params.wait_for_images as boolean | undefined, signal);
337
+ const config = loadCrawl4aiConfig(params as Record<string, unknown>, cwdFromContext(ctx), includeProjectEnv(ctx));
338
+ const result = await fetchCrawl4aiScreenshot(
339
+ config,
340
+ params.url as string,
341
+ params.wait_for as number | undefined,
342
+ params.wait_for_images as boolean | undefined,
343
+ signal,
344
+ );
497
345
  const screenshot = result.screenshot as string | undefined;
498
346
  const artifactUrl = result.url as string | undefined;
499
347
  const mime = result.mime as string | undefined;
@@ -507,23 +355,23 @@ export default function piWebExtension(pi: ExtensionAPI) {
507
355
  },
508
356
  });
509
357
 
510
- // ── Crawl4AI PDF ────────────────────────────────────────────────────
358
+ // ── web_pdf ──────────────────────────────────────────────────────────
511
359
  pi.registerTool({
512
- name: "crawl4ai_pdf",
513
- label: "Crawl4AI PDF",
360
+ name: "web_pdf",
361
+ label: "Web Page PDF",
514
362
  description:
515
- "Generate a PDF document of a URL using Crawl4AI's headless browser. Returns a base64-encoded PDF suitable for saving or further processing.",
516
- promptSnippet: "PDF with Crawl4AI",
363
+ "Generate a PDF document of a URL using Crawl4AI's headless browser. Returns a base64-encoded PDF suitable for saving or archiving.",
364
+ promptSnippet: "PDF a webpage",
517
365
  promptGuidelines: [
518
- "Use crawl4ai_pdf when you need a printable or archivable snapshot of a page.",
366
+ "Use web_pdf when a printable or archivable snapshot of a page is needed.",
519
367
  "The PDF is returned as a base64-encoded string.",
520
368
  ],
521
369
  parameters: Type.Object({
522
- ...crawl4aiControlSchema,
523
370
  url: Type.String(),
371
+ ...crawl4aiControlSchema,
524
372
  }),
525
373
  async execute(_id: string, params: Record<string, unknown>, signal: AbortSignal, _onUpdate: unknown, ctx: any) {
526
- const config = loadCrawl4aiConfig(params, cwdFromContext(ctx), includeProjectEnv(ctx));
374
+ const config = loadCrawl4aiConfig(params as Record<string, unknown>, cwdFromContext(ctx), includeProjectEnv(ctx));
527
375
  const result = await fetchCrawl4aiPdf(config, params.url as string, signal);
528
376
  const pdf = result.pdf as string | undefined;
529
377
  const artifactUrl = result.url as string | undefined;
@@ -536,33 +384,35 @@ export default function piWebExtension(pi: ExtensionAPI) {
536
384
  },
537
385
  });
538
386
 
539
- // ── Web Status ───────────────────────────────────────────────────────
387
+ // ── web_status ───────────────────────────────────────────────────────
540
388
  pi.registerTool({
541
389
  name: "web_status",
542
390
  label: "Web Provider Status",
543
391
  description:
544
- "Show Brave, SearXNG, Firecrawl, and Crawl4AI configuration status without printing secrets.",
545
- promptSnippet: "Check web provider configuration",
392
+ "Show all web provider configuration status (Brave, SearXNG, Firecrawl, Crawl4AI) without printing secrets. Includes Crawl4AI server health check. Use when web tools fail due to missing credentials or configuration issues.",
393
+ promptSnippet: "Check web provider configuration and server status",
546
394
  promptGuidelines: [
547
- "Use when web tools fail due to missing credentials/config or before comparing Brave, SearXNG, Firecrawl, and Crawl4AI availability.",
395
+ "Use web_status when web tools fail due to missing credentials/config or to verify which backends are available.",
396
+ "The output shows which backends are configured; if a backend is missing, configure its env vars.",
548
397
  "Never print API key values; this tool reports only presence and source.",
398
+ "Shows Crawl4AI server health, version, and auth status when the server is reachable.",
549
399
  ],
550
400
  parameters: Type.Object({}),
551
- async execute(_id: string, _params: Record<string, unknown>, _signal: AbortSignal, _onUpdate: unknown, ctx: any) {
401
+ async execute(_id: string, _params: Record<string, unknown>, signal: AbortSignal, _onUpdate: unknown, ctx: any) {
552
402
  const cwd = cwdFromContext(ctx);
553
403
  const trusted = includeProjectEnv(ctx);
404
+
405
+ // Provider config status
554
406
  const braveKey = findEnvValue("BRAVE_API_KEY", cwd, trusted);
555
407
  const searxngUrl = findEnvValue("SEARXNG_BASE_URL", cwd, trusted);
556
408
  const fireKey = findEnvValue("FIRECRAWL_API_KEY", cwd, trusted);
557
409
  const fireUrl = findEnvValue("FIRECRAWL_API_URL", cwd, trusted);
558
- const c4aiUrl = findEnvValue("CRAWL4AI_BASE_URL", cwd, trusted);
410
+ const c4aiUrl = findEnvValue("CRAWL4AI_API_URL", cwd, trusted);
559
411
  const c4aiToken = findEnvValue("CRAWL4AI_API_TOKEN", cwd, trusted);
560
- const status = {
412
+
413
+ const status: Record<string, unknown> = {
561
414
  brave: { apiKeyFound: Boolean(braveKey.value), apiKeySource: braveKey.value ? braveKey.source : "not set" },
562
- searxng: {
563
- baseUrl: normalizeSearxngBaseUrl(searxngUrl.value),
564
- baseUrlSource: searxngUrl.source || "default local",
565
- },
415
+ searxng: { baseUrl: normalizeSearxngBaseUrl(searxngUrl.value), baseUrlSource: searxngUrl.source || "default local" },
566
416
  firecrawl: {
567
417
  baseUrl: normalizeFirecrawlBaseUrl(fireUrl.value),
568
418
  apiUrlSource: fireUrl.source || "default hosted",
@@ -571,42 +421,24 @@ export default function piWebExtension(pi: ExtensionAPI) {
571
421
  hostedMode: !fireUrl.value || normalizeFirecrawlBaseUrl(fireUrl.value).startsWith(HOSTED_FIRECRAWL_BASE_URL),
572
422
  },
573
423
  crawl4ai: {
574
- baseUrl: normalizeCrawl4aiBaseUrl(c4aiUrl.value),
424
+ baseUrl: normalizeCrawl4aiApiUrl(c4aiUrl.value),
575
425
  baseUrlSource: c4aiUrl.source || "default",
576
426
  apiTokenFound: Boolean(c4aiToken.value),
577
427
  apiTokenSource: c4aiToken.value ? c4aiToken.source : "not set",
578
428
  },
579
429
  };
580
- return { content: [{ type: "text" as const, text: JSON.stringify(status, null, 2) }], details: status };
581
- },
582
- });
583
430
 
584
- // ── Crawl4AI Status ─────────────────────────────────────────────────
585
- pi.registerTool({
586
- name: "crawl4ai_status",
587
- label: "Crawl4AI Server Status",
588
- description:
589
- "Check Crawl4AI server health and version. Connects to the configured Crawl4AI endpoint to verify availability.",
590
- promptSnippet: "Check Crawl4AI server status",
591
- promptGuidelines: [
592
- "Use crawl4ai_status when Crawl4AI tools return connection errors.",
593
- "Shows server version, response timestamp, and whether auth is configured.",
594
- ],
595
- parameters: Type.Object({
596
- ...crawl4aiControlSchema,
597
- }),
598
- async execute(_id: string, params: Record<string, unknown>, signal: AbortSignal, _onUpdate: unknown, ctx: any) {
599
- const config = loadCrawl4aiConfig(params, cwdFromContext(ctx), includeProjectEnv(ctx));
600
- const health = await fetchCrawl4aiHealth(config, signal);
601
- const text = [
602
- `Crawl4AI Server Status`,
603
- `Endpoint: ${config.baseUrl}`,
604
- `Status: ${health.status || "unknown"}`,
605
- `Version: ${health.version || "unknown"}`,
606
- `Timestamp: ${health.timestamp ? new Date(health.timestamp * 1000).toISOString() : "unknown"}`,
607
- `Auth token: ${config.apiToken ? "configured" : "not set (loopback only)"}`,
608
- ].join("\n");
609
- return { content: [{ type: "text" as const, text: text }], details: { config: { baseUrl: config.baseUrl, apiTokenConfigured: Boolean(config.apiToken) }, health } };
431
+ // Crawl4AI health check
432
+ let c4aiHealth: Record<string, unknown> | undefined;
433
+ try {
434
+ const c4aiCfg = loadCrawl4aiConfig({}, cwd, trusted);
435
+ c4aiHealth = await fetchCrawl4aiHealth(c4aiCfg, signal);
436
+ } catch (e: any) {
437
+ c4aiHealth = { status: "unreachable", error: e?.message ?? String(e) };
438
+ }
439
+ status.crawl4ai = { ...(status.crawl4ai as Record<string, unknown>), health: c4aiHealth };
440
+
441
+ return { content: [{ type: "text" as const, text: JSON.stringify(status, null, 2) }], details: status };
610
442
  },
611
443
  });
612
444
 
@@ -620,7 +452,7 @@ export default function piWebExtension(pi: ExtensionAPI) {
620
452
  const searxngUrl = findEnvValue("SEARXNG_BASE_URL", cwd, trusted);
621
453
  const fireKey = findEnvValue("FIRECRAWL_API_KEY", cwd, trusted);
622
454
  const fireUrl = findEnvValue("FIRECRAWL_API_URL", cwd, trusted);
623
- const c4aiUrl = findEnvValue("CRAWL4AI_BASE_URL", cwd, trusted);
455
+ const c4aiUrl = findEnvValue("CRAWL4AI_API_URL", cwd, trusted);
624
456
  const c4aiToken = findEnvValue("CRAWL4AI_API_TOKEN", cwd, trusted);
625
457
  ctx.ui.notify(
626
458
  [
@@ -628,7 +460,7 @@ export default function piWebExtension(pi: ExtensionAPI) {
628
460
  `SearXNG base URL: ${normalizeSearxngBaseUrl(searxngUrl.value)} (${searxngUrl.source || "default local"})`,
629
461
  `Firecrawl base URL: ${normalizeFirecrawlBaseUrl(fireUrl.value)}`,
630
462
  `Firecrawl API key: ${fireKey.value ? `found (${fireKey.source})` : "not set"}`,
631
- `Crawl4AI base URL: ${normalizeCrawl4aiBaseUrl(c4aiUrl.value)} (${c4aiUrl.source || "default"})`,
463
+ `Crawl4AI API URL: ${normalizeCrawl4aiApiUrl(c4aiUrl.value)} (${c4aiUrl.source || "default"})`,
632
464
  `Crawl4AI API token: ${c4aiToken.value ? `found (${c4aiToken.source})` : "not set"}`,
633
465
  ].join("\n"),
634
466
  "info",