@bacnh85/pi-web 0.5.3 → 0.5.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/extensions/index.ts +368 -368
- package/extensions/lib/brave.ts +30 -30
- package/extensions/lib/config.ts +97 -97
- package/extensions/lib/content.ts +67 -67
- package/extensions/lib/crawl4ai.ts +97 -97
- package/extensions/lib/extract.ts +137 -137
- package/extensions/lib/firecrawl.ts +28 -28
- package/extensions/lib/format.ts +138 -138
- package/extensions/lib/retry.ts +32 -32
- package/extensions/lib/search.ts +174 -174
- package/extensions/lib/searxng.ts +49 -49
- package/extensions/types.d.ts +1 -1
- package/package.json +1 -1
package/extensions/index.ts
CHANGED
|
@@ -4,30 +4,30 @@ import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
|
|
|
4
4
|
import { Type } from "typebox";
|
|
5
5
|
|
|
6
6
|
import {
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
7
|
+
findEnvValue,
|
|
8
|
+
cwdFromContext,
|
|
9
|
+
includeProjectEnv,
|
|
10
|
+
normalizeSearxngBaseUrl,
|
|
11
|
+
normalizeFirecrawlBaseUrl,
|
|
12
|
+
normalizeCrawl4aiApiUrl,
|
|
13
|
+
loadFirecrawlConfig,
|
|
14
|
+
loadCrawl4aiConfig,
|
|
15
|
+
HOSTED_FIRECRAWL_BASE_URL,
|
|
16
16
|
} from "./lib/config";
|
|
17
17
|
import {
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
18
|
+
truncateText,
|
|
19
|
+
formatFirecrawlScrape,
|
|
20
|
+
formatCrawl4aiResult,
|
|
21
|
+
formatUnifiedSearchResults,
|
|
22
22
|
} from "./lib/format";
|
|
23
23
|
import { searchWithDiagnostics } from "./lib/search";
|
|
24
24
|
import { extractWithDiagnostics, type ExtractMode } from "./lib/extract";
|
|
25
25
|
import { firecrawlRequest, type FirecrawlResult } from "./lib/firecrawl";
|
|
26
26
|
import {
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
27
|
+
fetchCrawl4aiCrawl,
|
|
28
|
+
fetchCrawl4aiScreenshot,
|
|
29
|
+
fetchCrawl4aiPdf,
|
|
30
|
+
fetchCrawl4aiHealth,
|
|
31
31
|
} from "./lib/crawl4ai";
|
|
32
32
|
|
|
33
33
|
// ---------------------------------------------------------------------------
|
|
@@ -35,17 +35,17 @@ import {
|
|
|
35
35
|
// ---------------------------------------------------------------------------
|
|
36
36
|
|
|
37
37
|
const sharedControlSchema = {
|
|
38
|
-
|
|
38
|
+
timeout_ms: Type.Optional(Type.Number({ description: "Request timeout in milliseconds." })),
|
|
39
39
|
};
|
|
40
40
|
|
|
41
41
|
const firecrawlControlSchema = {
|
|
42
|
-
|
|
43
|
-
|
|
42
|
+
firecrawl_api_key: Type.Optional(Type.String({ description: "Override $FIRECRAWL_API_KEY." })),
|
|
43
|
+
firecrawl_api_url: Type.Optional(Type.String({ description: "Override $FIRECRAWL_API_URL." })),
|
|
44
44
|
};
|
|
45
45
|
|
|
46
46
|
const crawl4aiControlSchema = {
|
|
47
|
-
|
|
48
|
-
|
|
47
|
+
crawl4ai_api_url: Type.Optional(Type.String({ description: "Override $CRAWL4AI_API_URL." })),
|
|
48
|
+
crawl4ai_api_token: Type.Optional(Type.String({ description: "Override $CRAWL4AI_API_TOKEN." })),
|
|
49
49
|
};
|
|
50
50
|
|
|
51
51
|
|
|
@@ -54,364 +54,364 @@ const crawl4aiControlSchema = {
|
|
|
54
54
|
// ---------------------------------------------------------------------------
|
|
55
55
|
|
|
56
56
|
export default function piWebExtension(pi: ExtensionAPI) {
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
57
|
+
// ── web_search ────────────────────────────────────────────────────────
|
|
58
|
+
pi.registerTool({
|
|
59
|
+
name: "web_search",
|
|
60
|
+
label: "Web Search",
|
|
61
|
+
description:
|
|
62
|
+
"Search the web. Auto-selects backends: SearXNG, Brave, Firecrawl.",
|
|
63
|
+
promptSnippet: "Search current web results",
|
|
64
|
+
promptGuidelines: [
|
|
65
|
+
"Source discovery, docs, facts, and general search.",
|
|
66
|
+
"Broad discovery uses SearXNG; precision/site/docs use Brave; Firecrawl is fallback.",
|
|
67
|
+
"Force backend via backend:'brave'|'searxng' for poor auto results.",
|
|
68
|
+
"Use engines='google,github' for SearXNG tuning.",
|
|
69
|
+
"Cite source URLs.",
|
|
70
|
+
],
|
|
71
|
+
parameters: Type.Object({
|
|
72
|
+
query: Type.String(),
|
|
73
|
+
count: Type.Optional(Type.Number({ default: 5 })),
|
|
74
|
+
freshness: Type.Optional(Type.String({ description: "Time filter: pw/pm/py or YYYY-MM-DDtoYYYY-MM-DD." })),
|
|
75
|
+
country: Type.Optional(Type.String({ default: "US" })),
|
|
76
|
+
backend: Type.Optional(Type.Union(
|
|
77
|
+
[Type.Literal("auto"), Type.Literal("searxng"), Type.Literal("brave"), Type.Literal("firecrawl")],
|
|
78
|
+
{ default: "auto", description: "auto, searxng, brave, firecrawl." },
|
|
79
|
+
)),
|
|
80
|
+
engines: Type.Optional(Type.String({ description: "SearXNG engine list (google,github). Only for searxng/auto backend." })),
|
|
81
|
+
include_content: Type.Optional(Type.Boolean({ default: false, description: "Fetch inline page content (slower)." })),
|
|
82
|
+
content_chars: Type.Optional(Type.Number({ default: 5000 })),
|
|
83
|
+
...sharedControlSchema,
|
|
84
|
+
}),
|
|
85
|
+
async execute(_id: string, params: Record<string, unknown>, signal: AbortSignal, _onUpdate: unknown, ctx: any) {
|
|
86
|
+
const diagnostics = await searchWithDiagnostics({
|
|
87
|
+
query: params.query as string,
|
|
88
|
+
count: params.count as number | undefined,
|
|
89
|
+
freshness: params.freshness as string | undefined,
|
|
90
|
+
country: params.country as string | undefined,
|
|
91
|
+
backend: params.backend as "auto" | "searxng" | "brave" | "firecrawl" | undefined,
|
|
92
|
+
engines: params.engines as string | undefined,
|
|
93
|
+
include_content: params.include_content as boolean | undefined,
|
|
94
|
+
content_chars: params.content_chars as number | undefined,
|
|
95
|
+
timeout_ms: params.timeout_ms as number | undefined,
|
|
96
|
+
signal,
|
|
97
|
+
_ctx: ctx,
|
|
98
|
+
});
|
|
99
|
+
const attempts = diagnostics.attempts.map((a) => `${a.backend}: ${a.status}${a.message ? ` (${a.message})` : ""}`).join("\n");
|
|
100
|
+
const text = `${formatUnifiedSearchResults(diagnostics.results)}\n\n--- Search diagnostics ---\nSelected backend: ${diagnostics.selectedBackend}\n${attempts}`;
|
|
101
|
+
return { content: [{ type: "text" as const, text: truncateText(text) }], details: diagnostics };
|
|
102
|
+
},
|
|
103
|
+
});
|
|
104
104
|
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
105
|
+
// ── web_extract ──────────────────────────────────────────────────────
|
|
106
|
+
pi.registerTool({
|
|
107
|
+
name: "web_extract",
|
|
108
|
+
label: "Web Content Extraction",
|
|
109
|
+
description:
|
|
110
|
+
"Extract readable content from a URL. Auto mode: static\u2192dynamic\u2192full.",
|
|
111
|
+
promptSnippet: "Extract readable webpage content as markdown",
|
|
112
|
+
promptGuidelines: [
|
|
113
|
+
"Clean markdown from a known URL.",
|
|
114
|
+
"mode: 'static' (no API key, JSDOM), 'dynamic' (Firecrawl JS), 'full' (Crawl4AI).",
|
|
115
|
+
"'auto' tries static\u2192dynamic\u2192full; see diagnostics for fallback chain.",
|
|
116
|
+
"Use prompt+schema for structured JSON extraction (dynamic mode only).",
|
|
117
|
+
"Cite the source URL.",
|
|
118
|
+
],
|
|
119
|
+
parameters: Type.Object({
|
|
120
|
+
url: Type.String(),
|
|
121
|
+
mode: Type.Optional(Type.Union(
|
|
122
|
+
[Type.Literal("auto"), Type.Literal("static"), Type.Literal("dynamic"), Type.Literal("full")],
|
|
123
|
+
{ default: "auto", description: "auto, static, dynamic, full." },
|
|
124
|
+
)),
|
|
125
|
+
prompt: Type.Optional(Type.String({ description: "Prompt for structured JSON extraction (dynamic mode only)." })),
|
|
126
|
+
schema: Type.Optional(Type.Any({ description: "JSON schema for structured extraction (dynamic mode only)." })),
|
|
127
|
+
content_chars: Type.Optional(Type.Number({ default: 20000 })),
|
|
128
|
+
wait_for: Type.Optional(Type.Number({ description: "Ms to wait for Firecrawl render before extraction." })),
|
|
129
|
+
mobile: Type.Optional(Type.Boolean({ default: false, description: "Mobile viewport (dynamic mode only)." })),
|
|
130
|
+
...crawl4aiControlSchema,
|
|
131
|
+
...sharedControlSchema,
|
|
132
|
+
}),
|
|
133
|
+
async execute(_id: string, params: Record<string, unknown>, signal: AbortSignal, _onUpdate: unknown, ctx: any) {
|
|
134
|
+
const diagnostics = await extractWithDiagnostics({
|
|
135
|
+
url: params.url as string,
|
|
136
|
+
mode: params.mode as ExtractMode | undefined,
|
|
137
|
+
prompt: params.prompt as string | undefined,
|
|
138
|
+
schema: params.schema,
|
|
139
|
+
content_chars: params.content_chars as number | undefined,
|
|
140
|
+
timeout_ms: params.timeout_ms as number | undefined,
|
|
141
|
+
wait_for: params.wait_for as number | undefined,
|
|
142
|
+
mobile: params.mobile as boolean | undefined,
|
|
143
|
+
crawl4ai_api_token: params.crawl4ai_api_token as string | undefined,
|
|
144
|
+
crawl4ai_api_url: params.crawl4ai_api_url as string | undefined,
|
|
145
|
+
signal,
|
|
146
|
+
_ctx: ctx,
|
|
147
|
+
});
|
|
148
|
+
const result = diagnostics.result;
|
|
149
|
+
const attempts = diagnostics.attempts.map((a) => `${a.mode}: ${a.status}${a.message ? ` (${a.message})` : ""}`).join("\n");
|
|
150
|
+
const text = `${result.title ? `# ${result.title}\n\n` : ""}${result.markdown}\n\n--- Extraction diagnostics ---\nSelected mode: ${diagnostics.selectedMode}\nFallback used: ${diagnostics.fallbackUsed}\n${attempts}`;
|
|
151
|
+
return { content: [{ type: "text" as const, text: truncateText(text) }], details: { url: params.url, ...diagnostics } };
|
|
152
|
+
},
|
|
153
|
+
});
|
|
154
154
|
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
155
|
+
// ── web_map ──────────────────────────────────────────────────────────
|
|
156
|
+
pi.registerTool({
|
|
157
|
+
name: "web_map",
|
|
158
|
+
label: "Site URL Discovery",
|
|
159
|
+
description:
|
|
160
|
+
"Discover site URLs via Firecrawl Map.",
|
|
161
|
+
promptSnippet: "Map site URLs",
|
|
162
|
+
promptGuidelines: [
|
|
163
|
+
"Discover site URLs before crawling. Prefer web_extract for small jobs.",
|
|
164
|
+
"Best on base domains. sitemap:'only' for sub-path discovery.",
|
|
165
|
+
"Keep limits small unless broad discovery is requested.",
|
|
166
|
+
],
|
|
167
|
+
parameters: Type.Object({
|
|
168
|
+
url: Type.String(),
|
|
169
|
+
limit: Type.Optional(Type.Number({ default: 100 })),
|
|
170
|
+
include_subdomains: Type.Optional(Type.Boolean({ default: false })),
|
|
171
|
+
search: Type.Optional(Type.String({ description: "Search query to guide URL discovery (semantic map)." })),
|
|
172
|
+
sitemap: Type.Optional(Type.Union([Type.Literal("only"), Type.Literal("include"), Type.Literal("skip")], { description: "only, include(default), skip." })),
|
|
173
|
+
use_index: Type.Optional(Type.Boolean({ default: true, description: "Use Firecrawl index for discovery." })),
|
|
174
|
+
ignore_cache: Type.Optional(Type.Boolean({ default: false, description: "Ignore cached results." })),
|
|
175
|
+
...firecrawlControlSchema,
|
|
176
|
+
...sharedControlSchema,
|
|
177
|
+
}),
|
|
178
|
+
async execute(_id: string, params: Record<string, unknown>, signal: AbortSignal, _onUpdate: unknown, ctx: any) {
|
|
179
|
+
const body: Record<string, unknown> = {
|
|
180
|
+
url: params.url,
|
|
181
|
+
limit: (params.limit as number) ?? 100,
|
|
182
|
+
includeSubdomains: Boolean(params.include_subdomains),
|
|
183
|
+
};
|
|
184
|
+
if (params.search !== undefined) body.search = params.search;
|
|
185
|
+
if (params.sitemap !== undefined) body.sitemap = params.sitemap;
|
|
186
|
+
if (params.use_index !== undefined) body.useIndex = params.use_index;
|
|
187
|
+
if (params.ignore_cache !== undefined) body.ignoreCache = params.ignore_cache;
|
|
188
|
+
const config = loadFirecrawlConfig(params as Record<string, unknown>, cwdFromContext(ctx), includeProjectEnv(ctx));
|
|
189
|
+
const result = await firecrawlRequest(config, "POST", "/map", body, signal);
|
|
190
|
+
const urls = result.data || result.links || result.urls || [];
|
|
191
|
+
const text = Array.isArray(urls) && urls.length > 0
|
|
192
|
+
? (urls as Array<Record<string, unknown> | string>).map((u: any) => u.url || u).join("\n")
|
|
193
|
+
: JSON.stringify(result, null, 2);
|
|
194
|
+
return { content: [{ type: "text" as const, text: truncateText(text) }], details: result };
|
|
195
|
+
},
|
|
196
|
+
});
|
|
197
197
|
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
198
|
+
// ── web_crawl ────────────────────────────────────────────────────────
|
|
199
|
+
pi.registerTool({
|
|
200
|
+
name: "web_crawl",
|
|
201
|
+
label: "Site Crawl",
|
|
202
|
+
description:
|
|
203
|
+
"Crawl pages. Firecrawl 'light' or Crawl4AI 'full' headless mode.",
|
|
204
|
+
promptSnippet: "Crawl a small site section",
|
|
205
|
+
promptGuidelines: [
|
|
206
|
+
"Prefer web_map + web_extract over crawl for small jobs.",
|
|
207
|
+
"'light'=Firecrawl (url param), 'full'=Crawl4AI (urls[] param).",
|
|
208
|
+
"Keep limit low (default 10).",
|
|
209
|
+
],
|
|
210
|
+
parameters: Type.Object({
|
|
211
|
+
url: Type.Optional(Type.String({ description: "URL for Firecrawl-style crawl (mode:'light')." })),
|
|
212
|
+
urls: Type.Optional(Type.Array(Type.String(), { description: "URLs for Crawl4AI-style crawl (mode:'full'), up to 100." })),
|
|
213
|
+
mode: Type.Optional(Type.Union([Type.Literal("light"), Type.Literal("full")], { default: "light", description: "'light'(Firecrawl) or 'full'(Crawl4AI)." })),
|
|
214
|
+
limit: Type.Optional(Type.Number({ default: 10 })),
|
|
215
|
+
include_paths: Type.Optional(Type.String({ description: "Comma-separated paths to include (Firecrawl mode)." })),
|
|
216
|
+
exclude_paths: Type.Optional(Type.String({ description: "Comma-separated paths to exclude (Firecrawl mode)." })),
|
|
217
|
+
poll: Type.Optional(Type.Boolean({ default: false, description: "Poll for completion (Firecrawl mode)." })),
|
|
218
|
+
browser_config: Type.Optional(Type.Any({ description: "BrowserConfig JSON (full mode only)." })),
|
|
219
|
+
crawler_config: Type.Optional(Type.Any({ description: "CrawlerRunConfig JSON (full mode only)." })),
|
|
220
|
+
content_chars: Type.Optional(Type.Number({ default: 20000 })),
|
|
221
|
+
...firecrawlControlSchema,
|
|
222
|
+
...crawl4aiControlSchema,
|
|
223
|
+
...sharedControlSchema,
|
|
224
|
+
}),
|
|
225
|
+
async execute(_id: string, params: Record<string, unknown>, signal: AbortSignal, _onUpdate: unknown, ctx: any) {
|
|
226
|
+
const mode = (params.mode as string) || "light";
|
|
227
|
+
const cwd = cwdFromContext(ctx);
|
|
228
|
+
const trusted = includeProjectEnv(ctx);
|
|
229
|
+
const maxChars = (params.content_chars as number) ?? 20000;
|
|
230
230
|
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
231
|
+
if (mode === "full") {
|
|
232
|
+
// Crawl4AI mode
|
|
233
|
+
const urls = (params.urls as string[]) || (params.url ? [params.url as string] : []);
|
|
234
|
+
if (!urls.length) throw new Error("Either url or urls parameter is required for crawl.");
|
|
235
|
+
const config = loadCrawl4aiConfig(params as Record<string, unknown>, cwd, trusted);
|
|
236
|
+
const browserConfig = params.browser_config as Record<string, unknown> | undefined;
|
|
237
|
+
const crawlerConfig = params.crawler_config as Record<string, unknown> | undefined;
|
|
238
|
+
const result = await fetchCrawl4aiCrawl(config, urls, browserConfig, crawlerConfig, signal);
|
|
239
|
+
const text = formatCrawl4aiResult(result as unknown as Record<string, unknown>, maxChars);
|
|
240
|
+
return { content: [{ type: "text" as const, text: truncateText(text) }], details: result };
|
|
241
|
+
}
|
|
242
242
|
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
243
|
+
// Firecrawl mode ("light")
|
|
244
|
+
if (!params.url) throw new Error("The url parameter is required for Firecrawl mode ('light').");
|
|
245
|
+
const fcConfig = loadFirecrawlConfig(params as Record<string, unknown>, cwd, trusted);
|
|
246
|
+
let result = await firecrawlRequest(
|
|
247
|
+
fcConfig,
|
|
248
|
+
"POST",
|
|
249
|
+
"/crawl",
|
|
250
|
+
{
|
|
251
|
+
url: params.url as string,
|
|
252
|
+
limit: Math.min(10000, Math.max(1, (params.limit as number) ?? 10)),
|
|
253
|
+
includePaths: params.include_paths
|
|
254
|
+
? String(params.include_paths).split(",").map((s: string) => s.trim()).filter(Boolean)
|
|
255
|
+
: [],
|
|
256
|
+
excludePaths: params.exclude_paths
|
|
257
|
+
? String(params.exclude_paths).split(",").map((s: string) => s.trim()).filter(Boolean)
|
|
258
|
+
: [],
|
|
259
|
+
scrapeOptions: { formats: ["markdown"], onlyMainContent: true },
|
|
260
|
+
},
|
|
261
|
+
signal,
|
|
262
|
+
);
|
|
263
|
+
const id = result.id || (result.data as Record<string, unknown> | undefined)?.id;
|
|
264
|
+
if (params.poll && id && !Array.isArray(result.data)) {
|
|
265
|
+
const { abortableSleep } = await import("./lib/retry");
|
|
266
|
+
for (let i = 0; i < 60; i++) {
|
|
267
|
+
result = await firecrawlRequest(fcConfig, "GET", `/crawl/${id}`, undefined, signal);
|
|
268
|
+
if (["completed", "failed", "cancelled"].includes(
|
|
269
|
+
String(result.status || (result.data as Record<string, unknown> | undefined)?.status || ""),
|
|
270
|
+
)) break;
|
|
271
|
+
await abortableSleep(2000, signal);
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
const pages = Array.isArray(result.data)
|
|
275
|
+
? (result.data as Record<string, unknown>[])
|
|
276
|
+
: ((result.data as Record<string, unknown>)?.data as Record<string, unknown>[]) || [];
|
|
277
|
+
const text = pages.length
|
|
278
|
+
? pages.map((p: Record<string, unknown>) => formatFirecrawlScrape({ data: p } as Record<string, unknown>, maxChars)).join("\n\n---\n\n")
|
|
279
|
+
: id
|
|
280
|
+
? `Crawl started: ${id}\nUse poll=true or check Firecrawl status/dashboard.`
|
|
281
|
+
: JSON.stringify(result, null, 2);
|
|
282
|
+
return { content: [{ type: "text" as const, text: truncateText(text) }], details: result };
|
|
283
|
+
},
|
|
284
|
+
});
|
|
285
285
|
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
286
|
+
// ── web_screenshot ───────────────────────────────────────────────────
|
|
287
|
+
pi.registerTool({
|
|
288
|
+
name: "web_screenshot",
|
|
289
|
+
label: "Web Page Screenshot",
|
|
290
|
+
description:
|
|
291
|
+
"Full-page PNG screenshot via Crawl4AI.",
|
|
292
|
+
promptSnippet: "Screenshot a webpage",
|
|
293
|
+
promptGuidelines: [
|
|
294
|
+
"Use when web_extract fails on JS-heavy or bot-protected pages.",
|
|
295
|
+
"wait_for (default 2s) delays capture for dynamic content.",
|
|
296
|
+
],
|
|
297
|
+
parameters: Type.Object({
|
|
298
|
+
url: Type.String(),
|
|
299
|
+
wait_for: Type.Optional(Type.Number({ default: 2, description: "Seconds to wait before capture." })),
|
|
300
|
+
wait_for_images: Type.Optional(Type.Boolean({ default: false, description: "Wait for images before capture." })),
|
|
301
|
+
...crawl4aiControlSchema,
|
|
302
|
+
...sharedControlSchema,
|
|
303
|
+
}),
|
|
304
|
+
async execute(_id: string, params: Record<string, unknown>, signal: AbortSignal, _onUpdate: unknown, ctx: any) {
|
|
305
|
+
const config = loadCrawl4aiConfig(params as Record<string, unknown>, cwdFromContext(ctx), includeProjectEnv(ctx));
|
|
306
|
+
const result = await fetchCrawl4aiScreenshot(
|
|
307
|
+
config,
|
|
308
|
+
params.url as string,
|
|
309
|
+
params.wait_for as number | undefined,
|
|
310
|
+
params.wait_for_images as boolean | undefined,
|
|
311
|
+
signal,
|
|
312
|
+
);
|
|
313
|
+
const screenshot = result.screenshot as string | undefined;
|
|
314
|
+
const artifactUrl = result.url as string | undefined;
|
|
315
|
+
const mime = result.mime as string | undefined;
|
|
316
|
+
const size = result.size as number | undefined;
|
|
317
|
+
let text = `Screenshot: ${params.url}\n`;
|
|
318
|
+
if (screenshot) text += `Data: base64 PNG (${screenshot.length} chars)\n`;
|
|
319
|
+
if (artifactUrl) text += `Artifact: ${artifactUrl}\n`;
|
|
320
|
+
if (mime) text += `MIME: ${mime}\n`;
|
|
321
|
+
if (size) text += `Size: ${size} bytes\n`;
|
|
322
|
+
return { content: [{ type: "text" as const, text: truncateText(text) }], details: { ...result, url: params.url } };
|
|
323
|
+
},
|
|
324
|
+
});
|
|
325
325
|
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
326
|
+
// ── web_pdf ──────────────────────────────────────────────────────────
|
|
327
|
+
pi.registerTool({
|
|
328
|
+
name: "web_pdf",
|
|
329
|
+
label: "Web Page PDF",
|
|
330
|
+
description:
|
|
331
|
+
"PDF document via Crawl4AI.",
|
|
332
|
+
promptSnippet: "PDF a webpage",
|
|
333
|
+
promptGuidelines: [
|
|
334
|
+
"Printable or archivable page snapshot.",
|
|
335
|
+
"Returns base64 PDF string.",
|
|
336
|
+
],
|
|
337
|
+
parameters: Type.Object({
|
|
338
|
+
url: Type.String(),
|
|
339
|
+
...crawl4aiControlSchema,
|
|
340
|
+
...sharedControlSchema,
|
|
341
|
+
}),
|
|
342
|
+
async execute(_id: string, params: Record<string, unknown>, signal: AbortSignal, _onUpdate: unknown, ctx: any) {
|
|
343
|
+
const config = loadCrawl4aiConfig(params as Record<string, unknown>, cwdFromContext(ctx), includeProjectEnv(ctx));
|
|
344
|
+
const result = await fetchCrawl4aiPdf(config, params.url as string, signal);
|
|
345
|
+
const pdf = result.pdf as string | undefined;
|
|
346
|
+
const artifactUrl = result.url as string | undefined;
|
|
347
|
+
const size = result.size as number | undefined;
|
|
348
|
+
let text = `PDF: ${params.url}\n`;
|
|
349
|
+
if (pdf) text += `Data: base64 PDF (${pdf.length} chars)\n`;
|
|
350
|
+
if (artifactUrl) text += `Artifact: ${artifactUrl}\n`;
|
|
351
|
+
if (size) text += `Size: ${size} bytes\n`;
|
|
352
|
+
return { content: [{ type: "text" as const, text: truncateText(text) }], details: { ...result, url: params.url } };
|
|
353
|
+
},
|
|
354
|
+
});
|
|
355
355
|
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
356
|
+
// ── web_status ───────────────────────────────────────────────────────
|
|
357
|
+
pi.registerTool({
|
|
358
|
+
name: "web_status",
|
|
359
|
+
label: "Web Provider Status",
|
|
360
|
+
description:
|
|
361
|
+
"Show web provider config status without printing secrets.",
|
|
362
|
+
promptSnippet: "Check web provider config and server status",
|
|
363
|
+
promptGuidelines: [
|
|
364
|
+
"Check which backends are configured and their server status.",
|
|
365
|
+
"Never prints secrets — reports only presence and source.",
|
|
366
|
+
"apiKeyFound:false is normal for self-hosted Firecrawl; check ready field.",
|
|
367
|
+
],
|
|
368
|
+
parameters: Type.Object({}),
|
|
369
|
+
async execute(_id: string, _params: Record<string, unknown>, signal: AbortSignal, _onUpdate: unknown, ctx: any) {
|
|
370
|
+
const cwd = cwdFromContext(ctx);
|
|
371
|
+
const trusted = includeProjectEnv(ctx);
|
|
372
372
|
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
373
|
+
// Provider config status
|
|
374
|
+
const braveKey = findEnvValue("BRAVE_API_KEY", cwd, trusted);
|
|
375
|
+
const searxngUrl = findEnvValue("SEARXNG_BASE_URL", cwd, trusted);
|
|
376
|
+
const fireKey = findEnvValue("FIRECRAWL_API_KEY", cwd, trusted);
|
|
377
|
+
const fireUrl = findEnvValue("FIRECRAWL_API_URL", cwd, trusted);
|
|
378
|
+
const c4aiUrl = findEnvValue("CRAWL4AI_API_URL", cwd, trusted);
|
|
379
|
+
const c4aiToken = findEnvValue("CRAWL4AI_API_TOKEN", cwd, trusted);
|
|
380
380
|
|
|
381
|
-
|
|
382
|
-
|
|
381
|
+
const fcBaseUrl = normalizeFirecrawlBaseUrl(fireUrl.value);
|
|
382
|
+
const fcHosted = !fireUrl.value || fcBaseUrl.startsWith(HOSTED_FIRECRAWL_BASE_URL);
|
|
383
383
|
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
384
|
+
const status: Record<string, unknown> = {
|
|
385
|
+
brave: { apiKeyFound: Boolean(braveKey.value), apiKeySource: braveKey.value ? braveKey.source : "not set" },
|
|
386
|
+
searxng: { baseUrl: normalizeSearxngBaseUrl(searxngUrl.value), baseUrlSource: searxngUrl.source || "default local" },
|
|
387
|
+
firecrawl: {
|
|
388
|
+
baseUrl: fcBaseUrl,
|
|
389
|
+
apiUrlSource: fireUrl.source || "default hosted",
|
|
390
|
+
apiKeyFound: Boolean(fireKey.value),
|
|
391
|
+
apiKeySource: fireKey.value ? fireKey.source : "not set",
|
|
392
|
+
hostedMode: fcHosted,
|
|
393
|
+
ready: fcHosted ? Boolean(fireKey.value) : Boolean(fireUrl.value?.trim()),
|
|
394
|
+
},
|
|
395
|
+
crawl4ai: {
|
|
396
|
+
baseUrl: normalizeCrawl4aiApiUrl(c4aiUrl.value),
|
|
397
|
+
baseUrlSource: c4aiUrl.source || "default",
|
|
398
|
+
apiTokenFound: Boolean(c4aiToken.value),
|
|
399
|
+
apiTokenSource: c4aiToken.value ? c4aiToken.source : "not set",
|
|
400
|
+
},
|
|
401
|
+
};
|
|
402
402
|
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
403
|
+
// Crawl4AI health check
|
|
404
|
+
let c4aiHealth: Record<string, unknown> | undefined;
|
|
405
|
+
try {
|
|
406
|
+
const c4aiCfg = loadCrawl4aiConfig({}, cwd, trusted);
|
|
407
|
+
c4aiHealth = await fetchCrawl4aiHealth(c4aiCfg, signal);
|
|
408
|
+
} catch (e: any) {
|
|
409
|
+
c4aiHealth = { status: "unreachable", error: e?.message ?? String(e) };
|
|
410
|
+
}
|
|
411
|
+
status.crawl4ai = { ...(status.crawl4ai as Record<string, unknown>), health: c4aiHealth };
|
|
412
412
|
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
413
|
+
return { content: [{ type: "text" as const, text: JSON.stringify(status, null, 2) }], details: status };
|
|
414
|
+
},
|
|
415
|
+
});
|
|
416
416
|
|
|
417
417
|
}
|