firecrawl-mcp 3.22.4 → 3.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -0
- package/dist/index.js +826 -859
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -3,17 +3,89 @@
|
|
|
3
3
|
// src/index.ts
|
|
4
4
|
import FirecrawlApp from "@mendable/firecrawl-js";
|
|
5
5
|
import dotenv from "dotenv";
|
|
6
|
-
import { FastMCP } from "fastmcp";
|
|
6
|
+
import { FastMCP, UserError } from "fastmcp";
|
|
7
7
|
import { readFile } from "fs/promises";
|
|
8
8
|
import { createRequire } from "module";
|
|
9
|
+
import { randomUUID } from "crypto";
|
|
9
10
|
import path from "path";
|
|
10
11
|
import { z as z3 } from "zod";
|
|
11
12
|
|
|
13
|
+
// src/keyless-client-ip.ts
|
|
14
|
+
import net from "net";
|
|
15
|
+
function extractSingleTrustedClientIp(rawForwardedFor) {
|
|
16
|
+
if (Array.isArray(rawForwardedFor) && rawForwardedFor.length !== 1) {
|
|
17
|
+
return void 0;
|
|
18
|
+
}
|
|
19
|
+
const raw = Array.isArray(rawForwardedFor) ? rawForwardedFor[0] : rawForwardedFor;
|
|
20
|
+
if (typeof raw !== "string") return void 0;
|
|
21
|
+
const parts = raw.split(",").map((part) => part.trim()).filter(Boolean);
|
|
22
|
+
if (parts.length !== 1) return void 0;
|
|
23
|
+
const candidate = parts[0].replace(/^\[(.*)\]$/, "$1").toLowerCase();
|
|
24
|
+
return net.isIP(candidate) ? candidate : void 0;
|
|
25
|
+
}
|
|
26
|
+
|
|
12
27
|
// src/monitor.ts
|
|
13
28
|
import { z } from "zod";
|
|
29
|
+
|
|
30
|
+
// src/session-credential.ts
|
|
31
|
+
import { createHmac } from "crypto";
|
|
32
|
+
var managedOAuthApiKey = /* @__PURE__ */ Symbol("firecrawlManagedOAuthApiKey");
|
|
33
|
+
var CredentialValidationUnavailableError = class extends Error {
|
|
34
|
+
constructor() {
|
|
35
|
+
super("Firecrawl credential validation is temporarily unavailable");
|
|
36
|
+
this.name = "CredentialValidationUnavailableError";
|
|
37
|
+
}
|
|
38
|
+
};
|
|
39
|
+
function delegationSecret() {
|
|
40
|
+
const secret = process.env.MCP_DELEGATED_CREDENTIAL_SECRET?.trim();
|
|
41
|
+
if (!secret) throw new CredentialValidationUnavailableError();
|
|
42
|
+
return secret;
|
|
43
|
+
}
|
|
44
|
+
function requireDelegatedCredentialSigning() {
|
|
45
|
+
delegationSecret();
|
|
46
|
+
}
|
|
47
|
+
function setManagedOAuthApiKey(session, apiKey) {
|
|
48
|
+
Object.defineProperty(session, managedOAuthApiKey, {
|
|
49
|
+
configurable: false,
|
|
50
|
+
enumerable: false,
|
|
51
|
+
value: apiKey,
|
|
52
|
+
writable: false
|
|
53
|
+
});
|
|
54
|
+
return session;
|
|
55
|
+
}
|
|
56
|
+
function copyManagedOAuthApiKey(source, target) {
|
|
57
|
+
const apiKey = source?.[managedOAuthApiKey];
|
|
58
|
+
if (apiKey) setManagedOAuthApiKey(target, apiKey);
|
|
59
|
+
}
|
|
60
|
+
function hasCredential(session) {
|
|
61
|
+
return Boolean(session?.firecrawlApiKey || session?.[managedOAuthApiKey]);
|
|
62
|
+
}
|
|
63
|
+
function hasManagedOAuthCredential(session) {
|
|
64
|
+
return Boolean(session?.[managedOAuthApiKey]);
|
|
65
|
+
}
|
|
66
|
+
function credentialForOutboundRequest(session) {
|
|
67
|
+
const managedApiKey = session?.[managedOAuthApiKey];
|
|
68
|
+
if (!managedApiKey) return session?.firecrawlApiKey;
|
|
69
|
+
const iat = Math.floor(Date.now() / 1e3);
|
|
70
|
+
const payload = {
|
|
71
|
+
v: 1,
|
|
72
|
+
aud: "firecrawl-core",
|
|
73
|
+
purpose: "hosted_mcp_oauth",
|
|
74
|
+
api_key: managedApiKey,
|
|
75
|
+
iat,
|
|
76
|
+
exp: iat + 60
|
|
77
|
+
};
|
|
78
|
+
const encodedPayload = Buffer.from(JSON.stringify(payload)).toString(
|
|
79
|
+
"base64url"
|
|
80
|
+
);
|
|
81
|
+
const signature = createHmac("sha256", delegationSecret()).update(encodedPayload).digest("base64url");
|
|
82
|
+
return `fcmcp_${encodedPayload}.${signature}`;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
// src/monitor.ts
|
|
14
86
|
var DEFAULT_API_URL = "https://api.firecrawl.dev";
|
|
15
87
|
function resolveAuth(session) {
|
|
16
|
-
const apiKey = session
|
|
88
|
+
const apiKey = session === void 0 ? process.env.FIRECRAWL_API_KEY : credentialForOutboundRequest(session);
|
|
17
89
|
const baseUrl = (process.env.FIRECRAWL_API_URL ?? DEFAULT_API_URL).replace(
|
|
18
90
|
/\/$/,
|
|
19
91
|
""
|
|
@@ -142,124 +214,9 @@ function registerMonitorTools(server2) {
|
|
|
142
214
|
// Additive; creates a new monitor without deleting existing monitors or external content.
|
|
143
215
|
},
|
|
144
216
|
description: `
|
|
145
|
-
Create a
|
|
146
|
-
|
|
147
|
-
Prefer the simple path: pass \`page\` or \`pages\` plus \`goal\` to monitor specific URLs, OR pass \`queries\` plus \`goal\` to monitor web search results for new/changed hits. The tool will create the monitor with a 30-minute schedule and meaningful-change judging enabled by the API. Use \`body\` only for advanced requests such as crawl targets, JSON change tracking, custom retention, or manual \`judgeEnabled\` control.
|
|
148
|
-
|
|
149
|
-
Meaningful-change judge: set \`goal\` to a plain-language description of what the user actually cares about. \`judgeEnabled\` defaults to true when \`goal\` is set, so providing \`goal\` is enough. Page webhooks expose \`isMeaningful\` and \`judgment\` on \`monitor.page\` events.
|
|
150
|
-
|
|
151
|
-
Simple fields:
|
|
152
|
-
- \`page\`: one page URL to monitor.
|
|
153
|
-
- \`pages\`: multiple page URLs to monitor.
|
|
154
|
-
- \`queries\`: one or more search queries (1-12) to monitor instead of fixed URLs. Each check runs the searches and diffs the result set, so you get alerted when new or changed results appear. Mutually exclusive with \`page\`/\`pages\` in the simple path.
|
|
155
|
-
- \`searchWindow\`: optional recency window for search targets \u2014 one of \`5m\`, \`15m\`, \`1h\`, \`6h\`, \`24h\`, \`7d\` (default \`24h\`).
|
|
156
|
-
- \`maxResults\`: optional max results per search, 1-50 (default 10).
|
|
157
|
-
- \`includeDomains\` / \`excludeDomains\`: optional domain allow/deny lists for search targets.
|
|
158
|
-
- \`goal\`: plain-English instruction for what changes matter. Required for the simple path (and always required when \`queries\` are set \u2014 web monitors must have a goal).
|
|
159
|
-
- \`scheduleText\`: optional natural-language schedule, default \`every 30 minutes\`.
|
|
160
|
-
- \`email\`: optional email recipient for summaries.
|
|
161
|
-
- \`webhookUrl\`: optional webhook URL. Configures \`monitor.page\` and \`monitor.check.completed\`.
|
|
162
|
-
|
|
163
|
-
**Search-mode example:**
|
|
164
|
-
|
|
165
|
-
\`\`\`json
|
|
166
|
-
{
|
|
167
|
-
"name": "firecrawl_monitor_create",
|
|
168
|
-
"arguments": {
|
|
169
|
-
"queries": ["new LLM release", "frontier model launch"],
|
|
170
|
-
"goal": "Notify me about major new LLM model releases.",
|
|
171
|
-
"searchWindow": "24h",
|
|
172
|
-
"maxResults": 10
|
|
173
|
-
}
|
|
174
|
-
}
|
|
175
|
-
\`\`\`
|
|
176
|
-
|
|
177
|
-
Goal guidance:
|
|
178
|
-
- Expand the user's one-line monitoring intent into a concise 2-3 sentence monitor goal.
|
|
179
|
-
- State what should trigger an alert, restate any scope the user gave, and include intent-specific exclusions only when obvious from the user's request.
|
|
180
|
-
- Generic noise such as whitespace, formatting-only changes, request IDs, tracking params, generic metadata, and unrelated page chrome is already handled by the judge; do not repeat it in every goal.
|
|
181
|
-
- If the user is vague, keep the goal broad rather than guessing exclusions. If the user asks for broad monitoring or "any change", preserve that and do not add exclusions that hide changes.
|
|
182
|
-
- If the user says they do not care about something, include that explicitly. It is okay to ask whether they want to ignore specific noise when it is likely to matter.
|
|
183
|
-
- Do not invent page-specific sections, thresholds, entities, or business rules unless the user mentioned them.
|
|
217
|
+
Create a recurring scrape, crawl, or search monitor that compares each check with its retained predecessor. The simple form accepts \`page\`/\`pages\` or \`queries\` plus a plain-language \`goal\`; the advanced \`body\` form controls targets, schedule, change-tracking formats, judging, retention, webhook, and notifications.
|
|
184
218
|
|
|
185
|
-
|
|
186
|
-
- Write keywords, not sentences: \`OpenAI new model release\`, not \`tell me when OpenAI releases a new model\`.
|
|
187
|
-
- Quote multi-word entities (\`"Llama 4"\`); group synonyms with \`OR\` (\`launch OR release OR announcement\`).
|
|
188
|
-
- Keep each query tight (~2-6 terms). One broad query usually beats several narrow ones \u2014 extra queries split the \`maxResults\` budget. Use one query per distinct entity; do not emit one per facet of a single subject.
|
|
189
|
-
- Keep \`site:\` operators out of queries \u2014 use \`includeDomains\` / \`excludeDomains\`.
|
|
190
|
-
- A healthy web monitor mostly returns \`new: 0\` and alerts only on genuinely new, on-goal results. Many \`ignored\` results \u21D2 queries too broad (tighten them); nothing for long stretches \u21D2 queries too narrow or window too tight (broaden); dismissed alerts \u21D2 goal too broad (add an intent-specific Ignore). Aim for high precision with enough recall.
|
|
191
|
-
|
|
192
|
-
Full \`body\` requests require: \`name\`, \`schedule\` (with \`cron\` or \`text\`), and \`targets\` (one or more \`{ type: 'scrape', urls: [...] }\`, \`{ type: 'crawl', url: '...' }\`, or \`{ type: 'search', queries: [...], searchWindow?, maxResults?, includeDomains?, excludeDomains? }\`). Optional: \`goal\` (required when any search target is present), \`judgeEnabled\`, \`webhook\`, \`notification\`, \`retentionDays\`.
|
|
193
|
-
|
|
194
|
-
**Markdown-mode (default):** Each check produces a unified text diff of the page's markdown. No extra configuration needed.
|
|
195
|
-
|
|
196
|
-
\`\`\`json
|
|
197
|
-
{
|
|
198
|
-
"name": "firecrawl_monitor_create",
|
|
199
|
-
"arguments": {
|
|
200
|
-
"page": "https://example.com/blog",
|
|
201
|
-
"goal": "Alert when a new blog post is published or an existing headline changes.",
|
|
202
|
-
"email": "alerts@example.com"
|
|
203
|
-
}
|
|
204
|
-
}
|
|
205
|
-
\`\`\`
|
|
206
|
-
|
|
207
|
-
**Multiple pages:**
|
|
208
|
-
|
|
209
|
-
\`\`\`json
|
|
210
|
-
{
|
|
211
|
-
"name": "firecrawl_monitor_create",
|
|
212
|
-
"arguments": {
|
|
213
|
-
"pages": ["https://example.com/pricing", "https://example.com/changelog"],
|
|
214
|
-
"goal": "Alert when pricing, packaging, or launch messaging changes.",
|
|
215
|
-
"webhookUrl": "https://example.com/webhooks/firecrawl"
|
|
216
|
-
}
|
|
217
|
-
}
|
|
218
|
-
\`\`\`
|
|
219
|
-
|
|
220
|
-
**JSON-mode change tracking:** To detect changes in **specific structured fields** (price, headline, in-stock flag, list items) instead of the whole page, add a \`changeTracking\` format with \`modes: ["json"]\` and a JSON schema to the target's \`scrapeOptions.formats\`. The check response will then carry a per-field diff (keyed by JSON path, e.g. \`plans[0].price\`) and a \`snapshot.json\` with the full current extraction. See \`firecrawl_monitor_check\` for the response shape.
|
|
221
|
-
|
|
222
|
-
\`\`\`json
|
|
223
|
-
{
|
|
224
|
-
"name": "firecrawl_monitor_create",
|
|
225
|
-
"arguments": {
|
|
226
|
-
"body": {
|
|
227
|
-
"name": "Pricing watch",
|
|
228
|
-
"schedule": { "text": "hourly", "timezone": "UTC" },
|
|
229
|
-
"goal": "Alert when a pricing tier, price, billing period, limit, or headline feature changes. Ignore unrelated marketing copy unless it changes the pricing offer.",
|
|
230
|
-
"targets": [{
|
|
231
|
-
"type": "scrape",
|
|
232
|
-
"urls": ["https://example.com/pricing"],
|
|
233
|
-
"scrapeOptions": {
|
|
234
|
-
"formats": [{
|
|
235
|
-
"type": "changeTracking",
|
|
236
|
-
"modes": ["json"],
|
|
237
|
-
"prompt": "Extract pricing tiers and headline features for each plan.",
|
|
238
|
-
"schema": {
|
|
239
|
-
"type": "object",
|
|
240
|
-
"properties": {
|
|
241
|
-
"plans": {
|
|
242
|
-
"type": "array",
|
|
243
|
-
"items": {
|
|
244
|
-
"type": "object",
|
|
245
|
-
"properties": {
|
|
246
|
-
"name": { "type": "string" },
|
|
247
|
-
"price": { "type": "string" },
|
|
248
|
-
"features": { "type": "array", "items": { "type": "string" } }
|
|
249
|
-
}
|
|
250
|
-
}
|
|
251
|
-
}
|
|
252
|
-
}
|
|
253
|
-
}
|
|
254
|
-
}]
|
|
255
|
-
}
|
|
256
|
-
}]
|
|
257
|
-
}
|
|
258
|
-
}
|
|
259
|
-
}
|
|
260
|
-
\`\`\`
|
|
261
|
-
|
|
262
|
-
**Mixed mode (JSON + git-diff):** Use \`modes: ["json", "git-diff"]\` to get both per-field diffs and a markdown sidecar. The page is marked \`changed\` whenever either surface changed.
|
|
219
|
+
In the simple form, a \`goal\` is required. If \`queries\` contains one or more non-empty values and is supplied with \`page\`/\`pages\`, \`queries\` create the search target and page targets are ignored. A monitor schedules future network checks and can send configured email or webhook notifications. Returns the created monitor.
|
|
263
220
|
`,
|
|
264
221
|
parameters: z.object({
|
|
265
222
|
body: z.record(z.string(), z.any()).optional(),
|
|
@@ -300,12 +257,7 @@ Full \`body\` requests require: \`name\`, \`schedule\` (with \`cron\` or \`text\
|
|
|
300
257
|
// Read-only listing.
|
|
301
258
|
},
|
|
302
259
|
description: `
|
|
303
|
-
List
|
|
304
|
-
|
|
305
|
-
**Usage Example:**
|
|
306
|
-
\`\`\`json
|
|
307
|
-
{ "name": "firecrawl_monitor_list", "arguments": { "limit": 20 } }
|
|
308
|
-
\`\`\`
|
|
260
|
+
List monitors for the authenticated account with optional pagination controls. Returns one page of monitor records and pagination metadata.
|
|
309
261
|
`,
|
|
310
262
|
parameters: z.object({
|
|
311
263
|
limit: z.number().int().positive().optional(),
|
|
@@ -331,12 +283,7 @@ List all Firecrawl monitors for the authenticated account.
|
|
|
331
283
|
// Read-only retrieval.
|
|
332
284
|
},
|
|
333
285
|
description: `
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
**Usage Example:**
|
|
337
|
-
\`\`\`json
|
|
338
|
-
{ "name": "firecrawl_monitor_get", "arguments": { "id": "mon_abc123" } }
|
|
339
|
-
\`\`\`
|
|
286
|
+
Retrieve one monitor by ID, including its configuration and current state. This does not run or modify the monitor.
|
|
340
287
|
`,
|
|
341
288
|
parameters: z.object({ id: z.string() }),
|
|
342
289
|
execute: async (args2, { session }) => {
|
|
@@ -360,18 +307,9 @@ Get a single monitor by ID.
|
|
|
360
307
|
// Can pause, replace, or remove monitor configuration; changes overwrite prior settings.
|
|
361
308
|
},
|
|
362
309
|
description: `
|
|
363
|
-
|
|
310
|
+
Patch an existing monitor by ID. The body can change its name, active/paused status, schedule, targets, goal, judging, webhook, notifications, or retention; these changes affect future scheduled checks.
|
|
364
311
|
|
|
365
|
-
|
|
366
|
-
\`\`\`json
|
|
367
|
-
{
|
|
368
|
-
"name": "firecrawl_monitor_update",
|
|
369
|
-
"arguments": {
|
|
370
|
-
"id": "mon_abc123",
|
|
371
|
-
"body": { "status": "paused" }
|
|
372
|
-
}
|
|
373
|
-
}
|
|
374
|
-
\`\`\`
|
|
312
|
+
Returns the updated monitor.
|
|
375
313
|
`,
|
|
376
314
|
parameters: z.object({
|
|
377
315
|
id: z.string(),
|
|
@@ -399,12 +337,7 @@ Update a monitor. Pass any subset of fields to patch: \`name\`, \`status\` ("act
|
|
|
399
337
|
// Irreversibly removes the monitor and stops its schedule.
|
|
400
338
|
},
|
|
401
339
|
description: `
|
|
402
|
-
Permanently delete a monitor and stop its schedule. This cannot be undone.
|
|
403
|
-
|
|
404
|
-
**Usage Example:**
|
|
405
|
-
\`\`\`json
|
|
406
|
-
{ "name": "firecrawl_monitor_delete", "arguments": { "id": "mon_abc123" } }
|
|
407
|
-
\`\`\`
|
|
340
|
+
Permanently delete a monitor by ID and stop its future schedule. This operation cannot be undone and returns deletion status.
|
|
408
341
|
`,
|
|
409
342
|
parameters: z.object({ id: z.string() }),
|
|
410
343
|
execute: async (args2, { session, log }) => {
|
|
@@ -430,12 +363,7 @@ Permanently delete a monitor and stop its schedule. This cannot be undone.
|
|
|
430
363
|
// Starts a read-only check job; does not delete the monitor or external sites.
|
|
431
364
|
},
|
|
432
365
|
description: `
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
**Usage Example:**
|
|
436
|
-
\`\`\`json
|
|
437
|
-
{ "name": "firecrawl_monitor_run", "arguments": { "id": "mon_abc123" } }
|
|
438
|
-
\`\`\`
|
|
366
|
+
Queue an immediate check for a monitor outside its normal schedule. This starts network work for the monitor's configured targets and returns the queued check.
|
|
439
367
|
`,
|
|
440
368
|
parameters: z.object({ id: z.string() }),
|
|
441
369
|
execute: async (args2, { session }) => {
|
|
@@ -460,12 +388,7 @@ Trigger a monitor check immediately, outside its normal schedule. Returns the qu
|
|
|
460
388
|
// Read-only listing.
|
|
461
389
|
},
|
|
462
390
|
description: `
|
|
463
|
-
List historical checks for a monitor.
|
|
464
|
-
|
|
465
|
-
**Usage Example:**
|
|
466
|
-
\`\`\`json
|
|
467
|
-
{ "name": "firecrawl_monitor_checks", "arguments": { "id": "mon_abc123", "limit": 10, "status": "completed" } }
|
|
468
|
-
\`\`\`
|
|
391
|
+
List historical checks for a monitor, optionally filtered by status and bounded by a result limit. Returns one page of check summaries and pagination metadata.
|
|
469
392
|
`,
|
|
470
393
|
parameters: z.object({
|
|
471
394
|
id: z.string(),
|
|
@@ -495,60 +418,9 @@ List historical checks for a monitor.
|
|
|
495
418
|
// Read-only retrieval of diff snapshots and judgments.
|
|
496
419
|
},
|
|
497
420
|
description: `
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
Each entry in \`data.pages[]\` has \`url\`, \`status\` (\`same\` | \`new\` | \`changed\` | \`removed\` | \`error\`), optional \`judgment\` when goal-based judging ran, and \u2014 when changed \u2014 a \`diff\` and possibly a \`snapshot\`. The shape of \`diff\` depends on the monitor's \`formats\` configuration:
|
|
421
|
+
Retrieve one monitor check and its page-level results, optionally filtered by page status. Pages report \`same\`, \`new\`, \`changed\`, \`removed\`, or \`error\`; configured goal judging can add a meaningful-change decision.
|
|
501
422
|
|
|
502
|
-
|
|
503
|
-
- **JSON mode** (\`changeTracking\` with \`modes: ["json"]\`). \`diff.json\` is a per-field map keyed by JSON path into the extraction, e.g. \`plans[0].price\`, with each value being \`{ previous, current }\`. \`snapshot.json\` is the full current extraction. No \`diff.text\`.
|
|
504
|
-
- **Mixed mode** (\`modes: ["json", "git-diff"]\`). Both \`diff.text\` (markdown sidecar) AND \`diff.json\` (per-field map) are present, plus \`snapshot.json\`.
|
|
505
|
-
|
|
506
|
-
**Example JSON-mode response \`pages[]\` entry:**
|
|
507
|
-
|
|
508
|
-
\`\`\`json
|
|
509
|
-
{
|
|
510
|
-
"url": "https://example.com/pricing",
|
|
511
|
-
"status": "changed",
|
|
512
|
-
"diff": {
|
|
513
|
-
"json": {
|
|
514
|
-
"plans[0].price": { "previous": "$19/mo", "current": "$24/mo" },
|
|
515
|
-
"plans[1].features[2]": { "previous": "10 GB storage", "current": "25 GB storage" }
|
|
516
|
-
}
|
|
517
|
-
},
|
|
518
|
-
"snapshot": { "json": { "plans": [/* current full extraction matching the monitor's schema */] } },
|
|
519
|
-
"judgment": {
|
|
520
|
-
"meaningful": true,
|
|
521
|
-
"confidence": "high",
|
|
522
|
-
"reason": "The pricing changed, which matches the monitor goal.",
|
|
523
|
-
"meaningfulChanges": [
|
|
524
|
-
{
|
|
525
|
-
"type": "changed",
|
|
526
|
-
"before": "$19/mo",
|
|
527
|
-
"after": "$24/mo",
|
|
528
|
-
"reason": "The tracked plan price changed."
|
|
529
|
-
}
|
|
530
|
-
]
|
|
531
|
-
}
|
|
532
|
-
}
|
|
533
|
-
\`\`\`
|
|
534
|
-
|
|
535
|
-
When summarizing a check for the user, prefer \`diff.json\` paths (e.g. "plans[0].price changed from $19/mo to $24/mo") over re-printing the markdown diff \u2014 it's more concise and grounded in the schema fields they asked for.
|
|
536
|
-
|
|
537
|
-
When \`judgment\` is present, use it to decide what to surface. \`judgment.meaningful: false\` means the change was classified as noise for the monitor's goal. When \`judgment.meaningfulChanges\` is present, prefer those goal-relevant changes over raw diff hunks; each item includes \`type\`, \`before\`, \`after\`, and \`reason\`.
|
|
538
|
-
|
|
539
|
-
The endpoint paginates via a top-level \`next\` URL; this tool returns one page at a time. Increase \`limit\` (max 100) to fetch fewer pages.
|
|
540
|
-
|
|
541
|
-
**Usage Example:**
|
|
542
|
-
\`\`\`json
|
|
543
|
-
{
|
|
544
|
-
"name": "firecrawl_monitor_check",
|
|
545
|
-
"arguments": {
|
|
546
|
-
"id": "mon_abc123",
|
|
547
|
-
"checkId": "chk_xyz",
|
|
548
|
-
"pageStatus": "changed"
|
|
549
|
-
}
|
|
550
|
-
}
|
|
551
|
-
\`\`\`
|
|
423
|
+
Markdown tracking returns a unified text diff, JSON tracking returns field paths with previous/current values and a current snapshot, and mixed tracking returns both. Returns one page of results plus a \`next\` URL when more pages exist.
|
|
552
424
|
`,
|
|
553
425
|
parameters: z.object({
|
|
554
426
|
id: z.string(),
|
|
@@ -635,7 +507,7 @@ function fmtPaperMetadata(paper) {
|
|
|
635
507
|
lines.push("");
|
|
636
508
|
lines.push(`Paper ID: ${paper.paperId ?? "?"}`);
|
|
637
509
|
const ids = Object.entries(paper.ids ?? {}).flatMap(
|
|
638
|
-
([namespace,
|
|
510
|
+
([namespace, values2]) => values2.map((value) => `${namespace}:${value}`)
|
|
639
511
|
).join(", ");
|
|
640
512
|
if (ids) lines.push(`IDs: ${ids}`);
|
|
641
513
|
const authors = fmtAuthors(paper.authors);
|
|
@@ -689,7 +561,11 @@ function registerResearchTools(server2, getClient2) {
|
|
|
689
561
|
destructiveHint: false
|
|
690
562
|
// Query-only; no writes to external sources or the research index.
|
|
691
563
|
},
|
|
692
|
-
description:
|
|
564
|
+
description: `
|
|
565
|
+
For topics represented in the indexed corpus, search paper metadata and abstracts with a natural-language query. Optional author, category, and date filters constrain results.
|
|
566
|
+
|
|
567
|
+
Returns ranked papers with canonical IDs, titles, authors, and abstracts.
|
|
568
|
+
`,
|
|
693
569
|
parameters: z2.object({
|
|
694
570
|
query: z2.string().min(1).describe(
|
|
695
571
|
"Natural-language research topic or question, including methods, systems, conditions, populations, interventions, or outcomes when relevant."
|
|
@@ -736,7 +612,9 @@ function registerResearchTools(server2, getClient2) {
|
|
|
736
612
|
destructiveHint: false
|
|
737
613
|
// Read-only metadata lookup.
|
|
738
614
|
},
|
|
739
|
-
description:
|
|
615
|
+
description: `
|
|
616
|
+
Retrieve canonical metadata for one paper ID, such as an arXiv, PMC, PMID, or DOI identifier. Returns the title, abstract, authors, categories, source IDs, and dates as markdown.
|
|
617
|
+
`,
|
|
740
618
|
parameters: z2.object({
|
|
741
619
|
paperId: z2.string().min(1).describe(
|
|
742
620
|
"Canonical paperId or primaryId such as `arxiv:1706.03762`, `pmcid:PMC12530322`, `pmid:40953549`, or `doi:10.1016/j.neunet.2025.108095`."
|
|
@@ -763,7 +641,11 @@ function registerResearchTools(server2, getClient2) {
|
|
|
763
641
|
destructiveHint: false
|
|
764
642
|
// Read-only graph query; no modifications.
|
|
765
643
|
},
|
|
766
|
-
description:
|
|
644
|
+
description: `
|
|
645
|
+
Find citation-graph candidates from one to ten \`seed_ids\`; the first ID is the primary seed and later IDs are anchors. \`mode\` defaults to \`similar\` (co-citation/bibliographic coupling); \`citers\` returns papers citing a seed and \`references\` papers cited by a seed. \`intent\` ranks candidates.
|
|
646
|
+
|
|
647
|
+
Returns ranked candidates and the evaluated pool size.
|
|
648
|
+
`,
|
|
767
649
|
parameters: z2.object({
|
|
768
650
|
seed_ids: z2.array(z2.string()).min(1).max(10),
|
|
769
651
|
intent: z2.string().min(1),
|
|
@@ -805,7 +687,11 @@ note: ${res.data.note}` : "";
|
|
|
805
687
|
destructiveHint: false
|
|
806
688
|
// Read-only passage retrieval.
|
|
807
689
|
},
|
|
808
|
-
description:
|
|
690
|
+
description: `
|
|
691
|
+
Retrieve in-body passages from one paper that are relevant to a specific question. Full text is available only for indexed papers; \`k\` controls the number of passages.
|
|
692
|
+
|
|
693
|
+
Returns matching passages or a notice when full text is unavailable.
|
|
694
|
+
`,
|
|
809
695
|
parameters: z2.object({
|
|
810
696
|
paperId: z2.string().min(1).describe(
|
|
811
697
|
"Canonical paperId or primaryId such as `arxiv:1706.03762`, `pmcid:PMC12530322`, `pmid:40953549`, or `doi:10.1016/j.neunet.2025.108095`."
|
|
@@ -838,7 +724,9 @@ note: ${res.data.note}` : "";
|
|
|
838
724
|
destructiveHint: false
|
|
839
725
|
// Query-only; does not create issues, PRs, or modify repositories.
|
|
840
726
|
},
|
|
841
|
-
description:
|
|
727
|
+
description: `
|
|
728
|
+
Search indexed public GitHub issue, pull-request, and README content. Returns ranked matches with repository, URL, snippet, and full matched markdown when available.
|
|
729
|
+
`,
|
|
842
730
|
parameters: z2.object({
|
|
843
731
|
query: z2.string().min(1),
|
|
844
732
|
k: z2.number().int().min(1).max(100).optional()
|
|
@@ -878,6 +766,22 @@ function extractBearerToken(headers) {
|
|
|
878
766
|
function isFirecrawlOAuthAccessToken(token) {
|
|
879
767
|
return token.startsWith("fco_");
|
|
880
768
|
}
|
|
769
|
+
function isFirecrawlApiKey(token) {
|
|
770
|
+
return token.startsWith("fc-");
|
|
771
|
+
}
|
|
772
|
+
function isLegacyKeyPathRequest(request) {
|
|
773
|
+
return normalizeHeader(request?.headers?.["x-firecrawl-key-transport"]) === "path";
|
|
774
|
+
}
|
|
775
|
+
function requestShouldReceiveOAuthChallenge(request, profile) {
|
|
776
|
+
if (!profile.acceptApiKeys) return true;
|
|
777
|
+
if (!request?.headers) return true;
|
|
778
|
+
const headerApiKey = normalizeHeader(
|
|
779
|
+
request.headers["x-firecrawl-api-key"] ?? request.headers["x-api-key"]
|
|
780
|
+
);
|
|
781
|
+
if (headerApiKey) return false;
|
|
782
|
+
const bearer = extractBearerToken(request.headers);
|
|
783
|
+
return !bearer || isFirecrawlOAuthAccessToken(bearer);
|
|
784
|
+
}
|
|
881
785
|
function resolveCredentialFromEnv() {
|
|
882
786
|
return normalizeHeader(process.env.FIRECRAWL_OAUTH_TOKEN) ?? normalizeHeader(process.env.FIRECRAWL_API_KEY);
|
|
883
787
|
}
|
|
@@ -886,6 +790,9 @@ function isHttpStreamingTransport() {
|
|
|
886
790
|
}
|
|
887
791
|
var DEFAULT_OAUTH_ISSUER = "https://www.firecrawl.dev";
|
|
888
792
|
var DEFAULT_MCP_RESOURCE_URL = "https://mcp.firecrawl.dev/v2/mcp";
|
|
793
|
+
var DEFAULT_MCP_OAUTH_RESOURCE_URL = "https://mcp.firecrawl.dev/v2/mcp-oauth";
|
|
794
|
+
var DEFAULT_MCP_SEARCH_RESOURCE_URL = "https://mcp.firecrawl.dev/v2/mcp-search";
|
|
795
|
+
var DEFAULT_MCP_SEARCH_ENDPOINT = "/v2/mcp-search";
|
|
889
796
|
function withoutTrailingSlash(value) {
|
|
890
797
|
return value.replace(/\/+$/, "");
|
|
891
798
|
}
|
|
@@ -897,19 +804,40 @@ function getOAuthIssuer() {
|
|
|
897
804
|
function getMcpResourceUrl() {
|
|
898
805
|
return normalizeHeader(process.env.FIRECRAWL_MCP_RESOURCE_URL) ?? DEFAULT_MCP_RESOURCE_URL;
|
|
899
806
|
}
|
|
900
|
-
function
|
|
901
|
-
|
|
807
|
+
function getPrimaryEndpoint() {
|
|
808
|
+
const endpoint = normalizeHeader(process.env.FASTMCP_ENDPOINT) ?? "/v2/mcp";
|
|
809
|
+
if (endpoint === "/v2/mcp" || endpoint === "/v2/mcp-oauth" || endpoint === "/v2/mcp-search") {
|
|
810
|
+
return endpoint;
|
|
811
|
+
}
|
|
812
|
+
throw new Error(
|
|
813
|
+
`Unsupported FASTMCP_ENDPOINT: ${endpoint}. Expected /v2/mcp, /v2/mcp-oauth, or /v2/mcp-search.`
|
|
814
|
+
);
|
|
815
|
+
}
|
|
816
|
+
function getSearchMcpResourceUrl() {
|
|
817
|
+
return normalizeHeader(process.env.FIRECRAWL_MCP_SEARCH_RESOURCE_URL) ?? DEFAULT_MCP_SEARCH_RESOURCE_URL;
|
|
818
|
+
}
|
|
819
|
+
function getSearchMcpEndpoint() {
|
|
820
|
+
const configured = normalizeHeader(process.env.FIRECRAWL_MCP_SEARCH_ENDPOINT);
|
|
821
|
+
if (configured && configured.startsWith("/")) {
|
|
822
|
+
return configured;
|
|
823
|
+
}
|
|
824
|
+
return DEFAULT_MCP_SEARCH_ENDPOINT;
|
|
825
|
+
}
|
|
826
|
+
function getOAuthProtectedResourceMetadataUrl(profile) {
|
|
827
|
+
const resource = new URL(profile.resourceUrl);
|
|
828
|
+
const base = `${resource.origin}/.well-known/oauth-protected-resource`;
|
|
829
|
+
return profile.id === "full" ? base : `${base}${resource.pathname}`;
|
|
902
830
|
}
|
|
903
831
|
function escapeWWWAuthenticateValue(value) {
|
|
904
832
|
return value.replace(/\\/g, "\\\\").replace(/"/g, '\\"');
|
|
905
833
|
}
|
|
906
|
-
function createOAuthChallengeResponse(error) {
|
|
834
|
+
function createOAuthChallengeResponse(error, profile) {
|
|
907
835
|
if (!isMcpOAuthEnabled()) {
|
|
908
836
|
return void 0;
|
|
909
837
|
}
|
|
910
838
|
const errorMessage = error instanceof Error ? error.message : String(error || "Unauthorized");
|
|
911
839
|
const wwwAuthenticate = [
|
|
912
|
-
`resource_metadata="${escapeWWWAuthenticateValue(getOAuthProtectedResourceMetadataUrl())}"`,
|
|
840
|
+
`resource_metadata="${escapeWWWAuthenticateValue(getOAuthProtectedResourceMetadataUrl(profile))}"`,
|
|
913
841
|
'error="invalid_token"',
|
|
914
842
|
`error_description="${escapeWWWAuthenticateValue(errorMessage)}"`
|
|
915
843
|
].join(", ");
|
|
@@ -927,6 +855,18 @@ function createOAuthChallengeResponse(error) {
|
|
|
927
855
|
}
|
|
928
856
|
);
|
|
929
857
|
}
|
|
858
|
+
function createInvalidCredentialResponse(error) {
|
|
859
|
+
return new Response(
|
|
860
|
+
JSON.stringify({
|
|
861
|
+
error: "invalid_api_key",
|
|
862
|
+
error_description: error.message
|
|
863
|
+
}),
|
|
864
|
+
{
|
|
865
|
+
headers: { "Content-Type": "application/json" },
|
|
866
|
+
status: 401
|
|
867
|
+
}
|
|
868
|
+
);
|
|
869
|
+
}
|
|
930
870
|
function getOAuthIntrospectionEndpoint() {
|
|
931
871
|
return `${getOAuthIssuer()}/api/oauth/introspect`;
|
|
932
872
|
}
|
|
@@ -936,63 +876,168 @@ function getOAuthIntrospectionSecret() {
|
|
|
936
876
|
function isMcpOAuthEnabled() {
|
|
937
877
|
return process.env.CLOUD_SERVICE === "true";
|
|
938
878
|
}
|
|
939
|
-
|
|
879
|
+
function isOAuthCredentialPurpose(value) {
|
|
880
|
+
return value === "general" || value === "hosted_mcp_oauth";
|
|
881
|
+
}
|
|
882
|
+
var InvalidFirecrawlCredentialError = class extends Error {
|
|
883
|
+
constructor() {
|
|
884
|
+
super("The supplied Firecrawl credential is invalid or revoked. Replace it and retry.");
|
|
885
|
+
this.name = "InvalidFirecrawlCredentialError";
|
|
886
|
+
}
|
|
887
|
+
};
|
|
888
|
+
var MCP_GLOBAL_SCOPE = "firecrawl:global";
|
|
889
|
+
function values(value) {
|
|
890
|
+
if (typeof value === "string") return value.split(/\s+/).filter(Boolean);
|
|
891
|
+
return Array.isArray(value) ? value.flatMap((item) => item.split(/\s+/).filter(Boolean)) : [];
|
|
892
|
+
}
|
|
893
|
+
function audienceMatchesResource(aud, resourceUrl) {
|
|
894
|
+
const target = withoutTrailingSlash(resourceUrl);
|
|
895
|
+
return values(aud).some((entry) => withoutTrailingSlash(entry) === target);
|
|
896
|
+
}
|
|
897
|
+
function credentialMetadata(data) {
|
|
898
|
+
return {
|
|
899
|
+
teamId: typeof data.team_id === "string" ? data.team_id : void 0,
|
|
900
|
+
userId: typeof data.sub === "string" ? data.sub : void 0,
|
|
901
|
+
apiKeyId: typeof data.api_key_id === "string" ? data.api_key_id : void 0,
|
|
902
|
+
oauthClientId: typeof data.client_id === "string" ? data.client_id : void 0,
|
|
903
|
+
resource: typeof data.aud === "string" ? data.aud : void 0
|
|
904
|
+
};
|
|
905
|
+
}
|
|
906
|
+
async function introspectToken(token, expectedResource) {
|
|
940
907
|
const introspectionSecret = getOAuthIntrospectionSecret();
|
|
941
|
-
if (!introspectionSecret)
|
|
942
|
-
|
|
908
|
+
if (!introspectionSecret) throw new CredentialValidationUnavailableError();
|
|
909
|
+
const controller = new AbortController();
|
|
910
|
+
const timeout = setTimeout(() => controller.abort(), 1500);
|
|
911
|
+
let response;
|
|
912
|
+
try {
|
|
913
|
+
response = await fetch(getOAuthIntrospectionEndpoint(), {
|
|
914
|
+
method: "POST",
|
|
915
|
+
headers: {
|
|
916
|
+
"Content-Type": "application/x-www-form-urlencoded",
|
|
917
|
+
Authorization: `Bearer ${introspectionSecret}`
|
|
918
|
+
},
|
|
919
|
+
body: new URLSearchParams({
|
|
920
|
+
resource: expectedResource,
|
|
921
|
+
token,
|
|
922
|
+
token_type_hint: "access_token"
|
|
923
|
+
}),
|
|
924
|
+
signal: controller.signal
|
|
925
|
+
});
|
|
926
|
+
} catch {
|
|
927
|
+
throw new CredentialValidationUnavailableError();
|
|
928
|
+
} finally {
|
|
929
|
+
clearTimeout(timeout);
|
|
943
930
|
}
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
Authorization: `Bearer ${introspectionSecret}`
|
|
949
|
-
},
|
|
950
|
-
body: new URLSearchParams({
|
|
951
|
-
token,
|
|
952
|
-
token_type_hint: "access_token"
|
|
953
|
-
})
|
|
954
|
-
});
|
|
955
|
-
if (!response.ok) {
|
|
956
|
-
throw new Error(`OAuth token introspection failed: ${response.status}`);
|
|
931
|
+
if (!response.ok) throw new CredentialValidationUnavailableError();
|
|
932
|
+
const contentType = response.headers.get("content-type")?.toLowerCase() ?? "";
|
|
933
|
+
if (!contentType.includes("application/json")) {
|
|
934
|
+
throw new CredentialValidationUnavailableError();
|
|
957
935
|
}
|
|
958
936
|
const data = await response.json();
|
|
959
|
-
if (
|
|
960
|
-
throw new
|
|
937
|
+
if (typeof data.active !== "boolean") {
|
|
938
|
+
throw new CredentialValidationUnavailableError();
|
|
961
939
|
}
|
|
962
|
-
|
|
940
|
+
if (data.active && (!data.api_key || !isOAuthCredentialPurpose(data.credential_purpose) || !values(data.scope).includes(MCP_GLOBAL_SCOPE))) {
|
|
941
|
+
throw new CredentialValidationUnavailableError();
|
|
942
|
+
}
|
|
943
|
+
return data;
|
|
963
944
|
}
|
|
964
|
-
async function resolveCredentialFromHeaders(headers) {
|
|
945
|
+
async function resolveCredentialFromHeaders(headers, profile) {
|
|
965
946
|
const bearer = extractBearerToken(headers);
|
|
966
947
|
const headerApiKey = normalizeHeader(
|
|
967
948
|
headers["x-firecrawl-api-key"] ?? headers["x-api-key"]
|
|
968
949
|
);
|
|
969
|
-
|
|
970
|
-
|
|
950
|
+
const token = headerApiKey ?? bearer;
|
|
951
|
+
if (!token) return void 0;
|
|
952
|
+
if (!profile.acceptApiKeys && !isFirecrawlOAuthAccessToken(token)) {
|
|
953
|
+
throw new Error(
|
|
954
|
+
`OAuth access token required for the Firecrawl MCP resource ${profile.endpoint}`
|
|
955
|
+
);
|
|
956
|
+
}
|
|
957
|
+
if (!isFirecrawlOAuthAccessToken(token) && !isFirecrawlApiKey(token)) {
|
|
958
|
+
return { invalid: true };
|
|
959
|
+
}
|
|
960
|
+
let data = await introspectToken(token, profile.resourceUrl);
|
|
961
|
+
if (isFirecrawlOAuthAccessToken(token) && !data.active && profile.acceptLegacyAudience) {
|
|
962
|
+
data = await introspectToken(token, DEFAULT_MCP_RESOURCE_URL);
|
|
963
|
+
}
|
|
964
|
+
if (!data.active || !data.api_key) {
|
|
965
|
+
if (isFirecrawlOAuthAccessToken(token)) {
|
|
966
|
+
throw new Error("Invalid OAuth access token");
|
|
967
|
+
}
|
|
968
|
+
return { invalid: true };
|
|
969
|
+
}
|
|
970
|
+
if (isFirecrawlApiKey(token)) {
|
|
971
|
+
return data.credential_purpose === "general" ? {
|
|
972
|
+
credential: data.api_key,
|
|
973
|
+
source: "api-key",
|
|
974
|
+
metadata: credentialMetadata(data)
|
|
975
|
+
} : { invalid: true };
|
|
976
|
+
}
|
|
977
|
+
const expectedAudience = profile.acceptLegacyAudience && audienceMatchesResource(data.aud, DEFAULT_MCP_RESOURCE_URL) ? DEFAULT_MCP_RESOURCE_URL : profile.resourceUrl;
|
|
978
|
+
if (!audienceMatchesResource(data.aud, expectedAudience)) {
|
|
979
|
+
throw new Error("OAuth token audience does not match this resource");
|
|
971
980
|
}
|
|
972
|
-
if (
|
|
973
|
-
|
|
981
|
+
if (profile.requireManagedOAuth && data.credential_purpose !== "hosted_mcp_oauth") {
|
|
982
|
+
throw new Error("OAuth token is not a managed Firecrawl MCP credential");
|
|
974
983
|
}
|
|
975
|
-
if (
|
|
976
|
-
|
|
984
|
+
if (data.credential_purpose === "hosted_mcp_oauth") {
|
|
985
|
+
requireDelegatedCredentialSigning();
|
|
986
|
+
return {
|
|
987
|
+
managedOAuthApiKey: data.api_key,
|
|
988
|
+
source: "oauth",
|
|
989
|
+
metadata: credentialMetadata(data)
|
|
990
|
+
};
|
|
977
991
|
}
|
|
978
|
-
return
|
|
992
|
+
return {
|
|
993
|
+
credential: data.api_key,
|
|
994
|
+
source: "oauth",
|
|
995
|
+
metadata: credentialMetadata(data)
|
|
996
|
+
};
|
|
979
997
|
}
|
|
980
|
-
async function authenticateRequest(request) {
|
|
981
|
-
const
|
|
998
|
+
async function authenticateRequest(request, profile) {
|
|
999
|
+
const resolved = request?.headers ? await resolveCredentialFromHeaders(request.headers, profile) : void 0;
|
|
1000
|
+
const headerCred = resolved?.credential;
|
|
1001
|
+
const managedCred = resolved?.managedOAuthApiKey;
|
|
982
1002
|
const envCred = resolveCredentialFromEnv();
|
|
983
1003
|
if (process.env.CLOUD_SERVICE === "true") {
|
|
984
|
-
if (!headerCred) {
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
1004
|
+
if (!headerCred && !managedCred) {
|
|
1005
|
+
if (resolved?.invalid) {
|
|
1006
|
+
if (!profile.acceptApiKeys) {
|
|
1007
|
+
throw new Error(
|
|
1008
|
+
`OAuth access token required for the Firecrawl MCP resource ${profile.endpoint}`
|
|
1009
|
+
);
|
|
1010
|
+
}
|
|
1011
|
+
if (isLegacyKeyPathRequest(request)) {
|
|
1012
|
+
throw new InvalidFirecrawlCredentialError();
|
|
1013
|
+
}
|
|
1014
|
+
return { authType: "none", credentialError: "CREDENTIAL_INVALID" };
|
|
1015
|
+
}
|
|
1016
|
+
if (profile.allowKeyless) {
|
|
1017
|
+
return {
|
|
1018
|
+
authType: "keyless",
|
|
1019
|
+
firecrawlApiKey: void 0,
|
|
1020
|
+
keylessClientIp: extractClientIp(request)
|
|
1021
|
+
};
|
|
1022
|
+
}
|
|
1023
|
+
if (!profile.acceptApiKeys) {
|
|
1024
|
+
throw new Error(
|
|
1025
|
+
`OAuth access token required for the Firecrawl MCP resource ${profile.endpoint}`
|
|
1026
|
+
);
|
|
988
1027
|
}
|
|
989
1028
|
throw new Error(
|
|
990
1029
|
"Firecrawl credentials required: OAuth access token (Authorization: Bearer fco_...) or API key (x-firecrawl-api-key)"
|
|
991
1030
|
);
|
|
992
1031
|
}
|
|
993
|
-
|
|
1032
|
+
const session2 = {
|
|
1033
|
+
authType: resolved?.source === "oauth" ? "oauth" : "api-key",
|
|
1034
|
+
firecrawlApiKey: headerCred,
|
|
1035
|
+
...isLegacyKeyPathRequest(request) ? { keyTransport: "path" } : {},
|
|
1036
|
+
...resolved?.metadata
|
|
1037
|
+
};
|
|
1038
|
+
return managedCred ? setManagedOAuthApiKey(session2, managedCred) : session2;
|
|
994
1039
|
}
|
|
995
|
-
const credential = headerCred ?? envCred;
|
|
1040
|
+
const credential = headerCred ?? managedCred ?? envCred;
|
|
996
1041
|
const httpStreaming = isHttpStreamingTransport();
|
|
997
1042
|
if (!httpStreaming && !process.env.FIRECRAWL_API_KEY && !process.env.FIRECRAWL_API_URL) {
|
|
998
1043
|
console.error(
|
|
@@ -1005,23 +1050,103 @@ async function authenticateRequest(request) {
|
|
|
1005
1050
|
);
|
|
1006
1051
|
process.exit(1);
|
|
1007
1052
|
}
|
|
1008
|
-
|
|
1053
|
+
const session = {
|
|
1054
|
+
authType: resolved?.source === "oauth" ? "oauth" : credential ? "env" : "none",
|
|
1055
|
+
firecrawlApiKey: headerCred ?? envCred,
|
|
1056
|
+
...resolved?.metadata
|
|
1057
|
+
};
|
|
1058
|
+
return managedCred ? setManagedOAuthApiKey(session, managedCred) : session;
|
|
1009
1059
|
}
|
|
1010
|
-
|
|
1011
|
-
if (
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1060
|
+
function searchCompanionAuthMode(request, session) {
|
|
1061
|
+
if (session?.authType === "oauth") return "oauth";
|
|
1062
|
+
if (session?.authType === "api-key") return "api-key";
|
|
1063
|
+
const headerApiKey = normalizeHeader(
|
|
1064
|
+
request?.headers?.["x-firecrawl-api-key"] ?? request?.headers?.["x-api-key"]
|
|
1065
|
+
);
|
|
1066
|
+
if (headerApiKey) return "api-key";
|
|
1067
|
+
const bearer = request?.headers ? extractBearerToken(request.headers) : void 0;
|
|
1068
|
+
if (bearer?.startsWith("fco_")) return "oauth";
|
|
1069
|
+
if (bearer) return "api-key";
|
|
1070
|
+
return "none";
|
|
1071
|
+
}
|
|
1072
|
+
function emitSearchCompanionAuthTelemetry(profile, request, outcome, session) {
|
|
1073
|
+
if (process.env.CLOUD_SERVICE !== "true" || profile.id !== "search" || profile.primary === true) {
|
|
1074
|
+
return;
|
|
1075
|
+
}
|
|
1076
|
+
console.log(
|
|
1077
|
+
"[MCP_SEARCH_AUTH]",
|
|
1078
|
+
JSON.stringify({
|
|
1079
|
+
auth_mode: searchCompanionAuthMode(request, session),
|
|
1080
|
+
outcome,
|
|
1081
|
+
profile: "companion",
|
|
1082
|
+
// Unique only to this telemetry record; it is not a cross-service
|
|
1083
|
+
// correlation ID and does not accept client-controlled identifiers.
|
|
1084
|
+
event_id: randomUUID(),
|
|
1085
|
+
route: DEFAULT_MCP_SEARCH_ENDPOINT
|
|
1086
|
+
})
|
|
1087
|
+
);
|
|
1088
|
+
}
|
|
1089
|
+
function emitLegacyKeyPathTelemetry(profile, request, outcome, session) {
|
|
1090
|
+
if (profile.id !== "full" || !isLegacyKeyPathRequest(request)) return;
|
|
1091
|
+
console.log(
|
|
1092
|
+
"[MCP_LEGACY_KEY_PATH]",
|
|
1093
|
+
JSON.stringify({
|
|
1094
|
+
auth_type: session?.authType ?? "none",
|
|
1095
|
+
key_transport: "path",
|
|
1096
|
+
outcome,
|
|
1097
|
+
resource: profile.resourceUrl
|
|
1098
|
+
})
|
|
1099
|
+
);
|
|
1100
|
+
}
|
|
1101
|
+
function makeAuthenticate(profile) {
|
|
1102
|
+
return async function authenticateWithOAuthChallenge(request) {
|
|
1103
|
+
if (request?.[authResultByRequest]) {
|
|
1104
|
+
return request[authResultByRequest];
|
|
1018
1105
|
}
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1106
|
+
const authResult = authenticateRequest(request, profile).then((session) => {
|
|
1107
|
+
emitSearchCompanionAuthTelemetry(
|
|
1108
|
+
profile,
|
|
1109
|
+
request,
|
|
1110
|
+
session.credentialError ? "rejected" : "accepted",
|
|
1111
|
+
session
|
|
1112
|
+
);
|
|
1113
|
+
emitLegacyKeyPathTelemetry(
|
|
1114
|
+
profile,
|
|
1115
|
+
request,
|
|
1116
|
+
session.credentialError ? "rejected" : "accepted",
|
|
1117
|
+
session
|
|
1118
|
+
);
|
|
1119
|
+
return session;
|
|
1120
|
+
}).catch((error) => {
|
|
1121
|
+
emitSearchCompanionAuthTelemetry(profile, request, "rejected");
|
|
1122
|
+
emitLegacyKeyPathTelemetry(profile, request, "rejected");
|
|
1123
|
+
if (error instanceof InvalidFirecrawlCredentialError) {
|
|
1124
|
+
throw createInvalidCredentialResponse(error);
|
|
1125
|
+
}
|
|
1126
|
+
if (error instanceof CredentialValidationUnavailableError) {
|
|
1127
|
+
throw new Response(
|
|
1128
|
+
JSON.stringify({
|
|
1129
|
+
error: "temporarily_unavailable",
|
|
1130
|
+
error_description: error.message
|
|
1131
|
+
}),
|
|
1132
|
+
{
|
|
1133
|
+
headers: { "Content-Type": "application/json" },
|
|
1134
|
+
status: 503
|
|
1135
|
+
}
|
|
1136
|
+
);
|
|
1137
|
+
}
|
|
1138
|
+
const shouldChallenge = requestShouldReceiveOAuthChallenge(request, profile);
|
|
1139
|
+
const oauthChallenge = shouldChallenge ? createOAuthChallengeResponse(error, profile) : void 0;
|
|
1140
|
+
if (oauthChallenge) {
|
|
1141
|
+
throw oauthChallenge;
|
|
1142
|
+
}
|
|
1143
|
+
throw error;
|
|
1144
|
+
});
|
|
1145
|
+
if (request) {
|
|
1146
|
+
request[authResultByRequest] = authResult;
|
|
1147
|
+
}
|
|
1148
|
+
return authResult;
|
|
1149
|
+
};
|
|
1025
1150
|
}
|
|
1026
1151
|
function removeEmptyTopLevel(obj) {
|
|
1027
1152
|
const out = {};
|
|
@@ -1048,6 +1173,27 @@ function buildSearchQueryWithDomains(query, includeDomains, excludeDomains) {
|
|
|
1048
1173
|
}
|
|
1049
1174
|
return query;
|
|
1050
1175
|
}
|
|
1176
|
+
var searchToolBaseFields = {
|
|
1177
|
+
query: z3.string().min(1),
|
|
1178
|
+
highlights: z3.boolean().optional().describe(
|
|
1179
|
+
"Return query-relevant highlights for each search result. Set to false to keep the original search snippets."
|
|
1180
|
+
),
|
|
1181
|
+
limit: z3.number().optional(),
|
|
1182
|
+
tbs: z3.string().optional(),
|
|
1183
|
+
filter: z3.string().optional(),
|
|
1184
|
+
location: z3.string().optional(),
|
|
1185
|
+
includeDomains: z3.array(searchDomainSchema).optional(),
|
|
1186
|
+
excludeDomains: z3.array(searchDomainSchema).optional(),
|
|
1187
|
+
sources: z3.array(z3.object({ type: z3.enum(["web", "images", "news"]) })).optional(),
|
|
1188
|
+
categories: z3.array(z3.enum(["github", "research", "pdf"])).optional().describe(
|
|
1189
|
+
"Limit results to specific source types. `github` searches GitHub repositories, code, issues, and docs; `research` searches academic and research sources; `pdf` searches PDF results."
|
|
1190
|
+
),
|
|
1191
|
+
enterprise: z3.array(z3.enum(["default", "anon", "zdr"])).optional()
|
|
1192
|
+
};
|
|
1193
|
+
function searchDomainsAreExclusive(args2) {
|
|
1194
|
+
return !(args2.includeDomains?.length && args2.excludeDomains?.length);
|
|
1195
|
+
}
|
|
1196
|
+
var SEARCH_DOMAINS_CONFLICT_MESSAGE = "includeDomains and excludeDomains cannot both be specified";
|
|
1051
1197
|
var ConsoleLogger = class {
|
|
1052
1198
|
shouldLog = process.env.CLOUD_SERVICE === "true" || process.env.SSE_LOCAL === "true" || process.env.HTTP_STREAMABLE_SERVER === "true";
|
|
1053
1199
|
debug(...args2) {
|
|
@@ -1079,32 +1225,202 @@ var ConsoleLogger = class {
|
|
|
1079
1225
|
var openAiAppsChallengeToken = normalizeHeader(
|
|
1080
1226
|
process.env.OPENAI_APPS_CHALLENGE_TOKEN
|
|
1081
1227
|
);
|
|
1082
|
-
var
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1228
|
+
var FULL_PROFILE_INSTRUCTIONS = `Firecrawl provides web search, page retrieval, site URL discovery, multi-page collection, structured extraction, monitoring, and asynchronous research. Match the requested operation to the tool boundary: firecrawl_scrape retrieves one supplied page, firecrawl_map enumerates URLs under a site without retrieving their content, and firecrawl_agent starts multi-source research whose result is read with firecrawl_agent_status. Provide only the required inputs and account for stated network or external side effects.`;
|
|
1229
|
+
var KEYLESS_PROFILE_INSTRUCTIONS = `Without authentication, this endpoint exposes Search, Scrape, and Parse with usage limits. An OAuth connection or Authorization bearer API key exposes account tools; unavailable tools return connection guidance. Firecrawl provides web search, page retrieval, site URL discovery, multi-page collection, structured extraction, monitoring, and asynchronous research. Match the requested operation to the tool boundary: firecrawl_scrape retrieves one supplied page, firecrawl_map enumerates URLs under a site without retrieving their content, and firecrawl_agent starts multi-source research whose result is read with firecrawl_agent_status. Provide only the required inputs.`;
|
|
1230
|
+
var SEARCH_PROFILE_INSTRUCTIONS = `Firecrawl provides web and research search. Use firecrawl_search to find relevant results across the web and specialized indexes. Use the firecrawl_research_* tools to search academic and research literature, expand from anchor papers via the citation graph, read full-text passages from a specific paper, and search public code repositories. All tools are read-only and return ranked results.`;
|
|
1231
|
+
var SEARCH_PROFILE_TOOLS = /* @__PURE__ */ new Set([
|
|
1232
|
+
"firecrawl_search",
|
|
1233
|
+
"firecrawl_research_search_papers",
|
|
1234
|
+
"firecrawl_research_inspect_paper",
|
|
1235
|
+
"firecrawl_research_related_papers",
|
|
1236
|
+
"firecrawl_research_read_paper",
|
|
1237
|
+
"firecrawl_research_search_github"
|
|
1238
|
+
]);
|
|
1239
|
+
function makeFullProfile() {
|
|
1240
|
+
const account = getPrimaryEndpoint() === "/v2/mcp-oauth";
|
|
1241
|
+
return {
|
|
1242
|
+
id: account ? "account" : "full",
|
|
1243
|
+
resourceName: account ? "Firecrawl MCP Account" : "Firecrawl MCP",
|
|
1244
|
+
instructions: account ? FULL_PROFILE_INSTRUCTIONS : KEYLESS_PROFILE_INSTRUCTIONS,
|
|
1245
|
+
resourceUrl: account ? normalizeHeader(process.env.FIRECRAWL_MCP_RESOURCE_URL) ?? DEFAULT_MCP_OAUTH_RESOURCE_URL : getMcpResourceUrl(),
|
|
1246
|
+
endpoint: account ? "/v2/mcp-oauth" : void 0,
|
|
1247
|
+
port: Number(process.env.PORT || 3e3),
|
|
1248
|
+
allowKeyless: !account,
|
|
1249
|
+
acceptApiKeys: true,
|
|
1250
|
+
acceptLegacyAudience: account && process.env.MCP_OAUTH_ACCEPT_LEGACY_V2_MCP_AUD !== "false",
|
|
1251
|
+
primary: true
|
|
1252
|
+
};
|
|
1253
|
+
}
|
|
1254
|
+
function searchOAuthOnly() {
|
|
1255
|
+
return process.env.FIRECRAWL_MCP_SEARCH_OAUTH_ONLY === "true";
|
|
1256
|
+
}
|
|
1257
|
+
function makeSearchProfile({ primary = false } = {}) {
|
|
1258
|
+
const oauthOnly = searchOAuthOnly();
|
|
1259
|
+
if (primary && !oauthOnly) {
|
|
1260
|
+
throw new Error(
|
|
1261
|
+
"FASTMCP_ENDPOINT=/v2/mcp-search requires FIRECRAWL_MCP_SEARCH_OAUTH_ONLY=true"
|
|
1262
|
+
);
|
|
1263
|
+
}
|
|
1264
|
+
return {
|
|
1265
|
+
id: "search",
|
|
1266
|
+
resourceName: "Firecrawl Search",
|
|
1267
|
+
instructions: SEARCH_PROFILE_INSTRUCTIONS,
|
|
1268
|
+
resourceUrl: getSearchMcpResourceUrl(),
|
|
1269
|
+
endpoint: primary ? DEFAULT_MCP_SEARCH_ENDPOINT : getSearchMcpEndpoint(),
|
|
1270
|
+
port: primary ? Number(process.env.PORT || 3e3) : Number(process.env.FIRECRAWL_MCP_SEARCH_PORT || 3001),
|
|
1271
|
+
toolAllowlist: SEARCH_PROFILE_TOOLS,
|
|
1272
|
+
allowKeyless: false,
|
|
1273
|
+
// This is deliberately default-false because the image auto-deploys: the
|
|
1274
|
+
// existing in-process companion remains API-key compatible unless its
|
|
1275
|
+
// deployment explicitly enables the same profile flag used by primary.
|
|
1276
|
+
acceptApiKeys: !oauthOnly,
|
|
1277
|
+
requireManagedOAuth: oauthOnly,
|
|
1278
|
+
primary
|
|
1279
|
+
};
|
|
1280
|
+
}
|
|
1281
|
+
function makePrimaryProfile() {
|
|
1282
|
+
return getPrimaryEndpoint() === "/v2/mcp-search" ? makeSearchProfile({ primary: true }) : makeFullProfile();
|
|
1283
|
+
}
|
|
1284
|
+
function createServer(profile) {
|
|
1285
|
+
return new FastMCP({
|
|
1286
|
+
name: "firecrawl-fastmcp",
|
|
1287
|
+
version: packageVersion,
|
|
1288
|
+
instructions: profile.instructions,
|
|
1289
|
+
logger: new ConsoleLogger(),
|
|
1290
|
+
roots: { enabled: false },
|
|
1291
|
+
oauth: {
|
|
1292
|
+
enabled: isMcpOAuthEnabled(),
|
|
1293
|
+
protectedResource: {
|
|
1294
|
+
authorizationServers: [getOAuthIssuer()],
|
|
1295
|
+
bearerMethodsSupported: ["header"],
|
|
1296
|
+
resource: profile.resourceUrl,
|
|
1297
|
+
resourceName: profile.resourceName,
|
|
1298
|
+
scopesSupported: ["firecrawl:global"]
|
|
1299
|
+
}
|
|
1300
|
+
},
|
|
1301
|
+
authenticate: makeAuthenticate(profile),
|
|
1302
|
+
// Lightweight health endpoint for LB checks
|
|
1303
|
+
health: {
|
|
1304
|
+
enabled: true,
|
|
1305
|
+
message: "ok",
|
|
1306
|
+
path: "/health",
|
|
1307
|
+
status: 200
|
|
1098
1308
|
}
|
|
1099
|
-
}
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1309
|
+
});
|
|
1310
|
+
}
|
|
1311
|
+
var primaryProfile = makePrimaryProfile();
|
|
1312
|
+
var server = createServer(primaryProfile);
|
|
1313
|
+
var KEYLESS_TOOL_NAMES = /* @__PURE__ */ new Set([
|
|
1314
|
+
"firecrawl_scrape",
|
|
1315
|
+
"firecrawl_search",
|
|
1316
|
+
"firecrawl_parse"
|
|
1317
|
+
]);
|
|
1318
|
+
function isHostedKeylessSession(session) {
|
|
1319
|
+
return process.env.CLOUD_SERVICE === "true" && session?.authType === "keyless" && !session.firecrawlApiKey;
|
|
1320
|
+
}
|
|
1321
|
+
function recoveryPayload(code, requestId = randomUUID()) {
|
|
1322
|
+
return {
|
|
1323
|
+
code,
|
|
1324
|
+
request_id: requestId,
|
|
1325
|
+
auth_mode: code === "CREDENTIAL_INVALID" ? "credential_error" : "keyless",
|
|
1326
|
+
message: code === "CREDENTIAL_INVALID" ? "The supplied Firecrawl credential is invalid or revoked. Replace it or reconnect the account, then retry." : "This tool requires a Firecrawl account or API key. Connect an account or configure Authorization: Bearer <FIRECRAWL_API_KEY>, then retry.",
|
|
1327
|
+
available_tools: [...KEYLESS_TOOL_NAMES],
|
|
1328
|
+
docs_url: "https://docs.firecrawl.dev/mcp-server",
|
|
1329
|
+
next_actions: [
|
|
1330
|
+
{ kind: "connect_account", url: "https://firecrawl.dev/connect/mcp" },
|
|
1331
|
+
{
|
|
1332
|
+
kind: "configure_api_key",
|
|
1333
|
+
header: "Authorization: Bearer <FIRECRAWL_API_KEY>"
|
|
1334
|
+
}
|
|
1335
|
+
]
|
|
1336
|
+
};
|
|
1337
|
+
}
|
|
1338
|
+
function emitActionLog(toolName, status, session, error, requestId = randomUUID()) {
|
|
1339
|
+
if (process.env.CLOUD_SERVICE !== "true") return;
|
|
1340
|
+
const payload = {
|
|
1341
|
+
team_id: session?.teamId,
|
|
1342
|
+
user_id: session?.userId,
|
|
1343
|
+
api_key_id: session?.apiKeyId,
|
|
1344
|
+
oauth_client_id: session?.oauthClientId,
|
|
1345
|
+
auth_type: session?.authType ?? "none",
|
|
1346
|
+
tool_name: toolName,
|
|
1347
|
+
status,
|
|
1348
|
+
request_id: requestId,
|
|
1349
|
+
resource: primaryProfile.resourceUrl,
|
|
1350
|
+
...error ? { error_class: error instanceof Error ? error.name : typeof error } : {}
|
|
1351
|
+
};
|
|
1352
|
+
console.error("[MCP_ACTION]", JSON.stringify(payload));
|
|
1353
|
+
const secret = normalizeHeader(process.env.FIRECRAWL_MCP_ACTION_LOG_SECRET);
|
|
1354
|
+
const apiUrl = normalizeHeader(process.env.FIRECRAWL_API_URL);
|
|
1355
|
+
const endpoint = normalizeHeader(process.env.FIRECRAWL_MCP_ACTION_LOG_URL) ?? (apiUrl ? `${withoutTrailingSlash(apiUrl)}/v2/mcp/action-logs` : void 0);
|
|
1356
|
+
if (!secret || !endpoint || !payload.team_id || status === "started") return;
|
|
1357
|
+
void fetch(endpoint, {
|
|
1358
|
+
method: "POST",
|
|
1359
|
+
headers: {
|
|
1360
|
+
Authorization: `Bearer ${secret}`,
|
|
1361
|
+
"Content-Type": "application/json"
|
|
1362
|
+
},
|
|
1363
|
+
body: JSON.stringify(payload),
|
|
1364
|
+
signal: AbortSignal.timeout(1500)
|
|
1365
|
+
}).catch(() => void 0);
|
|
1366
|
+
}
|
|
1367
|
+
function guardHostedTool(tool, { logActions }) {
|
|
1368
|
+
const keylessTool = KEYLESS_TOOL_NAMES.has(tool.name);
|
|
1369
|
+
const execute = tool.execute;
|
|
1370
|
+
return {
|
|
1371
|
+
...tool,
|
|
1372
|
+
canList: (session) => !session?.credentialError && (!isHostedKeylessSession(session) || keylessTool),
|
|
1373
|
+
beforeValidate: (_args, session) => {
|
|
1374
|
+
const code = session?.credentialError ? "CREDENTIAL_INVALID" : isHostedKeylessSession(session) && !keylessTool ? "KEYLESS_TOOL_NOT_AVAILABLE" : void 0;
|
|
1375
|
+
if (!code) return void 0;
|
|
1376
|
+
const payload = recoveryPayload(code);
|
|
1377
|
+
return {
|
|
1378
|
+
content: [{ type: "text", text: String(payload.message) }],
|
|
1379
|
+
isError: true,
|
|
1380
|
+
structuredContent: payload
|
|
1381
|
+
};
|
|
1382
|
+
},
|
|
1383
|
+
execute: async (args2, context) => {
|
|
1384
|
+
const requestId = randomUUID();
|
|
1385
|
+
const invocationSession = {
|
|
1386
|
+
...context.session,
|
|
1387
|
+
requestId
|
|
1388
|
+
};
|
|
1389
|
+
copyManagedOAuthApiKey(context.session, invocationSession);
|
|
1390
|
+
const invocationContext = {
|
|
1391
|
+
...context,
|
|
1392
|
+
session: invocationSession
|
|
1393
|
+
};
|
|
1394
|
+
if (invocationSession.credentialError) {
|
|
1395
|
+
const payload = recoveryPayload("CREDENTIAL_INVALID", requestId);
|
|
1396
|
+
throw new UserError(String(payload.message), payload);
|
|
1397
|
+
}
|
|
1398
|
+
if (isHostedKeylessSession(invocationSession) && !keylessTool) {
|
|
1399
|
+
const payload = recoveryPayload("KEYLESS_TOOL_NOT_AVAILABLE", requestId);
|
|
1400
|
+
throw new UserError(String(payload.message), payload);
|
|
1401
|
+
}
|
|
1402
|
+
if (!logActions) return execute(args2, invocationContext);
|
|
1403
|
+
emitActionLog(tool.name, "started", invocationSession, void 0, requestId);
|
|
1404
|
+
try {
|
|
1405
|
+
const result = await execute(args2, invocationContext);
|
|
1406
|
+
emitActionLog(tool.name, "success", invocationSession, void 0, requestId);
|
|
1407
|
+
return result;
|
|
1408
|
+
} catch (error) {
|
|
1409
|
+
emitActionLog(tool.name, "error", invocationSession, error, requestId);
|
|
1410
|
+
throw error;
|
|
1411
|
+
}
|
|
1412
|
+
}
|
|
1413
|
+
};
|
|
1414
|
+
}
|
|
1415
|
+
var addTool = server.addTool.bind(server);
|
|
1416
|
+
server.addTool = ((tool) => {
|
|
1417
|
+
if (primaryProfile.toolAllowlist && !primaryProfile.toolAllowlist.has(tool.name)) {
|
|
1418
|
+
return;
|
|
1419
|
+
}
|
|
1420
|
+
if (primaryProfile.id === "search" && tool.name === "firecrawl_search") {
|
|
1421
|
+
return;
|
|
1107
1422
|
}
|
|
1423
|
+
addTool(guardHostedTool(tool, { logActions: primaryProfile.id !== "search" }));
|
|
1108
1424
|
});
|
|
1109
1425
|
if (openAiAppsChallengeToken) {
|
|
1110
1426
|
server.getApp().get(
|
|
@@ -1112,6 +1428,31 @@ if (openAiAppsChallengeToken) {
|
|
|
1112
1428
|
(context) => context.text(openAiAppsChallengeToken)
|
|
1113
1429
|
);
|
|
1114
1430
|
}
|
|
1431
|
+
server.getApp().get("/ready", (context) => {
|
|
1432
|
+
if (process.env.CLOUD_SERVICE !== "true") {
|
|
1433
|
+
return context.json({ ok: true }, 200);
|
|
1434
|
+
}
|
|
1435
|
+
const searchPrimary = primaryProfile.id === "search";
|
|
1436
|
+
const required = [
|
|
1437
|
+
"FIRECRAWL_API_URL",
|
|
1438
|
+
"FIRECRAWL_OAUTH_INTROSPECT_SECRET",
|
|
1439
|
+
"MCP_DELEGATED_CREDENTIAL_SECRET"
|
|
1440
|
+
];
|
|
1441
|
+
if (primaryProfile.allowKeyless) {
|
|
1442
|
+
required.push("KEYLESS_PROXY_SECRET");
|
|
1443
|
+
}
|
|
1444
|
+
const missing = required.filter((name) => !normalizeHeader(process.env[name]));
|
|
1445
|
+
const configuredEndpoint = getPrimaryEndpoint();
|
|
1446
|
+
const resourceMatchesEndpoint = searchPrimary ? withoutTrailingSlash(primaryProfile.resourceUrl) === DEFAULT_MCP_SEARCH_RESOURCE_URL : withoutTrailingSlash(primaryProfile.resourceUrl).endsWith(
|
|
1447
|
+
configuredEndpoint
|
|
1448
|
+
);
|
|
1449
|
+
if (!resourceMatchesEndpoint) {
|
|
1450
|
+
missing.push(
|
|
1451
|
+
searchPrimary ? "FIRECRAWL_MCP_SEARCH_RESOURCE_URL (endpoint mismatch)" : "FIRECRAWL_MCP_RESOURCE_URL (endpoint mismatch)"
|
|
1452
|
+
);
|
|
1453
|
+
}
|
|
1454
|
+
return missing.length ? context.json({ ok: false, missing }, 503) : context.json({ ok: true }, 200);
|
|
1455
|
+
});
|
|
1115
1456
|
function createClient(apiKey) {
|
|
1116
1457
|
const config = {
|
|
1117
1458
|
...process.env.FIRECRAWL_API_URL && {
|
|
@@ -1127,18 +1468,32 @@ var ORIGIN = "mcp-fastmcp";
|
|
|
1127
1468
|
var ORIGIN_HEADERS2 = { "X-Origin": ORIGIN };
|
|
1128
1469
|
var SAFE_MODE = process.env.CLOUD_SERVICE === "true";
|
|
1129
1470
|
function getClient(session) {
|
|
1130
|
-
if (process.env.CLOUD_SERVICE === "true") {
|
|
1131
|
-
|
|
1132
|
-
throw new Error("Unauthorized");
|
|
1133
|
-
}
|
|
1134
|
-
return createClient(session.firecrawlApiKey);
|
|
1471
|
+
if (process.env.CLOUD_SERVICE === "true" && !hasCredential(session)) {
|
|
1472
|
+
throw new Error("Unauthorized");
|
|
1135
1473
|
}
|
|
1136
|
-
if (!process.env.FIRECRAWL_API_URL && (
|
|
1474
|
+
if (!process.env.FIRECRAWL_API_URL && !hasCredential(session)) {
|
|
1137
1475
|
throw new Error(
|
|
1138
1476
|
"Unauthorized: API key is required when not using a self-hosted instance"
|
|
1139
1477
|
);
|
|
1140
1478
|
}
|
|
1141
|
-
|
|
1479
|
+
if (!hasManagedOAuthCredential(session)) {
|
|
1480
|
+
return createClient(credentialForOutboundRequest(session));
|
|
1481
|
+
}
|
|
1482
|
+
const client = createClient("request-scoped-hosted-oauth");
|
|
1483
|
+
const axiosInstance = client.http?.instance;
|
|
1484
|
+
if (!axiosInstance?.interceptors?.request?.use) {
|
|
1485
|
+
throw new CredentialValidationUnavailableError();
|
|
1486
|
+
}
|
|
1487
|
+
axiosInstance.interceptors.request.use((config) => {
|
|
1488
|
+
const credential = credentialForOutboundRequest(session);
|
|
1489
|
+
if (!credential) throw new CredentialValidationUnavailableError();
|
|
1490
|
+
config.headers = {
|
|
1491
|
+
...config.headers ?? {},
|
|
1492
|
+
Authorization: `Bearer ${credential}`
|
|
1493
|
+
};
|
|
1494
|
+
return config;
|
|
1495
|
+
});
|
|
1496
|
+
return client;
|
|
1142
1497
|
}
|
|
1143
1498
|
function asText2(data) {
|
|
1144
1499
|
return JSON.stringify(data, null, 2);
|
|
@@ -1412,8 +1767,9 @@ async function apiPostJson(pathName, body, apiKey) {
|
|
|
1412
1767
|
return parsed;
|
|
1413
1768
|
}
|
|
1414
1769
|
async function apiPostJsonForSession(pathName, body, session) {
|
|
1415
|
-
|
|
1416
|
-
|
|
1770
|
+
const credential = credentialForOutboundRequest(session);
|
|
1771
|
+
if (credential) {
|
|
1772
|
+
return apiPostJson(pathName, body, credential);
|
|
1417
1773
|
}
|
|
1418
1774
|
if (isKeylessMode(session)) {
|
|
1419
1775
|
return keylessPost(pathName, body, session);
|
|
@@ -1456,13 +1812,21 @@ async function executeHostedParse(args2, session, log) {
|
|
|
1456
1812
|
"Hosted firecrawl_parse requires exactly one of filePath or uploadRef."
|
|
1457
1813
|
);
|
|
1458
1814
|
}
|
|
1459
|
-
if (!session
|
|
1815
|
+
if (!hasCredential(session) && !isKeylessMode(session)) {
|
|
1460
1816
|
return asText2({
|
|
1461
1817
|
success: false,
|
|
1462
1818
|
mode: "hosted-upload-ref-auth-required",
|
|
1463
1819
|
message: "Hosted firecrawl_parse requires an authenticated Firecrawl session or keyless eligibility before a local file upload URL can be minted. Connect a Firecrawl account, provide an API key, or use keyless hosted MCP while eligible, then call firecrawl_parse again."
|
|
1464
1820
|
});
|
|
1465
1821
|
}
|
|
1822
|
+
if (isHostedKeylessSession(session) && args2.zeroDataRetention === true) {
|
|
1823
|
+
const payload = {
|
|
1824
|
+
...recoveryPayload("KEYLESS_OPTION_NOT_AVAILABLE", session?.requestId),
|
|
1825
|
+
option: "zeroDataRetention",
|
|
1826
|
+
message: "Zero Data Retention is not available in anonymous keyless mode. Omit zeroDataRetention to parse with keyless access, or connect an account or configure an API key for a team where Zero Data Retention is enabled, then retry."
|
|
1827
|
+
};
|
|
1828
|
+
throw new UserError(String(payload.message), payload);
|
|
1829
|
+
}
|
|
1466
1830
|
const options = extractParseOptions(args2);
|
|
1467
1831
|
if (hasFilePath && args2.filePath) {
|
|
1468
1832
|
const filename = path.basename(args2.filePath);
|
|
@@ -1535,106 +1899,11 @@ server.addTool({
|
|
|
1535
1899
|
// Does not modify, delete, or write to external websites.
|
|
1536
1900
|
},
|
|
1537
1901
|
description: `
|
|
1538
|
-
|
|
1539
|
-
This is the most powerful, fastest and most reliable scraper tool, if available you should always default to using this tool for any web scraping needs.
|
|
1540
|
-
|
|
1541
|
-
**Best for:** Single page content extraction, when you know exactly which page contains the information.
|
|
1542
|
-
**Not recommended for:** Multiple pages (call scrape multiple times or use crawl), unknown page location (use search).
|
|
1543
|
-
**Common mistakes:** Using markdown format when extracting specific data points (use JSON instead).
|
|
1544
|
-
**Other Features:** Use 'branding' format to extract brand identity (colors, fonts, typography, spacing, UI components) for design analysis or style replication.
|
|
1545
|
-
|
|
1546
|
-
**CRITICAL - Format Selection (you MUST follow this):**
|
|
1547
|
-
When the user asks for SPECIFIC data points, you MUST use JSON format with a schema. Only use markdown when the user needs the ENTIRE page content.
|
|
1548
|
-
|
|
1549
|
-
**Use JSON format when user asks for:**
|
|
1550
|
-
- Parameters, fields, or specifications (e.g., "get the header parameters", "what are the required fields")
|
|
1551
|
-
- Prices, numbers, or structured data (e.g., "extract the pricing", "get the product details")
|
|
1552
|
-
- API details, endpoints, or technical specs (e.g., "find the authentication endpoint")
|
|
1553
|
-
- Lists of items or properties (e.g., "list the features", "get all the options")
|
|
1554
|
-
- Any specific piece of information from a page
|
|
1555
|
-
|
|
1556
|
-
**Use markdown format ONLY when:**
|
|
1557
|
-
- User wants to read/summarize an entire article or blog post
|
|
1558
|
-
- User needs to see all content on a page without specific extraction
|
|
1559
|
-
- User explicitly asks for the full page content
|
|
1560
|
-
|
|
1561
|
-
**Handling JavaScript-rendered pages (SPAs):**
|
|
1562
|
-
If JSON extraction returns empty, minimal, or just navigation content, the page is likely JavaScript-rendered or the content is on a different URL. Try these steps IN ORDER:
|
|
1563
|
-
1. **Add waitFor parameter:** Set \`waitFor: 5000\` to \`waitFor: 10000\` to allow JavaScript to render before extraction
|
|
1564
|
-
2. **Try a different URL:** If the URL has a hash fragment (#section), try the base URL or look for a direct page URL
|
|
1565
|
-
3. **Use firecrawl_map to find the correct page:** Large documentation sites or SPAs often spread content across multiple URLs. Use \`firecrawl_map\` with a \`search\` parameter to discover the specific page containing your target content, then scrape that URL directly.
|
|
1566
|
-
Example: If scraping "https://docs.example.com/reference" fails to find webhook parameters, use \`firecrawl_map\` with \`{"url": "https://docs.example.com/reference", "search": "webhook"}\` to find URLs like "/reference/webhook-events", then scrape that specific page.
|
|
1567
|
-
4. **Use firecrawl_agent:** As a last resort for heavily dynamic pages where map+scrape still fails, use the agent which can autonomously navigate and research
|
|
1568
|
-
|
|
1569
|
-
**Usage Example (JSON format - REQUIRED for specific data extraction):**
|
|
1570
|
-
\`\`\`json
|
|
1571
|
-
{
|
|
1572
|
-
"name": "firecrawl_scrape",
|
|
1573
|
-
"arguments": {
|
|
1574
|
-
"url": "https://example.com/api-docs",
|
|
1575
|
-
"formats": ["json"],
|
|
1576
|
-
"jsonOptions": {
|
|
1577
|
-
"prompt": "Extract the header parameters for the authentication endpoint",
|
|
1578
|
-
"schema": {
|
|
1579
|
-
"type": "object",
|
|
1580
|
-
"properties": {
|
|
1581
|
-
"parameters": {
|
|
1582
|
-
"type": "array",
|
|
1583
|
-
"items": {
|
|
1584
|
-
"type": "object",
|
|
1585
|
-
"properties": {
|
|
1586
|
-
"name": { "type": "string" },
|
|
1587
|
-
"type": { "type": "string" },
|
|
1588
|
-
"required": { "type": "boolean" },
|
|
1589
|
-
"description": { "type": "string" }
|
|
1590
|
-
}
|
|
1591
|
-
}
|
|
1592
|
-
}
|
|
1593
|
-
}
|
|
1594
|
-
}
|
|
1595
|
-
}
|
|
1596
|
-
}
|
|
1597
|
-
}
|
|
1598
|
-
\`\`\`
|
|
1902
|
+
Retrieve and extract content from one supplied URL through Firecrawl. Use this when the request identifies a page and needs its content or defined fields. It can return markdown, HTML, links, screenshots, branding data, a targeted answer, or JSON matching a supplied schema; JSON is useful when the requested result has defined fields, while markdown preserves readable page content.
|
|
1599
1903
|
|
|
1600
|
-
|
|
1904
|
+
This tool operates on a known page. For a set of pages use \`firecrawl_crawl\`, and to discover page URLs use \`firecrawl_map\` or \`firecrawl_search\`. Options include JavaScript render delay, cache age, main-content filtering, PII redaction, and lockdown cache-only retrieval. Browser actions may change the live page when interactive actions are enabled.
|
|
1601
1905
|
|
|
1602
|
-
|
|
1603
|
-
- Structured data with specific fields (extract all products with name, price, description)
|
|
1604
|
-
- Data in a specific schema for downstream processing
|
|
1605
|
-
|
|
1606
|
-
**Use query format only when:**
|
|
1607
|
-
- The page is extremely long and you need a single targeted answer without processing the full content
|
|
1608
|
-
- You want a quick factual answer and don't need to retain the page content
|
|
1609
|
-
- Set \`queryOptions.mode\` to \`"directQuote"\` when you need verbatim page text; otherwise it defaults to \`"freeform"\`
|
|
1610
|
-
|
|
1611
|
-
**Usage Example (markdown format - default for most tasks):**
|
|
1612
|
-
\`\`\`json
|
|
1613
|
-
{
|
|
1614
|
-
"name": "firecrawl_scrape",
|
|
1615
|
-
"arguments": {
|
|
1616
|
-
"url": "https://example.com/article",
|
|
1617
|
-
"formats": ["markdown"],
|
|
1618
|
-
"onlyMainContent": true
|
|
1619
|
-
}
|
|
1620
|
-
}
|
|
1621
|
-
\`\`\`
|
|
1622
|
-
**Usage Example (branding format - extract brand identity):**
|
|
1623
|
-
\`\`\`json
|
|
1624
|
-
{
|
|
1625
|
-
"name": "firecrawl_scrape",
|
|
1626
|
-
"arguments": {
|
|
1627
|
-
"url": "https://example.com",
|
|
1628
|
-
"formats": ["branding"]
|
|
1629
|
-
}
|
|
1630
|
-
}
|
|
1631
|
-
\`\`\`
|
|
1632
|
-
**Branding format:** Extracts comprehensive brand identity (colors, fonts, typography, spacing, logo, UI components) for design analysis or style replication.
|
|
1633
|
-
**Performance:** Add maxAge parameter for 500% faster scrapes using cached data.
|
|
1634
|
-
**Lockdown mode:** Set \`lockdown: true\` to serve the request only from the existing index/cache without any outbound network request. For air-gapped or compliance-constrained use where the request URL itself is considered sensitive. Errors on cache miss. Billed at 5 credits.
|
|
1635
|
-
**Privacy:** Set \`redactPII: true\` to return content with personally identifiable information redacted.
|
|
1636
|
-
**Returns:** JSON structured data, markdown, branding profile, or other formats as specified.
|
|
1637
|
-
${SAFE_MODE ? "**Safe Mode:** Read-only content extraction. Interactive actions (click, write, executeJavascript) are disabled for security." : ""}
|
|
1906
|
+
Returns the selected content formats and page metadata.
|
|
1638
1907
|
`,
|
|
1639
1908
|
parameters: scrapeParamsSchema,
|
|
1640
1909
|
execute: async (args2, { session, log }) => {
|
|
@@ -1680,35 +1949,9 @@ server.addTool({
|
|
|
1680
1949
|
// Read-only discovery; no deletion or destructive updates.
|
|
1681
1950
|
},
|
|
1682
1951
|
description: `
|
|
1683
|
-
|
|
1684
|
-
|
|
1685
|
-
**Best for:** Discovering URLs on a website before deciding what to scrape; finding specific sections or pages within a large site; locating the correct page when scrape returns empty or incomplete results.
|
|
1686
|
-
**Not recommended for:** When you already know which specific URL you need (use scrape); when you need the content of the pages (use scrape after mapping).
|
|
1687
|
-
**Common mistakes:** Using crawl to discover URLs instead of map; jumping straight to firecrawl_agent when scrape fails instead of using map first to find the right page.
|
|
1952
|
+
Enumerate URLs indexed under one website through Firecrawl without fetching each page's content. Use this when the request asks for a site's URL inventory, when several relevant pages must be located, or when the desired page URL is unknown. An optional \`search\` term narrows the URL list, while sitemap, subdomain, query-parameter, and result-limit options control coverage.
|
|
1688
1953
|
|
|
1689
|
-
|
|
1690
|
-
|
|
1691
|
-
**Prompt Example:** "Find the webhook documentation page on this API docs site."
|
|
1692
|
-
**Usage Example (discover all URLs):**
|
|
1693
|
-
\`\`\`json
|
|
1694
|
-
{
|
|
1695
|
-
"name": "firecrawl_map",
|
|
1696
|
-
"arguments": {
|
|
1697
|
-
"url": "https://example.com"
|
|
1698
|
-
}
|
|
1699
|
-
}
|
|
1700
|
-
\`\`\`
|
|
1701
|
-
**Usage Example (search for specific content - RECOMMENDED when scrape fails):**
|
|
1702
|
-
\`\`\`json
|
|
1703
|
-
{
|
|
1704
|
-
"name": "firecrawl_map",
|
|
1705
|
-
"arguments": {
|
|
1706
|
-
"url": "https://docs.example.com/api",
|
|
1707
|
-
"search": "webhook events"
|
|
1708
|
-
}
|
|
1709
|
-
}
|
|
1710
|
-
\`\`\`
|
|
1711
|
-
**Returns:** Array of URLs found on the site, filtered by search query if provided.
|
|
1954
|
+
Returns matching URLs rather than page bodies. Retrieve one page with \`firecrawl_scrape\`; collect content across multiple pages with \`firecrawl_crawl\`.
|
|
1712
1955
|
`,
|
|
1713
1956
|
parameters: z3.object({
|
|
1714
1957
|
url: z3.string().url(),
|
|
@@ -1742,92 +1985,14 @@ server.addTool({
|
|
|
1742
1985
|
// Query-only; no destructive side effects on external entities.
|
|
1743
1986
|
},
|
|
1744
1987
|
description: `
|
|
1745
|
-
Search
|
|
1746
|
-
|
|
1747
|
-
The query also supports search operators, that you can use if needed to refine the search:
|
|
1748
|
-
| Operator | Functionality | Examples |
|
|
1749
|
-
---|-|-|
|
|
1750
|
-
| \`"\` | Non-fuzzy matches a string of text | \`"Firecrawl"\`
|
|
1751
|
-
| \`-\` | Excludes certain keywords or negates other operators | \`-bad\`, \`-site:firecrawl.dev\`
|
|
1752
|
-
| \`site:\` | Only returns results from a specified website | \`site:firecrawl.dev\`
|
|
1753
|
-
| \`inurl:\` | Only returns results that include a word in the URL | \`inurl:firecrawl\`
|
|
1754
|
-
| \`allinurl:\` | Only returns results that include multiple words in the URL | \`allinurl:git firecrawl\`
|
|
1755
|
-
| \`intitle:\` | Only returns results that include a word in the title of the page | \`intitle:Firecrawl\`
|
|
1756
|
-
| \`allintitle:\` | Only returns results that include multiple words in the title of the page | \`allintitle:firecrawl playground\`
|
|
1757
|
-
| \`related:\` | Only returns results that are related to a specific domain | \`related:firecrawl.dev\`
|
|
1758
|
-
| \`imagesize:\` | Only returns images with exact dimensions | \`imagesize:1920x1080\`
|
|
1759
|
-
| \`larger:\` | Only returns images larger than specified dimensions | \`larger:1920x1080\`
|
|
1760
|
-
|
|
1761
|
-
**Best for:** Finding specific information across multiple websites, when you don't know which website has the information; when you need the most relevant content for a query.
|
|
1762
|
-
**Not recommended for:** When you need to search the filesystem. When you already know which website to scrape (use scrape); when you need comprehensive coverage of a single website (use map or crawl.
|
|
1763
|
-
**Common mistakes:** Using crawl or map for open-ended questions (use search instead).
|
|
1764
|
-
**Prompt Example:** "Find the latest research papers on AI published in 2023."
|
|
1765
|
-
**Sources:** web, images, news, default to web unless needed images or news.
|
|
1766
|
-
**Categories:** Optional filter to limit result types: \`github\` (GitHub repositories, code, issues, and docs), \`research\` (academic and research sources), \`pdf\` (PDF results). Example: \`categories: ["github", "research"]\`.
|
|
1767
|
-
**Domain filters:** Use includeDomains to restrict results to specific domains, or excludeDomains to remove domains. Do not use both in the same request. Domains must be hostnames only, without protocol or path.
|
|
1768
|
-
**Scrape Options:** Only use scrapeOptions when you think it is absolutely necessary. When you do so default to a lower limit to avoid timeouts, 5 or lower.
|
|
1769
|
-
**Optimal Workflow:** Search first using firecrawl_search without formats, then after fetching the results, use the scrape tool to get the content of the relevantpage(s) that you want to scrape
|
|
1770
|
-
**After the search:** Once you have processed the results (or decided they were not useful), call \`firecrawl_search_feedback\` with the \`id\` from this response. The first feedback per search refunds 1 credit and helps Firecrawl improve search quality.
|
|
1988
|
+
Search web, news, or image sources and return ranked results. Operators include quoted phrases, \`-term\`, \`site:host\`, \`inurl:term\`, \`intitle:term\`, and \`related:host\`; the set is non-exhaustive. \`includeDomains\` and \`excludeDomains\` are mutually exclusive hostname filters; categories limit results to GitHub, research, or PDF sources.
|
|
1771
1989
|
|
|
1772
|
-
|
|
1773
|
-
\`\`\`json
|
|
1774
|
-
{
|
|
1775
|
-
"name": "firecrawl_search",
|
|
1776
|
-
"arguments": {
|
|
1777
|
-
"query": "top AI companies",
|
|
1778
|
-
"limit": 5,
|
|
1779
|
-
"includeDomains": ["example.com"],
|
|
1780
|
-
"sources": [
|
|
1781
|
-
{ "type": "web" }
|
|
1782
|
-
]
|
|
1783
|
-
}
|
|
1784
|
-
}
|
|
1785
|
-
\`\`\`
|
|
1786
|
-
**Usage Example with formats:**
|
|
1787
|
-
\`\`\`json
|
|
1788
|
-
{
|
|
1789
|
-
"name": "firecrawl_search",
|
|
1790
|
-
"arguments": {
|
|
1791
|
-
"query": "latest AI research papers 2023",
|
|
1792
|
-
"limit": 5,
|
|
1793
|
-
"categories": ["github", "research"],
|
|
1794
|
-
"lang": "en",
|
|
1795
|
-
"country": "us",
|
|
1796
|
-
"sources": [
|
|
1797
|
-
{ "type": "web" },
|
|
1798
|
-
{ "type": "images" },
|
|
1799
|
-
{ "type": "news" }
|
|
1800
|
-
],
|
|
1801
|
-
"scrapeOptions": {
|
|
1802
|
-
"formats": ["markdown"],
|
|
1803
|
-
"onlyMainContent": true
|
|
1804
|
-
}
|
|
1805
|
-
}
|
|
1806
|
-
}
|
|
1807
|
-
\`\`\`
|
|
1808
|
-
**Returns:** A JSON envelope of the form \`{ success, data: { web?, images?, news? }, id, creditsUsed }\`. Each result array contains the search results (with optional scraped content). Pass the top-level \`id\` to \`firecrawl_search_feedback\` after you've used the results.
|
|
1990
|
+
\`scrapeOptions\` can attach extracted page content. Returns source-type result groups, an \`id\` for optional search feedback, and usage metadata.
|
|
1809
1991
|
`,
|
|
1810
1992
|
parameters: z3.object({
|
|
1811
|
-
|
|
1812
|
-
|
|
1813
|
-
|
|
1814
|
-
),
|
|
1815
|
-
limit: z3.number().optional(),
|
|
1816
|
-
tbs: z3.string().optional(),
|
|
1817
|
-
filter: z3.string().optional(),
|
|
1818
|
-
location: z3.string().optional(),
|
|
1819
|
-
includeDomains: z3.array(searchDomainSchema).optional(),
|
|
1820
|
-
excludeDomains: z3.array(searchDomainSchema).optional(),
|
|
1821
|
-
sources: z3.array(z3.object({ type: z3.enum(["web", "images", "news"]) })).optional(),
|
|
1822
|
-
categories: z3.array(z3.enum(["github", "research", "pdf"])).optional().describe(
|
|
1823
|
-
"Limit results to specific source types. `github` searches GitHub repositories, code, issues, and docs; `research` searches academic and research sources; `pdf` searches PDF results."
|
|
1824
|
-
),
|
|
1825
|
-
scrapeOptions: scrapeParamsSchema.omit({ url: true }).partial().optional(),
|
|
1826
|
-
enterprise: z3.array(z3.enum(["default", "anon", "zdr"])).optional()
|
|
1827
|
-
}).refine(
|
|
1828
|
-
(args2) => !(args2.includeDomains?.length && args2.excludeDomains?.length),
|
|
1829
|
-
"includeDomains and excludeDomains cannot both be specified"
|
|
1830
|
-
),
|
|
1993
|
+
...searchToolBaseFields,
|
|
1994
|
+
scrapeOptions: scrapeParamsSchema.omit({ url: true }).partial().optional()
|
|
1995
|
+
}).refine(searchDomainsAreExclusive, SEARCH_DOMAINS_CONFLICT_MESSAGE),
|
|
1831
1996
|
execute: async (args2, { session, log }) => {
|
|
1832
1997
|
const { query, ...opts } = args2;
|
|
1833
1998
|
const searchOpts = { ...opts };
|
|
@@ -1869,10 +2034,7 @@ function resolveApiBaseUrl() {
|
|
|
1869
2034
|
);
|
|
1870
2035
|
}
|
|
1871
2036
|
function extractClientIp(request) {
|
|
1872
|
-
|
|
1873
|
-
const raw = Array.isArray(xff) ? xff[0] : xff;
|
|
1874
|
-
const first = typeof raw === "string" ? raw.split(",")[0].trim() : void 0;
|
|
1875
|
-
return first || void 0;
|
|
2037
|
+
return extractSingleTrustedClientIp(request?.headers?.["x-forwarded-for"]);
|
|
1876
2038
|
}
|
|
1877
2039
|
async function keylessEligible(clientIp) {
|
|
1878
2040
|
const secret = process.env.KEYLESS_PROXY_SECRET;
|
|
@@ -1896,13 +2058,20 @@ async function keylessEligible(clientIp) {
|
|
|
1896
2058
|
}
|
|
1897
2059
|
}
|
|
1898
2060
|
function isKeylessMode(session) {
|
|
1899
|
-
if (session?.
|
|
2061
|
+
if (hasCredential(session) || session?.credentialError) return false;
|
|
1900
2062
|
if (process.env.CLOUD_SERVICE === "true") {
|
|
1901
|
-
return
|
|
2063
|
+
return session?.authType === "keyless";
|
|
1902
2064
|
}
|
|
1903
2065
|
return !process.env.FIRECRAWL_API_URL;
|
|
1904
2066
|
}
|
|
1905
2067
|
async function keylessPost(path2, body, session) {
|
|
2068
|
+
if (isHostedKeylessSession(session) && (!session?.keylessClientIp || !await keylessEligible(session.keylessClientIp))) {
|
|
2069
|
+
const payload = recoveryPayload(
|
|
2070
|
+
"KEYLESS_ACCESS_NOT_AVAILABLE",
|
|
2071
|
+
session?.requestId
|
|
2072
|
+
);
|
|
2073
|
+
throw new UserError(String(payload.message), payload);
|
|
2074
|
+
}
|
|
1906
2075
|
const headers = {
|
|
1907
2076
|
...ORIGIN_HEADERS2,
|
|
1908
2077
|
"Content-Type": "application/json"
|
|
@@ -1918,6 +2087,13 @@ async function keylessPost(path2, body, session) {
|
|
|
1918
2087
|
});
|
|
1919
2088
|
const json = await response.json().catch(() => ({}));
|
|
1920
2089
|
if (!response.ok) {
|
|
2090
|
+
if (isHostedKeylessSession(session) && [401, 402, 429].includes(response.status)) {
|
|
2091
|
+
const payload = recoveryPayload(
|
|
2092
|
+
"KEYLESS_QUOTA_EXHAUSTED",
|
|
2093
|
+
session?.requestId
|
|
2094
|
+
);
|
|
2095
|
+
throw new UserError(String(payload.message), payload);
|
|
2096
|
+
}
|
|
1921
2097
|
throw new Error(
|
|
1922
2098
|
json?.error || `Firecrawl request failed (HTTP ${response.status})`
|
|
1923
2099
|
);
|
|
@@ -2023,65 +2199,9 @@ if (!SEARCH_FEEDBACK_DISABLED) {
|
|
|
2023
2199
|
// Additive only; records feedback and may refund credits, does not delete data.
|
|
2024
2200
|
},
|
|
2025
2201
|
description: `
|
|
2026
|
-
|
|
2027
|
-
|
|
2028
|
-
Pass the \`searchId\` returned by \`firecrawl_search\` (the \`id\` field on the response) and tell us:
|
|
2029
|
-
|
|
2030
|
-
- **rating** \u2014 overall result quality: \`good\`, \`partial\`, or \`bad\`.
|
|
2031
|
-
- **valuableSources** \u2014 which result URLs were actually useful, and a short reason why.
|
|
2032
|
-
- **missingContent** \u2014 **the most important field.** An ARRAY of specific pieces of content you expected to find but didn't. One entry per missing piece, each with a short \`topic\` and an optional longer \`description\`. Examples: \`{"topic":"enterprise pricing","description":"no pricing tier table for the Enterprise plan was returned"}\`, \`{"topic":"API rate limits"}\`, \`{"topic":"comparison vs competitors"}\`. **Be specific** \u2014 these aggregate across teams and tell us what to index next. Do not pack multiple topics into one entry.
|
|
2033
|
-
- **querySuggestions** \u2014 how the query or response shape could be improved (e.g. "would have liked official docs first", "should boost github.com").
|
|
2202
|
+
Records schema-validated quality feedback for a prior \`firecrawl_search\` UUID \`searchId\`. A \`good\` rating requires a valuable source, \`partial\` a valuable source or missing topic, and \`bad\` a missing topic or query suggestion; caps are 50 \`valuableSources\` and 20 \`missingContent\` entries.
|
|
2034
2203
|
|
|
2035
|
-
|
|
2036
|
-
- \`good\` \u2014 must include at least one \`valuableSources\` entry
|
|
2037
|
-
- \`partial\` \u2014 must include \`valuableSources\` or at least one \`missingContent\` entry
|
|
2038
|
-
- \`bad\` \u2014 must include at least one \`missingContent\` entry or \`querySuggestions\`
|
|
2039
|
-
|
|
2040
|
-
**Time window:** Feedback must be submitted within ~2 minutes of the search. Beyond that, the call returns HTTP 409 with \`feedbackErrorCode: "FEEDBACK_WINDOW_EXPIRED"\` \u2014 do not retry, just move on. Same goes for any 4xx response: do not retry-loop.
|
|
2041
|
-
|
|
2042
|
-
**Behaviors:**
|
|
2043
|
-
- Idempotent per \`searchId\`. Re-submitting for the same id returns \`alreadySubmitted: true\` with \`creditsRefunded: 0\`.
|
|
2044
|
-
- Refund only applies to billable searches; preview teams are blocked.
|
|
2045
|
-
- Failed searches cannot receive feedback (the search itself already returned an error you can act on).
|
|
2046
|
-
- **Daily refund cap (per team, per UTC day, default 100 credits).** Once a team's \`creditsRefundedToday\` reaches \`dailyRefundCap\`, the response returns \`dailyCapReached: true\` with \`creditsRefunded: 0\`. The feedback is still recorded for search-quality improvement \u2014 only the credit refund is gated. **Stop calling this tool for the rest of the UTC day** when you see \`dailyCapReached: true\`.
|
|
2047
|
-
|
|
2048
|
-
**When to call:** Right after processing a search result. If the result didn't help, send rating \`bad\` with a clear \`missingContent\` \u2014 that is just as valuable as a \`good\` rating.
|
|
2049
|
-
|
|
2050
|
-
**Usage Example (good rating with valuable sources + missing content):**
|
|
2051
|
-
\`\`\`json
|
|
2052
|
-
{
|
|
2053
|
-
"name": "firecrawl_search_feedback",
|
|
2054
|
-
"arguments": {
|
|
2055
|
-
"searchId": "0193f6c5-1234-7890-abcd-1234567890ab",
|
|
2056
|
-
"rating": "good",
|
|
2057
|
-
"valuableSources": [
|
|
2058
|
-
{ "url": "https://docs.firecrawl.dev/features/search", "reason": "Most up-to-date description of /search." }
|
|
2059
|
-
],
|
|
2060
|
-
"missingContent": [
|
|
2061
|
-
{ "topic": "Pricing for the search endpoint", "description": "No pricing tier table for /search specifically." },
|
|
2062
|
-
{ "topic": "Rate limits", "description": "Per-team RPS for /search not documented." }
|
|
2063
|
-
],
|
|
2064
|
-
"querySuggestions": "Boost docs.firecrawl.dev for queries that mention 'firecrawl'"
|
|
2065
|
-
}
|
|
2066
|
-
}
|
|
2067
|
-
\`\`\`
|
|
2068
|
-
|
|
2069
|
-
**Usage Example (bad rating, what was missing):**
|
|
2070
|
-
\`\`\`json
|
|
2071
|
-
{
|
|
2072
|
-
"name": "firecrawl_search_feedback",
|
|
2073
|
-
"arguments": {
|
|
2074
|
-
"searchId": "0193f6c5-1234-7890-abcd-1234567890ab",
|
|
2075
|
-
"rating": "bad",
|
|
2076
|
-
"missingContent": [
|
|
2077
|
-
{ "topic": "Recent benchmarks", "description": "All results were >12 months old." },
|
|
2078
|
-
{ "topic": "Comparison vs Algolia" }
|
|
2079
|
-
]
|
|
2080
|
-
}
|
|
2081
|
-
}
|
|
2082
|
-
\`\`\`
|
|
2083
|
-
|
|
2084
|
-
**Returns:** \`{ success, feedbackId, creditsRefunded, creditsRefundedToday, dailyRefundCap, dailyCapReached?, alreadySubmitted?, warning? }\` JSON.
|
|
2204
|
+
Eligibility is limited to successful searches within the feedback age window. The record is idempotent per search ID. Returns submission and daily-cap status with accounting fields.
|
|
2085
2205
|
`,
|
|
2086
2206
|
parameters: z3.object({
|
|
2087
2207
|
searchId: z3.string().uuid("searchId must be the UUID returned by firecrawl_search"),
|
|
@@ -2129,9 +2249,9 @@ Pass the \`searchId\` returned by \`firecrawl_search\` (the \`id\` field on the
|
|
|
2129
2249
|
...ORIGIN_HEADERS2,
|
|
2130
2250
|
"Content-Type": "application/json"
|
|
2131
2251
|
};
|
|
2132
|
-
const
|
|
2133
|
-
if (
|
|
2134
|
-
headers["Authorization"] = `Bearer ${
|
|
2252
|
+
const credential = credentialForOutboundRequest(session);
|
|
2253
|
+
if (credential) {
|
|
2254
|
+
headers["Authorization"] = `Bearer ${credential}`;
|
|
2135
2255
|
} else if (process.env.CLOUD_SERVICE === "true") {
|
|
2136
2256
|
throw new Error("Unauthorized: missing API key for search feedback.");
|
|
2137
2257
|
}
|
|
@@ -2183,21 +2303,9 @@ if (!ENDPOINT_FEEDBACK_DISABLED) {
|
|
|
2183
2303
|
// Additive only; submits ratings and notes, does not delete jobs or external content.
|
|
2184
2304
|
},
|
|
2185
2305
|
description: `
|
|
2186
|
-
|
|
2187
|
-
|
|
2188
|
-
For search-result quality specifically, prefer \`firecrawl_search_feedback\` when available because it has search-focused guidance. This generic tool posts to \`/v2/feedback\` and accepts endpoint-wide signals:
|
|
2189
|
-
|
|
2190
|
-
- **endpoint** \u2014 one of \`search\`, \`scrape\`, \`parse\`, or \`map\`.
|
|
2191
|
-
- **jobId** \u2014 the id returned by that endpoint.
|
|
2192
|
-
- **rating** \u2014 overall result quality: \`good\`, \`partial\`, or \`bad\`.
|
|
2193
|
-
- **issues** \u2014 stable lowercase issue codes such as \`missing_markdown\`, \`bad_pdf_parse\`, or \`wrong_links\`.
|
|
2194
|
-
- **tags** \u2014 optional lowercase tags for grouping feedback.
|
|
2195
|
-
- **note** \u2014 short human-readable context. Do not include huge page contents or raw scrape results.
|
|
2196
|
-
- **url**, **pageNumbers**, and **metadata** \u2014 small contextual fields that identify what the feedback refers to.
|
|
2306
|
+
Submit concise quality feedback for a completed search, scrape, parse, or map job. Provide the endpoint, job ID, rating, and relevant issue codes or small contextual fields; omit large page contents and raw outputs.
|
|
2197
2307
|
|
|
2198
|
-
|
|
2199
|
-
|
|
2200
|
-
**Returns:** \`{ success, feedbackId, creditsRefunded, creditsRefundedToday?, dailyRefundCap?, dailyCapReached?, alreadySubmitted?, warning? }\` JSON.
|
|
2308
|
+
Returns submission status, feedback ID, and accounting fields.
|
|
2201
2309
|
`,
|
|
2202
2310
|
parameters: z3.object({
|
|
2203
2311
|
endpoint: z3.enum(["search", "scrape", "parse", "map"]),
|
|
@@ -2233,9 +2341,9 @@ Do not store multi-MB outputs in feedback. Use concise notes, issue codes, URLs,
|
|
|
2233
2341
|
...ORIGIN_HEADERS2,
|
|
2234
2342
|
"Content-Type": "application/json"
|
|
2235
2343
|
};
|
|
2236
|
-
const
|
|
2237
|
-
if (
|
|
2238
|
-
headers["Authorization"] = `Bearer ${
|
|
2344
|
+
const credential = credentialForOutboundRequest(session);
|
|
2345
|
+
if (credential) {
|
|
2346
|
+
headers["Authorization"] = `Bearer ${credential}`;
|
|
2239
2347
|
} else if (process.env.CLOUD_SERVICE === "true") {
|
|
2240
2348
|
throw new Error("Unauthorized: missing API key for feedback.");
|
|
2241
2349
|
}
|
|
@@ -2296,30 +2404,10 @@ server.addTool({
|
|
|
2296
2404
|
// Reads pages from target sites; does not delete or alter external websites.
|
|
2297
2405
|
},
|
|
2298
2406
|
description: `
|
|
2299
|
-
|
|
2300
|
-
|
|
2301
|
-
|
|
2302
|
-
|
|
2303
|
-
**Warning:** Crawl responses can be very large and may exceed token limits. Limit the crawl depth and number of pages, or use map + scrape for tighter control.
|
|
2304
|
-
**Common mistakes:** Setting limit or maxDiscoveryDepth too high (causes token overflow) or too low (causes missing pages); using crawl for a single page (use scrape instead). Using a /* wildcard is not recommended.
|
|
2305
|
-
**Prompt Example:** "Get all blog posts from the first two levels of example.com/blog."
|
|
2306
|
-
**Usage Example:**
|
|
2307
|
-
\`\`\`json
|
|
2308
|
-
{
|
|
2309
|
-
"name": "firecrawl_crawl",
|
|
2310
|
-
"arguments": {
|
|
2311
|
-
"url": "https://example.com/blog/*",
|
|
2312
|
-
"maxDiscoveryDepth": 5,
|
|
2313
|
-
"limit": 20,
|
|
2314
|
-
"allowExternalLinks": false,
|
|
2315
|
-
"deduplicateSimilarURLs": true,
|
|
2316
|
-
"sitemap": "include"
|
|
2317
|
-
}
|
|
2318
|
-
}
|
|
2319
|
-
\`\`\`
|
|
2320
|
-
**Returns:** Final crawl status and data after internal polling, including the crawl id. Use firecrawl_check_crawl_status only when you need to re-check an existing crawl ID later.
|
|
2321
|
-
${SAFE_MODE ? "**Safe Mode:** Read-only crawling. Webhooks and interactive actions are disabled for security." : ""}
|
|
2322
|
-
`,
|
|
2407
|
+
Start a multi-page crawl at a website URL, poll it to a terminal state, and return the final status and collected data. Scope can be bounded with include/exclude paths, depth, page limit, subdomain/external-link controls, sitemap handling, delay, and scrape options.
|
|
2408
|
+
|
|
2409
|
+
Crawl results can be large; use conservative limits when full-site coverage is unnecessary. Webhooks and interactive scrape actions are unavailable in safe mode. Returns the crawl ID, status, and page data.
|
|
2410
|
+
`,
|
|
2323
2411
|
parameters: z3.object({
|
|
2324
2412
|
url: z3.string(),
|
|
2325
2413
|
prompt: z3.string().optional(),
|
|
@@ -2389,18 +2477,7 @@ server.addTool({
|
|
|
2389
2477
|
// Status lookup only; no deletes or updates.
|
|
2390
2478
|
},
|
|
2391
2479
|
description: `
|
|
2392
|
-
|
|
2393
|
-
|
|
2394
|
-
**Usage Example:**
|
|
2395
|
-
\`\`\`json
|
|
2396
|
-
{
|
|
2397
|
-
"name": "firecrawl_check_crawl_status",
|
|
2398
|
-
"arguments": {
|
|
2399
|
-
"id": "550e8400-e29b-41d4-a716-446655440000"
|
|
2400
|
-
}
|
|
2401
|
-
}
|
|
2402
|
-
\`\`\`
|
|
2403
|
-
**Returns:** Status and progress of the crawl job, including results if available.
|
|
2480
|
+
Retrieve the current status, progress, and available results for an existing crawl ID. This only reads Firecrawl job state and does not start or modify the crawl.
|
|
2404
2481
|
`,
|
|
2405
2482
|
parameters: z3.object({ id: z3.string() }),
|
|
2406
2483
|
execute: async (args2, { session }) => {
|
|
@@ -2422,41 +2499,9 @@ server.addTool({
|
|
|
2422
2499
|
// Read-only extraction; no destructive changes to external content.
|
|
2423
2500
|
},
|
|
2424
2501
|
description: `
|
|
2425
|
-
Extract structured information from
|
|
2502
|
+
Extract structured information from one or more URLs with an optional natural-language prompt and JSON schema. It can include subdomains, follow external links, or use web search when those options are enabled.
|
|
2426
2503
|
|
|
2427
|
-
|
|
2428
|
-
**Not recommended for:** When you need the full content of a page (use scrape); when you're not looking for specific structured data.
|
|
2429
|
-
**Arguments:**
|
|
2430
|
-
- urls: Array of URLs to extract information from
|
|
2431
|
-
- prompt: Custom prompt for the LLM extraction
|
|
2432
|
-
- schema: JSON schema for structured data extraction
|
|
2433
|
-
- allowExternalLinks: Allow extraction from external links
|
|
2434
|
-
- enableWebSearch: Enable web search for additional context
|
|
2435
|
-
- includeSubdomains: Include subdomains in extraction
|
|
2436
|
-
**Prompt Example:** "Extract the product name, price, and description from these product pages."
|
|
2437
|
-
**Usage Example:**
|
|
2438
|
-
\`\`\`json
|
|
2439
|
-
{
|
|
2440
|
-
"name": "firecrawl_extract",
|
|
2441
|
-
"arguments": {
|
|
2442
|
-
"urls": ["https://example.com/page1", "https://example.com/page2"],
|
|
2443
|
-
"prompt": "Extract product information including name, price, and description",
|
|
2444
|
-
"schema": {
|
|
2445
|
-
"type": "object",
|
|
2446
|
-
"properties": {
|
|
2447
|
-
"name": { "type": "string" },
|
|
2448
|
-
"price": { "type": "number" },
|
|
2449
|
-
"description": { "type": "string" }
|
|
2450
|
-
},
|
|
2451
|
-
"required": ["name", "price"]
|
|
2452
|
-
},
|
|
2453
|
-
"allowExternalLinks": false,
|
|
2454
|
-
"enableWebSearch": false,
|
|
2455
|
-
"includeSubdomains": false
|
|
2456
|
-
}
|
|
2457
|
-
}
|
|
2458
|
-
\`\`\`
|
|
2459
|
-
**Returns:** Extracted structured data as defined by your schema.
|
|
2504
|
+
Use this for a defined structured result rather than full page content. Returns data shaped by the supplied schema or prompt.
|
|
2460
2505
|
`,
|
|
2461
2506
|
parameters: z3.object({
|
|
2462
2507
|
urls: z3.array(z3.string()),
|
|
@@ -2497,73 +2542,9 @@ server.addTool({
|
|
|
2497
2542
|
// Gathers information only; does not delete external data or user resources.
|
|
2498
2543
|
},
|
|
2499
2544
|
description: `
|
|
2500
|
-
|
|
2545
|
+
Start an asynchronous web research job from a prompt, optional seed URLs, and an optional JSON schema. Use this for a requested synthesis across multiple sources when the task can wait for asynchronous completion. The agent can search, navigate, read pages, and assemble a structured result.
|
|
2501
2546
|
|
|
2502
|
-
|
|
2503
|
-
|
|
2504
|
-
**IMPORTANT - Async workflow with patient polling:**
|
|
2505
|
-
1. Call \`firecrawl_agent\` with your prompt/schema \u2192 returns job ID immediately
|
|
2506
|
-
2. Poll \`firecrawl_agent_status\` with the job ID to check progress
|
|
2507
|
-
3. **Keep polling for at least 2-3 minutes** - agent research typically takes 1-5 minutes for complex queries
|
|
2508
|
-
4. Poll every 15-30 seconds until status is "completed" or "failed"
|
|
2509
|
-
5. Do NOT give up after just a few polling attempts - the agent needs time to research
|
|
2510
|
-
|
|
2511
|
-
**Expected wait times:**
|
|
2512
|
-
- Simple queries with provided URLs: 30 seconds - 1 minute
|
|
2513
|
-
- Complex research across multiple sites: 2-5 minutes
|
|
2514
|
-
- Deep research tasks: 5+ minutes
|
|
2515
|
-
|
|
2516
|
-
**Best for:** Complex research tasks where you don't know the exact URLs; multi-source data gathering; finding information scattered across the web; extracting data from JavaScript-heavy SPAs that fail with regular scrape.
|
|
2517
|
-
**Not recommended for:**
|
|
2518
|
-
- Single-page extraction when you have a URL (use firecrawl_scrape, faster and cheaper)
|
|
2519
|
-
- Web search (use firecrawl_search first)
|
|
2520
|
-
- Interactive page tasks like clicking, filling forms, login, or navigating JS-heavy SPAs (use firecrawl_scrape + firecrawl_interact)
|
|
2521
|
-
- Extracting specific data from a known page (use firecrawl_scrape with JSON format)
|
|
2522
|
-
|
|
2523
|
-
**Arguments:**
|
|
2524
|
-
- prompt: Natural language description of the data you want (required, max 10,000 characters)
|
|
2525
|
-
- urls: Optional array of URLs to focus the agent on specific pages
|
|
2526
|
-
- schema: Optional JSON schema for structured output
|
|
2527
|
-
|
|
2528
|
-
**Prompt Example:** "Find the founders of Firecrawl and their backgrounds"
|
|
2529
|
-
**Usage Example (start agent, then poll patiently for results):**
|
|
2530
|
-
\`\`\`json
|
|
2531
|
-
{
|
|
2532
|
-
"name": "firecrawl_agent",
|
|
2533
|
-
"arguments": {
|
|
2534
|
-
"prompt": "Find the top 5 AI startups founded in 2024 and their funding amounts",
|
|
2535
|
-
"schema": {
|
|
2536
|
-
"type": "object",
|
|
2537
|
-
"properties": {
|
|
2538
|
-
"startups": {
|
|
2539
|
-
"type": "array",
|
|
2540
|
-
"items": {
|
|
2541
|
-
"type": "object",
|
|
2542
|
-
"properties": {
|
|
2543
|
-
"name": { "type": "string" },
|
|
2544
|
-
"funding": { "type": "string" },
|
|
2545
|
-
"founded": { "type": "string" }
|
|
2546
|
-
}
|
|
2547
|
-
}
|
|
2548
|
-
}
|
|
2549
|
-
}
|
|
2550
|
-
}
|
|
2551
|
-
}
|
|
2552
|
-
}
|
|
2553
|
-
\`\`\`
|
|
2554
|
-
Then poll with \`firecrawl_agent_status\` every 15-30 seconds for at least 2-3 minutes.
|
|
2555
|
-
|
|
2556
|
-
**Usage Example (with URLs - agent focuses on specific pages):**
|
|
2557
|
-
\`\`\`json
|
|
2558
|
-
{
|
|
2559
|
-
"name": "firecrawl_agent",
|
|
2560
|
-
"arguments": {
|
|
2561
|
-
"urls": ["https://docs.firecrawl.dev", "https://firecrawl.dev/pricing"],
|
|
2562
|
-
"prompt": "Compare the features and pricing information from these pages"
|
|
2563
|
-
}
|
|
2564
|
-
}
|
|
2565
|
-
\`\`\`
|
|
2566
|
-
**Returns:** Job ID for status checking. Use \`firecrawl_agent_status\` to poll for results.
|
|
2547
|
+
This call returns only a job ID, not the research result. Read the job with \`firecrawl_agent_status\` until it reaches \`completed\` or \`failed\`; research commonly takes several minutes. If the job cannot finish within the task's available time, \`firecrawl_search\` and \`firecrawl_scrape\` can gather evidence synchronously.
|
|
2567
2548
|
`,
|
|
2568
2549
|
parameters: z3.object({
|
|
2569
2550
|
prompt: z3.string().min(1).max(1e4),
|
|
@@ -2601,29 +2582,9 @@ server.addTool({
|
|
|
2601
2582
|
// Read-only status check.
|
|
2602
2583
|
},
|
|
2603
2584
|
description: `
|
|
2604
|
-
Check the status
|
|
2605
|
-
|
|
2606
|
-
**IMPORTANT - Be patient with polling:**
|
|
2607
|
-
- Poll every 15-30 seconds
|
|
2608
|
-
- **Keep polling for at least 2-3 minutes** before considering the request failed
|
|
2609
|
-
- Complex research can take 5+ minutes - do not give up early
|
|
2610
|
-
- Only stop polling when status is "completed" or "failed"
|
|
2611
|
-
|
|
2612
|
-
**Usage Example:**
|
|
2613
|
-
\`\`\`json
|
|
2614
|
-
{
|
|
2615
|
-
"name": "firecrawl_agent_status",
|
|
2616
|
-
"arguments": {
|
|
2617
|
-
"id": "550e8400-e29b-41d4-a716-446655440000"
|
|
2618
|
-
}
|
|
2619
|
-
}
|
|
2620
|
-
\`\`\`
|
|
2621
|
-
**Possible statuses:**
|
|
2622
|
-
- processing: Agent is still researching - keep polling, do not give up
|
|
2623
|
-
- completed: Research finished - response includes the extracted data
|
|
2624
|
-
- failed: An error occurred (only stop polling on this status)
|
|
2585
|
+
Retrieve progress or final results for a \`firecrawl_agent\` job ID. A \`processing\` response is non-terminal and does not contain the final research result. Check again after 15\u201330 seconds until the status is \`completed\` or \`failed\`; complex jobs can take several minutes. If the job cannot finish within the task's available time, use \`firecrawl_search\` and \`firecrawl_scrape\` to complete the requested output.
|
|
2625
2586
|
|
|
2626
|
-
|
|
2587
|
+
Returns job status, progress information, and result data when completed.
|
|
2627
2588
|
`,
|
|
2628
2589
|
parameters: z3.object({ id: z3.string() }),
|
|
2629
2590
|
execute: async (args2, { session, log }) => {
|
|
@@ -2649,45 +2610,9 @@ server.addTool({
|
|
|
2649
2610
|
// Transient page interactions only; does not delete monitors, jobs, or external sites.
|
|
2650
2611
|
},
|
|
2651
2612
|
description: `
|
|
2652
|
-
|
|
2653
|
-
|
|
2654
|
-
**Best for:** Multi-step workflows on a single page \u2014 searching a site, clicking through results, filling forms, extracting data that requires interaction.
|
|
2655
|
-
**Two ways to target a page:**
|
|
2656
|
-
- Pass a \`url\` to interact directly. The session is opened for you in one call (use this for a fresh page).
|
|
2657
|
-
- Pass a \`scrapeId\` from a previous firecrawl_scrape to reuse that already-loaded page (cheaper when you just scraped it).
|
|
2658
|
-
|
|
2659
|
-
**Arguments:**
|
|
2660
|
-
- url: Page to interact with; opens a session for you (use this OR scrapeId)
|
|
2661
|
-
- scrapeId: Scrape job ID from a previous scrape, found in its metadata (use this OR url)
|
|
2662
|
-
- prompt: Natural language instruction describing the action to take (use this OR code)
|
|
2663
|
-
- code: Code to execute in the browser session (use this OR prompt)
|
|
2664
|
-
- language: "bash", "python", or "node" (optional, defaults to "node", only used with code)
|
|
2665
|
-
- timeout: Interact execution timeout in seconds, 1-300 (optional, defaults to 30)
|
|
2666
|
-
- scrapeOptions: Optional scrape controls used only with url mode, such as waitFor, maxAge, proxy, or zeroDataRetention
|
|
2667
|
-
|
|
2668
|
-
**Usage Example (prompt, direct via url):**
|
|
2669
|
-
\`\`\`json
|
|
2670
|
-
{
|
|
2671
|
-
"name": "firecrawl_interact",
|
|
2672
|
-
"arguments": {
|
|
2673
|
-
"url": "https://example.com/products",
|
|
2674
|
-
"prompt": "Click on the first product and tell me its price"
|
|
2675
|
-
}
|
|
2676
|
-
}
|
|
2677
|
-
\`\`\`
|
|
2613
|
+
Open or reuse a live browser session to navigate a page, click controls, fill fields, or run browser code. Provide either \`url\` or \`scrapeId\`, and either a natural-language \`prompt\` or executable \`code\`; code can run as Bash, Python, or Node with a bounded timeout.
|
|
2678
2614
|
|
|
2679
|
-
|
|
2680
|
-
\`\`\`json
|
|
2681
|
-
{
|
|
2682
|
-
"name": "firecrawl_interact",
|
|
2683
|
-
"arguments": {
|
|
2684
|
-
"scrapeId": "scrape-id-from-previous-scrape",
|
|
2685
|
-
"code": "agent-browser click @e5",
|
|
2686
|
-
"language": "bash"
|
|
2687
|
-
}
|
|
2688
|
-
}
|
|
2689
|
-
\`\`\`
|
|
2690
|
-
**Returns:** Execution result including output, stdout, stderr, exit code, and live view URLs.
|
|
2615
|
+
This acts on the live site, so actions such as form submission can create persistent external side effects. Returns execution output, stdout/stderr, exit status, and session viewing URLs.
|
|
2691
2616
|
`,
|
|
2692
2617
|
parameters: z3.object({
|
|
2693
2618
|
scrapeId: z3.string().trim().min(1).optional(),
|
|
@@ -2770,18 +2695,7 @@ server.addTool({
|
|
|
2770
2695
|
// Terminates the live browser session; this end state cannot be resumed.
|
|
2771
2696
|
},
|
|
2772
2697
|
description: `
|
|
2773
|
-
Stop
|
|
2774
|
-
|
|
2775
|
-
**Usage Example:**
|
|
2776
|
-
\`\`\`json
|
|
2777
|
-
{
|
|
2778
|
-
"name": "firecrawl_interact_stop",
|
|
2779
|
-
"arguments": {
|
|
2780
|
-
"scrapeId": "scrape-id-here"
|
|
2781
|
-
}
|
|
2782
|
-
}
|
|
2783
|
-
\`\`\`
|
|
2784
|
-
**Returns:** Success confirmation.
|
|
2698
|
+
Stop the live interact session associated with a \`scrapeId\` and release its resources. Returns a success confirmation.
|
|
2785
2699
|
`,
|
|
2786
2700
|
parameters: z3.object({
|
|
2787
2701
|
scrapeId: z3.string()
|
|
@@ -2809,56 +2723,11 @@ server.addTool({
|
|
|
2809
2723
|
// Read-only parsing; no deletion or writes to the source file.
|
|
2810
2724
|
},
|
|
2811
2725
|
description: `
|
|
2812
|
-
Parse a
|
|
2813
|
-
|
|
2814
|
-
In local/non-cloud MCP mode, this tool reads filePath from the MCP server filesystem and posts multipart data to the configured self-hosted FIRECRAWL_API_URL, preserving the existing direct-read behavior.
|
|
2815
|
-
|
|
2816
|
-
In hosted CLOUD_SERVICE mode, this tool is a two-call flow because hosted MCP cannot read your local filesystem:
|
|
2817
|
-
1. Call with filePath, contentType, parse options, and optional declaredSizeBytes. The hosted server mints a short-lived upload URL and returns a safe local curl PUT command plus nextToolCall.
|
|
2818
|
-
2. Run the returned curl command locally, then call firecrawl_parse again with uploadRef and the desired parse options. The hosted server calls /v2/parse server-side with your session credential.
|
|
2819
|
-
|
|
2820
|
-
**Best for:** Extracting content from a local document (PDF, Word, Excel, HTML, etc.); pulling structured data out of a file with JSON format; converting binary documents into markdown for downstream reasoning.
|
|
2821
|
-
**Not recommended for:** Remote URLs (use firecrawl_scrape); multiple files at once (call parse multiple times); documents that require interactive actions, screenshots, or change tracking \u2014 those aren't supported by the parse endpoint.
|
|
2822
|
-
**Common mistakes:** In hosted mode, do not pass both filePath and uploadRef. Phase 1 uses filePath only to generate upload instructions; phase 2 uses uploadRef only to parse server-side.
|
|
2726
|
+
Parse one supported document into markdown, HTML, links, summary, targeted answers, or JSON matching a schema. Supported inputs include common HTML, PDF, Word, RTF, OpenDocument, and spreadsheet files; PDF parsing can be bounded with \`pdfOptions.maxPages\`.
|
|
2823
2727
|
|
|
2824
|
-
|
|
2825
|
-
**Unsupported options:** actions, screenshot/branding/changeTracking formats, waitFor > 0, location, mobile, proxy values other than "auto" or "basic".
|
|
2826
|
-
**Privacy:** Set \`redactPII: true\` to return content with personally identifiable information redacted.
|
|
2728
|
+
Local MCP reads \`filePath\` from the server filesystem. Hosted MCP uses two calls: first provide \`filePath\` to receive upload instructions, upload locally, then call again with the returned \`uploadRef\`; do not send both fields together. Remote web URLs belong in \`firecrawl_scrape\`.
|
|
2827
2729
|
|
|
2828
|
-
|
|
2829
|
-
When the user asks for SPECIFIC data points from a document, you MUST use JSON format with a schema. Only use markdown when the user needs the ENTIRE document content.
|
|
2830
|
-
|
|
2831
|
-
**Handling PDFs:**
|
|
2832
|
-
Add \`"parsers": ["pdf"]\` (optionally with \`pdfOptions.maxPages\`) when parsing a PDF so the PDF engine is invoked explicitly. For very long documents, cap \`maxPages\` to keep the response within token limits.
|
|
2833
|
-
|
|
2834
|
-
**Hosted phase 1 example:**
|
|
2835
|
-
\`\`\`json
|
|
2836
|
-
{
|
|
2837
|
-
"name": "firecrawl_parse",
|
|
2838
|
-
"arguments": {
|
|
2839
|
-
"filePath": "/absolute/path/to/document.pdf",
|
|
2840
|
-
"contentType": "application/pdf",
|
|
2841
|
-
"formats": ["markdown"],
|
|
2842
|
-
"parsers": ["pdf"],
|
|
2843
|
-
"zeroDataRetention": true
|
|
2844
|
-
}
|
|
2845
|
-
}
|
|
2846
|
-
\`\`\`
|
|
2847
|
-
|
|
2848
|
-
**Hosted phase 2 example:**
|
|
2849
|
-
\`\`\`json
|
|
2850
|
-
{
|
|
2851
|
-
"name": "firecrawl_parse",
|
|
2852
|
-
"arguments": {
|
|
2853
|
-
"uploadRef": "upload-ref-from-phase-1",
|
|
2854
|
-
"formats": ["markdown"],
|
|
2855
|
-
"parsers": ["pdf"],
|
|
2856
|
-
"zeroDataRetention": true
|
|
2857
|
-
}
|
|
2858
|
-
}
|
|
2859
|
-
\`\`\`
|
|
2860
|
-
|
|
2861
|
-
**Returns:** Phase 1 hosted upload instructions or a parsed document with markdown, html, links, summary, json, or query results depending on the requested formats.
|
|
2730
|
+
Set \`redactPII\` to request redaction of personally identifiable information in the returned content. \`zeroDataRetention\` requires an eligible authenticated account; omit it for anonymous keyless use. Returns upload instructions for hosted phase one or parsed document content for the final call.
|
|
2862
2731
|
`,
|
|
2863
2732
|
parameters: parseParamsSchema,
|
|
2864
2733
|
execute: async (args2, { session, log }) => {
|
|
@@ -2890,9 +2759,9 @@ Add \`"parsers": ["pdf"]\` (optionally with \`pdfOptions.maxPages\`) when parsin
|
|
|
2890
2759
|
form.append("file", blob, filename);
|
|
2891
2760
|
form.append("options", JSON.stringify(optionsPayload));
|
|
2892
2761
|
const headers = { ...ORIGIN_HEADERS2 };
|
|
2893
|
-
const
|
|
2894
|
-
if (
|
|
2895
|
-
headers["Authorization"] = `Bearer ${
|
|
2762
|
+
const credential = credentialForOutboundRequest(session);
|
|
2763
|
+
if (credential) {
|
|
2764
|
+
headers["Authorization"] = `Bearer ${credential}`;
|
|
2896
2765
|
}
|
|
2897
2766
|
const endpoint = `${apiUrl.replace(/\/$/, "")}/v2/parse`;
|
|
2898
2767
|
log.info("Parsing local file", {
|
|
@@ -2918,6 +2787,61 @@ Add \`"parsers": ["pdf"]\` (optionally with \`pdfOptions.maxPages\`) when parsin
|
|
|
2918
2787
|
}
|
|
2919
2788
|
}
|
|
2920
2789
|
});
|
|
2790
|
+
function registerMarketplaceSearchTool(registrar, getClientFn) {
|
|
2791
|
+
registrar.addTool({
|
|
2792
|
+
name: "firecrawl_search",
|
|
2793
|
+
annotations: {
|
|
2794
|
+
title: "Search the web",
|
|
2795
|
+
readOnlyHint: true,
|
|
2796
|
+
openWorldHint: true,
|
|
2797
|
+
destructiveHint: false
|
|
2798
|
+
},
|
|
2799
|
+
description: `
|
|
2800
|
+
Search web and specialized indexes, returning ranked results. Operators include quoted phrases, \`-term\`, \`site:host\`, \`inurl:term\`, \`intitle:term\`, and \`related:host\`; the set is non-exhaustive. \`includeDomains\` and \`excludeDomains\` are mutually exclusive hostname filters; categories limit result types to \`github\`, \`research\`, or \`pdf\`.
|
|
2801
|
+
|
|
2802
|
+
Returns \`{ success, data, id, creditsUsed }\`, with source arrays in \`data\`.
|
|
2803
|
+
`,
|
|
2804
|
+
parameters: z3.object({ ...searchToolBaseFields }).strict().refine(searchDomainsAreExclusive, SEARCH_DOMAINS_CONFLICT_MESSAGE),
|
|
2805
|
+
execute: async (args2, { session, log }) => {
|
|
2806
|
+
const {
|
|
2807
|
+
query,
|
|
2808
|
+
includeDomains,
|
|
2809
|
+
excludeDomains,
|
|
2810
|
+
limit,
|
|
2811
|
+
tbs,
|
|
2812
|
+
filter,
|
|
2813
|
+
location,
|
|
2814
|
+
sources,
|
|
2815
|
+
categories,
|
|
2816
|
+
highlights,
|
|
2817
|
+
enterprise
|
|
2818
|
+
} = args2;
|
|
2819
|
+
const searchQuery = buildSearchQueryWithDomains(
|
|
2820
|
+
query,
|
|
2821
|
+
includeDomains,
|
|
2822
|
+
excludeDomains
|
|
2823
|
+
);
|
|
2824
|
+
const searchBody = {
|
|
2825
|
+
query: searchQuery,
|
|
2826
|
+
...removeEmptyTopLevel({
|
|
2827
|
+
limit,
|
|
2828
|
+
tbs,
|
|
2829
|
+
filter,
|
|
2830
|
+
location,
|
|
2831
|
+
sources,
|
|
2832
|
+
categories,
|
|
2833
|
+
highlights,
|
|
2834
|
+
enterprise
|
|
2835
|
+
}),
|
|
2836
|
+
origin: ORIGIN
|
|
2837
|
+
};
|
|
2838
|
+
log.info("Searching", { query: searchQuery });
|
|
2839
|
+
const client = getClientFn(session);
|
|
2840
|
+
const httpRes = await client.http.post("/v2/search", searchBody);
|
|
2841
|
+
return asText2(httpRes?.data ?? {});
|
|
2842
|
+
}
|
|
2843
|
+
});
|
|
2844
|
+
}
|
|
2921
2845
|
var PORT = Number(process.env.PORT || 3e3);
|
|
2922
2846
|
var HOST = process.env.CLOUD_SERVICE === "true" ? "0.0.0.0" : process.env.HOST || "localhost";
|
|
2923
2847
|
var args;
|
|
@@ -2927,6 +2851,7 @@ if (process.env.CLOUD_SERVICE === "true" || process.env.SSE_LOCAL === "true" ||
|
|
|
2927
2851
|
httpStream: {
|
|
2928
2852
|
port: PORT,
|
|
2929
2853
|
host: HOST,
|
|
2854
|
+
endpoint: primaryProfile.endpoint,
|
|
2930
2855
|
stateless: true
|
|
2931
2856
|
}
|
|
2932
2857
|
};
|
|
@@ -2937,4 +2862,46 @@ if (process.env.CLOUD_SERVICE === "true" || process.env.SSE_LOCAL === "true" ||
|
|
|
2937
2862
|
}
|
|
2938
2863
|
registerMonitorTools(server);
|
|
2939
2864
|
registerResearchTools(server, getClient);
|
|
2865
|
+
if (primaryProfile.id === "search") {
|
|
2866
|
+
const primarySearchRegistrar = {
|
|
2867
|
+
addTool: ((tool) => {
|
|
2868
|
+
if (primaryProfile.toolAllowlist?.has(tool.name)) {
|
|
2869
|
+
addTool(guardHostedTool(tool, { logActions: false }));
|
|
2870
|
+
}
|
|
2871
|
+
})
|
|
2872
|
+
};
|
|
2873
|
+
registerMarketplaceSearchTool(primarySearchRegistrar, getClient);
|
|
2874
|
+
}
|
|
2940
2875
|
await server.start(args);
|
|
2876
|
+
var searchProfileEnabled = process.env.CLOUD_SERVICE === "true" && primaryProfile.id === "full" && process.env.FIRECRAWL_MCP_SEARCH_ENABLED !== "false";
|
|
2877
|
+
if (searchProfileEnabled) {
|
|
2878
|
+
const searchProfile = makeSearchProfile();
|
|
2879
|
+
const searchServer = createServer(searchProfile);
|
|
2880
|
+
const searchRegistrar = {
|
|
2881
|
+
addTool: ((tool) => {
|
|
2882
|
+
if (searchProfile.toolAllowlist?.has(tool.name)) {
|
|
2883
|
+
searchServer.addTool(
|
|
2884
|
+
guardHostedTool(tool, { logActions: false })
|
|
2885
|
+
);
|
|
2886
|
+
}
|
|
2887
|
+
})
|
|
2888
|
+
};
|
|
2889
|
+
registerResearchTools(searchRegistrar, getClient);
|
|
2890
|
+
registerMarketplaceSearchTool(searchRegistrar, getClient);
|
|
2891
|
+
try {
|
|
2892
|
+
await searchServer.start({
|
|
2893
|
+
transportType: "httpStream",
|
|
2894
|
+
httpStream: {
|
|
2895
|
+
port: searchProfile.port,
|
|
2896
|
+
host: HOST,
|
|
2897
|
+
endpoint: searchProfile.endpoint,
|
|
2898
|
+
stateless: true
|
|
2899
|
+
}
|
|
2900
|
+
});
|
|
2901
|
+
} catch (error) {
|
|
2902
|
+
console.error(
|
|
2903
|
+
`[search-profile] failed to start on port ${searchProfile.port}; the full surface is unaffected`,
|
|
2904
|
+
error
|
|
2905
|
+
);
|
|
2906
|
+
}
|
|
2907
|
+
}
|