firecrawl-mcp 3.22.4 → 3.23.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -0
- package/dist/index.js +1173 -1104
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -3,17 +3,143 @@
|
|
|
3
3
|
// src/index.ts
|
|
4
4
|
import FirecrawlApp from "@mendable/firecrawl-js";
|
|
5
5
|
import dotenv from "dotenv";
|
|
6
|
-
import { FastMCP } from "fastmcp";
|
|
6
|
+
import { FastMCP, UserError } from "fastmcp";
|
|
7
7
|
import { readFile } from "fs/promises";
|
|
8
8
|
import { createRequire } from "module";
|
|
9
|
+
import { randomUUID } from "crypto";
|
|
9
10
|
import path from "path";
|
|
10
|
-
import { z as
|
|
11
|
+
import { z as z4 } from "zod";
|
|
11
12
|
|
|
12
|
-
// src/
|
|
13
|
+
// src/developer.ts
|
|
13
14
|
import { z } from "zod";
|
|
15
|
+
var BASE = "/v2/developer/search";
|
|
16
|
+
var ORIGIN_HEADERS = { "X-Origin": "mcp-fastmcp" };
|
|
17
|
+
var MAX_PASSAGE_CHARS = 1200;
|
|
18
|
+
function fmtDeveloper(results) {
|
|
19
|
+
if (!results || results.length === 0) return "(no results)";
|
|
20
|
+
return results.map((r) => {
|
|
21
|
+
const kind = r.type ? ` (${r.type})` : "";
|
|
22
|
+
const lines = [`## [${r.id ?? "?"}]${kind} ${r.title ?? "(untitled)"}`];
|
|
23
|
+
if (r.url) lines.push(r.url);
|
|
24
|
+
const body = (r.passages ?? []).map((p) => p.text ?? "").join("\n---\n").trim();
|
|
25
|
+
lines.push(body ? body.slice(0, MAX_PASSAGE_CHARS) : "(no content)");
|
|
26
|
+
return lines.join("\n");
|
|
27
|
+
}).join("\n\n");
|
|
28
|
+
}
|
|
29
|
+
function registerDeveloperTools(server2, getClient2) {
|
|
30
|
+
server2.addTool({
|
|
31
|
+
name: "firecrawl_developer_search",
|
|
32
|
+
annotations: {
|
|
33
|
+
title: "Search developer sources",
|
|
34
|
+
readOnlyHint: true,
|
|
35
|
+
// Semantic search over an indexed developer corpus; returns ranked results only.
|
|
36
|
+
openWorldHint: true,
|
|
37
|
+
// Searches the Firecrawl developer index of public GitHub and documentation content.
|
|
38
|
+
destructiveHint: false
|
|
39
|
+
// Query-only; no writes to external sources or the developer index.
|
|
40
|
+
},
|
|
41
|
+
description: `
|
|
42
|
+
For a developer question \u2014 code behaviour, a library or framework, an API contract, an error message, or a known bug \u2014 search an index built for coding agents. The index covers GitHub issues, merged pull requests, repository READMEs, and curated documentation sites.
|
|
43
|
+
|
|
44
|
+
Returns ranked results with an ID, source type, URL, title, and the matched passages in markdown.
|
|
45
|
+
`,
|
|
46
|
+
parameters: z.object({
|
|
47
|
+
query: z.string().min(1).describe(
|
|
48
|
+
"Natural-language developer question or search phrase, including the library, error message, or API involved when relevant."
|
|
49
|
+
),
|
|
50
|
+
k: z.number().int().min(1).max(100).optional().describe("Number of ranked results to return (default 20).")
|
|
51
|
+
}),
|
|
52
|
+
execute: async (args2, { session }) => {
|
|
53
|
+
const { query, k } = args2;
|
|
54
|
+
const params = new URLSearchParams();
|
|
55
|
+
params.append("query", query);
|
|
56
|
+
if (k != null) params.append("k", String(k));
|
|
57
|
+
const client = getClient2(session);
|
|
58
|
+
const res = await client.http.get(
|
|
59
|
+
`${BASE}?${params.toString()}`,
|
|
60
|
+
ORIGIN_HEADERS
|
|
61
|
+
);
|
|
62
|
+
return fmtDeveloper(res.data?.results);
|
|
63
|
+
}
|
|
64
|
+
});
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
// src/keyless-client-ip.ts
|
|
68
|
+
import net from "net";
|
|
69
|
+
function extractSingleTrustedClientIp(rawForwardedFor) {
|
|
70
|
+
if (Array.isArray(rawForwardedFor) && rawForwardedFor.length !== 1) {
|
|
71
|
+
return void 0;
|
|
72
|
+
}
|
|
73
|
+
const raw = Array.isArray(rawForwardedFor) ? rawForwardedFor[0] : rawForwardedFor;
|
|
74
|
+
if (typeof raw !== "string") return void 0;
|
|
75
|
+
const parts = raw.split(",").map((part) => part.trim()).filter(Boolean);
|
|
76
|
+
if (parts.length !== 1) return void 0;
|
|
77
|
+
const candidate = parts[0].replace(/^\[(.*)\]$/, "$1").toLowerCase();
|
|
78
|
+
return net.isIP(candidate) ? candidate : void 0;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
// src/monitor.ts
|
|
82
|
+
import { z as z2 } from "zod";
|
|
83
|
+
|
|
84
|
+
// src/session-credential.ts
|
|
85
|
+
import { createHmac } from "crypto";
|
|
86
|
+
var managedOAuthApiKey = /* @__PURE__ */ Symbol("firecrawlManagedOAuthApiKey");
|
|
87
|
+
var CredentialValidationUnavailableError = class extends Error {
|
|
88
|
+
constructor() {
|
|
89
|
+
super("Firecrawl credential validation is temporarily unavailable");
|
|
90
|
+
this.name = "CredentialValidationUnavailableError";
|
|
91
|
+
}
|
|
92
|
+
};
|
|
93
|
+
function delegationSecret() {
|
|
94
|
+
const secret = process.env.MCP_DELEGATED_CREDENTIAL_SECRET?.trim();
|
|
95
|
+
if (!secret) throw new CredentialValidationUnavailableError();
|
|
96
|
+
return secret;
|
|
97
|
+
}
|
|
98
|
+
function requireDelegatedCredentialSigning() {
|
|
99
|
+
delegationSecret();
|
|
100
|
+
}
|
|
101
|
+
function setManagedOAuthApiKey(session, apiKey) {
|
|
102
|
+
Object.defineProperty(session, managedOAuthApiKey, {
|
|
103
|
+
configurable: false,
|
|
104
|
+
enumerable: false,
|
|
105
|
+
value: apiKey,
|
|
106
|
+
writable: false
|
|
107
|
+
});
|
|
108
|
+
return session;
|
|
109
|
+
}
|
|
110
|
+
function copyManagedOAuthApiKey(source, target) {
|
|
111
|
+
const apiKey = source?.[managedOAuthApiKey];
|
|
112
|
+
if (apiKey) setManagedOAuthApiKey(target, apiKey);
|
|
113
|
+
}
|
|
114
|
+
function hasCredential(session) {
|
|
115
|
+
return Boolean(session?.firecrawlApiKey || session?.[managedOAuthApiKey]);
|
|
116
|
+
}
|
|
117
|
+
function hasManagedOAuthCredential(session) {
|
|
118
|
+
return Boolean(session?.[managedOAuthApiKey]);
|
|
119
|
+
}
|
|
120
|
+
function credentialForOutboundRequest(session) {
|
|
121
|
+
const managedApiKey = session?.[managedOAuthApiKey];
|
|
122
|
+
if (!managedApiKey) return session?.firecrawlApiKey;
|
|
123
|
+
const iat = Math.floor(Date.now() / 1e3);
|
|
124
|
+
const payload = {
|
|
125
|
+
v: 1,
|
|
126
|
+
aud: "firecrawl-core",
|
|
127
|
+
purpose: "hosted_mcp_oauth",
|
|
128
|
+
api_key: managedApiKey,
|
|
129
|
+
iat,
|
|
130
|
+
exp: iat + 60
|
|
131
|
+
};
|
|
132
|
+
const encodedPayload = Buffer.from(JSON.stringify(payload)).toString(
|
|
133
|
+
"base64url"
|
|
134
|
+
);
|
|
135
|
+
const signature = createHmac("sha256", delegationSecret()).update(encodedPayload).digest("base64url");
|
|
136
|
+
return `fcmcp_${encodedPayload}.${signature}`;
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
// src/monitor.ts
|
|
14
140
|
var DEFAULT_API_URL = "https://api.firecrawl.dev";
|
|
15
141
|
function resolveAuth(session) {
|
|
16
|
-
const apiKey = session
|
|
142
|
+
const apiKey = session === void 0 ? process.env.FIRECRAWL_API_KEY : credentialForOutboundRequest(session);
|
|
17
143
|
const baseUrl = (process.env.FIRECRAWL_API_URL ?? DEFAULT_API_URL).replace(
|
|
18
144
|
/\/$/,
|
|
19
145
|
""
|
|
@@ -52,8 +178,8 @@ async function monitorRequest(session, path2, init = {}) {
|
|
|
52
178
|
function asText(data) {
|
|
53
179
|
return JSON.stringify(data, null, 2);
|
|
54
180
|
}
|
|
55
|
-
var pageStatusSchema =
|
|
56
|
-
var checkStatusSchema =
|
|
181
|
+
var pageStatusSchema = z2.enum(["same", "new", "changed", "removed", "error"]);
|
|
182
|
+
var checkStatusSchema = z2.enum([
|
|
57
183
|
"queued",
|
|
58
184
|
"running",
|
|
59
185
|
"completed",
|
|
@@ -142,141 +268,26 @@ function registerMonitorTools(server2) {
|
|
|
142
268
|
// Additive; creates a new monitor without deleting existing monitors or external content.
|
|
143
269
|
},
|
|
144
270
|
description: `
|
|
145
|
-
Create a
|
|
146
|
-
|
|
147
|
-
Prefer the simple path: pass \`page\` or \`pages\` plus \`goal\` to monitor specific URLs, OR pass \`queries\` plus \`goal\` to monitor web search results for new/changed hits. The tool will create the monitor with a 30-minute schedule and meaningful-change judging enabled by the API. Use \`body\` only for advanced requests such as crawl targets, JSON change tracking, custom retention, or manual \`judgeEnabled\` control.
|
|
148
|
-
|
|
149
|
-
Meaningful-change judge: set \`goal\` to a plain-language description of what the user actually cares about. \`judgeEnabled\` defaults to true when \`goal\` is set, so providing \`goal\` is enough. Page webhooks expose \`isMeaningful\` and \`judgment\` on \`monitor.page\` events.
|
|
271
|
+
Create a recurring scrape, crawl, or search monitor that compares each check with its retained predecessor. The simple form accepts \`page\`/\`pages\` or \`queries\` plus a plain-language \`goal\`; the advanced \`body\` form controls targets, schedule, change-tracking formats, judging, retention, webhook, and notifications.
|
|
150
272
|
|
|
151
|
-
|
|
152
|
-
- \`page\`: one page URL to monitor.
|
|
153
|
-
- \`pages\`: multiple page URLs to monitor.
|
|
154
|
-
- \`queries\`: one or more search queries (1-12) to monitor instead of fixed URLs. Each check runs the searches and diffs the result set, so you get alerted when new or changed results appear. Mutually exclusive with \`page\`/\`pages\` in the simple path.
|
|
155
|
-
- \`searchWindow\`: optional recency window for search targets \u2014 one of \`5m\`, \`15m\`, \`1h\`, \`6h\`, \`24h\`, \`7d\` (default \`24h\`).
|
|
156
|
-
- \`maxResults\`: optional max results per search, 1-50 (default 10).
|
|
157
|
-
- \`includeDomains\` / \`excludeDomains\`: optional domain allow/deny lists for search targets.
|
|
158
|
-
- \`goal\`: plain-English instruction for what changes matter. Required for the simple path (and always required when \`queries\` are set \u2014 web monitors must have a goal).
|
|
159
|
-
- \`scheduleText\`: optional natural-language schedule, default \`every 30 minutes\`.
|
|
160
|
-
- \`email\`: optional email recipient for summaries.
|
|
161
|
-
- \`webhookUrl\`: optional webhook URL. Configures \`monitor.page\` and \`monitor.check.completed\`.
|
|
162
|
-
|
|
163
|
-
**Search-mode example:**
|
|
164
|
-
|
|
165
|
-
\`\`\`json
|
|
166
|
-
{
|
|
167
|
-
"name": "firecrawl_monitor_create",
|
|
168
|
-
"arguments": {
|
|
169
|
-
"queries": ["new LLM release", "frontier model launch"],
|
|
170
|
-
"goal": "Notify me about major new LLM model releases.",
|
|
171
|
-
"searchWindow": "24h",
|
|
172
|
-
"maxResults": 10
|
|
173
|
-
}
|
|
174
|
-
}
|
|
175
|
-
\`\`\`
|
|
176
|
-
|
|
177
|
-
Goal guidance:
|
|
178
|
-
- Expand the user's one-line monitoring intent into a concise 2-3 sentence monitor goal.
|
|
179
|
-
- State what should trigger an alert, restate any scope the user gave, and include intent-specific exclusions only when obvious from the user's request.
|
|
180
|
-
- Generic noise such as whitespace, formatting-only changes, request IDs, tracking params, generic metadata, and unrelated page chrome is already handled by the judge; do not repeat it in every goal.
|
|
181
|
-
- If the user is vague, keep the goal broad rather than guessing exclusions. If the user asks for broad monitoring or "any change", preserve that and do not add exclusions that hide changes.
|
|
182
|
-
- If the user says they do not care about something, include that explicitly. It is okay to ask whether they want to ignore specific noise when it is likely to matter.
|
|
183
|
-
- Do not invent page-specific sections, thresholds, entities, or business rules unless the user mentioned them.
|
|
184
|
-
|
|
185
|
-
Query guidance (web monitors): \`queries\` control recall (what search retrieves) and \`goal\` controls precision (which results alert) \u2014 tune both.
|
|
186
|
-
- Write keywords, not sentences: \`OpenAI new model release\`, not \`tell me when OpenAI releases a new model\`.
|
|
187
|
-
- Quote multi-word entities (\`"Llama 4"\`); group synonyms with \`OR\` (\`launch OR release OR announcement\`).
|
|
188
|
-
- Keep each query tight (~2-6 terms). One broad query usually beats several narrow ones \u2014 extra queries split the \`maxResults\` budget. Use one query per distinct entity; do not emit one per facet of a single subject.
|
|
189
|
-
- Keep \`site:\` operators out of queries \u2014 use \`includeDomains\` / \`excludeDomains\`.
|
|
190
|
-
- A healthy web monitor mostly returns \`new: 0\` and alerts only on genuinely new, on-goal results. Many \`ignored\` results \u21D2 queries too broad (tighten them); nothing for long stretches \u21D2 queries too narrow or window too tight (broaden); dismissed alerts \u21D2 goal too broad (add an intent-specific Ignore). Aim for high precision with enough recall.
|
|
191
|
-
|
|
192
|
-
Full \`body\` requests require: \`name\`, \`schedule\` (with \`cron\` or \`text\`), and \`targets\` (one or more \`{ type: 'scrape', urls: [...] }\`, \`{ type: 'crawl', url: '...' }\`, or \`{ type: 'search', queries: [...], searchWindow?, maxResults?, includeDomains?, excludeDomains? }\`). Optional: \`goal\` (required when any search target is present), \`judgeEnabled\`, \`webhook\`, \`notification\`, \`retentionDays\`.
|
|
193
|
-
|
|
194
|
-
**Markdown-mode (default):** Each check produces a unified text diff of the page's markdown. No extra configuration needed.
|
|
195
|
-
|
|
196
|
-
\`\`\`json
|
|
197
|
-
{
|
|
198
|
-
"name": "firecrawl_monitor_create",
|
|
199
|
-
"arguments": {
|
|
200
|
-
"page": "https://example.com/blog",
|
|
201
|
-
"goal": "Alert when a new blog post is published or an existing headline changes.",
|
|
202
|
-
"email": "alerts@example.com"
|
|
203
|
-
}
|
|
204
|
-
}
|
|
205
|
-
\`\`\`
|
|
206
|
-
|
|
207
|
-
**Multiple pages:**
|
|
208
|
-
|
|
209
|
-
\`\`\`json
|
|
210
|
-
{
|
|
211
|
-
"name": "firecrawl_monitor_create",
|
|
212
|
-
"arguments": {
|
|
213
|
-
"pages": ["https://example.com/pricing", "https://example.com/changelog"],
|
|
214
|
-
"goal": "Alert when pricing, packaging, or launch messaging changes.",
|
|
215
|
-
"webhookUrl": "https://example.com/webhooks/firecrawl"
|
|
216
|
-
}
|
|
217
|
-
}
|
|
218
|
-
\`\`\`
|
|
219
|
-
|
|
220
|
-
**JSON-mode change tracking:** To detect changes in **specific structured fields** (price, headline, in-stock flag, list items) instead of the whole page, add a \`changeTracking\` format with \`modes: ["json"]\` and a JSON schema to the target's \`scrapeOptions.formats\`. The check response will then carry a per-field diff (keyed by JSON path, e.g. \`plans[0].price\`) and a \`snapshot.json\` with the full current extraction. See \`firecrawl_monitor_check\` for the response shape.
|
|
221
|
-
|
|
222
|
-
\`\`\`json
|
|
223
|
-
{
|
|
224
|
-
"name": "firecrawl_monitor_create",
|
|
225
|
-
"arguments": {
|
|
226
|
-
"body": {
|
|
227
|
-
"name": "Pricing watch",
|
|
228
|
-
"schedule": { "text": "hourly", "timezone": "UTC" },
|
|
229
|
-
"goal": "Alert when a pricing tier, price, billing period, limit, or headline feature changes. Ignore unrelated marketing copy unless it changes the pricing offer.",
|
|
230
|
-
"targets": [{
|
|
231
|
-
"type": "scrape",
|
|
232
|
-
"urls": ["https://example.com/pricing"],
|
|
233
|
-
"scrapeOptions": {
|
|
234
|
-
"formats": [{
|
|
235
|
-
"type": "changeTracking",
|
|
236
|
-
"modes": ["json"],
|
|
237
|
-
"prompt": "Extract pricing tiers and headline features for each plan.",
|
|
238
|
-
"schema": {
|
|
239
|
-
"type": "object",
|
|
240
|
-
"properties": {
|
|
241
|
-
"plans": {
|
|
242
|
-
"type": "array",
|
|
243
|
-
"items": {
|
|
244
|
-
"type": "object",
|
|
245
|
-
"properties": {
|
|
246
|
-
"name": { "type": "string" },
|
|
247
|
-
"price": { "type": "string" },
|
|
248
|
-
"features": { "type": "array", "items": { "type": "string" } }
|
|
249
|
-
}
|
|
250
|
-
}
|
|
251
|
-
}
|
|
252
|
-
}
|
|
253
|
-
}
|
|
254
|
-
}]
|
|
255
|
-
}
|
|
256
|
-
}]
|
|
257
|
-
}
|
|
258
|
-
}
|
|
259
|
-
}
|
|
260
|
-
\`\`\`
|
|
261
|
-
|
|
262
|
-
**Mixed mode (JSON + git-diff):** Use \`modes: ["json", "git-diff"]\` to get both per-field diffs and a markdown sidecar. The page is marked \`changed\` whenever either surface changed.
|
|
273
|
+
In the simple form, a \`goal\` is required. If \`queries\` contains one or more non-empty values and is supplied with \`page\`/\`pages\`, \`queries\` create the search target and page targets are ignored. A monitor schedules future network checks and can send configured email or webhook notifications. Returns the created monitor.
|
|
263
274
|
`,
|
|
264
|
-
parameters:
|
|
265
|
-
body:
|
|
266
|
-
page:
|
|
267
|
-
pages:
|
|
268
|
-
queries:
|
|
269
|
-
searchWindow:
|
|
270
|
-
maxResults:
|
|
271
|
-
includeDomains:
|
|
272
|
-
excludeDomains:
|
|
273
|
-
goal:
|
|
274
|
-
name:
|
|
275
|
-
scheduleText:
|
|
276
|
-
timezone:
|
|
277
|
-
email:
|
|
278
|
-
includeDiffs:
|
|
279
|
-
webhookUrl:
|
|
275
|
+
parameters: z2.object({
|
|
276
|
+
body: z2.record(z2.string(), z2.any()).optional(),
|
|
277
|
+
page: z2.string().optional(),
|
|
278
|
+
pages: z2.array(z2.string()).optional(),
|
|
279
|
+
queries: z2.array(z2.string()).optional(),
|
|
280
|
+
searchWindow: z2.enum(["5m", "15m", "1h", "6h", "24h", "7d"]).optional(),
|
|
281
|
+
maxResults: z2.number().int().min(1).max(50).optional(),
|
|
282
|
+
includeDomains: z2.array(z2.string()).optional(),
|
|
283
|
+
excludeDomains: z2.array(z2.string()).optional(),
|
|
284
|
+
goal: z2.string().optional(),
|
|
285
|
+
name: z2.string().optional(),
|
|
286
|
+
scheduleText: z2.string().optional(),
|
|
287
|
+
timezone: z2.string().optional(),
|
|
288
|
+
email: z2.string().optional(),
|
|
289
|
+
includeDiffs: z2.boolean().optional(),
|
|
290
|
+
webhookUrl: z2.string().optional()
|
|
280
291
|
}),
|
|
281
292
|
execute: async (args2, { session, log }) => {
|
|
282
293
|
const body = buildMonitorCreateBody(args2);
|
|
@@ -300,16 +311,11 @@ Full \`body\` requests require: \`name\`, \`schedule\` (with \`cron\` or \`text\
|
|
|
300
311
|
// Read-only listing.
|
|
301
312
|
},
|
|
302
313
|
description: `
|
|
303
|
-
List
|
|
304
|
-
|
|
305
|
-
**Usage Example:**
|
|
306
|
-
\`\`\`json
|
|
307
|
-
{ "name": "firecrawl_monitor_list", "arguments": { "limit": 20 } }
|
|
308
|
-
\`\`\`
|
|
314
|
+
List monitors for the authenticated account with optional pagination controls. Returns one page of monitor records and pagination metadata.
|
|
309
315
|
`,
|
|
310
|
-
parameters:
|
|
311
|
-
limit:
|
|
312
|
-
offset:
|
|
316
|
+
parameters: z2.object({
|
|
317
|
+
limit: z2.number().int().positive().optional(),
|
|
318
|
+
offset: z2.number().int().nonnegative().optional()
|
|
313
319
|
}),
|
|
314
320
|
execute: async (args2, { session }) => {
|
|
315
321
|
const { limit, offset } = args2;
|
|
@@ -331,14 +337,9 @@ List all Firecrawl monitors for the authenticated account.
|
|
|
331
337
|
// Read-only retrieval.
|
|
332
338
|
},
|
|
333
339
|
description: `
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
**Usage Example:**
|
|
337
|
-
\`\`\`json
|
|
338
|
-
{ "name": "firecrawl_monitor_get", "arguments": { "id": "mon_abc123" } }
|
|
339
|
-
\`\`\`
|
|
340
|
+
Retrieve one monitor by ID, including its configuration and current state. This does not run or modify the monitor.
|
|
340
341
|
`,
|
|
341
|
-
parameters:
|
|
342
|
+
parameters: z2.object({ id: z2.string() }),
|
|
342
343
|
execute: async (args2, { session }) => {
|
|
343
344
|
const { id } = args2;
|
|
344
345
|
const res = await monitorRequest(
|
|
@@ -360,22 +361,13 @@ Get a single monitor by ID.
|
|
|
360
361
|
// Can pause, replace, or remove monitor configuration; changes overwrite prior settings.
|
|
361
362
|
},
|
|
362
363
|
description: `
|
|
363
|
-
|
|
364
|
+
Patch an existing monitor by ID. The body can change its name, active/paused status, schedule, targets, goal, judging, webhook, notifications, or retention; these changes affect future scheduled checks.
|
|
364
365
|
|
|
365
|
-
|
|
366
|
-
\`\`\`json
|
|
367
|
-
{
|
|
368
|
-
"name": "firecrawl_monitor_update",
|
|
369
|
-
"arguments": {
|
|
370
|
-
"id": "mon_abc123",
|
|
371
|
-
"body": { "status": "paused" }
|
|
372
|
-
}
|
|
373
|
-
}
|
|
374
|
-
\`\`\`
|
|
366
|
+
Returns the updated monitor.
|
|
375
367
|
`,
|
|
376
|
-
parameters:
|
|
377
|
-
id:
|
|
378
|
-
body:
|
|
368
|
+
parameters: z2.object({
|
|
369
|
+
id: z2.string(),
|
|
370
|
+
body: z2.record(z2.string(), z2.any())
|
|
379
371
|
}),
|
|
380
372
|
execute: async (args2, { session }) => {
|
|
381
373
|
const { id, body } = args2;
|
|
@@ -399,14 +391,9 @@ Update a monitor. Pass any subset of fields to patch: \`name\`, \`status\` ("act
|
|
|
399
391
|
// Irreversibly removes the monitor and stops its schedule.
|
|
400
392
|
},
|
|
401
393
|
description: `
|
|
402
|
-
Permanently delete a monitor and stop its schedule. This cannot be undone.
|
|
403
|
-
|
|
404
|
-
**Usage Example:**
|
|
405
|
-
\`\`\`json
|
|
406
|
-
{ "name": "firecrawl_monitor_delete", "arguments": { "id": "mon_abc123" } }
|
|
407
|
-
\`\`\`
|
|
394
|
+
Permanently delete a monitor by ID and stop its future schedule. This operation cannot be undone and returns deletion status.
|
|
408
395
|
`,
|
|
409
|
-
parameters:
|
|
396
|
+
parameters: z2.object({ id: z2.string() }),
|
|
410
397
|
execute: async (args2, { session, log }) => {
|
|
411
398
|
const { id } = args2;
|
|
412
399
|
log.info("Deleting monitor", { id });
|
|
@@ -430,14 +417,9 @@ Permanently delete a monitor and stop its schedule. This cannot be undone.
|
|
|
430
417
|
// Starts a read-only check job; does not delete the monitor or external sites.
|
|
431
418
|
},
|
|
432
419
|
description: `
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
**Usage Example:**
|
|
436
|
-
\`\`\`json
|
|
437
|
-
{ "name": "firecrawl_monitor_run", "arguments": { "id": "mon_abc123" } }
|
|
438
|
-
\`\`\`
|
|
420
|
+
Queue an immediate check for a monitor outside its normal schedule. This starts network work for the monitor's configured targets and returns the queued check.
|
|
439
421
|
`,
|
|
440
|
-
parameters:
|
|
422
|
+
parameters: z2.object({ id: z2.string() }),
|
|
441
423
|
execute: async (args2, { session }) => {
|
|
442
424
|
const { id } = args2;
|
|
443
425
|
const res = await monitorRequest(
|
|
@@ -460,17 +442,12 @@ Trigger a monitor check immediately, outside its normal schedule. Returns the qu
|
|
|
460
442
|
// Read-only listing.
|
|
461
443
|
},
|
|
462
444
|
description: `
|
|
463
|
-
List historical checks for a monitor.
|
|
464
|
-
|
|
465
|
-
**Usage Example:**
|
|
466
|
-
\`\`\`json
|
|
467
|
-
{ "name": "firecrawl_monitor_checks", "arguments": { "id": "mon_abc123", "limit": 10, "status": "completed" } }
|
|
468
|
-
\`\`\`
|
|
445
|
+
List historical checks for a monitor, optionally filtered by status and bounded by a result limit. Returns one page of check summaries and pagination metadata.
|
|
469
446
|
`,
|
|
470
|
-
parameters:
|
|
471
|
-
id:
|
|
472
|
-
limit:
|
|
473
|
-
offset:
|
|
447
|
+
parameters: z2.object({
|
|
448
|
+
id: z2.string(),
|
|
449
|
+
limit: z2.number().int().positive().optional(),
|
|
450
|
+
offset: z2.number().int().nonnegative().optional(),
|
|
474
451
|
status: checkStatusSchema.optional()
|
|
475
452
|
}),
|
|
476
453
|
execute: async (args2, { session }) => {
|
|
@@ -495,66 +472,15 @@ List historical checks for a monitor.
|
|
|
495
472
|
// Read-only retrieval of diff snapshots and judgments.
|
|
496
473
|
},
|
|
497
474
|
description: `
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
Each entry in \`data.pages[]\` has \`url\`, \`status\` (\`same\` | \`new\` | \`changed\` | \`removed\` | \`error\`), optional \`judgment\` when goal-based judging ran, and \u2014 when changed \u2014 a \`diff\` and possibly a \`snapshot\`. The shape of \`diff\` depends on the monitor's \`formats\` configuration:
|
|
475
|
+
Retrieve one monitor check and its page-level results, optionally filtered by page status. Pages report \`same\`, \`new\`, \`changed\`, \`removed\`, or \`error\`; configured goal judging can add a meaningful-change decision.
|
|
501
476
|
|
|
502
|
-
|
|
503
|
-
- **JSON mode** (\`changeTracking\` with \`modes: ["json"]\`). \`diff.json\` is a per-field map keyed by JSON path into the extraction, e.g. \`plans[0].price\`, with each value being \`{ previous, current }\`. \`snapshot.json\` is the full current extraction. No \`diff.text\`.
|
|
504
|
-
- **Mixed mode** (\`modes: ["json", "git-diff"]\`). Both \`diff.text\` (markdown sidecar) AND \`diff.json\` (per-field map) are present, plus \`snapshot.json\`.
|
|
505
|
-
|
|
506
|
-
**Example JSON-mode response \`pages[]\` entry:**
|
|
507
|
-
|
|
508
|
-
\`\`\`json
|
|
509
|
-
{
|
|
510
|
-
"url": "https://example.com/pricing",
|
|
511
|
-
"status": "changed",
|
|
512
|
-
"diff": {
|
|
513
|
-
"json": {
|
|
514
|
-
"plans[0].price": { "previous": "$19/mo", "current": "$24/mo" },
|
|
515
|
-
"plans[1].features[2]": { "previous": "10 GB storage", "current": "25 GB storage" }
|
|
516
|
-
}
|
|
517
|
-
},
|
|
518
|
-
"snapshot": { "json": { "plans": [/* current full extraction matching the monitor's schema */] } },
|
|
519
|
-
"judgment": {
|
|
520
|
-
"meaningful": true,
|
|
521
|
-
"confidence": "high",
|
|
522
|
-
"reason": "The pricing changed, which matches the monitor goal.",
|
|
523
|
-
"meaningfulChanges": [
|
|
524
|
-
{
|
|
525
|
-
"type": "changed",
|
|
526
|
-
"before": "$19/mo",
|
|
527
|
-
"after": "$24/mo",
|
|
528
|
-
"reason": "The tracked plan price changed."
|
|
529
|
-
}
|
|
530
|
-
]
|
|
531
|
-
}
|
|
532
|
-
}
|
|
533
|
-
\`\`\`
|
|
534
|
-
|
|
535
|
-
When summarizing a check for the user, prefer \`diff.json\` paths (e.g. "plans[0].price changed from $19/mo to $24/mo") over re-printing the markdown diff \u2014 it's more concise and grounded in the schema fields they asked for.
|
|
536
|
-
|
|
537
|
-
When \`judgment\` is present, use it to decide what to surface. \`judgment.meaningful: false\` means the change was classified as noise for the monitor's goal. When \`judgment.meaningfulChanges\` is present, prefer those goal-relevant changes over raw diff hunks; each item includes \`type\`, \`before\`, \`after\`, and \`reason\`.
|
|
538
|
-
|
|
539
|
-
The endpoint paginates via a top-level \`next\` URL; this tool returns one page at a time. Increase \`limit\` (max 100) to fetch fewer pages.
|
|
540
|
-
|
|
541
|
-
**Usage Example:**
|
|
542
|
-
\`\`\`json
|
|
543
|
-
{
|
|
544
|
-
"name": "firecrawl_monitor_check",
|
|
545
|
-
"arguments": {
|
|
546
|
-
"id": "mon_abc123",
|
|
547
|
-
"checkId": "chk_xyz",
|
|
548
|
-
"pageStatus": "changed"
|
|
549
|
-
}
|
|
550
|
-
}
|
|
551
|
-
\`\`\`
|
|
477
|
+
Markdown tracking returns a unified text diff, JSON tracking returns field paths with previous/current values and a current snapshot, and mixed tracking returns both. Returns one page of results plus a \`next\` URL when more pages exist.
|
|
552
478
|
`,
|
|
553
|
-
parameters:
|
|
554
|
-
id:
|
|
555
|
-
checkId:
|
|
556
|
-
limit:
|
|
557
|
-
skip:
|
|
479
|
+
parameters: z2.object({
|
|
480
|
+
id: z2.string(),
|
|
481
|
+
checkId: z2.string(),
|
|
482
|
+
limit: z2.number().int().positive().optional(),
|
|
483
|
+
skip: z2.number().int().nonnegative().optional(),
|
|
558
484
|
pageStatus: pageStatusSchema.optional()
|
|
559
485
|
}),
|
|
560
486
|
execute: async (args2, { session }) => {
|
|
@@ -570,9 +496,9 @@ The endpoint paginates via a top-level \`next\` URL; this tool returns one page
|
|
|
570
496
|
}
|
|
571
497
|
|
|
572
498
|
// src/research.ts
|
|
573
|
-
import { z as
|
|
574
|
-
var
|
|
575
|
-
var
|
|
499
|
+
import { z as z3 } from "zod";
|
|
500
|
+
var BASE2 = "/v2/search/research";
|
|
501
|
+
var ORIGIN_HEADERS2 = { "X-Origin": "mcp-fastmcp" };
|
|
576
502
|
function appendParam(params, key, value) {
|
|
577
503
|
if (value == null) return;
|
|
578
504
|
if (Array.isArray(value)) {
|
|
@@ -635,7 +561,7 @@ function fmtPaperMetadata(paper) {
|
|
|
635
561
|
lines.push("");
|
|
636
562
|
lines.push(`Paper ID: ${paper.paperId ?? "?"}`);
|
|
637
563
|
const ids = Object.entries(paper.ids ?? {}).flatMap(
|
|
638
|
-
([namespace,
|
|
564
|
+
([namespace, values2]) => values2.map((value) => `${namespace}:${value}`)
|
|
639
565
|
).join(", ");
|
|
640
566
|
if (ids) lines.push(`IDs: ${ids}`);
|
|
641
567
|
const authors = fmtAuthors(paper.authors);
|
|
@@ -689,22 +615,26 @@ function registerResearchTools(server2, getClient2) {
|
|
|
689
615
|
destructiveHint: false
|
|
690
616
|
// Query-only; no writes to external sources or the research index.
|
|
691
617
|
},
|
|
692
|
-
description:
|
|
693
|
-
|
|
694
|
-
|
|
618
|
+
description: `
|
|
619
|
+
For topics represented in the indexed corpus, search paper metadata and abstracts with a natural-language query. Optional author, category, and date filters constrain results.
|
|
620
|
+
|
|
621
|
+
Returns ranked papers with canonical IDs, titles, authors, and abstracts.
|
|
622
|
+
`,
|
|
623
|
+
parameters: z3.object({
|
|
624
|
+
query: z3.string().min(1).describe(
|
|
695
625
|
"Natural-language research topic or question, including methods, systems, conditions, populations, interventions, or outcomes when relevant."
|
|
696
626
|
),
|
|
697
|
-
k:
|
|
698
|
-
authors:
|
|
627
|
+
k: z3.number().int().min(1).max(500).optional().describe("Number of ranked papers to return (default 40)."),
|
|
628
|
+
authors: z3.array(z3.string()).optional().describe(
|
|
699
629
|
"Author substring filter(s); ALL must match (case-insensitive)."
|
|
700
630
|
),
|
|
701
|
-
categories:
|
|
631
|
+
categories: z3.array(z3.string()).optional().describe(
|
|
702
632
|
"Paper category filter(s) (e.g. `cs.LG`); ALL provided values must match."
|
|
703
633
|
),
|
|
704
|
-
from:
|
|
634
|
+
from: z3.string().optional().describe(
|
|
705
635
|
"Inclusive lower bound on created/updated date (`YYYY-MM-DD`)."
|
|
706
636
|
),
|
|
707
|
-
to:
|
|
637
|
+
to: z3.string().optional().describe(
|
|
708
638
|
"Inclusive upper bound on created/updated date (`YYYY-MM-DD`)."
|
|
709
639
|
)
|
|
710
640
|
}),
|
|
@@ -719,8 +649,8 @@ function registerResearchTools(server2, getClient2) {
|
|
|
719
649
|
appendParam(params, "to", to);
|
|
720
650
|
const client = getClient2(session);
|
|
721
651
|
const res = await client.http.get(
|
|
722
|
-
withQuery(`${
|
|
723
|
-
|
|
652
|
+
withQuery(`${BASE2}/papers`, params),
|
|
653
|
+
ORIGIN_HEADERS2
|
|
724
654
|
);
|
|
725
655
|
return fmtHits(res.data?.results);
|
|
726
656
|
}
|
|
@@ -736,9 +666,11 @@ function registerResearchTools(server2, getClient2) {
|
|
|
736
666
|
destructiveHint: false
|
|
737
667
|
// Read-only metadata lookup.
|
|
738
668
|
},
|
|
739
|
-
description:
|
|
740
|
-
|
|
741
|
-
|
|
669
|
+
description: `
|
|
670
|
+
Retrieve canonical metadata for one paper ID, such as an arXiv, PMC, PMID, or DOI identifier. Returns the title, abstract, authors, categories, source IDs, and dates as markdown.
|
|
671
|
+
`,
|
|
672
|
+
parameters: z3.object({
|
|
673
|
+
paperId: z3.string().min(1).describe(
|
|
742
674
|
"Canonical paperId or primaryId such as `arxiv:1706.03762`, `pmcid:PMC12530322`, `pmid:40953549`, or `doi:10.1016/j.neunet.2025.108095`."
|
|
743
675
|
)
|
|
744
676
|
}),
|
|
@@ -746,8 +678,8 @@ function registerResearchTools(server2, getClient2) {
|
|
|
746
678
|
const { paperId } = args2;
|
|
747
679
|
const client = getClient2(session);
|
|
748
680
|
const res = await client.http.get(
|
|
749
|
-
`${
|
|
750
|
-
|
|
681
|
+
`${BASE2}/papers/${encodeURIComponent(paperId)}`,
|
|
682
|
+
ORIGIN_HEADERS2
|
|
751
683
|
);
|
|
752
684
|
return fmtPaperMetadata(res.data?.paper);
|
|
753
685
|
}
|
|
@@ -763,13 +695,17 @@ function registerResearchTools(server2, getClient2) {
|
|
|
763
695
|
destructiveHint: false
|
|
764
696
|
// Read-only graph query; no modifications.
|
|
765
697
|
},
|
|
766
|
-
description:
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
698
|
+
description: `
|
|
699
|
+
Find citation-graph candidates from one to ten \`seed_ids\`; the first ID is the primary seed and later IDs are anchors. \`mode\` defaults to \`similar\` (co-citation/bibliographic coupling); \`citers\` returns papers citing a seed and \`references\` papers cited by a seed. \`intent\` ranks candidates.
|
|
700
|
+
|
|
701
|
+
Returns ranked candidates and the evaluated pool size.
|
|
702
|
+
`,
|
|
703
|
+
parameters: z3.object({
|
|
704
|
+
seed_ids: z3.array(z3.string()).min(1).max(10),
|
|
705
|
+
intent: z3.string().min(1),
|
|
706
|
+
mode: z3.enum(["similar", "citers", "references"]).optional(),
|
|
707
|
+
k: z3.number().int().min(1).max(500).optional(),
|
|
708
|
+
rerank: z3.boolean().optional().describe("Apply an additional rerank over the fused candidates.")
|
|
773
709
|
}),
|
|
774
710
|
execute: async (args2, { session }) => {
|
|
775
711
|
const { seed_ids, intent, mode, k, rerank } = args2;
|
|
@@ -783,10 +719,10 @@ function registerResearchTools(server2, getClient2) {
|
|
|
783
719
|
const client = getClient2(session);
|
|
784
720
|
const res = await client.http.get(
|
|
785
721
|
withQuery(
|
|
786
|
-
`${
|
|
722
|
+
`${BASE2}/papers/${encodeURIComponent(primary)}/similar`,
|
|
787
723
|
params
|
|
788
724
|
),
|
|
789
|
-
|
|
725
|
+
ORIGIN_HEADERS2
|
|
790
726
|
);
|
|
791
727
|
const note = res.data?.note ? `
|
|
792
728
|
note: ${res.data.note}` : "";
|
|
@@ -805,13 +741,17 @@ note: ${res.data.note}` : "";
|
|
|
805
741
|
destructiveHint: false
|
|
806
742
|
// Read-only passage retrieval.
|
|
807
743
|
},
|
|
808
|
-
description:
|
|
809
|
-
|
|
810
|
-
|
|
744
|
+
description: `
|
|
745
|
+
Retrieve in-body passages from one paper that are relevant to a specific question. Full text is available only for indexed papers; \`k\` controls the number of passages.
|
|
746
|
+
|
|
747
|
+
Returns matching passages or a notice when full text is unavailable.
|
|
748
|
+
`,
|
|
749
|
+
parameters: z3.object({
|
|
750
|
+
paperId: z3.string().min(1).describe(
|
|
811
751
|
"Canonical paperId or primaryId such as `arxiv:1706.03762`, `pmcid:PMC12530322`, `pmid:40953549`, or `doi:10.1016/j.neunet.2025.108095`."
|
|
812
752
|
),
|
|
813
|
-
question:
|
|
814
|
-
k:
|
|
753
|
+
question: z3.string().min(1),
|
|
754
|
+
k: z3.number().int().min(1).max(50).optional().describe("Number of passages to return (default 4).")
|
|
815
755
|
}),
|
|
816
756
|
execute: async (args2, { session }) => {
|
|
817
757
|
const { paperId, question, k } = args2;
|
|
@@ -820,8 +760,8 @@ note: ${res.data.note}` : "";
|
|
|
820
760
|
appendParam(params, "k", k);
|
|
821
761
|
const client = getClient2(session);
|
|
822
762
|
const res = await client.http.get(
|
|
823
|
-
withQuery(`${
|
|
824
|
-
|
|
763
|
+
withQuery(`${BASE2}/papers/${encodeURIComponent(paperId)}`, params),
|
|
764
|
+
ORIGIN_HEADERS2
|
|
825
765
|
);
|
|
826
766
|
const passages = res.data?.passages ?? [];
|
|
827
767
|
return passages.length ? passages.map((p) => p.text).join("\n---\n") : "(no full-text passages available for this paper)";
|
|
@@ -838,10 +778,12 @@ note: ${res.data.note}` : "";
|
|
|
838
778
|
destructiveHint: false
|
|
839
779
|
// Query-only; does not create issues, PRs, or modify repositories.
|
|
840
780
|
},
|
|
841
|
-
description:
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
781
|
+
description: `
|
|
782
|
+
Search indexed public GitHub issue, pull-request, and README content. Returns ranked matches with repository, URL, snippet, and full matched markdown when available.
|
|
783
|
+
`,
|
|
784
|
+
parameters: z3.object({
|
|
785
|
+
query: z3.string().min(1),
|
|
786
|
+
k: z3.number().int().min(1).max(100).optional()
|
|
845
787
|
}),
|
|
846
788
|
execute: async (args2, { session }) => {
|
|
847
789
|
const { query, k } = args2;
|
|
@@ -850,8 +792,8 @@ note: ${res.data.note}` : "";
|
|
|
850
792
|
appendParam(params, "k", k);
|
|
851
793
|
const client = getClient2(session);
|
|
852
794
|
const res = await client.http.get(
|
|
853
|
-
withQuery(`${
|
|
854
|
-
|
|
795
|
+
withQuery(`${BASE2}/github`, params),
|
|
796
|
+
ORIGIN_HEADERS2
|
|
855
797
|
);
|
|
856
798
|
return fmtGithub(res.data?.results);
|
|
857
799
|
}
|
|
@@ -878,6 +820,22 @@ function extractBearerToken(headers) {
|
|
|
878
820
|
function isFirecrawlOAuthAccessToken(token) {
|
|
879
821
|
return token.startsWith("fco_");
|
|
880
822
|
}
|
|
823
|
+
function isFirecrawlApiKey(token) {
|
|
824
|
+
return token.startsWith("fc-");
|
|
825
|
+
}
|
|
826
|
+
function isLegacyKeyPathRequest(request) {
|
|
827
|
+
return normalizeHeader(request?.headers?.["x-firecrawl-key-transport"]) === "path";
|
|
828
|
+
}
|
|
829
|
+
function requestShouldReceiveOAuthChallenge(request, profile) {
|
|
830
|
+
if (!profile.acceptApiKeys) return true;
|
|
831
|
+
if (!request?.headers) return true;
|
|
832
|
+
const headerApiKey = normalizeHeader(
|
|
833
|
+
request.headers["x-firecrawl-api-key"] ?? request.headers["x-api-key"]
|
|
834
|
+
);
|
|
835
|
+
if (headerApiKey) return false;
|
|
836
|
+
const bearer = extractBearerToken(request.headers);
|
|
837
|
+
return !bearer || isFirecrawlOAuthAccessToken(bearer);
|
|
838
|
+
}
|
|
881
839
|
function resolveCredentialFromEnv() {
|
|
882
840
|
return normalizeHeader(process.env.FIRECRAWL_OAUTH_TOKEN) ?? normalizeHeader(process.env.FIRECRAWL_API_KEY);
|
|
883
841
|
}
|
|
@@ -886,6 +844,9 @@ function isHttpStreamingTransport() {
|
|
|
886
844
|
}
|
|
887
845
|
var DEFAULT_OAUTH_ISSUER = "https://www.firecrawl.dev";
|
|
888
846
|
var DEFAULT_MCP_RESOURCE_URL = "https://mcp.firecrawl.dev/v2/mcp";
|
|
847
|
+
var DEFAULT_MCP_OAUTH_RESOURCE_URL = "https://mcp.firecrawl.dev/v2/mcp-oauth";
|
|
848
|
+
var DEFAULT_MCP_SEARCH_RESOURCE_URL = "https://mcp.firecrawl.dev/v2/mcp-search";
|
|
849
|
+
var DEFAULT_MCP_SEARCH_ENDPOINT = "/v2/mcp-search";
|
|
889
850
|
function withoutTrailingSlash(value) {
|
|
890
851
|
return value.replace(/\/+$/, "");
|
|
891
852
|
}
|
|
@@ -897,19 +858,40 @@ function getOAuthIssuer() {
|
|
|
897
858
|
function getMcpResourceUrl() {
|
|
898
859
|
return normalizeHeader(process.env.FIRECRAWL_MCP_RESOURCE_URL) ?? DEFAULT_MCP_RESOURCE_URL;
|
|
899
860
|
}
|
|
900
|
-
function
|
|
901
|
-
|
|
861
|
+
function getPrimaryEndpoint() {
|
|
862
|
+
const endpoint = normalizeHeader(process.env.FASTMCP_ENDPOINT) ?? "/v2/mcp";
|
|
863
|
+
if (endpoint === "/v2/mcp" || endpoint === "/v2/mcp-oauth" || endpoint === "/v2/mcp-search") {
|
|
864
|
+
return endpoint;
|
|
865
|
+
}
|
|
866
|
+
throw new Error(
|
|
867
|
+
`Unsupported FASTMCP_ENDPOINT: ${endpoint}. Expected /v2/mcp, /v2/mcp-oauth, or /v2/mcp-search.`
|
|
868
|
+
);
|
|
869
|
+
}
|
|
870
|
+
function getSearchMcpResourceUrl() {
|
|
871
|
+
return normalizeHeader(process.env.FIRECRAWL_MCP_SEARCH_RESOURCE_URL) ?? DEFAULT_MCP_SEARCH_RESOURCE_URL;
|
|
872
|
+
}
|
|
873
|
+
function getSearchMcpEndpoint() {
|
|
874
|
+
const configured = normalizeHeader(process.env.FIRECRAWL_MCP_SEARCH_ENDPOINT);
|
|
875
|
+
if (configured && configured.startsWith("/")) {
|
|
876
|
+
return configured;
|
|
877
|
+
}
|
|
878
|
+
return DEFAULT_MCP_SEARCH_ENDPOINT;
|
|
879
|
+
}
|
|
880
|
+
function getOAuthProtectedResourceMetadataUrl(profile) {
|
|
881
|
+
const resource = new URL(profile.resourceUrl);
|
|
882
|
+
const base = `${resource.origin}/.well-known/oauth-protected-resource`;
|
|
883
|
+
return profile.id === "full" ? base : `${base}${resource.pathname}`;
|
|
902
884
|
}
|
|
903
885
|
function escapeWWWAuthenticateValue(value) {
|
|
904
886
|
return value.replace(/\\/g, "\\\\").replace(/"/g, '\\"');
|
|
905
887
|
}
|
|
906
|
-
function createOAuthChallengeResponse(error) {
|
|
888
|
+
function createOAuthChallengeResponse(error, profile) {
|
|
907
889
|
if (!isMcpOAuthEnabled()) {
|
|
908
890
|
return void 0;
|
|
909
891
|
}
|
|
910
892
|
const errorMessage = error instanceof Error ? error.message : String(error || "Unauthorized");
|
|
911
893
|
const wwwAuthenticate = [
|
|
912
|
-
`resource_metadata="${escapeWWWAuthenticateValue(getOAuthProtectedResourceMetadataUrl())}"`,
|
|
894
|
+
`resource_metadata="${escapeWWWAuthenticateValue(getOAuthProtectedResourceMetadataUrl(profile))}"`,
|
|
913
895
|
'error="invalid_token"',
|
|
914
896
|
`error_description="${escapeWWWAuthenticateValue(errorMessage)}"`
|
|
915
897
|
].join(", ");
|
|
@@ -927,6 +909,18 @@ function createOAuthChallengeResponse(error) {
|
|
|
927
909
|
}
|
|
928
910
|
);
|
|
929
911
|
}
|
|
912
|
+
function createInvalidCredentialResponse(error) {
|
|
913
|
+
return new Response(
|
|
914
|
+
JSON.stringify({
|
|
915
|
+
error: "invalid_api_key",
|
|
916
|
+
error_description: error.message
|
|
917
|
+
}),
|
|
918
|
+
{
|
|
919
|
+
headers: { "Content-Type": "application/json" },
|
|
920
|
+
status: 401
|
|
921
|
+
}
|
|
922
|
+
);
|
|
923
|
+
}
|
|
930
924
|
function getOAuthIntrospectionEndpoint() {
|
|
931
925
|
return `${getOAuthIssuer()}/api/oauth/introspect`;
|
|
932
926
|
}
|
|
@@ -936,63 +930,168 @@ function getOAuthIntrospectionSecret() {
|
|
|
936
930
|
function isMcpOAuthEnabled() {
|
|
937
931
|
return process.env.CLOUD_SERVICE === "true";
|
|
938
932
|
}
|
|
939
|
-
|
|
933
|
+
function isOAuthCredentialPurpose(value) {
|
|
934
|
+
return value === "general" || value === "hosted_mcp_oauth";
|
|
935
|
+
}
|
|
936
|
+
var InvalidFirecrawlCredentialError = class extends Error {
|
|
937
|
+
constructor() {
|
|
938
|
+
super("The supplied Firecrawl credential is invalid or revoked. Replace it and retry.");
|
|
939
|
+
this.name = "InvalidFirecrawlCredentialError";
|
|
940
|
+
}
|
|
941
|
+
};
|
|
942
|
+
var MCP_GLOBAL_SCOPE = "firecrawl:global";
|
|
943
|
+
function values(value) {
|
|
944
|
+
if (typeof value === "string") return value.split(/\s+/).filter(Boolean);
|
|
945
|
+
return Array.isArray(value) ? value.flatMap((item) => item.split(/\s+/).filter(Boolean)) : [];
|
|
946
|
+
}
|
|
947
|
+
function audienceMatchesResource(aud, resourceUrl) {
|
|
948
|
+
const target = withoutTrailingSlash(resourceUrl);
|
|
949
|
+
return values(aud).some((entry) => withoutTrailingSlash(entry) === target);
|
|
950
|
+
}
|
|
951
|
+
function credentialMetadata(data) {
|
|
952
|
+
return {
|
|
953
|
+
teamId: typeof data.team_id === "string" ? data.team_id : void 0,
|
|
954
|
+
userId: typeof data.sub === "string" ? data.sub : void 0,
|
|
955
|
+
apiKeyId: typeof data.api_key_id === "string" ? data.api_key_id : void 0,
|
|
956
|
+
oauthClientId: typeof data.client_id === "string" ? data.client_id : void 0,
|
|
957
|
+
resource: typeof data.aud === "string" ? data.aud : void 0
|
|
958
|
+
};
|
|
959
|
+
}
|
|
960
|
+
async function introspectToken(token, expectedResource) {
|
|
940
961
|
const introspectionSecret = getOAuthIntrospectionSecret();
|
|
941
|
-
if (!introspectionSecret)
|
|
942
|
-
|
|
962
|
+
if (!introspectionSecret) throw new CredentialValidationUnavailableError();
|
|
963
|
+
const controller = new AbortController();
|
|
964
|
+
const timeout = setTimeout(() => controller.abort(), 1500);
|
|
965
|
+
let response;
|
|
966
|
+
try {
|
|
967
|
+
response = await fetch(getOAuthIntrospectionEndpoint(), {
|
|
968
|
+
method: "POST",
|
|
969
|
+
headers: {
|
|
970
|
+
"Content-Type": "application/x-www-form-urlencoded",
|
|
971
|
+
Authorization: `Bearer ${introspectionSecret}`
|
|
972
|
+
},
|
|
973
|
+
body: new URLSearchParams({
|
|
974
|
+
resource: expectedResource,
|
|
975
|
+
token,
|
|
976
|
+
token_type_hint: "access_token"
|
|
977
|
+
}),
|
|
978
|
+
signal: controller.signal
|
|
979
|
+
});
|
|
980
|
+
} catch {
|
|
981
|
+
throw new CredentialValidationUnavailableError();
|
|
982
|
+
} finally {
|
|
983
|
+
clearTimeout(timeout);
|
|
943
984
|
}
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
Authorization: `Bearer ${introspectionSecret}`
|
|
949
|
-
},
|
|
950
|
-
body: new URLSearchParams({
|
|
951
|
-
token,
|
|
952
|
-
token_type_hint: "access_token"
|
|
953
|
-
})
|
|
954
|
-
});
|
|
955
|
-
if (!response.ok) {
|
|
956
|
-
throw new Error(`OAuth token introspection failed: ${response.status}`);
|
|
985
|
+
if (!response.ok) throw new CredentialValidationUnavailableError();
|
|
986
|
+
const contentType = response.headers.get("content-type")?.toLowerCase() ?? "";
|
|
987
|
+
if (!contentType.includes("application/json")) {
|
|
988
|
+
throw new CredentialValidationUnavailableError();
|
|
957
989
|
}
|
|
958
990
|
const data = await response.json();
|
|
959
|
-
if (
|
|
960
|
-
throw new
|
|
991
|
+
if (typeof data.active !== "boolean") {
|
|
992
|
+
throw new CredentialValidationUnavailableError();
|
|
993
|
+
}
|
|
994
|
+
if (data.active && (!data.api_key || !isOAuthCredentialPurpose(data.credential_purpose) || !values(data.scope).includes(MCP_GLOBAL_SCOPE))) {
|
|
995
|
+
throw new CredentialValidationUnavailableError();
|
|
961
996
|
}
|
|
962
|
-
return data
|
|
997
|
+
return data;
|
|
963
998
|
}
|
|
964
|
-
async function resolveCredentialFromHeaders(headers) {
|
|
999
|
+
async function resolveCredentialFromHeaders(headers, profile) {
|
|
965
1000
|
const bearer = extractBearerToken(headers);
|
|
966
1001
|
const headerApiKey = normalizeHeader(
|
|
967
1002
|
headers["x-firecrawl-api-key"] ?? headers["x-api-key"]
|
|
968
1003
|
);
|
|
969
|
-
|
|
970
|
-
|
|
1004
|
+
const token = headerApiKey ?? bearer;
|
|
1005
|
+
if (!token) return void 0;
|
|
1006
|
+
if (!profile.acceptApiKeys && !isFirecrawlOAuthAccessToken(token)) {
|
|
1007
|
+
throw new Error(
|
|
1008
|
+
`OAuth access token required for the Firecrawl MCP resource ${profile.endpoint}`
|
|
1009
|
+
);
|
|
1010
|
+
}
|
|
1011
|
+
if (!isFirecrawlOAuthAccessToken(token) && !isFirecrawlApiKey(token)) {
|
|
1012
|
+
return { invalid: true };
|
|
1013
|
+
}
|
|
1014
|
+
let data = await introspectToken(token, profile.resourceUrl);
|
|
1015
|
+
if (isFirecrawlOAuthAccessToken(token) && !data.active && profile.acceptLegacyAudience) {
|
|
1016
|
+
data = await introspectToken(token, DEFAULT_MCP_RESOURCE_URL);
|
|
1017
|
+
}
|
|
1018
|
+
if (!data.active || !data.api_key) {
|
|
1019
|
+
if (isFirecrawlOAuthAccessToken(token)) {
|
|
1020
|
+
throw new Error("Invalid OAuth access token");
|
|
1021
|
+
}
|
|
1022
|
+
return { invalid: true };
|
|
971
1023
|
}
|
|
972
|
-
if (
|
|
973
|
-
return
|
|
1024
|
+
if (isFirecrawlApiKey(token)) {
|
|
1025
|
+
return data.credential_purpose === "general" ? {
|
|
1026
|
+
credential: data.api_key,
|
|
1027
|
+
source: "api-key",
|
|
1028
|
+
metadata: credentialMetadata(data)
|
|
1029
|
+
} : { invalid: true };
|
|
974
1030
|
}
|
|
975
|
-
|
|
976
|
-
|
|
1031
|
+
const expectedAudience = profile.acceptLegacyAudience && audienceMatchesResource(data.aud, DEFAULT_MCP_RESOURCE_URL) ? DEFAULT_MCP_RESOURCE_URL : profile.resourceUrl;
|
|
1032
|
+
if (!audienceMatchesResource(data.aud, expectedAudience)) {
|
|
1033
|
+
throw new Error("OAuth token audience does not match this resource");
|
|
977
1034
|
}
|
|
978
|
-
|
|
1035
|
+
if (profile.requireManagedOAuth && data.credential_purpose !== "hosted_mcp_oauth") {
|
|
1036
|
+
throw new Error("OAuth token is not a managed Firecrawl MCP credential");
|
|
1037
|
+
}
|
|
1038
|
+
if (data.credential_purpose === "hosted_mcp_oauth") {
|
|
1039
|
+
requireDelegatedCredentialSigning();
|
|
1040
|
+
return {
|
|
1041
|
+
managedOAuthApiKey: data.api_key,
|
|
1042
|
+
source: "oauth",
|
|
1043
|
+
metadata: credentialMetadata(data)
|
|
1044
|
+
};
|
|
1045
|
+
}
|
|
1046
|
+
return {
|
|
1047
|
+
credential: data.api_key,
|
|
1048
|
+
source: "oauth",
|
|
1049
|
+
metadata: credentialMetadata(data)
|
|
1050
|
+
};
|
|
979
1051
|
}
|
|
980
|
-
async function authenticateRequest(request) {
|
|
981
|
-
const
|
|
1052
|
+
async function authenticateRequest(request, profile) {
|
|
1053
|
+
const resolved = request?.headers ? await resolveCredentialFromHeaders(request.headers, profile) : void 0;
|
|
1054
|
+
const headerCred = resolved?.credential;
|
|
1055
|
+
const managedCred = resolved?.managedOAuthApiKey;
|
|
982
1056
|
const envCred = resolveCredentialFromEnv();
|
|
983
1057
|
if (process.env.CLOUD_SERVICE === "true") {
|
|
984
|
-
if (!headerCred) {
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
1058
|
+
if (!headerCred && !managedCred) {
|
|
1059
|
+
if (resolved?.invalid) {
|
|
1060
|
+
if (!profile.acceptApiKeys) {
|
|
1061
|
+
throw new Error(
|
|
1062
|
+
`OAuth access token required for the Firecrawl MCP resource ${profile.endpoint}`
|
|
1063
|
+
);
|
|
1064
|
+
}
|
|
1065
|
+
if (isLegacyKeyPathRequest(request)) {
|
|
1066
|
+
throw new InvalidFirecrawlCredentialError();
|
|
1067
|
+
}
|
|
1068
|
+
return { authType: "none", credentialError: "CREDENTIAL_INVALID" };
|
|
1069
|
+
}
|
|
1070
|
+
if (profile.allowKeyless) {
|
|
1071
|
+
return {
|
|
1072
|
+
authType: "keyless",
|
|
1073
|
+
firecrawlApiKey: void 0,
|
|
1074
|
+
keylessClientIp: extractClientIp(request)
|
|
1075
|
+
};
|
|
1076
|
+
}
|
|
1077
|
+
if (!profile.acceptApiKeys) {
|
|
1078
|
+
throw new Error(
|
|
1079
|
+
`OAuth access token required for the Firecrawl MCP resource ${profile.endpoint}`
|
|
1080
|
+
);
|
|
988
1081
|
}
|
|
989
1082
|
throw new Error(
|
|
990
1083
|
"Firecrawl credentials required: OAuth access token (Authorization: Bearer fco_...) or API key (x-firecrawl-api-key)"
|
|
991
1084
|
);
|
|
992
1085
|
}
|
|
993
|
-
|
|
1086
|
+
const session2 = {
|
|
1087
|
+
authType: resolved?.source === "oauth" ? "oauth" : "api-key",
|
|
1088
|
+
firecrawlApiKey: headerCred,
|
|
1089
|
+
...isLegacyKeyPathRequest(request) ? { keyTransport: "path" } : {},
|
|
1090
|
+
...resolved?.metadata
|
|
1091
|
+
};
|
|
1092
|
+
return managedCred ? setManagedOAuthApiKey(session2, managedCred) : session2;
|
|
994
1093
|
}
|
|
995
|
-
const credential = headerCred ?? envCred;
|
|
1094
|
+
const credential = headerCred ?? managedCred ?? envCred;
|
|
996
1095
|
const httpStreaming = isHttpStreamingTransport();
|
|
997
1096
|
if (!httpStreaming && !process.env.FIRECRAWL_API_KEY && !process.env.FIRECRAWL_API_URL) {
|
|
998
1097
|
console.error(
|
|
@@ -1005,23 +1104,103 @@ async function authenticateRequest(request) {
|
|
|
1005
1104
|
);
|
|
1006
1105
|
process.exit(1);
|
|
1007
1106
|
}
|
|
1008
|
-
|
|
1107
|
+
const session = {
|
|
1108
|
+
authType: resolved?.source === "oauth" ? "oauth" : credential ? "env" : "none",
|
|
1109
|
+
firecrawlApiKey: headerCred ?? envCred,
|
|
1110
|
+
...resolved?.metadata
|
|
1111
|
+
};
|
|
1112
|
+
return managedCred ? setManagedOAuthApiKey(session, managedCred) : session;
|
|
1009
1113
|
}
|
|
1010
|
-
|
|
1011
|
-
if (
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1114
|
+
function searchCompanionAuthMode(request, session) {
|
|
1115
|
+
if (session?.authType === "oauth") return "oauth";
|
|
1116
|
+
if (session?.authType === "api-key") return "api-key";
|
|
1117
|
+
const headerApiKey = normalizeHeader(
|
|
1118
|
+
request?.headers?.["x-firecrawl-api-key"] ?? request?.headers?.["x-api-key"]
|
|
1119
|
+
);
|
|
1120
|
+
if (headerApiKey) return "api-key";
|
|
1121
|
+
const bearer = request?.headers ? extractBearerToken(request.headers) : void 0;
|
|
1122
|
+
if (bearer?.startsWith("fco_")) return "oauth";
|
|
1123
|
+
if (bearer) return "api-key";
|
|
1124
|
+
return "none";
|
|
1125
|
+
}
|
|
1126
|
+
function emitSearchCompanionAuthTelemetry(profile, request, outcome, session) {
|
|
1127
|
+
if (process.env.CLOUD_SERVICE !== "true" || profile.id !== "search" || profile.primary === true) {
|
|
1128
|
+
return;
|
|
1129
|
+
}
|
|
1130
|
+
console.log(
|
|
1131
|
+
"[MCP_SEARCH_AUTH]",
|
|
1132
|
+
JSON.stringify({
|
|
1133
|
+
auth_mode: searchCompanionAuthMode(request, session),
|
|
1134
|
+
outcome,
|
|
1135
|
+
profile: "companion",
|
|
1136
|
+
// Unique only to this telemetry record; it is not a cross-service
|
|
1137
|
+
// correlation ID and does not accept client-controlled identifiers.
|
|
1138
|
+
event_id: randomUUID(),
|
|
1139
|
+
route: DEFAULT_MCP_SEARCH_ENDPOINT
|
|
1140
|
+
})
|
|
1141
|
+
);
|
|
1142
|
+
}
|
|
1143
|
+
function emitLegacyKeyPathTelemetry(profile, request, outcome, session) {
|
|
1144
|
+
if (profile.id !== "full" || !isLegacyKeyPathRequest(request)) return;
|
|
1145
|
+
console.log(
|
|
1146
|
+
"[MCP_LEGACY_KEY_PATH]",
|
|
1147
|
+
JSON.stringify({
|
|
1148
|
+
auth_type: session?.authType ?? "none",
|
|
1149
|
+
key_transport: "path",
|
|
1150
|
+
outcome,
|
|
1151
|
+
resource: profile.resourceUrl
|
|
1152
|
+
})
|
|
1153
|
+
);
|
|
1154
|
+
}
|
|
1155
|
+
function makeAuthenticate(profile) {
|
|
1156
|
+
return async function authenticateWithOAuthChallenge(request) {
|
|
1157
|
+
if (request?.[authResultByRequest]) {
|
|
1158
|
+
return request[authResultByRequest];
|
|
1018
1159
|
}
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1160
|
+
const authResult = authenticateRequest(request, profile).then((session) => {
|
|
1161
|
+
emitSearchCompanionAuthTelemetry(
|
|
1162
|
+
profile,
|
|
1163
|
+
request,
|
|
1164
|
+
session.credentialError ? "rejected" : "accepted",
|
|
1165
|
+
session
|
|
1166
|
+
);
|
|
1167
|
+
emitLegacyKeyPathTelemetry(
|
|
1168
|
+
profile,
|
|
1169
|
+
request,
|
|
1170
|
+
session.credentialError ? "rejected" : "accepted",
|
|
1171
|
+
session
|
|
1172
|
+
);
|
|
1173
|
+
return session;
|
|
1174
|
+
}).catch((error) => {
|
|
1175
|
+
emitSearchCompanionAuthTelemetry(profile, request, "rejected");
|
|
1176
|
+
emitLegacyKeyPathTelemetry(profile, request, "rejected");
|
|
1177
|
+
if (error instanceof InvalidFirecrawlCredentialError) {
|
|
1178
|
+
throw createInvalidCredentialResponse(error);
|
|
1179
|
+
}
|
|
1180
|
+
if (error instanceof CredentialValidationUnavailableError) {
|
|
1181
|
+
throw new Response(
|
|
1182
|
+
JSON.stringify({
|
|
1183
|
+
error: "temporarily_unavailable",
|
|
1184
|
+
error_description: error.message
|
|
1185
|
+
}),
|
|
1186
|
+
{
|
|
1187
|
+
headers: { "Content-Type": "application/json" },
|
|
1188
|
+
status: 503
|
|
1189
|
+
}
|
|
1190
|
+
);
|
|
1191
|
+
}
|
|
1192
|
+
const shouldChallenge = requestShouldReceiveOAuthChallenge(request, profile);
|
|
1193
|
+
const oauthChallenge = shouldChallenge ? createOAuthChallengeResponse(error, profile) : void 0;
|
|
1194
|
+
if (oauthChallenge) {
|
|
1195
|
+
throw oauthChallenge;
|
|
1196
|
+
}
|
|
1197
|
+
throw error;
|
|
1198
|
+
});
|
|
1199
|
+
if (request) {
|
|
1200
|
+
request[authResultByRequest] = authResult;
|
|
1201
|
+
}
|
|
1202
|
+
return authResult;
|
|
1203
|
+
};
|
|
1025
1204
|
}
|
|
1026
1205
|
function removeEmptyTopLevel(obj) {
|
|
1027
1206
|
const out = {};
|
|
@@ -1035,7 +1214,7 @@ function removeEmptyTopLevel(obj) {
|
|
|
1035
1214
|
}
|
|
1036
1215
|
return out;
|
|
1037
1216
|
}
|
|
1038
|
-
var searchDomainSchema =
|
|
1217
|
+
var searchDomainSchema = z4.string().trim().toLowerCase().min(1).max(253).regex(
|
|
1039
1218
|
/^(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z0-9][a-z0-9-]{0,61}[a-z0-9]$/,
|
|
1040
1219
|
"Domain must be a valid hostname without protocol or path"
|
|
1041
1220
|
);
|
|
@@ -1048,6 +1227,27 @@ function buildSearchQueryWithDomains(query, includeDomains, excludeDomains) {
|
|
|
1048
1227
|
}
|
|
1049
1228
|
return query;
|
|
1050
1229
|
}
|
|
1230
|
+
var searchToolBaseFields = {
|
|
1231
|
+
query: z4.string().min(1),
|
|
1232
|
+
highlights: z4.boolean().optional().describe(
|
|
1233
|
+
"Return query-relevant highlights for each search result. Set to false to keep the original search snippets."
|
|
1234
|
+
),
|
|
1235
|
+
limit: z4.number().optional(),
|
|
1236
|
+
tbs: z4.string().optional(),
|
|
1237
|
+
filter: z4.string().optional(),
|
|
1238
|
+
location: z4.string().optional(),
|
|
1239
|
+
includeDomains: z4.array(searchDomainSchema).optional(),
|
|
1240
|
+
excludeDomains: z4.array(searchDomainSchema).optional(),
|
|
1241
|
+
sources: z4.array(z4.object({ type: z4.enum(["web", "images", "news"]) })).optional(),
|
|
1242
|
+
categories: z4.array(z4.enum(["github", "research", "pdf", "developer"])).optional().describe(
|
|
1243
|
+
"Limit results to specific source types. `github` searches GitHub repositories, code, issues, and docs; `research` searches academic and research sources; `pdf` searches PDF results; `developer` searches an index built for coding agents over GitHub issues, merged pull requests, repository READMEs, and curated documentation sites. `developer` adds a `data.developer` group of `{ url, title, description }` results, where `description` holds the matched passage; the other categories filter `data.web`."
|
|
1244
|
+
),
|
|
1245
|
+
enterprise: z4.array(z4.enum(["default", "anon", "zdr"])).optional()
|
|
1246
|
+
};
|
|
1247
|
+
function searchDomainsAreExclusive(args2) {
|
|
1248
|
+
return !(args2.includeDomains?.length && args2.excludeDomains?.length);
|
|
1249
|
+
}
|
|
1250
|
+
var SEARCH_DOMAINS_CONFLICT_MESSAGE = "includeDomains and excludeDomains cannot both be specified";
|
|
1051
1251
|
var ConsoleLogger = class {
|
|
1052
1252
|
shouldLog = process.env.CLOUD_SERVICE === "true" || process.env.SSE_LOCAL === "true" || process.env.HTTP_STREAMABLE_SERVER === "true";
|
|
1053
1253
|
debug(...args2) {
|
|
@@ -1079,32 +1279,224 @@ var ConsoleLogger = class {
|
|
|
1079
1279
|
var openAiAppsChallengeToken = normalizeHeader(
|
|
1080
1280
|
process.env.OPENAI_APPS_CHALLENGE_TOKEN
|
|
1081
1281
|
);
|
|
1082
|
-
var
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1282
|
+
var FULL_PROFILE_INSTRUCTIONS = `Firecrawl provides web search, page retrieval, site URL discovery, multi-page collection, structured extraction, monitoring, and asynchronous research. Match the requested operation to the tool boundary: firecrawl_scrape retrieves one supplied page, firecrawl_map enumerates URLs under a site without retrieving their content, and firecrawl_agent starts multi-source research whose result is read with firecrawl_agent_status. Provide only the required inputs and account for stated network or external side effects.`;
|
|
1283
|
+
var KEYLESS_PROFILE_INSTRUCTIONS = `Without authentication, this endpoint exposes Search, Scrape, and Parse with usage limits. An OAuth connection or Authorization bearer API key exposes account tools; unavailable tools return connection guidance. Firecrawl provides web search, page retrieval, site URL discovery, multi-page collection, structured extraction, monitoring, and asynchronous research. Match the requested operation to the tool boundary: firecrawl_scrape retrieves one supplied page, firecrawl_map enumerates URLs under a site without retrieving their content, and firecrawl_agent starts multi-source research whose result is read with firecrawl_agent_status. Provide only the required inputs.`;
|
|
1284
|
+
var SEARCH_PROFILE_INSTRUCTIONS = `Firecrawl provides web, developer, and research search. Use firecrawl_search to find relevant results across the web and specialized indexes. For a programming question, call firecrawl_developer_search \u2014 or firecrawl_search with categories: ["developer"] \u2014 to search indexed GitHub issues, merged pull requests, READMEs, and documentation. Use the firecrawl_research_* tools to search academic and research literature, expand from anchor papers via the citation graph, read full-text passages from a specific paper, and search public code repositories. All tools are read-only and return ranked results.`;
|
|
1285
|
+
var SEARCH_PROFILE_TOOLS = /* @__PURE__ */ new Set([
|
|
1286
|
+
"firecrawl_search",
|
|
1287
|
+
"firecrawl_developer_search",
|
|
1288
|
+
"firecrawl_research_search_papers",
|
|
1289
|
+
"firecrawl_research_inspect_paper",
|
|
1290
|
+
"firecrawl_research_related_papers",
|
|
1291
|
+
"firecrawl_research_read_paper",
|
|
1292
|
+
"firecrawl_research_search_github"
|
|
1293
|
+
]);
|
|
1294
|
+
function makeFullProfile() {
|
|
1295
|
+
const account = getPrimaryEndpoint() === "/v2/mcp-oauth";
|
|
1296
|
+
return {
|
|
1297
|
+
id: account ? "account" : "full",
|
|
1298
|
+
resourceName: account ? "Firecrawl MCP Account" : "Firecrawl MCP",
|
|
1299
|
+
instructions: account ? FULL_PROFILE_INSTRUCTIONS : KEYLESS_PROFILE_INSTRUCTIONS,
|
|
1300
|
+
resourceUrl: account ? normalizeHeader(process.env.FIRECRAWL_MCP_RESOURCE_URL) ?? DEFAULT_MCP_OAUTH_RESOURCE_URL : getMcpResourceUrl(),
|
|
1301
|
+
endpoint: account ? "/v2/mcp-oauth" : void 0,
|
|
1302
|
+
port: Number(process.env.PORT || 3e3),
|
|
1303
|
+
allowKeyless: !account,
|
|
1304
|
+
acceptApiKeys: true,
|
|
1305
|
+
acceptLegacyAudience: account && process.env.MCP_OAUTH_ACCEPT_LEGACY_V2_MCP_AUD !== "false",
|
|
1306
|
+
primary: true
|
|
1307
|
+
};
|
|
1308
|
+
}
|
|
1309
|
+
function searchOAuthOnly() {
|
|
1310
|
+
return process.env.FIRECRAWL_MCP_SEARCH_OAUTH_ONLY === "true";
|
|
1311
|
+
}
|
|
1312
|
+
function makeSearchProfile({ primary = false } = {}) {
|
|
1313
|
+
const oauthOnly = searchOAuthOnly();
|
|
1314
|
+
if (primary && !oauthOnly) {
|
|
1315
|
+
throw new Error(
|
|
1316
|
+
"FASTMCP_ENDPOINT=/v2/mcp-search requires FIRECRAWL_MCP_SEARCH_OAUTH_ONLY=true"
|
|
1317
|
+
);
|
|
1318
|
+
}
|
|
1319
|
+
return {
|
|
1320
|
+
id: "search",
|
|
1321
|
+
resourceName: "Firecrawl Search",
|
|
1322
|
+
instructions: SEARCH_PROFILE_INSTRUCTIONS,
|
|
1323
|
+
resourceUrl: getSearchMcpResourceUrl(),
|
|
1324
|
+
endpoint: primary ? DEFAULT_MCP_SEARCH_ENDPOINT : getSearchMcpEndpoint(),
|
|
1325
|
+
port: primary ? Number(process.env.PORT || 3e3) : Number(process.env.FIRECRAWL_MCP_SEARCH_PORT || 3001),
|
|
1326
|
+
toolAllowlist: SEARCH_PROFILE_TOOLS,
|
|
1327
|
+
allowKeyless: false,
|
|
1328
|
+
// This is deliberately default-false because the image auto-deploys: the
|
|
1329
|
+
// existing in-process companion remains API-key compatible unless its
|
|
1330
|
+
// deployment explicitly enables the same profile flag used by primary.
|
|
1331
|
+
acceptApiKeys: !oauthOnly,
|
|
1332
|
+
requireManagedOAuth: oauthOnly,
|
|
1333
|
+
primary
|
|
1334
|
+
};
|
|
1335
|
+
}
|
|
1336
|
+
function makePrimaryProfile() {
|
|
1337
|
+
return getPrimaryEndpoint() === "/v2/mcp-search" ? makeSearchProfile({ primary: true }) : makeFullProfile();
|
|
1338
|
+
}
|
|
1339
|
+
function createServer(profile) {
|
|
1340
|
+
return new FastMCP({
|
|
1341
|
+
name: "firecrawl-fastmcp",
|
|
1342
|
+
version: packageVersion,
|
|
1343
|
+
instructions: profile.instructions,
|
|
1344
|
+
logger: new ConsoleLogger(),
|
|
1345
|
+
roots: { enabled: false },
|
|
1346
|
+
oauth: {
|
|
1347
|
+
enabled: isMcpOAuthEnabled(),
|
|
1348
|
+
protectedResource: {
|
|
1349
|
+
authorizationServers: [getOAuthIssuer()],
|
|
1350
|
+
bearerMethodsSupported: ["header"],
|
|
1351
|
+
resource: profile.resourceUrl,
|
|
1352
|
+
resourceName: profile.resourceName,
|
|
1353
|
+
scopesSupported: ["firecrawl:global"]
|
|
1354
|
+
}
|
|
1355
|
+
},
|
|
1356
|
+
authenticate: makeAuthenticate(profile),
|
|
1357
|
+
// Lightweight health endpoint for LB checks
|
|
1358
|
+
health: {
|
|
1359
|
+
enabled: true,
|
|
1360
|
+
message: "ok",
|
|
1361
|
+
path: "/health",
|
|
1362
|
+
status: 200
|
|
1098
1363
|
}
|
|
1099
|
-
}
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1364
|
+
});
|
|
1365
|
+
}
|
|
1366
|
+
var primaryProfile = makePrimaryProfile();
|
|
1367
|
+
var server = createServer(primaryProfile);
|
|
1368
|
+
var KEYLESS_TOOL_NAMES = /* @__PURE__ */ new Set([
|
|
1369
|
+
"firecrawl_scrape",
|
|
1370
|
+
"firecrawl_search",
|
|
1371
|
+
"firecrawl_parse"
|
|
1372
|
+
]);
|
|
1373
|
+
function isHostedKeylessSession(session) {
|
|
1374
|
+
return process.env.CLOUD_SERVICE === "true" && session?.authType === "keyless" && !session.firecrawlApiKey;
|
|
1375
|
+
}
|
|
1376
|
+
function isLocalKeylessStartup() {
|
|
1377
|
+
return process.env.CLOUD_SERVICE !== "true" && !isHttpStreamingTransport() && !resolveCredentialFromEnv() && !normalizeHeader(process.env.FIRECRAWL_API_URL);
|
|
1378
|
+
}
|
|
1379
|
+
function recoveryPayload(code, requestId = randomUUID(), options = {}) {
|
|
1380
|
+
const retryAfterSeconds = options.retryAfterSeconds;
|
|
1381
|
+
const isQuotaExhausted = code === "KEYLESS_QUOTA_EXHAUSTED" || code === "KEYLESS_LIMIT_REACHED";
|
|
1382
|
+
const isToolUnavailable = code === "KEYLESS_TOOL_NOT_AVAILABLE";
|
|
1383
|
+
const isKeylessAccessUnavailable = code === "KEYLESS_ACCESS_NOT_AVAILABLE";
|
|
1384
|
+
const isKeylessEligibilityUnavailable = code === "KEYLESS_ELIGIBILITY_UNAVAILABLE";
|
|
1385
|
+
const oauthUrl = "https://mcp.firecrawl.dev/v2/mcp-oauth";
|
|
1386
|
+
return {
|
|
1387
|
+
code,
|
|
1388
|
+
request_id: requestId,
|
|
1389
|
+
auth_mode: code === "CREDENTIAL_INVALID" ? "credential_error" : "keyless",
|
|
1390
|
+
message: code === "CREDENTIAL_INVALID" ? "The supplied Firecrawl credential is invalid or revoked. Replace it or reconnect the account, then retry." : isQuotaExhausted ? `The free daily limit for this network has been reached${retryAfterSeconds ? `; try again in about ${retryAfterSeconds} seconds` : ""}. To continue now, connect a Firecrawl account via OAuth or configure an API key.` : isToolUnavailable ? "The free tier includes Search, Scrape, and Parse. This tool needs a connected account; Search, Scrape, and Parse still work, so no action is required." : isKeylessAccessUnavailable ? "Anonymous keyless access is unavailable for this request. To continue, connect a Firecrawl account via OAuth or configure an API key." : isKeylessEligibilityUnavailable ? "The anonymous keyless eligibility check is temporarily unavailable. Retry shortly." : "This tool requires a Firecrawl account or API key. Connect an account or configure Authorization: Bearer <FIRECRAWL_API_KEY>, then retry.",
|
|
1391
|
+
...isKeylessAccessUnavailable ? {} : { available_tools: [...KEYLESS_TOOL_NAMES] },
|
|
1392
|
+
docs_url: "https://docs.firecrawl.dev/mcp-server",
|
|
1393
|
+
...retryAfterSeconds ? { retry_after_seconds: retryAfterSeconds } : {},
|
|
1394
|
+
next_actions: isKeylessEligibilityUnavailable ? [{ kind: "retry_later", after_seconds: 30 }] : isQuotaExhausted || isKeylessAccessUnavailable ? [
|
|
1395
|
+
{ kind: "connect_oauth", url: oauthUrl, client_commands: [{ client: "claude-code", command: "claude mcp add --transport http firecrawl https://mcp.firecrawl.dev/v2/mcp-oauth" }] },
|
|
1396
|
+
{ kind: "configure_api_key", header: "Authorization: Bearer <FIRECRAWL_API_KEY>" }
|
|
1397
|
+
] : isToolUnavailable ? [{ kind: "continue_keyless", tools: [...KEYLESS_TOOL_NAMES] }] : [
|
|
1398
|
+
{ kind: "connect_oauth", url: oauthUrl },
|
|
1399
|
+
{ kind: "configure_api_key", header: "Authorization: Bearer <FIRECRAWL_API_KEY>" }
|
|
1400
|
+
]
|
|
1401
|
+
};
|
|
1402
|
+
}
|
|
1403
|
+
function emitActionLog(toolName, status, session, error, requestId = randomUUID(), code) {
|
|
1404
|
+
if (process.env.CLOUD_SERVICE !== "true") return;
|
|
1405
|
+
const payload = {
|
|
1406
|
+
team_id: session?.teamId,
|
|
1407
|
+
user_id: session?.userId,
|
|
1408
|
+
api_key_id: session?.apiKeyId,
|
|
1409
|
+
oauth_client_id: session?.oauthClientId,
|
|
1410
|
+
auth_type: session?.authType ?? "none",
|
|
1411
|
+
tool_name: toolName,
|
|
1412
|
+
status,
|
|
1413
|
+
request_id: requestId,
|
|
1414
|
+
resource: primaryProfile.resourceUrl,
|
|
1415
|
+
...error ? { error_class: error instanceof Error ? error.name : typeof error } : {},
|
|
1416
|
+
...code ? { code } : {}
|
|
1417
|
+
};
|
|
1418
|
+
console.error("[MCP_ACTION]", JSON.stringify(payload));
|
|
1419
|
+
const secret = normalizeHeader(process.env.FIRECRAWL_MCP_ACTION_LOG_SECRET);
|
|
1420
|
+
const apiUrl = normalizeHeader(process.env.FIRECRAWL_API_URL);
|
|
1421
|
+
const endpoint = normalizeHeader(process.env.FIRECRAWL_MCP_ACTION_LOG_URL) ?? (apiUrl ? `${withoutTrailingSlash(apiUrl)}/v2/mcp/action-logs` : void 0);
|
|
1422
|
+
if (!secret || !endpoint || !payload.team_id || status === "started") return;
|
|
1423
|
+
const actionLogPayload = { ...payload };
|
|
1424
|
+
delete actionLogPayload.code;
|
|
1425
|
+
void fetch(endpoint, {
|
|
1426
|
+
method: "POST",
|
|
1427
|
+
headers: {
|
|
1428
|
+
Authorization: `Bearer ${secret}`,
|
|
1429
|
+
"Content-Type": "application/json"
|
|
1430
|
+
},
|
|
1431
|
+
body: JSON.stringify(actionLogPayload),
|
|
1432
|
+
signal: AbortSignal.timeout(1500)
|
|
1433
|
+
}).catch(() => void 0);
|
|
1434
|
+
}
|
|
1435
|
+
function guardHostedTool(tool, { logActions }) {
|
|
1436
|
+
const keylessTool = KEYLESS_TOOL_NAMES.has(tool.name);
|
|
1437
|
+
const execute = tool.execute;
|
|
1438
|
+
return {
|
|
1439
|
+
...tool,
|
|
1440
|
+
canList: (session) => !session?.credentialError && (!isHostedKeylessSession(session) || keylessTool),
|
|
1441
|
+
beforeValidate: (_args, session) => {
|
|
1442
|
+
const code = session?.credentialError ? "CREDENTIAL_INVALID" : isHostedKeylessSession(session) && !keylessTool ? "KEYLESS_TOOL_NOT_AVAILABLE" : void 0;
|
|
1443
|
+
if (!code) return void 0;
|
|
1444
|
+
const requestId = randomUUID();
|
|
1445
|
+
const payload = recoveryPayload(code, requestId);
|
|
1446
|
+
if (logActions) {
|
|
1447
|
+
emitActionLog(tool.name, "error", session, new UserError(String(payload.message), payload), requestId, code);
|
|
1448
|
+
}
|
|
1449
|
+
return {
|
|
1450
|
+
content: [{ type: "text", text: String(payload.message) }],
|
|
1451
|
+
isError: true,
|
|
1452
|
+
structuredContent: payload
|
|
1453
|
+
};
|
|
1454
|
+
},
|
|
1455
|
+
execute: async (args2, context) => {
|
|
1456
|
+
const requestId = randomUUID();
|
|
1457
|
+
const invocationSession = {
|
|
1458
|
+
...context.session,
|
|
1459
|
+
requestId
|
|
1460
|
+
};
|
|
1461
|
+
copyManagedOAuthApiKey(context.session, invocationSession);
|
|
1462
|
+
const invocationContext = {
|
|
1463
|
+
...context,
|
|
1464
|
+
session: invocationSession
|
|
1465
|
+
};
|
|
1466
|
+
if (invocationSession.credentialError) {
|
|
1467
|
+
const code = "CREDENTIAL_INVALID";
|
|
1468
|
+
const payload = recoveryPayload(code, requestId);
|
|
1469
|
+
if (logActions) emitActionLog(tool.name, "error", invocationSession, new UserError(String(payload.message), payload), requestId, code);
|
|
1470
|
+
throw new UserError(String(payload.message), payload);
|
|
1471
|
+
}
|
|
1472
|
+
if (isHostedKeylessSession(invocationSession) && !keylessTool) {
|
|
1473
|
+
const code = "KEYLESS_TOOL_NOT_AVAILABLE";
|
|
1474
|
+
const payload = recoveryPayload(code, requestId);
|
|
1475
|
+
if (logActions) emitActionLog(tool.name, "error", invocationSession, new UserError(String(payload.message), payload), requestId, code);
|
|
1476
|
+
throw new UserError(String(payload.message), payload);
|
|
1477
|
+
}
|
|
1478
|
+
if (!logActions) return execute(args2, invocationContext);
|
|
1479
|
+
emitActionLog(tool.name, "started", invocationSession, void 0, requestId);
|
|
1480
|
+
try {
|
|
1481
|
+
const result = await execute(args2, invocationContext);
|
|
1482
|
+
emitActionLog(tool.name, "success", invocationSession, void 0, requestId);
|
|
1483
|
+
return result;
|
|
1484
|
+
} catch (error) {
|
|
1485
|
+
emitActionLog(tool.name, "error", invocationSession, error, requestId);
|
|
1486
|
+
throw error;
|
|
1487
|
+
}
|
|
1488
|
+
}
|
|
1489
|
+
};
|
|
1490
|
+
}
|
|
1491
|
+
var addTool = server.addTool.bind(server);
|
|
1492
|
+
server.addTool = ((tool) => {
|
|
1493
|
+
if (primaryProfile.toolAllowlist && !primaryProfile.toolAllowlist.has(tool.name)) {
|
|
1494
|
+
return;
|
|
1107
1495
|
}
|
|
1496
|
+
if (primaryProfile.id === "search" && tool.name === "firecrawl_search") {
|
|
1497
|
+
return;
|
|
1498
|
+
}
|
|
1499
|
+
addTool(guardHostedTool(tool, { logActions: primaryProfile.id !== "search" }));
|
|
1108
1500
|
});
|
|
1109
1501
|
if (openAiAppsChallengeToken) {
|
|
1110
1502
|
server.getApp().get(
|
|
@@ -1112,6 +1504,31 @@ if (openAiAppsChallengeToken) {
|
|
|
1112
1504
|
(context) => context.text(openAiAppsChallengeToken)
|
|
1113
1505
|
);
|
|
1114
1506
|
}
|
|
1507
|
+
server.getApp().get("/ready", (context) => {
|
|
1508
|
+
if (process.env.CLOUD_SERVICE !== "true") {
|
|
1509
|
+
return context.json({ ok: true }, 200);
|
|
1510
|
+
}
|
|
1511
|
+
const searchPrimary = primaryProfile.id === "search";
|
|
1512
|
+
const required = [
|
|
1513
|
+
"FIRECRAWL_API_URL",
|
|
1514
|
+
"FIRECRAWL_OAUTH_INTROSPECT_SECRET",
|
|
1515
|
+
"MCP_DELEGATED_CREDENTIAL_SECRET"
|
|
1516
|
+
];
|
|
1517
|
+
if (primaryProfile.allowKeyless) {
|
|
1518
|
+
required.push("KEYLESS_PROXY_SECRET");
|
|
1519
|
+
}
|
|
1520
|
+
const missing = required.filter((name) => !normalizeHeader(process.env[name]));
|
|
1521
|
+
const configuredEndpoint = getPrimaryEndpoint();
|
|
1522
|
+
const resourceMatchesEndpoint = searchPrimary ? withoutTrailingSlash(primaryProfile.resourceUrl) === DEFAULT_MCP_SEARCH_RESOURCE_URL : withoutTrailingSlash(primaryProfile.resourceUrl).endsWith(
|
|
1523
|
+
configuredEndpoint
|
|
1524
|
+
);
|
|
1525
|
+
if (!resourceMatchesEndpoint) {
|
|
1526
|
+
missing.push(
|
|
1527
|
+
searchPrimary ? "FIRECRAWL_MCP_SEARCH_RESOURCE_URL (endpoint mismatch)" : "FIRECRAWL_MCP_RESOURCE_URL (endpoint mismatch)"
|
|
1528
|
+
);
|
|
1529
|
+
}
|
|
1530
|
+
return missing.length ? context.json({ ok: false, missing }, 503) : context.json({ ok: true }, 200);
|
|
1531
|
+
});
|
|
1115
1532
|
function createClient(apiKey) {
|
|
1116
1533
|
const config = {
|
|
1117
1534
|
...process.env.FIRECRAWL_API_URL && {
|
|
@@ -1124,21 +1541,35 @@ function createClient(apiKey) {
|
|
|
1124
1541
|
return new FirecrawlApp(config);
|
|
1125
1542
|
}
|
|
1126
1543
|
var ORIGIN = "mcp-fastmcp";
|
|
1127
|
-
var
|
|
1544
|
+
var ORIGIN_HEADERS3 = { "X-Origin": ORIGIN };
|
|
1128
1545
|
var SAFE_MODE = process.env.CLOUD_SERVICE === "true";
|
|
1129
1546
|
function getClient(session) {
|
|
1130
|
-
if (process.env.CLOUD_SERVICE === "true") {
|
|
1131
|
-
|
|
1132
|
-
throw new Error("Unauthorized");
|
|
1133
|
-
}
|
|
1134
|
-
return createClient(session.firecrawlApiKey);
|
|
1547
|
+
if (process.env.CLOUD_SERVICE === "true" && !hasCredential(session)) {
|
|
1548
|
+
throw new Error("Unauthorized");
|
|
1135
1549
|
}
|
|
1136
|
-
if (!process.env.FIRECRAWL_API_URL && (
|
|
1550
|
+
if (!process.env.FIRECRAWL_API_URL && !hasCredential(session)) {
|
|
1137
1551
|
throw new Error(
|
|
1138
1552
|
"Unauthorized: API key is required when not using a self-hosted instance"
|
|
1139
1553
|
);
|
|
1140
1554
|
}
|
|
1141
|
-
|
|
1555
|
+
if (!hasManagedOAuthCredential(session)) {
|
|
1556
|
+
return createClient(credentialForOutboundRequest(session));
|
|
1557
|
+
}
|
|
1558
|
+
const client = createClient("request-scoped-hosted-oauth");
|
|
1559
|
+
const axiosInstance = client.http?.instance;
|
|
1560
|
+
if (!axiosInstance?.interceptors?.request?.use) {
|
|
1561
|
+
throw new CredentialValidationUnavailableError();
|
|
1562
|
+
}
|
|
1563
|
+
axiosInstance.interceptors.request.use((config) => {
|
|
1564
|
+
const credential = credentialForOutboundRequest(session);
|
|
1565
|
+
if (!credential) throw new CredentialValidationUnavailableError();
|
|
1566
|
+
config.headers = {
|
|
1567
|
+
...config.headers ?? {},
|
|
1568
|
+
Authorization: `Bearer ${credential}`
|
|
1569
|
+
};
|
|
1570
|
+
return config;
|
|
1571
|
+
});
|
|
1572
|
+
return client;
|
|
1142
1573
|
}
|
|
1143
1574
|
function asText2(data) {
|
|
1144
1575
|
return JSON.stringify(data, null, 2);
|
|
@@ -1208,10 +1639,10 @@ function transformScrapeParams(args2) {
|
|
|
1208
1639
|
delete out.pdfOptions;
|
|
1209
1640
|
return out;
|
|
1210
1641
|
}
|
|
1211
|
-
var scrapeParamsSchema =
|
|
1212
|
-
url:
|
|
1213
|
-
formats:
|
|
1214
|
-
|
|
1642
|
+
var scrapeParamsSchema = z4.object({
|
|
1643
|
+
url: z4.string().url(),
|
|
1644
|
+
formats: z4.array(
|
|
1645
|
+
z4.enum([
|
|
1215
1646
|
"markdown",
|
|
1216
1647
|
"html",
|
|
1217
1648
|
"rawHtml",
|
|
@@ -1225,62 +1656,62 @@ var scrapeParamsSchema = z3.object({
|
|
|
1225
1656
|
"audio"
|
|
1226
1657
|
])
|
|
1227
1658
|
).optional(),
|
|
1228
|
-
jsonOptions:
|
|
1229
|
-
prompt:
|
|
1230
|
-
schema:
|
|
1659
|
+
jsonOptions: z4.object({
|
|
1660
|
+
prompt: z4.string().optional(),
|
|
1661
|
+
schema: z4.record(z4.string(), z4.any()).optional()
|
|
1231
1662
|
}).optional(),
|
|
1232
|
-
queryOptions:
|
|
1233
|
-
prompt:
|
|
1234
|
-
mode:
|
|
1663
|
+
queryOptions: z4.object({
|
|
1664
|
+
prompt: z4.string().max(1e4),
|
|
1665
|
+
mode: z4.enum(["directQuote", "freeform"]).default("freeform")
|
|
1235
1666
|
}).optional(),
|
|
1236
|
-
screenshotOptions:
|
|
1237
|
-
fullPage:
|
|
1238
|
-
quality:
|
|
1239
|
-
viewport:
|
|
1667
|
+
screenshotOptions: z4.object({
|
|
1668
|
+
fullPage: z4.boolean().optional(),
|
|
1669
|
+
quality: z4.number().optional(),
|
|
1670
|
+
viewport: z4.object({ width: z4.number(), height: z4.number() }).optional()
|
|
1240
1671
|
}).optional(),
|
|
1241
|
-
parsers:
|
|
1242
|
-
pdfOptions:
|
|
1243
|
-
maxPages:
|
|
1672
|
+
parsers: z4.array(z4.enum(["pdf"])).optional(),
|
|
1673
|
+
pdfOptions: z4.object({
|
|
1674
|
+
maxPages: z4.number().int().min(1).max(1e4).optional()
|
|
1244
1675
|
}).optional(),
|
|
1245
|
-
onlyMainContent:
|
|
1246
|
-
redactPII:
|
|
1247
|
-
includeTags:
|
|
1248
|
-
excludeTags:
|
|
1249
|
-
waitFor:
|
|
1676
|
+
onlyMainContent: z4.boolean().optional(),
|
|
1677
|
+
redactPII: z4.boolean().optional(),
|
|
1678
|
+
includeTags: z4.array(z4.string()).optional(),
|
|
1679
|
+
excludeTags: z4.array(z4.string()).optional(),
|
|
1680
|
+
waitFor: z4.number().optional(),
|
|
1250
1681
|
...SAFE_MODE ? {} : {
|
|
1251
|
-
actions:
|
|
1252
|
-
|
|
1253
|
-
type:
|
|
1254
|
-
selector:
|
|
1255
|
-
milliseconds:
|
|
1256
|
-
text:
|
|
1257
|
-
key:
|
|
1258
|
-
direction:
|
|
1259
|
-
script:
|
|
1260
|
-
fullPage:
|
|
1682
|
+
actions: z4.array(
|
|
1683
|
+
z4.object({
|
|
1684
|
+
type: z4.enum(allowedActionTypes),
|
|
1685
|
+
selector: z4.string().optional(),
|
|
1686
|
+
milliseconds: z4.number().optional(),
|
|
1687
|
+
text: z4.string().optional(),
|
|
1688
|
+
key: z4.string().optional(),
|
|
1689
|
+
direction: z4.enum(["up", "down"]).optional(),
|
|
1690
|
+
script: z4.string().optional(),
|
|
1691
|
+
fullPage: z4.boolean().optional()
|
|
1261
1692
|
})
|
|
1262
1693
|
).optional()
|
|
1263
1694
|
},
|
|
1264
|
-
mobile:
|
|
1265
|
-
skipTlsVerification:
|
|
1266
|
-
removeBase64Images:
|
|
1267
|
-
location:
|
|
1268
|
-
country:
|
|
1269
|
-
languages:
|
|
1695
|
+
mobile: z4.boolean().optional(),
|
|
1696
|
+
skipTlsVerification: z4.boolean().optional(),
|
|
1697
|
+
removeBase64Images: z4.boolean().optional(),
|
|
1698
|
+
location: z4.object({
|
|
1699
|
+
country: z4.string().optional(),
|
|
1700
|
+
languages: z4.array(z4.string()).optional()
|
|
1270
1701
|
}).optional(),
|
|
1271
|
-
storeInCache:
|
|
1272
|
-
zeroDataRetention:
|
|
1273
|
-
maxAge:
|
|
1274
|
-
lockdown:
|
|
1275
|
-
proxy:
|
|
1276
|
-
profile:
|
|
1277
|
-
name:
|
|
1278
|
-
saveChanges:
|
|
1702
|
+
storeInCache: z4.boolean().optional(),
|
|
1703
|
+
zeroDataRetention: z4.boolean().optional(),
|
|
1704
|
+
maxAge: z4.number().optional(),
|
|
1705
|
+
lockdown: z4.boolean().optional(),
|
|
1706
|
+
proxy: z4.enum(["basic", "stealth", "enhanced", "auto"]).optional(),
|
|
1707
|
+
profile: z4.object({
|
|
1708
|
+
name: z4.string(),
|
|
1709
|
+
saveChanges: z4.boolean().optional()
|
|
1279
1710
|
}).optional()
|
|
1280
1711
|
});
|
|
1281
|
-
var parseOptionParamsSchema =
|
|
1282
|
-
formats:
|
|
1283
|
-
|
|
1712
|
+
var parseOptionParamsSchema = z4.object({
|
|
1713
|
+
formats: z4.array(
|
|
1714
|
+
z4.enum([
|
|
1284
1715
|
"markdown",
|
|
1285
1716
|
"html",
|
|
1286
1717
|
"rawHtml",
|
|
@@ -1290,48 +1721,48 @@ var parseOptionParamsSchema = z3.object({
|
|
|
1290
1721
|
"query"
|
|
1291
1722
|
])
|
|
1292
1723
|
).optional(),
|
|
1293
|
-
jsonOptions:
|
|
1294
|
-
prompt:
|
|
1295
|
-
schema:
|
|
1724
|
+
jsonOptions: z4.object({
|
|
1725
|
+
prompt: z4.string().optional(),
|
|
1726
|
+
schema: z4.record(z4.string(), z4.any()).optional()
|
|
1296
1727
|
}).optional(),
|
|
1297
|
-
queryOptions:
|
|
1298
|
-
prompt:
|
|
1299
|
-
mode:
|
|
1728
|
+
queryOptions: z4.object({
|
|
1729
|
+
prompt: z4.string().max(1e4),
|
|
1730
|
+
mode: z4.enum(["directQuote", "freeform"]).default("freeform")
|
|
1300
1731
|
}).optional(),
|
|
1301
|
-
parsers:
|
|
1302
|
-
pdfOptions:
|
|
1303
|
-
maxPages:
|
|
1732
|
+
parsers: z4.array(z4.enum(["pdf"])).optional(),
|
|
1733
|
+
pdfOptions: z4.object({
|
|
1734
|
+
maxPages: z4.number().int().min(1).max(1e4).optional()
|
|
1304
1735
|
}).optional(),
|
|
1305
|
-
onlyMainContent:
|
|
1306
|
-
redactPII:
|
|
1307
|
-
includeTags:
|
|
1308
|
-
excludeTags:
|
|
1309
|
-
removeBase64Images:
|
|
1310
|
-
skipTlsVerification:
|
|
1311
|
-
storeInCache:
|
|
1312
|
-
zeroDataRetention:
|
|
1313
|
-
maxAge:
|
|
1314
|
-
proxy:
|
|
1736
|
+
onlyMainContent: z4.boolean().optional(),
|
|
1737
|
+
redactPII: z4.boolean().optional(),
|
|
1738
|
+
includeTags: z4.array(z4.string()).optional(),
|
|
1739
|
+
excludeTags: z4.array(z4.string()).optional(),
|
|
1740
|
+
removeBase64Images: z4.boolean().optional(),
|
|
1741
|
+
skipTlsVerification: z4.boolean().optional(),
|
|
1742
|
+
storeInCache: z4.boolean().optional(),
|
|
1743
|
+
zeroDataRetention: z4.boolean().optional(),
|
|
1744
|
+
maxAge: z4.number().optional(),
|
|
1745
|
+
proxy: z4.enum(["basic", "auto"]).optional()
|
|
1315
1746
|
});
|
|
1316
1747
|
var localParseParamsSchema = parseOptionParamsSchema.extend({
|
|
1317
|
-
filePath:
|
|
1748
|
+
filePath: z4.string().min(1).describe(
|
|
1318
1749
|
"Absolute or relative path to a local file to parse. Supported: .html, .htm, .pdf, .docx, .doc, .odt, .rtf, .xlsx, .xls"
|
|
1319
1750
|
),
|
|
1320
|
-
contentType:
|
|
1751
|
+
contentType: z4.string().optional().describe(
|
|
1321
1752
|
"Optional MIME type override. If omitted, the server infers the file kind from the extension."
|
|
1322
1753
|
)
|
|
1323
1754
|
});
|
|
1324
1755
|
var hostedParseParamsSchema = parseOptionParamsSchema.extend({
|
|
1325
|
-
filePath:
|
|
1756
|
+
filePath: z4.string().min(1).optional().describe(
|
|
1326
1757
|
"Phase 1 only: path to the local file on the caller/harness machine. Hosted MCP will not read or stat this path; it is used only to produce upload instructions."
|
|
1327
1758
|
),
|
|
1328
|
-
uploadRef:
|
|
1759
|
+
uploadRef: z4.string().min(1).optional().describe(
|
|
1329
1760
|
"Phase 2 only: short-lived upload reference returned by phase 1 after the local PUT upload completes."
|
|
1330
1761
|
),
|
|
1331
|
-
contentType:
|
|
1762
|
+
contentType: z4.string().optional().describe(
|
|
1332
1763
|
"Phase 1 MIME type override. If omitted, the server infers it from the file extension without reading the file."
|
|
1333
1764
|
),
|
|
1334
|
-
declaredSizeBytes:
|
|
1765
|
+
declaredSizeBytes: z4.number().int().positive().optional().describe(
|
|
1335
1766
|
"Optional phase 1 size declaration. Hosted MCP does not stat the file; provide this only if the caller already knows it."
|
|
1336
1767
|
)
|
|
1337
1768
|
}).superRefine((value, ctx) => {
|
|
@@ -1339,7 +1770,7 @@ var hostedParseParamsSchema = parseOptionParamsSchema.extend({
|
|
|
1339
1770
|
const hasUploadRef = typeof value.uploadRef === "string" && value.uploadRef.length > 0;
|
|
1340
1771
|
if (hasFilePath === hasUploadRef) {
|
|
1341
1772
|
ctx.addIssue({
|
|
1342
|
-
code:
|
|
1773
|
+
code: z4.ZodIssueCode.custom,
|
|
1343
1774
|
message: "Hosted firecrawl_parse requires exactly one of filePath (phase 1) or uploadRef (phase 2).",
|
|
1344
1775
|
path: hasFilePath && hasUploadRef ? ["uploadRef"] : ["filePath"]
|
|
1345
1776
|
});
|
|
@@ -1412,8 +1843,9 @@ async function apiPostJson(pathName, body, apiKey) {
|
|
|
1412
1843
|
return parsed;
|
|
1413
1844
|
}
|
|
1414
1845
|
async function apiPostJsonForSession(pathName, body, session) {
|
|
1415
|
-
|
|
1416
|
-
|
|
1846
|
+
const credential = credentialForOutboundRequest(session);
|
|
1847
|
+
if (credential) {
|
|
1848
|
+
return apiPostJson(pathName, body, credential);
|
|
1417
1849
|
}
|
|
1418
1850
|
if (isKeylessMode(session)) {
|
|
1419
1851
|
return keylessPost(pathName, body, session);
|
|
@@ -1456,13 +1888,21 @@ async function executeHostedParse(args2, session, log) {
|
|
|
1456
1888
|
"Hosted firecrawl_parse requires exactly one of filePath or uploadRef."
|
|
1457
1889
|
);
|
|
1458
1890
|
}
|
|
1459
|
-
if (!session
|
|
1891
|
+
if (!hasCredential(session) && !isKeylessMode(session)) {
|
|
1460
1892
|
return asText2({
|
|
1461
1893
|
success: false,
|
|
1462
1894
|
mode: "hosted-upload-ref-auth-required",
|
|
1463
1895
|
message: "Hosted firecrawl_parse requires an authenticated Firecrawl session or keyless eligibility before a local file upload URL can be minted. Connect a Firecrawl account, provide an API key, or use keyless hosted MCP while eligible, then call firecrawl_parse again."
|
|
1464
1896
|
});
|
|
1465
1897
|
}
|
|
1898
|
+
if (isHostedKeylessSession(session) && args2.zeroDataRetention === true) {
|
|
1899
|
+
const payload = {
|
|
1900
|
+
...recoveryPayload("KEYLESS_OPTION_NOT_AVAILABLE", session?.requestId),
|
|
1901
|
+
option: "zeroDataRetention",
|
|
1902
|
+
message: "Zero Data Retention is not available in anonymous keyless mode. Omit zeroDataRetention to parse with keyless access, or connect an account or configure an API key for a team where Zero Data Retention is enabled, then retry."
|
|
1903
|
+
};
|
|
1904
|
+
throw new UserError(String(payload.message), payload);
|
|
1905
|
+
}
|
|
1466
1906
|
const options = extractParseOptions(args2);
|
|
1467
1907
|
if (hasFilePath && args2.filePath) {
|
|
1468
1908
|
const filename = path.basename(args2.filePath);
|
|
@@ -1535,106 +1975,11 @@ server.addTool({
|
|
|
1535
1975
|
// Does not modify, delete, or write to external websites.
|
|
1536
1976
|
},
|
|
1537
1977
|
description: `
|
|
1538
|
-
|
|
1539
|
-
This is the most powerful, fastest and most reliable scraper tool, if available you should always default to using this tool for any web scraping needs.
|
|
1540
|
-
|
|
1541
|
-
**Best for:** Single page content extraction, when you know exactly which page contains the information.
|
|
1542
|
-
**Not recommended for:** Multiple pages (call scrape multiple times or use crawl), unknown page location (use search).
|
|
1543
|
-
**Common mistakes:** Using markdown format when extracting specific data points (use JSON instead).
|
|
1544
|
-
**Other Features:** Use 'branding' format to extract brand identity (colors, fonts, typography, spacing, UI components) for design analysis or style replication.
|
|
1545
|
-
|
|
1546
|
-
**CRITICAL - Format Selection (you MUST follow this):**
|
|
1547
|
-
When the user asks for SPECIFIC data points, you MUST use JSON format with a schema. Only use markdown when the user needs the ENTIRE page content.
|
|
1548
|
-
|
|
1549
|
-
**Use JSON format when user asks for:**
|
|
1550
|
-
- Parameters, fields, or specifications (e.g., "get the header parameters", "what are the required fields")
|
|
1551
|
-
- Prices, numbers, or structured data (e.g., "extract the pricing", "get the product details")
|
|
1552
|
-
- API details, endpoints, or technical specs (e.g., "find the authentication endpoint")
|
|
1553
|
-
- Lists of items or properties (e.g., "list the features", "get all the options")
|
|
1554
|
-
- Any specific piece of information from a page
|
|
1555
|
-
|
|
1556
|
-
**Use markdown format ONLY when:**
|
|
1557
|
-
- User wants to read/summarize an entire article or blog post
|
|
1558
|
-
- User needs to see all content on a page without specific extraction
|
|
1559
|
-
- User explicitly asks for the full page content
|
|
1560
|
-
|
|
1561
|
-
**Handling JavaScript-rendered pages (SPAs):**
|
|
1562
|
-
If JSON extraction returns empty, minimal, or just navigation content, the page is likely JavaScript-rendered or the content is on a different URL. Try these steps IN ORDER:
|
|
1563
|
-
1. **Add waitFor parameter:** Set \`waitFor: 5000\` to \`waitFor: 10000\` to allow JavaScript to render before extraction
|
|
1564
|
-
2. **Try a different URL:** If the URL has a hash fragment (#section), try the base URL or look for a direct page URL
|
|
1565
|
-
3. **Use firecrawl_map to find the correct page:** Large documentation sites or SPAs often spread content across multiple URLs. Use \`firecrawl_map\` with a \`search\` parameter to discover the specific page containing your target content, then scrape that URL directly.
|
|
1566
|
-
Example: If scraping "https://docs.example.com/reference" fails to find webhook parameters, use \`firecrawl_map\` with \`{"url": "https://docs.example.com/reference", "search": "webhook"}\` to find URLs like "/reference/webhook-events", then scrape that specific page.
|
|
1567
|
-
4. **Use firecrawl_agent:** As a last resort for heavily dynamic pages where map+scrape still fails, use the agent which can autonomously navigate and research
|
|
1568
|
-
|
|
1569
|
-
**Usage Example (JSON format - REQUIRED for specific data extraction):**
|
|
1570
|
-
\`\`\`json
|
|
1571
|
-
{
|
|
1572
|
-
"name": "firecrawl_scrape",
|
|
1573
|
-
"arguments": {
|
|
1574
|
-
"url": "https://example.com/api-docs",
|
|
1575
|
-
"formats": ["json"],
|
|
1576
|
-
"jsonOptions": {
|
|
1577
|
-
"prompt": "Extract the header parameters for the authentication endpoint",
|
|
1578
|
-
"schema": {
|
|
1579
|
-
"type": "object",
|
|
1580
|
-
"properties": {
|
|
1581
|
-
"parameters": {
|
|
1582
|
-
"type": "array",
|
|
1583
|
-
"items": {
|
|
1584
|
-
"type": "object",
|
|
1585
|
-
"properties": {
|
|
1586
|
-
"name": { "type": "string" },
|
|
1587
|
-
"type": { "type": "string" },
|
|
1588
|
-
"required": { "type": "boolean" },
|
|
1589
|
-
"description": { "type": "string" }
|
|
1590
|
-
}
|
|
1591
|
-
}
|
|
1592
|
-
}
|
|
1593
|
-
}
|
|
1594
|
-
}
|
|
1595
|
-
}
|
|
1596
|
-
}
|
|
1597
|
-
}
|
|
1598
|
-
\`\`\`
|
|
1978
|
+
Retrieve and extract content from one supplied URL through Firecrawl. Use this when the request identifies a page and needs its content or defined fields. It can return markdown, HTML, links, screenshots, branding data, a targeted answer, or JSON matching a supplied schema; JSON is useful when the requested result has defined fields, while markdown preserves readable page content.
|
|
1599
1979
|
|
|
1600
|
-
|
|
1980
|
+
This tool operates on a known page. For a set of pages use \`firecrawl_crawl\`, and to discover page URLs use \`firecrawl_map\` or \`firecrawl_search\`. Options include JavaScript render delay, cache age, main-content filtering, PII redaction, and lockdown cache-only retrieval. Browser actions may change the live page when interactive actions are enabled.
|
|
1601
1981
|
|
|
1602
|
-
|
|
1603
|
-
- Structured data with specific fields (extract all products with name, price, description)
|
|
1604
|
-
- Data in a specific schema for downstream processing
|
|
1605
|
-
|
|
1606
|
-
**Use query format only when:**
|
|
1607
|
-
- The page is extremely long and you need a single targeted answer without processing the full content
|
|
1608
|
-
- You want a quick factual answer and don't need to retain the page content
|
|
1609
|
-
- Set \`queryOptions.mode\` to \`"directQuote"\` when you need verbatim page text; otherwise it defaults to \`"freeform"\`
|
|
1610
|
-
|
|
1611
|
-
**Usage Example (markdown format - default for most tasks):**
|
|
1612
|
-
\`\`\`json
|
|
1613
|
-
{
|
|
1614
|
-
"name": "firecrawl_scrape",
|
|
1615
|
-
"arguments": {
|
|
1616
|
-
"url": "https://example.com/article",
|
|
1617
|
-
"formats": ["markdown"],
|
|
1618
|
-
"onlyMainContent": true
|
|
1619
|
-
}
|
|
1620
|
-
}
|
|
1621
|
-
\`\`\`
|
|
1622
|
-
**Usage Example (branding format - extract brand identity):**
|
|
1623
|
-
\`\`\`json
|
|
1624
|
-
{
|
|
1625
|
-
"name": "firecrawl_scrape",
|
|
1626
|
-
"arguments": {
|
|
1627
|
-
"url": "https://example.com",
|
|
1628
|
-
"formats": ["branding"]
|
|
1629
|
-
}
|
|
1630
|
-
}
|
|
1631
|
-
\`\`\`
|
|
1632
|
-
**Branding format:** Extracts comprehensive brand identity (colors, fonts, typography, spacing, logo, UI components) for design analysis or style replication.
|
|
1633
|
-
**Performance:** Add maxAge parameter for 500% faster scrapes using cached data.
|
|
1634
|
-
**Lockdown mode:** Set \`lockdown: true\` to serve the request only from the existing index/cache without any outbound network request. For air-gapped or compliance-constrained use where the request URL itself is considered sensitive. Errors on cache miss. Billed at 5 credits.
|
|
1635
|
-
**Privacy:** Set \`redactPII: true\` to return content with personally identifiable information redacted.
|
|
1636
|
-
**Returns:** JSON structured data, markdown, branding profile, or other formats as specified.
|
|
1637
|
-
${SAFE_MODE ? "**Safe Mode:** Read-only content extraction. Interactive actions (click, write, executeJavascript) are disabled for security." : ""}
|
|
1982
|
+
Returns the selected content formats and page metadata.
|
|
1638
1983
|
`,
|
|
1639
1984
|
parameters: scrapeParamsSchema,
|
|
1640
1985
|
execute: async (args2, { session, log }) => {
|
|
@@ -1680,43 +2025,17 @@ server.addTool({
|
|
|
1680
2025
|
// Read-only discovery; no deletion or destructive updates.
|
|
1681
2026
|
},
|
|
1682
2027
|
description: `
|
|
1683
|
-
|
|
1684
|
-
|
|
1685
|
-
**Best for:** Discovering URLs on a website before deciding what to scrape; finding specific sections or pages within a large site; locating the correct page when scrape returns empty or incomplete results.
|
|
1686
|
-
**Not recommended for:** When you already know which specific URL you need (use scrape); when you need the content of the pages (use scrape after mapping).
|
|
1687
|
-
**Common mistakes:** Using crawl to discover URLs instead of map; jumping straight to firecrawl_agent when scrape fails instead of using map first to find the right page.
|
|
2028
|
+
Enumerate URLs indexed under one website through Firecrawl without fetching each page's content. Use this when the request asks for a site's URL inventory, when several relevant pages must be located, or when the desired page URL is unknown. An optional \`search\` term narrows the URL list, while sitemap, subdomain, query-parameter, and result-limit options control coverage.
|
|
1688
2029
|
|
|
1689
|
-
|
|
1690
|
-
|
|
1691
|
-
**Prompt Example:** "Find the webhook documentation page on this API docs site."
|
|
1692
|
-
**Usage Example (discover all URLs):**
|
|
1693
|
-
\`\`\`json
|
|
1694
|
-
{
|
|
1695
|
-
"name": "firecrawl_map",
|
|
1696
|
-
"arguments": {
|
|
1697
|
-
"url": "https://example.com"
|
|
1698
|
-
}
|
|
1699
|
-
}
|
|
1700
|
-
\`\`\`
|
|
1701
|
-
**Usage Example (search for specific content - RECOMMENDED when scrape fails):**
|
|
1702
|
-
\`\`\`json
|
|
1703
|
-
{
|
|
1704
|
-
"name": "firecrawl_map",
|
|
1705
|
-
"arguments": {
|
|
1706
|
-
"url": "https://docs.example.com/api",
|
|
1707
|
-
"search": "webhook events"
|
|
1708
|
-
}
|
|
1709
|
-
}
|
|
1710
|
-
\`\`\`
|
|
1711
|
-
**Returns:** Array of URLs found on the site, filtered by search query if provided.
|
|
2030
|
+
Returns matching URLs rather than page bodies. Retrieve one page with \`firecrawl_scrape\`; collect content across multiple pages with \`firecrawl_crawl\`.
|
|
1712
2031
|
`,
|
|
1713
|
-
parameters:
|
|
1714
|
-
url:
|
|
1715
|
-
search:
|
|
1716
|
-
sitemap:
|
|
1717
|
-
includeSubdomains:
|
|
1718
|
-
limit:
|
|
1719
|
-
ignoreQueryParameters:
|
|
2032
|
+
parameters: z4.object({
|
|
2033
|
+
url: z4.string().url(),
|
|
2034
|
+
search: z4.string().optional(),
|
|
2035
|
+
sitemap: z4.enum(["include", "skip", "only"]).optional(),
|
|
2036
|
+
includeSubdomains: z4.boolean().optional(),
|
|
2037
|
+
limit: z4.number().optional(),
|
|
2038
|
+
ignoreQueryParameters: z4.boolean().optional()
|
|
1720
2039
|
}),
|
|
1721
2040
|
execute: async (args2, { session, log }) => {
|
|
1722
2041
|
const { url, ...options } = args2;
|
|
@@ -1742,92 +2061,16 @@ server.addTool({
|
|
|
1742
2061
|
// Query-only; no destructive side effects on external entities.
|
|
1743
2062
|
},
|
|
1744
2063
|
description: `
|
|
1745
|
-
Search
|
|
1746
|
-
|
|
1747
|
-
The query also supports search operators, that you can use if needed to refine the search:
|
|
1748
|
-
| Operator | Functionality | Examples |
|
|
1749
|
-
---|-|-|
|
|
1750
|
-
| \`"\` | Non-fuzzy matches a string of text | \`"Firecrawl"\`
|
|
1751
|
-
| \`-\` | Excludes certain keywords or negates other operators | \`-bad\`, \`-site:firecrawl.dev\`
|
|
1752
|
-
| \`site:\` | Only returns results from a specified website | \`site:firecrawl.dev\`
|
|
1753
|
-
| \`inurl:\` | Only returns results that include a word in the URL | \`inurl:firecrawl\`
|
|
1754
|
-
| \`allinurl:\` | Only returns results that include multiple words in the URL | \`allinurl:git firecrawl\`
|
|
1755
|
-
| \`intitle:\` | Only returns results that include a word in the title of the page | \`intitle:Firecrawl\`
|
|
1756
|
-
| \`allintitle:\` | Only returns results that include multiple words in the title of the page | \`allintitle:firecrawl playground\`
|
|
1757
|
-
| \`related:\` | Only returns results that are related to a specific domain | \`related:firecrawl.dev\`
|
|
1758
|
-
| \`imagesize:\` | Only returns images with exact dimensions | \`imagesize:1920x1080\`
|
|
1759
|
-
| \`larger:\` | Only returns images larger than specified dimensions | \`larger:1920x1080\`
|
|
2064
|
+
Search web, news, or image sources and return ranked results. Operators include quoted phrases, \`-term\`, \`site:host\`, \`inurl:term\`, \`intitle:term\`, and \`related:host\`; the set is non-exhaustive. \`includeDomains\` and \`excludeDomains\` are mutually exclusive hostname filters; categories limit results to GitHub, research, PDF, or developer sources.
|
|
1760
2065
|
|
|
1761
|
-
|
|
1762
|
-
**Not recommended for:** When you need to search the filesystem. When you already know which website to scrape (use scrape); when you need comprehensive coverage of a single website (use map or crawl.
|
|
1763
|
-
**Common mistakes:** Using crawl or map for open-ended questions (use search instead).
|
|
1764
|
-
**Prompt Example:** "Find the latest research papers on AI published in 2023."
|
|
1765
|
-
**Sources:** web, images, news, default to web unless needed images or news.
|
|
1766
|
-
**Categories:** Optional filter to limit result types: \`github\` (GitHub repositories, code, issues, and docs), \`research\` (academic and research sources), \`pdf\` (PDF results). Example: \`categories: ["github", "research"]\`.
|
|
1767
|
-
**Domain filters:** Use includeDomains to restrict results to specific domains, or excludeDomains to remove domains. Do not use both in the same request. Domains must be hostnames only, without protocol or path.
|
|
1768
|
-
**Scrape Options:** Only use scrapeOptions when you think it is absolutely necessary. When you do so default to a lower limit to avoid timeouts, 5 or lower.
|
|
1769
|
-
**Optimal Workflow:** Search first using firecrawl_search without formats, then after fetching the results, use the scrape tool to get the content of the relevantpage(s) that you want to scrape
|
|
1770
|
-
**After the search:** Once you have processed the results (or decided they were not useful), call \`firecrawl_search_feedback\` with the \`id\` from this response. The first feedback per search refunds 1 credit and helps Firecrawl improve search quality.
|
|
2066
|
+
For a programming question, add \`categories: ["developer"]\`. It searches an index of GitHub issues, merged pull requests, repository READMEs, and curated documentation sites, and returns the hits in \`data.developer\` beside the web results.
|
|
1771
2067
|
|
|
1772
|
-
|
|
1773
|
-
\`\`\`json
|
|
1774
|
-
{
|
|
1775
|
-
"name": "firecrawl_search",
|
|
1776
|
-
"arguments": {
|
|
1777
|
-
"query": "top AI companies",
|
|
1778
|
-
"limit": 5,
|
|
1779
|
-
"includeDomains": ["example.com"],
|
|
1780
|
-
"sources": [
|
|
1781
|
-
{ "type": "web" }
|
|
1782
|
-
]
|
|
1783
|
-
}
|
|
1784
|
-
}
|
|
1785
|
-
\`\`\`
|
|
1786
|
-
**Usage Example with formats:**
|
|
1787
|
-
\`\`\`json
|
|
1788
|
-
{
|
|
1789
|
-
"name": "firecrawl_search",
|
|
1790
|
-
"arguments": {
|
|
1791
|
-
"query": "latest AI research papers 2023",
|
|
1792
|
-
"limit": 5,
|
|
1793
|
-
"categories": ["github", "research"],
|
|
1794
|
-
"lang": "en",
|
|
1795
|
-
"country": "us",
|
|
1796
|
-
"sources": [
|
|
1797
|
-
{ "type": "web" },
|
|
1798
|
-
{ "type": "images" },
|
|
1799
|
-
{ "type": "news" }
|
|
1800
|
-
],
|
|
1801
|
-
"scrapeOptions": {
|
|
1802
|
-
"formats": ["markdown"],
|
|
1803
|
-
"onlyMainContent": true
|
|
1804
|
-
}
|
|
1805
|
-
}
|
|
1806
|
-
}
|
|
1807
|
-
\`\`\`
|
|
1808
|
-
**Returns:** A JSON envelope of the form \`{ success, data: { web?, images?, news? }, id, creditsUsed }\`. Each result array contains the search results (with optional scraped content). Pass the top-level \`id\` to \`firecrawl_search_feedback\` after you've used the results.
|
|
2068
|
+
\`scrapeOptions\` can attach extracted page content. Returns source-type result groups and usage metadata. Authenticated responses can include an \`id\` for optional search feedback.
|
|
1809
2069
|
`,
|
|
1810
|
-
parameters:
|
|
1811
|
-
|
|
1812
|
-
|
|
1813
|
-
|
|
1814
|
-
),
|
|
1815
|
-
limit: z3.number().optional(),
|
|
1816
|
-
tbs: z3.string().optional(),
|
|
1817
|
-
filter: z3.string().optional(),
|
|
1818
|
-
location: z3.string().optional(),
|
|
1819
|
-
includeDomains: z3.array(searchDomainSchema).optional(),
|
|
1820
|
-
excludeDomains: z3.array(searchDomainSchema).optional(),
|
|
1821
|
-
sources: z3.array(z3.object({ type: z3.enum(["web", "images", "news"]) })).optional(),
|
|
1822
|
-
categories: z3.array(z3.enum(["github", "research", "pdf"])).optional().describe(
|
|
1823
|
-
"Limit results to specific source types. `github` searches GitHub repositories, code, issues, and docs; `research` searches academic and research sources; `pdf` searches PDF results."
|
|
1824
|
-
),
|
|
1825
|
-
scrapeOptions: scrapeParamsSchema.omit({ url: true }).partial().optional(),
|
|
1826
|
-
enterprise: z3.array(z3.enum(["default", "anon", "zdr"])).optional()
|
|
1827
|
-
}).refine(
|
|
1828
|
-
(args2) => !(args2.includeDomains?.length && args2.excludeDomains?.length),
|
|
1829
|
-
"includeDomains and excludeDomains cannot both be specified"
|
|
1830
|
-
),
|
|
2070
|
+
parameters: z4.object({
|
|
2071
|
+
...searchToolBaseFields,
|
|
2072
|
+
scrapeOptions: scrapeParamsSchema.omit({ url: true }).partial().optional()
|
|
2073
|
+
}).refine(searchDomainsAreExclusive, SEARCH_DOMAINS_CONFLICT_MESSAGE),
|
|
1831
2074
|
execute: async (args2, { session, log }) => {
|
|
1832
2075
|
const { query, ...opts } = args2;
|
|
1833
2076
|
const searchOpts = { ...opts };
|
|
@@ -1854,7 +2097,9 @@ The query also supports search operators, that you can use if needed to refine t
|
|
|
1854
2097
|
};
|
|
1855
2098
|
if (isKeylessMode(session)) {
|
|
1856
2099
|
const json = await keylessPost("/v2/search", searchBody, session);
|
|
1857
|
-
|
|
2100
|
+
const keylessResponse = { ...json ?? {} };
|
|
2101
|
+
delete keylessResponse.id;
|
|
2102
|
+
return asText2(keylessResponse);
|
|
1858
2103
|
}
|
|
1859
2104
|
const client = getClient(session);
|
|
1860
2105
|
const httpRes = await client.http.post("/v2/search", searchBody);
|
|
@@ -1869,42 +2114,59 @@ function resolveApiBaseUrl() {
|
|
|
1869
2114
|
);
|
|
1870
2115
|
}
|
|
1871
2116
|
function extractClientIp(request) {
|
|
1872
|
-
|
|
1873
|
-
|
|
1874
|
-
|
|
1875
|
-
return
|
|
2117
|
+
return extractSingleTrustedClientIp(request?.headers?.["x-forwarded-for"]);
|
|
2118
|
+
}
|
|
2119
|
+
function keylessQuotaReason(reason) {
|
|
2120
|
+
return reason === "requests" || reason === "credits";
|
|
1876
2121
|
}
|
|
1877
2122
|
async function keylessEligible(clientIp) {
|
|
1878
2123
|
const secret = process.env.KEYLESS_PROXY_SECRET;
|
|
1879
|
-
if (!secret) return false;
|
|
2124
|
+
if (!secret) return { eligible: false, unavailable: true };
|
|
1880
2125
|
try {
|
|
1881
2126
|
const response = await fetch(
|
|
1882
2127
|
`${resolveApiBaseUrl()}/v2/keyless/eligibility`,
|
|
1883
2128
|
{
|
|
1884
2129
|
headers: {
|
|
1885
|
-
...
|
|
2130
|
+
...ORIGIN_HEADERS3,
|
|
1886
2131
|
"x-firecrawl-keyless-ip": clientIp,
|
|
1887
2132
|
"x-firecrawl-keyless-secret": secret
|
|
1888
2133
|
}
|
|
1889
2134
|
}
|
|
1890
2135
|
);
|
|
1891
|
-
if (!response.ok) return false;
|
|
1892
|
-
const json = await response.json().catch(() =>
|
|
1893
|
-
|
|
2136
|
+
if (!response.ok) return { eligible: false, unavailable: true };
|
|
2137
|
+
const json = await response.json().catch(() => null);
|
|
2138
|
+
if (typeof json?.eligible !== "boolean") {
|
|
2139
|
+
return { eligible: false, unavailable: true };
|
|
2140
|
+
}
|
|
2141
|
+
return {
|
|
2142
|
+
eligible: json?.eligible === true,
|
|
2143
|
+
...typeof json?.reason === "string" ? { reason: json.reason } : {},
|
|
2144
|
+
...Number.isFinite(json?.retryAfterSeconds) && json.retryAfterSeconds > 0 ? { retryAfterSeconds: json.retryAfterSeconds } : {}
|
|
2145
|
+
};
|
|
1894
2146
|
} catch {
|
|
1895
|
-
return false;
|
|
2147
|
+
return { eligible: false, unavailable: true };
|
|
1896
2148
|
}
|
|
1897
2149
|
}
|
|
1898
2150
|
function isKeylessMode(session) {
|
|
1899
|
-
if (session?.
|
|
2151
|
+
if (hasCredential(session) || session?.credentialError) return false;
|
|
1900
2152
|
if (process.env.CLOUD_SERVICE === "true") {
|
|
1901
|
-
return
|
|
2153
|
+
return session?.authType === "keyless";
|
|
1902
2154
|
}
|
|
1903
2155
|
return !process.env.FIRECRAWL_API_URL;
|
|
1904
2156
|
}
|
|
1905
2157
|
async function keylessPost(path2, body, session) {
|
|
2158
|
+
if (isHostedKeylessSession(session)) {
|
|
2159
|
+
const eligibility = session?.keylessClientIp ? await keylessEligible(session.keylessClientIp) : { eligible: false };
|
|
2160
|
+
if (!eligibility.eligible) {
|
|
2161
|
+
const code = eligibility.unavailable ? "KEYLESS_ELIGIBILITY_UNAVAILABLE" : keylessQuotaReason(eligibility.reason) ? "KEYLESS_QUOTA_EXHAUSTED" : "KEYLESS_ACCESS_NOT_AVAILABLE";
|
|
2162
|
+
const payload = recoveryPayload(code, session?.requestId, {
|
|
2163
|
+
retryAfterSeconds: eligibility.retryAfterSeconds
|
|
2164
|
+
});
|
|
2165
|
+
throw new UserError(String(payload.message), payload);
|
|
2166
|
+
}
|
|
2167
|
+
}
|
|
1906
2168
|
const headers = {
|
|
1907
|
-
...
|
|
2169
|
+
...ORIGIN_HEADERS3,
|
|
1908
2170
|
"Content-Type": "application/json"
|
|
1909
2171
|
};
|
|
1910
2172
|
if (session?.keylessClientIp && process.env.KEYLESS_PROXY_SECRET) {
|
|
@@ -1918,6 +2180,13 @@ async function keylessPost(path2, body, session) {
|
|
|
1918
2180
|
});
|
|
1919
2181
|
const json = await response.json().catch(() => ({}));
|
|
1920
2182
|
if (!response.ok) {
|
|
2183
|
+
if (isKeylessMode(session) && response.status === 429) {
|
|
2184
|
+
const code = keylessQuotaReason(json?.reason) ? "KEYLESS_QUOTA_EXHAUSTED" : "KEYLESS_LIMIT_REACHED";
|
|
2185
|
+
const payload = recoveryPayload(code, session?.requestId, {
|
|
2186
|
+
retryAfterSeconds: Number.isFinite(json?.retry_after_seconds) && json.retry_after_seconds > 0 ? json.retry_after_seconds : void 0
|
|
2187
|
+
});
|
|
2188
|
+
throw new UserError(String(payload.message), payload);
|
|
2189
|
+
}
|
|
1921
2190
|
throw new Error(
|
|
1922
2191
|
json?.error || `Firecrawl request failed (HTTP ${response.status})`
|
|
1923
2192
|
);
|
|
@@ -1927,7 +2196,7 @@ async function keylessPost(path2, body, session) {
|
|
|
1927
2196
|
async function getCrawlStatusWithOrigin(client, jobId) {
|
|
1928
2197
|
const res = await client.http.get(
|
|
1929
2198
|
`/v2/crawl/${encodeURIComponent(jobId)}`,
|
|
1930
|
-
|
|
2199
|
+
ORIGIN_HEADERS3
|
|
1931
2200
|
);
|
|
1932
2201
|
const body = res?.data ?? {};
|
|
1933
2202
|
const initialDocs = Array.isArray(body.data) ? body.data : [];
|
|
@@ -1946,7 +2215,7 @@ async function getCrawlStatusWithOrigin(client, jobId) {
|
|
|
1946
2215
|
const docs = initialDocs.slice();
|
|
1947
2216
|
let current = body.next;
|
|
1948
2217
|
while (current) {
|
|
1949
|
-
const pageRes = await client.http.get(current,
|
|
2218
|
+
const pageRes = await client.http.get(current, ORIGIN_HEADERS3);
|
|
1950
2219
|
const payload = pageRes?.data ?? {};
|
|
1951
2220
|
if (!payload.success) break;
|
|
1952
2221
|
const pageData = Array.isArray(payload.data) ? payload.data : payload.data?.pages || [];
|
|
@@ -1979,17 +2248,17 @@ async function waitForCrawlCompletionWithOrigin(client, jobId, pollInterval = 2,
|
|
|
1979
2248
|
);
|
|
1980
2249
|
}
|
|
1981
2250
|
}
|
|
1982
|
-
var feedbackIssueSchema =
|
|
2251
|
+
var feedbackIssueSchema = z4.string().trim().min(1).max(80).regex(
|
|
1983
2252
|
/^[a-z0-9][a-z0-9_-]*$/,
|
|
1984
2253
|
"Issue codes must use lowercase letters, numbers, underscores, or hyphens"
|
|
1985
2254
|
);
|
|
1986
|
-
var valuableSourceSchema =
|
|
1987
|
-
url:
|
|
1988
|
-
reason:
|
|
2255
|
+
var valuableSourceSchema = z4.object({
|
|
2256
|
+
url: z4.string().url(),
|
|
2257
|
+
reason: z4.string().max(1e3).optional()
|
|
1989
2258
|
});
|
|
1990
|
-
var missingContentSchema =
|
|
1991
|
-
topic:
|
|
1992
|
-
description:
|
|
2259
|
+
var missingContentSchema = z4.object({
|
|
2260
|
+
topic: z4.string().min(1, "topic must not be empty").max(200, "topic must be 200 characters or fewer"),
|
|
2261
|
+
description: z4.string().max(2e3).optional()
|
|
1993
2262
|
});
|
|
1994
2263
|
var FEEDBACK_DISABLED_VALUES = /* @__PURE__ */ new Set(["1", "true", "yes", "on"]);
|
|
1995
2264
|
function feedbackEnvEnabled(...keys) {
|
|
@@ -2010,7 +2279,7 @@ if (SEARCH_FEEDBACK_DISABLED) {
|
|
|
2010
2279
|
"[firecrawl-mcp] Search feedback tool disabled by FIRECRAWL_NO_SEARCH_FEEDBACK; firecrawl_search_feedback will not be registered."
|
|
2011
2280
|
);
|
|
2012
2281
|
}
|
|
2013
|
-
if (!SEARCH_FEEDBACK_DISABLED) {
|
|
2282
|
+
if (!SEARCH_FEEDBACK_DISABLED && !isLocalKeylessStartup()) {
|
|
2014
2283
|
server.addTool({
|
|
2015
2284
|
name: "firecrawl_search_feedback",
|
|
2016
2285
|
annotations: {
|
|
@@ -2023,84 +2292,28 @@ if (!SEARCH_FEEDBACK_DISABLED) {
|
|
|
2023
2292
|
// Additive only; records feedback and may refund credits, does not delete data.
|
|
2024
2293
|
},
|
|
2025
2294
|
description: `
|
|
2026
|
-
|
|
2027
|
-
|
|
2028
|
-
Pass the \`searchId\` returned by \`firecrawl_search\` (the \`id\` field on the response) and tell us:
|
|
2029
|
-
|
|
2030
|
-
- **rating** \u2014 overall result quality: \`good\`, \`partial\`, or \`bad\`.
|
|
2031
|
-
- **valuableSources** \u2014 which result URLs were actually useful, and a short reason why.
|
|
2032
|
-
- **missingContent** \u2014 **the most important field.** An ARRAY of specific pieces of content you expected to find but didn't. One entry per missing piece, each with a short \`topic\` and an optional longer \`description\`. Examples: \`{"topic":"enterprise pricing","description":"no pricing tier table for the Enterprise plan was returned"}\`, \`{"topic":"API rate limits"}\`, \`{"topic":"comparison vs competitors"}\`. **Be specific** \u2014 these aggregate across teams and tell us what to index next. Do not pack multiple topics into one entry.
|
|
2033
|
-
- **querySuggestions** \u2014 how the query or response shape could be improved (e.g. "would have liked official docs first", "should boost github.com").
|
|
2034
|
-
|
|
2035
|
-
**Substantive-feedback requirement** (zero-effort feedback is rejected with HTTP 400):
|
|
2036
|
-
- \`good\` \u2014 must include at least one \`valuableSources\` entry
|
|
2037
|
-
- \`partial\` \u2014 must include \`valuableSources\` or at least one \`missingContent\` entry
|
|
2038
|
-
- \`bad\` \u2014 must include at least one \`missingContent\` entry or \`querySuggestions\`
|
|
2295
|
+
Records schema-validated quality feedback for a prior \`firecrawl_search\` UUID \`searchId\`. A \`good\` rating requires a valuable source, \`partial\` a valuable source or at least one \`missingContent\` entry, and \`bad\` at least one \`missingContent\` entry or a query suggestion; caps are 50 \`valuableSources\` and 20 \`missingContent\` entries.
|
|
2039
2296
|
|
|
2040
|
-
|
|
2041
|
-
|
|
2042
|
-
**Behaviors:**
|
|
2043
|
-
- Idempotent per \`searchId\`. Re-submitting for the same id returns \`alreadySubmitted: true\` with \`creditsRefunded: 0\`.
|
|
2044
|
-
- Refund only applies to billable searches; preview teams are blocked.
|
|
2045
|
-
- Failed searches cannot receive feedback (the search itself already returned an error you can act on).
|
|
2046
|
-
- **Daily refund cap (per team, per UTC day, default 100 credits).** Once a team's \`creditsRefundedToday\` reaches \`dailyRefundCap\`, the response returns \`dailyCapReached: true\` with \`creditsRefunded: 0\`. The feedback is still recorded for search-quality improvement \u2014 only the credit refund is gated. **Stop calling this tool for the rest of the UTC day** when you see \`dailyCapReached: true\`.
|
|
2047
|
-
|
|
2048
|
-
**When to call:** Right after processing a search result. If the result didn't help, send rating \`bad\` with a clear \`missingContent\` \u2014 that is just as valuable as a \`good\` rating.
|
|
2049
|
-
|
|
2050
|
-
**Usage Example (good rating with valuable sources + missing content):**
|
|
2051
|
-
\`\`\`json
|
|
2052
|
-
{
|
|
2053
|
-
"name": "firecrawl_search_feedback",
|
|
2054
|
-
"arguments": {
|
|
2055
|
-
"searchId": "0193f6c5-1234-7890-abcd-1234567890ab",
|
|
2056
|
-
"rating": "good",
|
|
2057
|
-
"valuableSources": [
|
|
2058
|
-
{ "url": "https://docs.firecrawl.dev/features/search", "reason": "Most up-to-date description of /search." }
|
|
2059
|
-
],
|
|
2060
|
-
"missingContent": [
|
|
2061
|
-
{ "topic": "Pricing for the search endpoint", "description": "No pricing tier table for /search specifically." },
|
|
2062
|
-
{ "topic": "Rate limits", "description": "Per-team RPS for /search not documented." }
|
|
2063
|
-
],
|
|
2064
|
-
"querySuggestions": "Boost docs.firecrawl.dev for queries that mention 'firecrawl'"
|
|
2065
|
-
}
|
|
2066
|
-
}
|
|
2067
|
-
\`\`\`
|
|
2068
|
-
|
|
2069
|
-
**Usage Example (bad rating, what was missing):**
|
|
2070
|
-
\`\`\`json
|
|
2071
|
-
{
|
|
2072
|
-
"name": "firecrawl_search_feedback",
|
|
2073
|
-
"arguments": {
|
|
2074
|
-
"searchId": "0193f6c5-1234-7890-abcd-1234567890ab",
|
|
2075
|
-
"rating": "bad",
|
|
2076
|
-
"missingContent": [
|
|
2077
|
-
{ "topic": "Recent benchmarks", "description": "All results were >12 months old." },
|
|
2078
|
-
{ "topic": "Comparison vs Algolia" }
|
|
2079
|
-
]
|
|
2080
|
-
}
|
|
2081
|
-
}
|
|
2082
|
-
\`\`\`
|
|
2083
|
-
|
|
2084
|
-
**Returns:** \`{ success, feedbackId, creditsRefunded, creditsRefundedToday, dailyRefundCap, dailyCapReached?, alreadySubmitted?, warning? }\` JSON.
|
|
2297
|
+
Eligibility is limited to successful searches within the feedback age window. The record is idempotent per search ID. Eligible first feedback for a search can refund 1 credit; refunds are subject to the team's daily cap. The response reports whether a refund was applied, along with submission and daily-cap status.
|
|
2085
2298
|
`,
|
|
2086
|
-
parameters:
|
|
2087
|
-
searchId:
|
|
2088
|
-
rating:
|
|
2089
|
-
valuableSources:
|
|
2090
|
-
|
|
2091
|
-
url:
|
|
2092
|
-
reason:
|
|
2299
|
+
parameters: z4.object({
|
|
2300
|
+
searchId: z4.string().uuid("searchId must be the UUID returned by firecrawl_search"),
|
|
2301
|
+
rating: z4.enum(["good", "bad", "partial"]),
|
|
2302
|
+
valuableSources: z4.array(
|
|
2303
|
+
z4.object({
|
|
2304
|
+
url: z4.string().url(),
|
|
2305
|
+
reason: z4.string().max(1e3).optional()
|
|
2093
2306
|
})
|
|
2094
2307
|
).max(50).optional(),
|
|
2095
|
-
missingContent:
|
|
2096
|
-
|
|
2097
|
-
topic:
|
|
2098
|
-
description:
|
|
2308
|
+
missingContent: z4.array(
|
|
2309
|
+
z4.object({
|
|
2310
|
+
topic: z4.string().min(1, "topic must not be empty").max(200, "topic must be 200 characters or fewer"),
|
|
2311
|
+
description: z4.string().max(2e3).optional()
|
|
2099
2312
|
})
|
|
2100
2313
|
).max(20).optional().describe(
|
|
2101
2314
|
"Array of specific pieces of content the agent expected to find but did not. One entry per distinct topic. Each entry has a short `topic` and optional longer `description`."
|
|
2102
2315
|
),
|
|
2103
|
-
querySuggestions:
|
|
2316
|
+
querySuggestions: z4.string().max(2e3).optional()
|
|
2104
2317
|
}),
|
|
2105
2318
|
execute: async (args2, { session, log }) => {
|
|
2106
2319
|
const {
|
|
@@ -2126,12 +2339,12 @@ Pass the \`searchId\` returned by \`firecrawl_search\` (the \`id\` field on the
|
|
|
2126
2339
|
}
|
|
2127
2340
|
if (querySuggestions) body.querySuggestions = querySuggestions;
|
|
2128
2341
|
const headers = {
|
|
2129
|
-
...
|
|
2342
|
+
...ORIGIN_HEADERS3,
|
|
2130
2343
|
"Content-Type": "application/json"
|
|
2131
2344
|
};
|
|
2132
|
-
const
|
|
2133
|
-
if (
|
|
2134
|
-
headers["Authorization"] = `Bearer ${
|
|
2345
|
+
const credential = credentialForOutboundRequest(session);
|
|
2346
|
+
if (credential) {
|
|
2347
|
+
headers["Authorization"] = `Bearer ${credential}`;
|
|
2135
2348
|
} else if (process.env.CLOUD_SERVICE === "true") {
|
|
2136
2349
|
throw new Error("Unauthorized: missing API key for search feedback.");
|
|
2137
2350
|
}
|
|
@@ -2170,7 +2383,7 @@ if (ENDPOINT_FEEDBACK_DISABLED) {
|
|
|
2170
2383
|
"[firecrawl-mcp] Endpoint feedback tool disabled by FIRECRAWL_NO_ENDPOINT_FEEDBACK; firecrawl_feedback will not be registered."
|
|
2171
2384
|
);
|
|
2172
2385
|
}
|
|
2173
|
-
if (!ENDPOINT_FEEDBACK_DISABLED) {
|
|
2386
|
+
if (!ENDPOINT_FEEDBACK_DISABLED && !isLocalKeylessStartup()) {
|
|
2174
2387
|
server.addTool({
|
|
2175
2388
|
name: "firecrawl_feedback",
|
|
2176
2389
|
annotations: {
|
|
@@ -2183,35 +2396,23 @@ if (!ENDPOINT_FEEDBACK_DISABLED) {
|
|
|
2183
2396
|
// Additive only; submits ratings and notes, does not delete jobs or external content.
|
|
2184
2397
|
},
|
|
2185
2398
|
description: `
|
|
2186
|
-
|
|
2187
|
-
|
|
2188
|
-
For search-result quality specifically, prefer \`firecrawl_search_feedback\` when available because it has search-focused guidance. This generic tool posts to \`/v2/feedback\` and accepts endpoint-wide signals:
|
|
2189
|
-
|
|
2190
|
-
- **endpoint** \u2014 one of \`search\`, \`scrape\`, \`parse\`, or \`map\`.
|
|
2191
|
-
- **jobId** \u2014 the id returned by that endpoint.
|
|
2192
|
-
- **rating** \u2014 overall result quality: \`good\`, \`partial\`, or \`bad\`.
|
|
2193
|
-
- **issues** \u2014 stable lowercase issue codes such as \`missing_markdown\`, \`bad_pdf_parse\`, or \`wrong_links\`.
|
|
2194
|
-
- **tags** \u2014 optional lowercase tags for grouping feedback.
|
|
2195
|
-
- **note** \u2014 short human-readable context. Do not include huge page contents or raw scrape results.
|
|
2196
|
-
- **url**, **pageNumbers**, and **metadata** \u2014 small contextual fields that identify what the feedback refers to.
|
|
2197
|
-
|
|
2198
|
-
Do not store multi-MB outputs in feedback. Use concise notes, issue codes, URLs, and page numbers.
|
|
2399
|
+
Submit concise quality feedback for a completed search, scrape, parse, or map job. Provide the endpoint, job ID, rating, and relevant issue codes or small contextual fields; omit large page contents and raw outputs.
|
|
2199
2400
|
|
|
2200
|
-
|
|
2401
|
+
Returns submission status, feedback ID, and accounting fields.
|
|
2201
2402
|
`,
|
|
2202
|
-
parameters:
|
|
2203
|
-
endpoint:
|
|
2204
|
-
jobId:
|
|
2205
|
-
rating:
|
|
2206
|
-
issues:
|
|
2207
|
-
tags:
|
|
2208
|
-
note:
|
|
2209
|
-
valuableSources:
|
|
2210
|
-
missingContent:
|
|
2211
|
-
querySuggestions:
|
|
2212
|
-
url:
|
|
2213
|
-
pageNumbers:
|
|
2214
|
-
metadata:
|
|
2403
|
+
parameters: z4.object({
|
|
2404
|
+
endpoint: z4.enum(["search", "scrape", "parse", "map"]),
|
|
2405
|
+
jobId: z4.string().uuid("jobId must be the UUID returned by Firecrawl"),
|
|
2406
|
+
rating: z4.enum(["good", "bad", "partial"]),
|
|
2407
|
+
issues: z4.array(feedbackIssueSchema).max(20).optional(),
|
|
2408
|
+
tags: z4.array(feedbackIssueSchema).max(20).optional(),
|
|
2409
|
+
note: z4.string().max(4e3).optional(),
|
|
2410
|
+
valuableSources: z4.array(valuableSourceSchema).max(50).optional(),
|
|
2411
|
+
missingContent: z4.array(missingContentSchema).max(50).optional(),
|
|
2412
|
+
querySuggestions: z4.string().max(2e3).optional(),
|
|
2413
|
+
url: z4.string().url().optional(),
|
|
2414
|
+
pageNumbers: z4.array(z4.number().int().positive()).max(100).optional(),
|
|
2415
|
+
metadata: z4.record(z4.string(), z4.unknown()).optional()
|
|
2215
2416
|
}),
|
|
2216
2417
|
execute: async (args2, { session, log }) => {
|
|
2217
2418
|
const {
|
|
@@ -2230,12 +2431,12 @@ Do not store multi-MB outputs in feedback. Use concise notes, issue codes, URLs,
|
|
|
2230
2431
|
} = args2;
|
|
2231
2432
|
const apiBase = resolveApiBaseUrl();
|
|
2232
2433
|
const headers = {
|
|
2233
|
-
...
|
|
2434
|
+
...ORIGIN_HEADERS3,
|
|
2234
2435
|
"Content-Type": "application/json"
|
|
2235
2436
|
};
|
|
2236
|
-
const
|
|
2237
|
-
if (
|
|
2238
|
-
headers["Authorization"] = `Bearer ${
|
|
2437
|
+
const credential = credentialForOutboundRequest(session);
|
|
2438
|
+
if (credential) {
|
|
2439
|
+
headers["Authorization"] = `Bearer ${credential}`;
|
|
2239
2440
|
} else if (process.env.CLOUD_SERVICE === "true") {
|
|
2240
2441
|
throw new Error("Unauthorized: missing API key for feedback.");
|
|
2241
2442
|
}
|
|
@@ -2296,49 +2497,29 @@ server.addTool({
|
|
|
2296
2497
|
// Reads pages from target sites; does not delete or alter external websites.
|
|
2297
2498
|
},
|
|
2298
2499
|
description: `
|
|
2299
|
-
|
|
2300
|
-
|
|
2301
|
-
|
|
2302
|
-
|
|
2303
|
-
|
|
2304
|
-
|
|
2305
|
-
|
|
2306
|
-
|
|
2307
|
-
|
|
2308
|
-
|
|
2309
|
-
|
|
2310
|
-
|
|
2311
|
-
|
|
2312
|
-
|
|
2313
|
-
|
|
2314
|
-
|
|
2315
|
-
|
|
2316
|
-
"sitemap": "include"
|
|
2317
|
-
}
|
|
2318
|
-
}
|
|
2319
|
-
\`\`\`
|
|
2320
|
-
**Returns:** Final crawl status and data after internal polling, including the crawl id. Use firecrawl_check_crawl_status only when you need to re-check an existing crawl ID later.
|
|
2321
|
-
${SAFE_MODE ? "**Safe Mode:** Read-only crawling. Webhooks and interactive actions are disabled for security." : ""}
|
|
2322
|
-
`,
|
|
2323
|
-
parameters: z3.object({
|
|
2324
|
-
url: z3.string(),
|
|
2325
|
-
prompt: z3.string().optional(),
|
|
2326
|
-
excludePaths: z3.array(z3.string()).optional(),
|
|
2327
|
-
includePaths: z3.array(z3.string()).optional(),
|
|
2328
|
-
maxDiscoveryDepth: z3.number().optional(),
|
|
2329
|
-
sitemap: z3.enum(["skip", "include", "only"]).optional(),
|
|
2330
|
-
limit: z3.number().optional(),
|
|
2331
|
-
allowExternalLinks: z3.boolean().optional(),
|
|
2332
|
-
allowSubdomains: z3.boolean().optional(),
|
|
2333
|
-
crawlEntireDomain: z3.boolean().optional(),
|
|
2334
|
-
delay: z3.number().optional(),
|
|
2335
|
-
maxConcurrency: z3.number().optional(),
|
|
2500
|
+
Start a multi-page crawl at a website URL, poll it to a terminal state, and return the final status and collected data. Scope can be bounded with include/exclude paths, depth, page limit, subdomain/external-link controls, sitemap handling, delay, and scrape options.
|
|
2501
|
+
|
|
2502
|
+
Crawl results can be large; use conservative limits when full-site coverage is unnecessary. Webhooks and interactive scrape actions are unavailable in safe mode. Returns the crawl ID, status, and page data.
|
|
2503
|
+
`,
|
|
2504
|
+
parameters: z4.object({
|
|
2505
|
+
url: z4.string(),
|
|
2506
|
+
prompt: z4.string().optional(),
|
|
2507
|
+
excludePaths: z4.array(z4.string()).optional(),
|
|
2508
|
+
includePaths: z4.array(z4.string()).optional(),
|
|
2509
|
+
maxDiscoveryDepth: z4.number().optional(),
|
|
2510
|
+
sitemap: z4.enum(["skip", "include", "only"]).optional(),
|
|
2511
|
+
limit: z4.number().optional(),
|
|
2512
|
+
allowExternalLinks: z4.boolean().optional(),
|
|
2513
|
+
allowSubdomains: z4.boolean().optional(),
|
|
2514
|
+
crawlEntireDomain: z4.boolean().optional(),
|
|
2515
|
+
delay: z4.number().optional(),
|
|
2516
|
+
maxConcurrency: z4.number().optional(),
|
|
2336
2517
|
...SAFE_MODE ? {} : {
|
|
2337
|
-
webhook:
|
|
2338
|
-
webhookHeaders:
|
|
2518
|
+
webhook: z4.string().optional(),
|
|
2519
|
+
webhookHeaders: z4.record(z4.string(), z4.string()).optional()
|
|
2339
2520
|
},
|
|
2340
|
-
deduplicateSimilarURLs:
|
|
2341
|
-
ignoreQueryParameters:
|
|
2521
|
+
deduplicateSimilarURLs: z4.boolean().optional(),
|
|
2522
|
+
ignoreQueryParameters: z4.boolean().optional(),
|
|
2342
2523
|
scrapeOptions: scrapeParamsSchema.omit({ url: true }).partial().optional()
|
|
2343
2524
|
}),
|
|
2344
2525
|
execute: async (args2, { session, log }) => {
|
|
@@ -2389,20 +2570,9 @@ server.addTool({
|
|
|
2389
2570
|
// Status lookup only; no deletes or updates.
|
|
2390
2571
|
},
|
|
2391
2572
|
description: `
|
|
2392
|
-
|
|
2393
|
-
|
|
2394
|
-
**Usage Example:**
|
|
2395
|
-
\`\`\`json
|
|
2396
|
-
{
|
|
2397
|
-
"name": "firecrawl_check_crawl_status",
|
|
2398
|
-
"arguments": {
|
|
2399
|
-
"id": "550e8400-e29b-41d4-a716-446655440000"
|
|
2400
|
-
}
|
|
2401
|
-
}
|
|
2402
|
-
\`\`\`
|
|
2403
|
-
**Returns:** Status and progress of the crawl job, including results if available.
|
|
2573
|
+
Retrieve the current status, progress, and available results for an existing crawl ID. This only reads Firecrawl job state and does not start or modify the crawl.
|
|
2404
2574
|
`,
|
|
2405
|
-
parameters:
|
|
2575
|
+
parameters: z4.object({ id: z4.string() }),
|
|
2406
2576
|
execute: async (args2, { session }) => {
|
|
2407
2577
|
const client = getClient(session);
|
|
2408
2578
|
const id = args2.id;
|
|
@@ -2422,49 +2592,17 @@ server.addTool({
|
|
|
2422
2592
|
// Read-only extraction; no destructive changes to external content.
|
|
2423
2593
|
},
|
|
2424
2594
|
description: `
|
|
2425
|
-
Extract structured information from
|
|
2595
|
+
Extract structured information from one or more URLs with an optional natural-language prompt and JSON schema. It can include subdomains, follow external links, or use web search when those options are enabled.
|
|
2426
2596
|
|
|
2427
|
-
|
|
2428
|
-
**Not recommended for:** When you need the full content of a page (use scrape); when you're not looking for specific structured data.
|
|
2429
|
-
**Arguments:**
|
|
2430
|
-
- urls: Array of URLs to extract information from
|
|
2431
|
-
- prompt: Custom prompt for the LLM extraction
|
|
2432
|
-
- schema: JSON schema for structured data extraction
|
|
2433
|
-
- allowExternalLinks: Allow extraction from external links
|
|
2434
|
-
- enableWebSearch: Enable web search for additional context
|
|
2435
|
-
- includeSubdomains: Include subdomains in extraction
|
|
2436
|
-
**Prompt Example:** "Extract the product name, price, and description from these product pages."
|
|
2437
|
-
**Usage Example:**
|
|
2438
|
-
\`\`\`json
|
|
2439
|
-
{
|
|
2440
|
-
"name": "firecrawl_extract",
|
|
2441
|
-
"arguments": {
|
|
2442
|
-
"urls": ["https://example.com/page1", "https://example.com/page2"],
|
|
2443
|
-
"prompt": "Extract product information including name, price, and description",
|
|
2444
|
-
"schema": {
|
|
2445
|
-
"type": "object",
|
|
2446
|
-
"properties": {
|
|
2447
|
-
"name": { "type": "string" },
|
|
2448
|
-
"price": { "type": "number" },
|
|
2449
|
-
"description": { "type": "string" }
|
|
2450
|
-
},
|
|
2451
|
-
"required": ["name", "price"]
|
|
2452
|
-
},
|
|
2453
|
-
"allowExternalLinks": false,
|
|
2454
|
-
"enableWebSearch": false,
|
|
2455
|
-
"includeSubdomains": false
|
|
2456
|
-
}
|
|
2457
|
-
}
|
|
2458
|
-
\`\`\`
|
|
2459
|
-
**Returns:** Extracted structured data as defined by your schema.
|
|
2597
|
+
Use this for a defined structured result rather than full page content. Returns data shaped by the supplied schema or prompt.
|
|
2460
2598
|
`,
|
|
2461
|
-
parameters:
|
|
2462
|
-
urls:
|
|
2463
|
-
prompt:
|
|
2464
|
-
schema:
|
|
2465
|
-
allowExternalLinks:
|
|
2466
|
-
enableWebSearch:
|
|
2467
|
-
includeSubdomains:
|
|
2599
|
+
parameters: z4.object({
|
|
2600
|
+
urls: z4.array(z4.string()),
|
|
2601
|
+
prompt: z4.string().optional(),
|
|
2602
|
+
schema: z4.record(z4.string(), z4.any()).optional(),
|
|
2603
|
+
allowExternalLinks: z4.boolean().optional(),
|
|
2604
|
+
enableWebSearch: z4.boolean().optional(),
|
|
2605
|
+
includeSubdomains: z4.boolean().optional()
|
|
2468
2606
|
}),
|
|
2469
2607
|
execute: async (args2, { session, log }) => {
|
|
2470
2608
|
const client = getClient(session);
|
|
@@ -2497,78 +2635,14 @@ server.addTool({
|
|
|
2497
2635
|
// Gathers information only; does not delete external data or user resources.
|
|
2498
2636
|
},
|
|
2499
2637
|
description: `
|
|
2500
|
-
|
|
2501
|
-
|
|
2502
|
-
**How it works:** The agent performs web searches, follows links, reads pages, and gathers data autonomously. This runs **asynchronously** - it returns a job ID immediately, and you poll \`firecrawl_agent_status\` to check when complete and retrieve results.
|
|
2503
|
-
|
|
2504
|
-
**IMPORTANT - Async workflow with patient polling:**
|
|
2505
|
-
1. Call \`firecrawl_agent\` with your prompt/schema \u2192 returns job ID immediately
|
|
2506
|
-
2. Poll \`firecrawl_agent_status\` with the job ID to check progress
|
|
2507
|
-
3. **Keep polling for at least 2-3 minutes** - agent research typically takes 1-5 minutes for complex queries
|
|
2508
|
-
4. Poll every 15-30 seconds until status is "completed" or "failed"
|
|
2509
|
-
5. Do NOT give up after just a few polling attempts - the agent needs time to research
|
|
2510
|
-
|
|
2511
|
-
**Expected wait times:**
|
|
2512
|
-
- Simple queries with provided URLs: 30 seconds - 1 minute
|
|
2513
|
-
- Complex research across multiple sites: 2-5 minutes
|
|
2514
|
-
- Deep research tasks: 5+ minutes
|
|
2638
|
+
Start an asynchronous web research job from a prompt, optional seed URLs, and an optional JSON schema. Use this for a requested synthesis across multiple sources when the task can wait for asynchronous completion. The agent can search, navigate, read pages, and assemble a structured result.
|
|
2515
2639
|
|
|
2516
|
-
|
|
2517
|
-
**Not recommended for:**
|
|
2518
|
-
- Single-page extraction when you have a URL (use firecrawl_scrape, faster and cheaper)
|
|
2519
|
-
- Web search (use firecrawl_search first)
|
|
2520
|
-
- Interactive page tasks like clicking, filling forms, login, or navigating JS-heavy SPAs (use firecrawl_scrape + firecrawl_interact)
|
|
2521
|
-
- Extracting specific data from a known page (use firecrawl_scrape with JSON format)
|
|
2522
|
-
|
|
2523
|
-
**Arguments:**
|
|
2524
|
-
- prompt: Natural language description of the data you want (required, max 10,000 characters)
|
|
2525
|
-
- urls: Optional array of URLs to focus the agent on specific pages
|
|
2526
|
-
- schema: Optional JSON schema for structured output
|
|
2527
|
-
|
|
2528
|
-
**Prompt Example:** "Find the founders of Firecrawl and their backgrounds"
|
|
2529
|
-
**Usage Example (start agent, then poll patiently for results):**
|
|
2530
|
-
\`\`\`json
|
|
2531
|
-
{
|
|
2532
|
-
"name": "firecrawl_agent",
|
|
2533
|
-
"arguments": {
|
|
2534
|
-
"prompt": "Find the top 5 AI startups founded in 2024 and their funding amounts",
|
|
2535
|
-
"schema": {
|
|
2536
|
-
"type": "object",
|
|
2537
|
-
"properties": {
|
|
2538
|
-
"startups": {
|
|
2539
|
-
"type": "array",
|
|
2540
|
-
"items": {
|
|
2541
|
-
"type": "object",
|
|
2542
|
-
"properties": {
|
|
2543
|
-
"name": { "type": "string" },
|
|
2544
|
-
"funding": { "type": "string" },
|
|
2545
|
-
"founded": { "type": "string" }
|
|
2546
|
-
}
|
|
2547
|
-
}
|
|
2548
|
-
}
|
|
2549
|
-
}
|
|
2550
|
-
}
|
|
2551
|
-
}
|
|
2552
|
-
}
|
|
2553
|
-
\`\`\`
|
|
2554
|
-
Then poll with \`firecrawl_agent_status\` every 15-30 seconds for at least 2-3 minutes.
|
|
2555
|
-
|
|
2556
|
-
**Usage Example (with URLs - agent focuses on specific pages):**
|
|
2557
|
-
\`\`\`json
|
|
2558
|
-
{
|
|
2559
|
-
"name": "firecrawl_agent",
|
|
2560
|
-
"arguments": {
|
|
2561
|
-
"urls": ["https://docs.firecrawl.dev", "https://firecrawl.dev/pricing"],
|
|
2562
|
-
"prompt": "Compare the features and pricing information from these pages"
|
|
2563
|
-
}
|
|
2564
|
-
}
|
|
2565
|
-
\`\`\`
|
|
2566
|
-
**Returns:** Job ID for status checking. Use \`firecrawl_agent_status\` to poll for results.
|
|
2640
|
+
This call returns only a job ID, not the research result. Read the job with \`firecrawl_agent_status\` until it reaches \`completed\` or \`failed\`; research commonly takes several minutes. If the job cannot finish within the task's available time, \`firecrawl_search\` and \`firecrawl_scrape\` can gather evidence synchronously.
|
|
2567
2641
|
`,
|
|
2568
|
-
parameters:
|
|
2569
|
-
prompt:
|
|
2570
|
-
urls:
|
|
2571
|
-
schema:
|
|
2642
|
+
parameters: z4.object({
|
|
2643
|
+
prompt: z4.string().min(1).max(1e4),
|
|
2644
|
+
urls: z4.array(z4.string().url()).optional(),
|
|
2645
|
+
schema: z4.record(z4.string(), z4.any()).optional()
|
|
2572
2646
|
}),
|
|
2573
2647
|
execute: async (args2, { session, log }) => {
|
|
2574
2648
|
const client = getClient(session);
|
|
@@ -2601,38 +2675,18 @@ server.addTool({
|
|
|
2601
2675
|
// Read-only status check.
|
|
2602
2676
|
},
|
|
2603
2677
|
description: `
|
|
2604
|
-
Check the status
|
|
2605
|
-
|
|
2606
|
-
**IMPORTANT - Be patient with polling:**
|
|
2607
|
-
- Poll every 15-30 seconds
|
|
2608
|
-
- **Keep polling for at least 2-3 minutes** before considering the request failed
|
|
2609
|
-
- Complex research can take 5+ minutes - do not give up early
|
|
2610
|
-
- Only stop polling when status is "completed" or "failed"
|
|
2678
|
+
Retrieve progress or final results for a \`firecrawl_agent\` job ID. A \`processing\` response is non-terminal and does not contain the final research result. Check again after 15\u201330 seconds until the status is \`completed\` or \`failed\`; complex jobs can take several minutes. If the job cannot finish within the task's available time, use \`firecrawl_search\` and \`firecrawl_scrape\` to complete the requested output.
|
|
2611
2679
|
|
|
2612
|
-
|
|
2613
|
-
\`\`\`json
|
|
2614
|
-
{
|
|
2615
|
-
"name": "firecrawl_agent_status",
|
|
2616
|
-
"arguments": {
|
|
2617
|
-
"id": "550e8400-e29b-41d4-a716-446655440000"
|
|
2618
|
-
}
|
|
2619
|
-
}
|
|
2620
|
-
\`\`\`
|
|
2621
|
-
**Possible statuses:**
|
|
2622
|
-
- processing: Agent is still researching - keep polling, do not give up
|
|
2623
|
-
- completed: Research finished - response includes the extracted data
|
|
2624
|
-
- failed: An error occurred (only stop polling on this status)
|
|
2625
|
-
|
|
2626
|
-
**Returns:** Status, progress, and results (if completed) of the agent job.
|
|
2680
|
+
Returns job status, progress information, and result data when completed.
|
|
2627
2681
|
`,
|
|
2628
|
-
parameters:
|
|
2682
|
+
parameters: z4.object({ id: z4.string() }),
|
|
2629
2683
|
execute: async (args2, { session, log }) => {
|
|
2630
2684
|
const client = getClient(session);
|
|
2631
2685
|
const { id } = args2;
|
|
2632
2686
|
log.info("Checking agent status", { id });
|
|
2633
2687
|
const res = await client.http.get(
|
|
2634
2688
|
`/v2/agent/${encodeURIComponent(id)}`,
|
|
2635
|
-
|
|
2689
|
+
ORIGIN_HEADERS3
|
|
2636
2690
|
);
|
|
2637
2691
|
return asText2(res?.data ?? {});
|
|
2638
2692
|
}
|
|
@@ -2649,53 +2703,17 @@ server.addTool({
|
|
|
2649
2703
|
// Transient page interactions only; does not delete monitors, jobs, or external sites.
|
|
2650
2704
|
},
|
|
2651
2705
|
description: `
|
|
2652
|
-
|
|
2653
|
-
|
|
2654
|
-
**Best for:** Multi-step workflows on a single page \u2014 searching a site, clicking through results, filling forms, extracting data that requires interaction.
|
|
2655
|
-
**Two ways to target a page:**
|
|
2656
|
-
- Pass a \`url\` to interact directly. The session is opened for you in one call (use this for a fresh page).
|
|
2657
|
-
- Pass a \`scrapeId\` from a previous firecrawl_scrape to reuse that already-loaded page (cheaper when you just scraped it).
|
|
2706
|
+
Open or reuse a live browser session to navigate a page, click controls, fill fields, or run browser code. Provide either \`url\` or \`scrapeId\`, and either a natural-language \`prompt\` or executable \`code\`; code can run as Bash, Python, or Node with a bounded timeout.
|
|
2658
2707
|
|
|
2659
|
-
|
|
2660
|
-
- url: Page to interact with; opens a session for you (use this OR scrapeId)
|
|
2661
|
-
- scrapeId: Scrape job ID from a previous scrape, found in its metadata (use this OR url)
|
|
2662
|
-
- prompt: Natural language instruction describing the action to take (use this OR code)
|
|
2663
|
-
- code: Code to execute in the browser session (use this OR prompt)
|
|
2664
|
-
- language: "bash", "python", or "node" (optional, defaults to "node", only used with code)
|
|
2665
|
-
- timeout: Interact execution timeout in seconds, 1-300 (optional, defaults to 30)
|
|
2666
|
-
- scrapeOptions: Optional scrape controls used only with url mode, such as waitFor, maxAge, proxy, or zeroDataRetention
|
|
2667
|
-
|
|
2668
|
-
**Usage Example (prompt, direct via url):**
|
|
2669
|
-
\`\`\`json
|
|
2670
|
-
{
|
|
2671
|
-
"name": "firecrawl_interact",
|
|
2672
|
-
"arguments": {
|
|
2673
|
-
"url": "https://example.com/products",
|
|
2674
|
-
"prompt": "Click on the first product and tell me its price"
|
|
2675
|
-
}
|
|
2676
|
-
}
|
|
2677
|
-
\`\`\`
|
|
2678
|
-
|
|
2679
|
-
**Usage Example (code):**
|
|
2680
|
-
\`\`\`json
|
|
2681
|
-
{
|
|
2682
|
-
"name": "firecrawl_interact",
|
|
2683
|
-
"arguments": {
|
|
2684
|
-
"scrapeId": "scrape-id-from-previous-scrape",
|
|
2685
|
-
"code": "agent-browser click @e5",
|
|
2686
|
-
"language": "bash"
|
|
2687
|
-
}
|
|
2688
|
-
}
|
|
2689
|
-
\`\`\`
|
|
2690
|
-
**Returns:** Execution result including output, stdout, stderr, exit code, and live view URLs.
|
|
2708
|
+
This acts on the live site, so actions such as form submission can create persistent external side effects. Returns execution output, stdout/stderr, exit status, and session viewing URLs.
|
|
2691
2709
|
`,
|
|
2692
|
-
parameters:
|
|
2693
|
-
scrapeId:
|
|
2694
|
-
url:
|
|
2695
|
-
prompt:
|
|
2696
|
-
code:
|
|
2697
|
-
language:
|
|
2698
|
-
timeout:
|
|
2710
|
+
parameters: z4.object({
|
|
2711
|
+
scrapeId: z4.string().trim().min(1).optional(),
|
|
2712
|
+
url: z4.string().trim().url().optional(),
|
|
2713
|
+
prompt: z4.string().trim().min(1).optional(),
|
|
2714
|
+
code: z4.string().trim().min(1).optional(),
|
|
2715
|
+
language: z4.enum(["bash", "python", "node"]).optional(),
|
|
2716
|
+
timeout: z4.number().min(1).max(300).optional(),
|
|
2699
2717
|
scrapeOptions: scrapeParamsSchema.omit({ url: true }).partial().optional()
|
|
2700
2718
|
}).refine((data) => Boolean(data.scrapeId) !== Boolean(data.url), {
|
|
2701
2719
|
message: "Provide either 'url' (interact directly) or 'scrapeId' (reuse a previous scrape), not both."
|
|
@@ -2770,21 +2788,10 @@ server.addTool({
|
|
|
2770
2788
|
// Terminates the live browser session; this end state cannot be resumed.
|
|
2771
2789
|
},
|
|
2772
2790
|
description: `
|
|
2773
|
-
Stop
|
|
2774
|
-
|
|
2775
|
-
**Usage Example:**
|
|
2776
|
-
\`\`\`json
|
|
2777
|
-
{
|
|
2778
|
-
"name": "firecrawl_interact_stop",
|
|
2779
|
-
"arguments": {
|
|
2780
|
-
"scrapeId": "scrape-id-here"
|
|
2781
|
-
}
|
|
2782
|
-
}
|
|
2783
|
-
\`\`\`
|
|
2784
|
-
**Returns:** Success confirmation.
|
|
2791
|
+
Stop the live interact session associated with a \`scrapeId\` and release its resources. Returns a success confirmation.
|
|
2785
2792
|
`,
|
|
2786
|
-
parameters:
|
|
2787
|
-
scrapeId:
|
|
2793
|
+
parameters: z4.object({
|
|
2794
|
+
scrapeId: z4.string()
|
|
2788
2795
|
}),
|
|
2789
2796
|
execute: async (args2, { session, log }) => {
|
|
2790
2797
|
const client = getClient(session);
|
|
@@ -2792,7 +2799,7 @@ Stop an interact session for a scraped page. Call this when you are done interac
|
|
|
2792
2799
|
log.info("Stopping interact session", { scrapeId });
|
|
2793
2800
|
const res = await client.http.delete(
|
|
2794
2801
|
`/v2/scrape/${encodeURIComponent(scrapeId)}/interact`,
|
|
2795
|
-
|
|
2802
|
+
ORIGIN_HEADERS3
|
|
2796
2803
|
);
|
|
2797
2804
|
return asText2(res?.data ?? {});
|
|
2798
2805
|
}
|
|
@@ -2809,56 +2816,11 @@ server.addTool({
|
|
|
2809
2816
|
// Read-only parsing; no deletion or writes to the source file.
|
|
2810
2817
|
},
|
|
2811
2818
|
description: `
|
|
2812
|
-
Parse a
|
|
2813
|
-
|
|
2814
|
-
In local/non-cloud MCP mode, this tool reads filePath from the MCP server filesystem and posts multipart data to the configured self-hosted FIRECRAWL_API_URL, preserving the existing direct-read behavior.
|
|
2815
|
-
|
|
2816
|
-
In hosted CLOUD_SERVICE mode, this tool is a two-call flow because hosted MCP cannot read your local filesystem:
|
|
2817
|
-
1. Call with filePath, contentType, parse options, and optional declaredSizeBytes. The hosted server mints a short-lived upload URL and returns a safe local curl PUT command plus nextToolCall.
|
|
2818
|
-
2. Run the returned curl command locally, then call firecrawl_parse again with uploadRef and the desired parse options. The hosted server calls /v2/parse server-side with your session credential.
|
|
2819
|
-
|
|
2820
|
-
**Best for:** Extracting content from a local document (PDF, Word, Excel, HTML, etc.); pulling structured data out of a file with JSON format; converting binary documents into markdown for downstream reasoning.
|
|
2821
|
-
**Not recommended for:** Remote URLs (use firecrawl_scrape); multiple files at once (call parse multiple times); documents that require interactive actions, screenshots, or change tracking \u2014 those aren't supported by the parse endpoint.
|
|
2822
|
-
**Common mistakes:** In hosted mode, do not pass both filePath and uploadRef. Phase 1 uses filePath only to generate upload instructions; phase 2 uses uploadRef only to parse server-side.
|
|
2823
|
-
|
|
2824
|
-
**Supported file types:** .html, .htm, .xhtml, .pdf, .docx, .doc, .odt, .rtf, .xlsx, .xls
|
|
2825
|
-
**Unsupported options:** actions, screenshot/branding/changeTracking formats, waitFor > 0, location, mobile, proxy values other than "auto" or "basic".
|
|
2826
|
-
**Privacy:** Set \`redactPII: true\` to return content with personally identifiable information redacted.
|
|
2819
|
+
Parse one supported document into markdown, HTML, links, summary, targeted answers, or JSON matching a schema. Supported inputs include common HTML, PDF, Word, RTF, OpenDocument, and spreadsheet files; PDF parsing can be bounded with \`pdfOptions.maxPages\`.
|
|
2827
2820
|
|
|
2828
|
-
|
|
2829
|
-
When the user asks for SPECIFIC data points from a document, you MUST use JSON format with a schema. Only use markdown when the user needs the ENTIRE document content.
|
|
2821
|
+
Local MCP reads \`filePath\` from the server filesystem. Hosted MCP uses two calls: first provide \`filePath\` to receive upload instructions, upload locally, then call again with the returned \`uploadRef\`; do not send both fields together. Remote web URLs belong in \`firecrawl_scrape\`.
|
|
2830
2822
|
|
|
2831
|
-
|
|
2832
|
-
Add \`"parsers": ["pdf"]\` (optionally with \`pdfOptions.maxPages\`) when parsing a PDF so the PDF engine is invoked explicitly. For very long documents, cap \`maxPages\` to keep the response within token limits.
|
|
2833
|
-
|
|
2834
|
-
**Hosted phase 1 example:**
|
|
2835
|
-
\`\`\`json
|
|
2836
|
-
{
|
|
2837
|
-
"name": "firecrawl_parse",
|
|
2838
|
-
"arguments": {
|
|
2839
|
-
"filePath": "/absolute/path/to/document.pdf",
|
|
2840
|
-
"contentType": "application/pdf",
|
|
2841
|
-
"formats": ["markdown"],
|
|
2842
|
-
"parsers": ["pdf"],
|
|
2843
|
-
"zeroDataRetention": true
|
|
2844
|
-
}
|
|
2845
|
-
}
|
|
2846
|
-
\`\`\`
|
|
2847
|
-
|
|
2848
|
-
**Hosted phase 2 example:**
|
|
2849
|
-
\`\`\`json
|
|
2850
|
-
{
|
|
2851
|
-
"name": "firecrawl_parse",
|
|
2852
|
-
"arguments": {
|
|
2853
|
-
"uploadRef": "upload-ref-from-phase-1",
|
|
2854
|
-
"formats": ["markdown"],
|
|
2855
|
-
"parsers": ["pdf"],
|
|
2856
|
-
"zeroDataRetention": true
|
|
2857
|
-
}
|
|
2858
|
-
}
|
|
2859
|
-
\`\`\`
|
|
2860
|
-
|
|
2861
|
-
**Returns:** Phase 1 hosted upload instructions or a parsed document with markdown, html, links, summary, json, or query results depending on the requested formats.
|
|
2823
|
+
Set \`redactPII\` to request redaction of personally identifiable information in the returned content. \`zeroDataRetention\` requires an eligible authenticated account; omit it for anonymous keyless use. Returns upload instructions for hosted phase one or parsed document content for the final call.
|
|
2862
2824
|
`,
|
|
2863
2825
|
parameters: parseParamsSchema,
|
|
2864
2826
|
execute: async (args2, { session, log }) => {
|
|
@@ -2889,10 +2851,10 @@ Add \`"parsers": ["pdf"]\` (optionally with \`pdfOptions.maxPages\`) when parsin
|
|
|
2889
2851
|
});
|
|
2890
2852
|
form.append("file", blob, filename);
|
|
2891
2853
|
form.append("options", JSON.stringify(optionsPayload));
|
|
2892
|
-
const headers = { ...
|
|
2893
|
-
const
|
|
2894
|
-
if (
|
|
2895
|
-
headers["Authorization"] = `Bearer ${
|
|
2854
|
+
const headers = { ...ORIGIN_HEADERS3 };
|
|
2855
|
+
const credential = credentialForOutboundRequest(session);
|
|
2856
|
+
if (credential) {
|
|
2857
|
+
headers["Authorization"] = `Bearer ${credential}`;
|
|
2896
2858
|
}
|
|
2897
2859
|
const endpoint = `${apiUrl.replace(/\/$/, "")}/v2/parse`;
|
|
2898
2860
|
log.info("Parsing local file", {
|
|
@@ -2918,6 +2880,63 @@ Add \`"parsers": ["pdf"]\` (optionally with \`pdfOptions.maxPages\`) when parsin
|
|
|
2918
2880
|
}
|
|
2919
2881
|
}
|
|
2920
2882
|
});
|
|
2883
|
+
function registerMarketplaceSearchTool(registrar, getClientFn) {
|
|
2884
|
+
registrar.addTool({
|
|
2885
|
+
name: "firecrawl_search",
|
|
2886
|
+
annotations: {
|
|
2887
|
+
title: "Search the web",
|
|
2888
|
+
readOnlyHint: true,
|
|
2889
|
+
openWorldHint: true,
|
|
2890
|
+
destructiveHint: false
|
|
2891
|
+
},
|
|
2892
|
+
description: `
|
|
2893
|
+
Search web and specialized indexes, returning ranked results. Operators include quoted phrases, \`-term\`, \`site:host\`, \`inurl:term\`, \`intitle:term\`, and \`related:host\`; the set is non-exhaustive. \`includeDomains\` and \`excludeDomains\` are mutually exclusive hostname filters; categories limit result types to \`github\`, \`research\`, \`pdf\`, or \`developer\`.
|
|
2894
|
+
|
|
2895
|
+
For a programming question, add \`categories: ["developer"]\`. It searches an index of GitHub issues, merged pull requests, repository READMEs, and curated documentation sites, and returns the hits in \`data.developer\` beside the web results.
|
|
2896
|
+
|
|
2897
|
+
Returns \`{ success, data, id, creditsUsed }\`, with source arrays in \`data\`.
|
|
2898
|
+
`,
|
|
2899
|
+
parameters: z4.object({ ...searchToolBaseFields }).strict().refine(searchDomainsAreExclusive, SEARCH_DOMAINS_CONFLICT_MESSAGE),
|
|
2900
|
+
execute: async (args2, { session, log }) => {
|
|
2901
|
+
const {
|
|
2902
|
+
query,
|
|
2903
|
+
includeDomains,
|
|
2904
|
+
excludeDomains,
|
|
2905
|
+
limit,
|
|
2906
|
+
tbs,
|
|
2907
|
+
filter,
|
|
2908
|
+
location,
|
|
2909
|
+
sources,
|
|
2910
|
+
categories,
|
|
2911
|
+
highlights,
|
|
2912
|
+
enterprise
|
|
2913
|
+
} = args2;
|
|
2914
|
+
const searchQuery = buildSearchQueryWithDomains(
|
|
2915
|
+
query,
|
|
2916
|
+
includeDomains,
|
|
2917
|
+
excludeDomains
|
|
2918
|
+
);
|
|
2919
|
+
const searchBody = {
|
|
2920
|
+
query: searchQuery,
|
|
2921
|
+
...removeEmptyTopLevel({
|
|
2922
|
+
limit,
|
|
2923
|
+
tbs,
|
|
2924
|
+
filter,
|
|
2925
|
+
location,
|
|
2926
|
+
sources,
|
|
2927
|
+
categories,
|
|
2928
|
+
highlights,
|
|
2929
|
+
enterprise
|
|
2930
|
+
}),
|
|
2931
|
+
origin: ORIGIN
|
|
2932
|
+
};
|
|
2933
|
+
log.info("Searching", { query: searchQuery });
|
|
2934
|
+
const client = getClientFn(session);
|
|
2935
|
+
const httpRes = await client.http.post("/v2/search", searchBody);
|
|
2936
|
+
return asText2(httpRes?.data ?? {});
|
|
2937
|
+
}
|
|
2938
|
+
});
|
|
2939
|
+
}
|
|
2921
2940
|
var PORT = Number(process.env.PORT || 3e3);
|
|
2922
2941
|
var HOST = process.env.CLOUD_SERVICE === "true" ? "0.0.0.0" : process.env.HOST || "localhost";
|
|
2923
2942
|
var args;
|
|
@@ -2927,6 +2946,7 @@ if (process.env.CLOUD_SERVICE === "true" || process.env.SSE_LOCAL === "true" ||
|
|
|
2927
2946
|
httpStream: {
|
|
2928
2947
|
port: PORT,
|
|
2929
2948
|
host: HOST,
|
|
2949
|
+
endpoint: primaryProfile.endpoint,
|
|
2930
2950
|
stateless: true
|
|
2931
2951
|
}
|
|
2932
2952
|
};
|
|
@@ -2937,4 +2957,53 @@ if (process.env.CLOUD_SERVICE === "true" || process.env.SSE_LOCAL === "true" ||
|
|
|
2937
2957
|
}
|
|
2938
2958
|
registerMonitorTools(server);
|
|
2939
2959
|
registerResearchTools(server, getClient);
|
|
2960
|
+
registerDeveloperTools(server, getClient);
|
|
2961
|
+
if (process.env.CLOUD_SERVICE === "true" && primaryProfile.allowKeyless && !normalizeHeader(process.env.KEYLESS_PROXY_SECRET)) {
|
|
2962
|
+
console.warn(
|
|
2963
|
+
"[firecrawl-mcp] KEYLESS_PROXY_SECRET is missing; keyless requests will be unavailable and /ready will fail."
|
|
2964
|
+
);
|
|
2965
|
+
}
|
|
2966
|
+
if (primaryProfile.id === "search") {
|
|
2967
|
+
const primarySearchRegistrar = {
|
|
2968
|
+
addTool: ((tool) => {
|
|
2969
|
+
if (primaryProfile.toolAllowlist?.has(tool.name)) {
|
|
2970
|
+
addTool(guardHostedTool(tool, { logActions: false }));
|
|
2971
|
+
}
|
|
2972
|
+
})
|
|
2973
|
+
};
|
|
2974
|
+
registerMarketplaceSearchTool(primarySearchRegistrar, getClient);
|
|
2975
|
+
}
|
|
2940
2976
|
await server.start(args);
|
|
2977
|
+
var searchProfileEnabled = process.env.CLOUD_SERVICE === "true" && primaryProfile.id === "full" && process.env.FIRECRAWL_MCP_SEARCH_ENABLED !== "false";
|
|
2978
|
+
if (searchProfileEnabled) {
|
|
2979
|
+
const searchProfile = makeSearchProfile();
|
|
2980
|
+
const searchServer = createServer(searchProfile);
|
|
2981
|
+
const searchRegistrar = {
|
|
2982
|
+
addTool: ((tool) => {
|
|
2983
|
+
if (searchProfile.toolAllowlist?.has(tool.name)) {
|
|
2984
|
+
searchServer.addTool(
|
|
2985
|
+
guardHostedTool(tool, { logActions: false })
|
|
2986
|
+
);
|
|
2987
|
+
}
|
|
2988
|
+
})
|
|
2989
|
+
};
|
|
2990
|
+
registerResearchTools(searchRegistrar, getClient);
|
|
2991
|
+
registerDeveloperTools(searchRegistrar, getClient);
|
|
2992
|
+
registerMarketplaceSearchTool(searchRegistrar, getClient);
|
|
2993
|
+
try {
|
|
2994
|
+
await searchServer.start({
|
|
2995
|
+
transportType: "httpStream",
|
|
2996
|
+
httpStream: {
|
|
2997
|
+
port: searchProfile.port,
|
|
2998
|
+
host: HOST,
|
|
2999
|
+
endpoint: searchProfile.endpoint,
|
|
3000
|
+
stateless: true
|
|
3001
|
+
}
|
|
3002
|
+
});
|
|
3003
|
+
} catch (error) {
|
|
3004
|
+
console.error(
|
|
3005
|
+
`[search-profile] failed to start on port ${searchProfile.port}; the full surface is unaffected`,
|
|
3006
|
+
error
|
|
3007
|
+
);
|
|
3008
|
+
}
|
|
3009
|
+
}
|