firecrawl-mcp 3.25.5 → 3.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +78 -1
- package/dist/agent-hints.js +46 -0
- package/dist/index.js +532 -84
- package/dist/introspection-cache.js +67 -0
- package/dist/keyless-signup-link.js +34 -0
- package/package.json +15 -14
package/dist/index.js
CHANGED
|
@@ -87,6 +87,45 @@ import { randomUUID } from "crypto";
|
|
|
87
87
|
import path from "path";
|
|
88
88
|
import { z as z8 } from "zod";
|
|
89
89
|
|
|
90
|
+
// src/agent-hints.ts
|
|
91
|
+
var AGENT_HINTS_HEADERS = { "X-Firecrawl-Agent-Hints": "true" };
|
|
92
|
+
function readAgentHints(value) {
|
|
93
|
+
if (!value || typeof value !== "object") return void 0;
|
|
94
|
+
const hints = value.agent_hints;
|
|
95
|
+
return Array.isArray(hints) && hints.every((hint) => typeof hint === "string") ? hints : void 0;
|
|
96
|
+
}
|
|
97
|
+
function preserveAgentHints(data, envelope) {
|
|
98
|
+
const hints = readAgentHints(envelope);
|
|
99
|
+
if (!hints) return data;
|
|
100
|
+
return data && typeof data === "object" && !Array.isArray(data) ? { ...data, agent_hints: hints } : { data, agent_hints: hints };
|
|
101
|
+
}
|
|
102
|
+
function agentHintsText(hints) {
|
|
103
|
+
return `Firecrawl API agent_hints (response guidance, separate from page content):
|
|
104
|
+
${JSON.stringify(hints, null, 2)}`;
|
|
105
|
+
}
|
|
106
|
+
function withAgentHints(result, response, separateText = false) {
|
|
107
|
+
const hints = readAgentHints(response);
|
|
108
|
+
if (!hints) return result;
|
|
109
|
+
return {
|
|
110
|
+
...result,
|
|
111
|
+
content: separateText && hints.length ? [...result.content, { type: "text", text: agentHintsText(hints) }] : result.content,
|
|
112
|
+
structuredContent: { ...result.structuredContent, agent_hints: hints }
|
|
113
|
+
};
|
|
114
|
+
}
|
|
115
|
+
function readErrorAgentHints(error2) {
|
|
116
|
+
const hints = readAgentHints(error2);
|
|
117
|
+
if (hints) return hints;
|
|
118
|
+
if (!error2 || typeof error2 !== "object") return void 0;
|
|
119
|
+
const extras = error2.extras;
|
|
120
|
+
const extrasHints = readAgentHints(extras);
|
|
121
|
+
if (extrasHints) return extrasHints;
|
|
122
|
+
const details = error2.details;
|
|
123
|
+
const detailsHints = readAgentHints(details);
|
|
124
|
+
if (detailsHints) return detailsHints;
|
|
125
|
+
const response = error2.response;
|
|
126
|
+
return readAgentHints(response?.data);
|
|
127
|
+
}
|
|
128
|
+
|
|
90
129
|
// src/tool-output.ts
|
|
91
130
|
import { z as z2 } from "zod";
|
|
92
131
|
function isRecord(value) {
|
|
@@ -115,8 +154,11 @@ var unknown = (description) => z2.unknown().optional().describe(description);
|
|
|
115
154
|
var success = bool("Whether the API call succeeded.");
|
|
116
155
|
var error = unknown("Error message or error object when the call did not succeed.");
|
|
117
156
|
var warning = str("Non-fatal warning about the result.");
|
|
157
|
+
var agentHints = {
|
|
158
|
+
agent_hints: z2.array(z2.string()).optional().describe("Optional response guidance from the Firecrawl API.")
|
|
159
|
+
};
|
|
118
160
|
var alexandriaEnvelope = {
|
|
119
|
-
requestId: str("Identifier of this
|
|
161
|
+
requestId: str("Identifier of this Alexandria execution."),
|
|
120
162
|
scrape_id: str("Identifier of the underlying scrape."),
|
|
121
163
|
receipt: unknown("Billing receipt for the execution."),
|
|
122
164
|
creditsCost: num("Credits this call consumed."),
|
|
@@ -132,6 +174,7 @@ var alexandriaEnvelope = {
|
|
|
132
174
|
feedbackTool: unknown("Pointer to the feedback tool for reporting how this result served the task.")
|
|
133
175
|
};
|
|
134
176
|
var scrapeOutputSchema = z2.object({
|
|
177
|
+
...agentHints,
|
|
135
178
|
// URL mode: the scraped document, with the formats that were requested.
|
|
136
179
|
markdown: str("Page content as markdown."),
|
|
137
180
|
html: str("Processed HTML of the page."),
|
|
@@ -163,6 +206,7 @@ var scrapeOutputSchema = z2.object({
|
|
|
163
206
|
...alexandriaEnvelope
|
|
164
207
|
}).describe("A scraped document, or the Alexandria execution envelope when `alexandria` was passed.");
|
|
165
208
|
var mapOutputSchema = z2.object({
|
|
209
|
+
...agentHints,
|
|
166
210
|
links: unknown("URLs discovered under the website."),
|
|
167
211
|
success,
|
|
168
212
|
error,
|
|
@@ -170,6 +214,7 @@ var mapOutputSchema = z2.object({
|
|
|
170
214
|
id: str("Identifier of the map run, for optional map feedback.")
|
|
171
215
|
}).describe("URLs indexed under the requested website.");
|
|
172
216
|
var searchOutputSchema = z2.object({
|
|
217
|
+
...agentHints,
|
|
173
218
|
success,
|
|
174
219
|
data: unknown("Ranked results grouped by source, such as `web`, `news`, `images`, and `alexandria`."),
|
|
175
220
|
error,
|
|
@@ -181,12 +226,14 @@ var searchOutputSchema = z2.object({
|
|
|
181
226
|
feedbackTool: unknown("Pointer to the feedback tool for this search.")
|
|
182
227
|
}).describe("Ranked search results grouped by source.");
|
|
183
228
|
var findToolsOutputSchema = z2.object({
|
|
229
|
+
...agentHints,
|
|
184
230
|
success,
|
|
185
231
|
data: unknown("Discovery page in `data.alexandria[0].data`, with `level`, `items`, `total`, and an optional `nextTool`."),
|
|
186
232
|
error,
|
|
187
233
|
...alexandriaEnvelope
|
|
188
234
|
}).describe("One page of the Alexandria catalogue: categories, providers, tools, or one contract.");
|
|
189
235
|
var feedbackOutputSchema = z2.object({
|
|
236
|
+
...agentHints,
|
|
190
237
|
success,
|
|
191
238
|
error,
|
|
192
239
|
status: num("HTTP status when the submission was rejected."),
|
|
@@ -204,6 +251,7 @@ var feedbackOutputSchema = z2.object({
|
|
|
204
251
|
raw: str("Response body when it was not JSON.")
|
|
205
252
|
}).describe("Result of submitting feedback.");
|
|
206
253
|
var crawlOutputSchema = z2.object({
|
|
254
|
+
...agentHints,
|
|
207
255
|
success,
|
|
208
256
|
error,
|
|
209
257
|
warning,
|
|
@@ -218,6 +266,7 @@ var crawlOutputSchema = z2.object({
|
|
|
218
266
|
data: unknown("Scraped documents collected by the crawl.")
|
|
219
267
|
}).describe("Crawl job state and the documents collected so far.");
|
|
220
268
|
var agentOutputSchema = z2.object({
|
|
269
|
+
...agentHints,
|
|
221
270
|
success,
|
|
222
271
|
error,
|
|
223
272
|
id: str("Agent job identifier, for `firecrawl_agent_status`."),
|
|
@@ -226,6 +275,7 @@ var agentOutputSchema = z2.object({
|
|
|
226
275
|
threadTurn: num("Turn number of this job within its thread.")
|
|
227
276
|
}).describe("The queued research job. The result is read with `firecrawl_agent_status`.");
|
|
228
277
|
var agentStatusOutputSchema = z2.object({
|
|
278
|
+
...agentHints,
|
|
229
279
|
success,
|
|
230
280
|
error,
|
|
231
281
|
id: str("Agent job identifier."),
|
|
@@ -238,9 +288,18 @@ var agentStatusOutputSchema = z2.object({
|
|
|
238
288
|
model: str("Agent model that ran the job."),
|
|
239
289
|
mode: str("Agent mode the job ran in."),
|
|
240
290
|
threadId: str("Research thread this job belongs to."),
|
|
241
|
-
threadTurn: num("Turn number of this job within its thread.")
|
|
291
|
+
threadTurn: num("Turn number of this job within its thread."),
|
|
292
|
+
message: unknown("The agent's reply; in chat mode, the short answer to a follow-up."),
|
|
293
|
+
suggestions: unknown("Follow-ups the agent offers; send one as the prompt of the next turn with this threadId."),
|
|
294
|
+
exchange: unknown(
|
|
295
|
+
"What the job did with Alexandria providers: onTermsRequired, paidCalls, creditsUsed, skippedProviders (gated providers that would have helped), and requiresAction (approvalId plus the terms/show and terms/accept calls, each provider digest string | null and always present; call accept only with the user's explicit consent, then continue the thread with exchange.approve: {approvalId})."
|
|
296
|
+
),
|
|
297
|
+
pendingApproval: unknown(
|
|
298
|
+
'Set when the job ended waiting on the caller. Answer it by calling firecrawl_agent with this threadId and exchange.approve or exchange.decline carrying its id. kind "terms" lists providers whose data terms need accepting; otherwise calls lists paid calls waiting for approval.'
|
|
299
|
+
)
|
|
242
300
|
}).describe("Progress or final result of a research agent job.");
|
|
243
301
|
var interactOutputSchema = z2.object({
|
|
302
|
+
...agentHints,
|
|
244
303
|
success,
|
|
245
304
|
error: unknown("Why the interaction or the session could not run."),
|
|
246
305
|
url: str("The URL the session was opened from, when opening failed."),
|
|
@@ -255,12 +314,14 @@ var interactOutputSchema = z2.object({
|
|
|
255
314
|
liveUrl: str("URL for viewing the live session.")
|
|
256
315
|
}).describe("Result of a browser interaction, with the session identifier to continue or stop it.");
|
|
257
316
|
var interactStopOutputSchema = z2.object({
|
|
317
|
+
...agentHints,
|
|
258
318
|
success,
|
|
259
319
|
error,
|
|
260
320
|
message: str("Confirmation that the session was stopped."),
|
|
261
321
|
scrapeId: str("Session that was stopped.")
|
|
262
322
|
}).describe("Confirmation that the interact session was stopped.");
|
|
263
323
|
var parseOutputSchema = z2.object({
|
|
324
|
+
...agentHints,
|
|
264
325
|
success,
|
|
265
326
|
error,
|
|
266
327
|
warning,
|
|
@@ -288,33 +349,39 @@ var deprecatedToolOutputSchema = z2.object({
|
|
|
288
349
|
docs_url: str("Documentation for the replacement.")
|
|
289
350
|
}).describe("A pointer to the replacement tool. This entry point is deprecated and always errors.");
|
|
290
351
|
var monitorOutputSchema = z2.object({
|
|
352
|
+
...agentHints,
|
|
291
353
|
success,
|
|
292
354
|
error,
|
|
293
355
|
data: unknown("The monitor record: configuration, schedule, targets, and current state.")
|
|
294
356
|
}).describe("One monitor record.");
|
|
295
357
|
var monitorListOutputSchema = z2.object({
|
|
358
|
+
...agentHints,
|
|
296
359
|
success,
|
|
297
360
|
error,
|
|
298
361
|
data: unknown("One page of monitor records."),
|
|
299
362
|
pagination: unknown("Pagination metadata for the listing.")
|
|
300
363
|
}).describe("One page of monitors for the authenticated account.");
|
|
301
364
|
var monitorDeleteOutputSchema = z2.object({
|
|
365
|
+
...agentHints,
|
|
302
366
|
success,
|
|
303
367
|
error,
|
|
304
368
|
data: unknown("Deletion details, when the API returns any.")
|
|
305
369
|
}).describe("Deletion status for the monitor.");
|
|
306
370
|
var monitorRunOutputSchema = z2.object({
|
|
371
|
+
...agentHints,
|
|
307
372
|
success,
|
|
308
373
|
error,
|
|
309
374
|
data: unknown("The queued check.")
|
|
310
375
|
}).describe("The check queued outside the monitor schedule.");
|
|
311
376
|
var monitorChecksOutputSchema = z2.object({
|
|
377
|
+
...agentHints,
|
|
312
378
|
success,
|
|
313
379
|
error,
|
|
314
380
|
data: unknown("One page of check summaries."),
|
|
315
381
|
pagination: unknown("Pagination metadata for the listing.")
|
|
316
382
|
}).describe("One page of historical checks for the monitor.");
|
|
317
383
|
var monitorCheckOutputSchema = z2.object({
|
|
384
|
+
...agentHints,
|
|
318
385
|
success,
|
|
319
386
|
error,
|
|
320
387
|
data: unknown("The check and its page-level results: each page reports `same`, `new`, `changed`, `removed`, or `error`, with diffs and any goal judgment."),
|
|
@@ -333,20 +400,25 @@ var paperSchema = z2.looseObject({
|
|
|
333
400
|
updateDate: str("Date the paper was last updated.")
|
|
334
401
|
});
|
|
335
402
|
var researchSearchOutputSchema = z2.object({
|
|
403
|
+
...agentHints,
|
|
336
404
|
results: z2.array(paperSchema).describe("Ranked papers, in the order the text block lists them.")
|
|
337
405
|
}).describe("Ranked papers with canonical IDs, titles, authors, and abstracts.");
|
|
338
406
|
var researchPaperOutputSchema = z2.object({
|
|
407
|
+
...agentHints,
|
|
339
408
|
paper: paperSchema.optional().describe("Canonical metadata for the requested paper, absent when it was not found.")
|
|
340
409
|
}).describe("Canonical metadata for one paper.");
|
|
341
410
|
var researchRelatedOutputSchema = z2.object({
|
|
411
|
+
...agentHints,
|
|
342
412
|
results: z2.array(paperSchema).describe("Ranked citation-graph candidates."),
|
|
343
413
|
poolSize: z2.number().describe("Number of candidates evaluated before ranking."),
|
|
344
414
|
note: str("Note about how the candidates were produced.")
|
|
345
415
|
}).describe("Ranked citation-graph candidates and the size of the evaluated pool.");
|
|
346
416
|
var researchReadOutputSchema = z2.object({
|
|
417
|
+
...agentHints,
|
|
347
418
|
passages: z2.array(z2.looseObject({ text: str("Passage text.") })).describe("In-body passages relevant to the question; empty when no full text is indexed.")
|
|
348
419
|
}).describe("Full-text passages from one paper.");
|
|
349
420
|
var developerSearchOutputSchema = z2.object({
|
|
421
|
+
...agentHints,
|
|
350
422
|
results: z2.array(
|
|
351
423
|
z2.looseObject({
|
|
352
424
|
id: str("Stable result id, such as `issue:owner/repo#123` or `doc:<hash>`; its prefix gives the source type."),
|
|
@@ -435,7 +507,8 @@ function withFindToolsNavigation(envelope) {
|
|
|
435
507
|
const link = (next) => next?.provider === "firecrawl" && next?.capability === "find-tools" && next.options ? { name: "firecrawl_find_tools", arguments: next.options } : void 0;
|
|
436
508
|
for (const item of page.items) {
|
|
437
509
|
if (page.level === "providers" && link(item.next)) {
|
|
438
|
-
const
|
|
510
|
+
const selectors = { ...item.next.options };
|
|
511
|
+
delete selectors.expand;
|
|
439
512
|
item.next = { ...item.next, options: { ...selectors, level: "tools" } };
|
|
440
513
|
}
|
|
441
514
|
const nextTool = link(item.next);
|
|
@@ -594,7 +667,7 @@ Returns ranked results with an ID, source type, URL, title, and the matched pass
|
|
|
594
667
|
const client = getClient2(session);
|
|
595
668
|
const res = await client.http.get(`${BASE}?${params.toString()}`, originHeaders(requestOrigin(mcpClient, session)));
|
|
596
669
|
const results = res.data?.results ?? [];
|
|
597
|
-
return withStructured(fmtDeveloper(results), { results });
|
|
670
|
+
return withAgentHints(withStructured(fmtDeveloper(results), { results }), res.data, true);
|
|
598
671
|
}
|
|
599
672
|
});
|
|
600
673
|
}
|
|
@@ -613,6 +686,38 @@ function extractSingleTrustedClientIp(rawForwardedFor) {
|
|
|
613
686
|
return net.isIP(candidate) ? candidate : void 0;
|
|
614
687
|
}
|
|
615
688
|
|
|
689
|
+
// src/keyless-signup-link.ts
|
|
690
|
+
var SIGNUP_HOSTS = /* @__PURE__ */ new Set(["firecrawl.dev", "www.firecrawl.dev"]);
|
|
691
|
+
var TOKEN_PATH = /^\/k\/[0-9abcdefghjkmnpqrstvwxyz]{12}$/;
|
|
692
|
+
var SURFACES = /* @__PURE__ */ new Set(["api", "mcp", "cli"]);
|
|
693
|
+
var SIGNIN_PARAMS = /* @__PURE__ */ new Set(["utm_source", "utm_medium", "redirect"]);
|
|
694
|
+
var SIGNIN_REDIRECT = "/app/api-keys";
|
|
695
|
+
function checkKeylessSignupUrl(value) {
|
|
696
|
+
if (typeof value !== "string") return { ok: false, firecrawlHost: false };
|
|
697
|
+
let url;
|
|
698
|
+
try {
|
|
699
|
+
url = new URL(value);
|
|
700
|
+
} catch {
|
|
701
|
+
return { ok: false, firecrawlHost: false };
|
|
702
|
+
}
|
|
703
|
+
const firecrawlHost = SIGNUP_HOSTS.has(url.hostname);
|
|
704
|
+
if (url.protocol !== "https:" || !firecrawlHost || url.port || url.username || url.password || url.hash) {
|
|
705
|
+
return { ok: false, firecrawlHost };
|
|
706
|
+
}
|
|
707
|
+
if (TOKEN_PATH.test(url.pathname) && !url.search) {
|
|
708
|
+
return { ok: true, url: value };
|
|
709
|
+
}
|
|
710
|
+
if (url.pathname === "/signin") {
|
|
711
|
+
const params = url.searchParams;
|
|
712
|
+
const keys = [...params.keys()];
|
|
713
|
+
const known = keys.every((key) => SIGNIN_PARAMS.has(key)) && new Set(keys).size === keys.length;
|
|
714
|
+
if (known && params.get("utm_source") === "keyless" && SURFACES.has(params.get("utm_medium") ?? "") && (!params.has("redirect") || params.get("redirect") === SIGNIN_REDIRECT)) {
|
|
715
|
+
return { ok: true, url: value };
|
|
716
|
+
}
|
|
717
|
+
}
|
|
718
|
+
return { ok: false, firecrawlHost };
|
|
719
|
+
}
|
|
720
|
+
|
|
616
721
|
// src/monitor.ts
|
|
617
722
|
import { z as z5 } from "zod";
|
|
618
723
|
|
|
@@ -621,10 +726,12 @@ import { createHmac } from "crypto";
|
|
|
621
726
|
var managedOAuthApiKey = /* @__PURE__ */ Symbol("firecrawlManagedOAuthApiKey");
|
|
622
727
|
var CoreHttpError = class extends Error {
|
|
623
728
|
status;
|
|
624
|
-
|
|
729
|
+
agent_hints;
|
|
730
|
+
constructor(message, status, agentHints2) {
|
|
625
731
|
super(message);
|
|
626
732
|
this.name = "CoreHttpError";
|
|
627
733
|
this.status = status;
|
|
734
|
+
if (agentHints2 !== void 0) this.agent_hints = agentHints2;
|
|
628
735
|
}
|
|
629
736
|
};
|
|
630
737
|
var CredentialValidationUnavailableError = class extends Error {
|
|
@@ -636,11 +743,12 @@ var CredentialValidationUnavailableError = class extends Error {
|
|
|
636
743
|
}
|
|
637
744
|
};
|
|
638
745
|
function credentialValidationUnavailable(diagnostics) {
|
|
639
|
-
const { aborted, elapsedMs, reason, resource, status } = diagnostics;
|
|
746
|
+
const { aborted, edgeMitigation: edgeMitigation2, elapsedMs, reason, resource, status } = diagnostics;
|
|
640
747
|
console.error(
|
|
641
748
|
"[MCP_CREDENTIAL_VALIDATION]",
|
|
642
749
|
JSON.stringify({
|
|
643
750
|
aborted: aborted ?? null,
|
|
751
|
+
edge_mitigation: edgeMitigation2 ?? null,
|
|
644
752
|
elapsed_ms: elapsedMs ?? null,
|
|
645
753
|
introspect_status: status ?? null,
|
|
646
754
|
reason,
|
|
@@ -724,7 +832,10 @@ async function monitorRequest(session, origin, path2, init = {}) {
|
|
|
724
832
|
const s = qs.toString();
|
|
725
833
|
if (s) url += `?${s}`;
|
|
726
834
|
}
|
|
727
|
-
const headers =
|
|
835
|
+
const headers = {
|
|
836
|
+
...originHeaders(origin),
|
|
837
|
+
...AGENT_HINTS_HEADERS
|
|
838
|
+
};
|
|
728
839
|
if (apiKey) headers.Authorization = `Bearer ${apiKey}`;
|
|
729
840
|
if (init.body !== void 0) headers["Content-Type"] = "application/json";
|
|
730
841
|
const response = await fetch(url, {
|
|
@@ -735,10 +846,13 @@ async function monitorRequest(session, origin, path2, init = {}) {
|
|
|
735
846
|
const payload = await response.json().catch(() => ({}));
|
|
736
847
|
if (!response.ok || payload?.success === false) {
|
|
737
848
|
const message = payload?.error || `HTTP ${response.status}: ${response.statusText || "Request failed"}`;
|
|
738
|
-
throw new CoreHttpError(message, response.status);
|
|
849
|
+
throw new CoreHttpError(message, response.status, readAgentHints(payload));
|
|
739
850
|
}
|
|
740
851
|
return payload;
|
|
741
852
|
}
|
|
853
|
+
function monitorResult(data) {
|
|
854
|
+
return withAgentHints(structuredText(data), data);
|
|
855
|
+
}
|
|
742
856
|
var pageStatusSchema = z5.enum(["same", "new", "changed", "removed", "error"]);
|
|
743
857
|
var checkStatusSchema = z5.enum([
|
|
744
858
|
"queued",
|
|
@@ -863,7 +977,7 @@ In the simple form, a \`goal\` is required. If \`queries\` contains one or more
|
|
|
863
977
|
body
|
|
864
978
|
}
|
|
865
979
|
);
|
|
866
|
-
return
|
|
980
|
+
return monitorResult(res);
|
|
867
981
|
}
|
|
868
982
|
});
|
|
869
983
|
server2.addTool({
|
|
@@ -895,7 +1009,7 @@ List monitors for the authenticated account with optional pagination controls. R
|
|
|
895
1009
|
query: { limit, offset }
|
|
896
1010
|
}
|
|
897
1011
|
);
|
|
898
|
-
return
|
|
1012
|
+
return monitorResult(res);
|
|
899
1013
|
}
|
|
900
1014
|
});
|
|
901
1015
|
server2.addTool({
|
|
@@ -921,7 +1035,7 @@ Retrieve one monitor by ID, including its configuration and current state. This
|
|
|
921
1035
|
requestOrigin(mcpClient, session),
|
|
922
1036
|
`/monitor/${encodeURIComponent(id)}`
|
|
923
1037
|
);
|
|
924
|
-
return
|
|
1038
|
+
return monitorResult(res);
|
|
925
1039
|
}
|
|
926
1040
|
});
|
|
927
1041
|
server2.addTool({
|
|
@@ -953,7 +1067,7 @@ Returns the updated monitor.
|
|
|
953
1067
|
`/monitor/${encodeURIComponent(id)}`,
|
|
954
1068
|
{ method: "PATCH", body }
|
|
955
1069
|
);
|
|
956
|
-
return
|
|
1070
|
+
return monitorResult(res);
|
|
957
1071
|
}
|
|
958
1072
|
});
|
|
959
1073
|
server2.addTool({
|
|
@@ -981,7 +1095,7 @@ Permanently delete a monitor by ID and stop its future schedule. This operation
|
|
|
981
1095
|
`/monitor/${encodeURIComponent(id)}`,
|
|
982
1096
|
{ method: "DELETE" }
|
|
983
1097
|
);
|
|
984
|
-
return
|
|
1098
|
+
return monitorResult(res);
|
|
985
1099
|
}
|
|
986
1100
|
});
|
|
987
1101
|
server2.addTool({
|
|
@@ -1008,7 +1122,7 @@ Queue an immediate check for a monitor outside its normal schedule. This starts
|
|
|
1008
1122
|
`/monitor/${encodeURIComponent(id)}/run`,
|
|
1009
1123
|
{ method: "POST" }
|
|
1010
1124
|
);
|
|
1011
|
-
return
|
|
1125
|
+
return monitorResult(res);
|
|
1012
1126
|
}
|
|
1013
1127
|
});
|
|
1014
1128
|
server2.addTool({
|
|
@@ -1040,7 +1154,7 @@ List historical checks for a monitor, optionally filtered by status and bounded
|
|
|
1040
1154
|
`/monitor/${encodeURIComponent(id)}/checks`,
|
|
1041
1155
|
{ query: { limit, offset, status } }
|
|
1042
1156
|
);
|
|
1043
|
-
return
|
|
1157
|
+
return monitorResult(res);
|
|
1044
1158
|
}
|
|
1045
1159
|
});
|
|
1046
1160
|
server2.addTool({
|
|
@@ -1075,7 +1189,7 @@ Markdown tracking returns a unified text diff, JSON tracking returns field paths
|
|
|
1075
1189
|
`/monitor/${encodeURIComponent(id)}/checks/${encodeURIComponent(checkId)}`,
|
|
1076
1190
|
{ query: { limit, skip, status: pageStatus } }
|
|
1077
1191
|
);
|
|
1078
|
-
return
|
|
1192
|
+
return monitorResult(res);
|
|
1079
1193
|
}
|
|
1080
1194
|
});
|
|
1081
1195
|
}
|
|
@@ -1084,6 +1198,9 @@ Markdown tracking returns a unified text diff, JSON tracking returns field paths
|
|
|
1084
1198
|
import { z as z6 } from "zod";
|
|
1085
1199
|
import { UserError } from "fastmcp";
|
|
1086
1200
|
var BASE2 = "/v2/search/research";
|
|
1201
|
+
function researchHeaders(origin) {
|
|
1202
|
+
return { ...originHeaders(origin), ...AGENT_HINTS_HEADERS };
|
|
1203
|
+
}
|
|
1087
1204
|
function appendParam(params, key, value) {
|
|
1088
1205
|
if (value == null) return;
|
|
1089
1206
|
if (Array.isArray(value)) {
|
|
@@ -1228,10 +1345,10 @@ Returns ranked papers with canonical IDs, titles, authors, and abstracts.
|
|
|
1228
1345
|
const client = getClient2(session);
|
|
1229
1346
|
const res = await client.http.get(
|
|
1230
1347
|
withQuery(`${BASE2}/papers`, params),
|
|
1231
|
-
|
|
1348
|
+
researchHeaders(requestOrigin(mcpClient, session))
|
|
1232
1349
|
);
|
|
1233
1350
|
const results = res.data?.results ?? [];
|
|
1234
|
-
return withStructured(fmtHits(results), { results });
|
|
1351
|
+
return withAgentHints(withStructured(fmtHits(results), { results }), res.data, true);
|
|
1235
1352
|
}
|
|
1236
1353
|
});
|
|
1237
1354
|
server2.addTool({
|
|
@@ -1259,13 +1376,13 @@ Retrieve canonical metadata for one paper ID, such as an arXiv, PMC, PMID, or DO
|
|
|
1259
1376
|
const client = getClient2(session);
|
|
1260
1377
|
const res = await client.http.get(
|
|
1261
1378
|
`${BASE2}/papers/${encodeURIComponent(paperId)}`,
|
|
1262
|
-
|
|
1379
|
+
researchHeaders(requestOrigin(mcpClient, session))
|
|
1263
1380
|
);
|
|
1264
1381
|
const paper = res.data?.paper;
|
|
1265
|
-
return withStructured(
|
|
1382
|
+
return withAgentHints(withStructured(
|
|
1266
1383
|
fmtPaperMetadata(paper),
|
|
1267
1384
|
paper ? { paper } : {}
|
|
1268
|
-
);
|
|
1385
|
+
), res.data, true);
|
|
1269
1386
|
}
|
|
1270
1387
|
});
|
|
1271
1388
|
server2.addTool({
|
|
@@ -1307,17 +1424,17 @@ Returns ranked candidates and the evaluated pool size.
|
|
|
1307
1424
|
`${BASE2}/papers/${encodeURIComponent(primary)}/similar`,
|
|
1308
1425
|
params
|
|
1309
1426
|
),
|
|
1310
|
-
|
|
1427
|
+
researchHeaders(requestOrigin(mcpClient, session))
|
|
1311
1428
|
);
|
|
1312
1429
|
const results = res.data?.results ?? [];
|
|
1313
1430
|
const poolSize = res.data?.poolSize ?? 0;
|
|
1314
1431
|
const note = res.data?.note ? `
|
|
1315
1432
|
note: ${res.data.note}` : "";
|
|
1316
|
-
return withStructured(
|
|
1433
|
+
return withAgentHints(withStructured(
|
|
1317
1434
|
`${fmtHits(results)}
|
|
1318
1435
|
(poolSize=${poolSize})${note}`,
|
|
1319
1436
|
{ results, poolSize, ...res.data?.note ? { note: res.data.note } : {} }
|
|
1320
|
-
);
|
|
1437
|
+
), res.data, true);
|
|
1321
1438
|
}
|
|
1322
1439
|
});
|
|
1323
1440
|
server2.addTool({
|
|
@@ -1352,13 +1469,13 @@ Returns matching passages or a notice when full text is unavailable.
|
|
|
1352
1469
|
const client = getClient2(session);
|
|
1353
1470
|
const res = await client.http.get(
|
|
1354
1471
|
withQuery(`${BASE2}/papers/${encodeURIComponent(paperId)}`, params),
|
|
1355
|
-
|
|
1472
|
+
researchHeaders(requestOrigin(mcpClient, session))
|
|
1356
1473
|
);
|
|
1357
1474
|
const passages = res.data?.passages ?? [];
|
|
1358
|
-
return withStructured(
|
|
1475
|
+
return withAgentHints(withStructured(
|
|
1359
1476
|
passages.length ? passages.map((p) => p.text).join("\n---\n") : "(no full-text passages available for this paper)",
|
|
1360
1477
|
{ passages }
|
|
1361
|
-
);
|
|
1478
|
+
), res.data, true);
|
|
1362
1479
|
}
|
|
1363
1480
|
});
|
|
1364
1481
|
server2.addTool({
|
|
@@ -1468,6 +1585,68 @@ function escapeWWWAuthenticateValue(value) {
|
|
|
1468
1585
|
return value.replace(/\\/g, "\\\\").replace(/"/g, '\\"');
|
|
1469
1586
|
}
|
|
1470
1587
|
|
|
1588
|
+
// src/introspection-cache.ts
|
|
1589
|
+
import { createHash } from "crypto";
|
|
1590
|
+
function createIntrospectionCache(options) {
|
|
1591
|
+
const now = options.now ?? Date.now;
|
|
1592
|
+
const entries = /* @__PURE__ */ new Map();
|
|
1593
|
+
const inflight = /* @__PURE__ */ new Map();
|
|
1594
|
+
function store(key, value) {
|
|
1595
|
+
const at = now();
|
|
1596
|
+
const ttl = options.ttlMs(value, at);
|
|
1597
|
+
if (!(ttl > 0)) return;
|
|
1598
|
+
entries.delete(key);
|
|
1599
|
+
entries.set(key, { expiresAt: at + ttl, value });
|
|
1600
|
+
while (entries.size > options.maxEntries) {
|
|
1601
|
+
const oldest = entries.keys().next().value;
|
|
1602
|
+
if (oldest === void 0) break;
|
|
1603
|
+
entries.delete(oldest);
|
|
1604
|
+
}
|
|
1605
|
+
}
|
|
1606
|
+
return {
|
|
1607
|
+
async get(parts, load) {
|
|
1608
|
+
const key = createHash("sha256").update(parts.join("\0")).digest("hex");
|
|
1609
|
+
const hit = entries.get(key);
|
|
1610
|
+
if (hit) {
|
|
1611
|
+
entries.delete(key);
|
|
1612
|
+
if (hit.expiresAt > now()) {
|
|
1613
|
+
entries.set(key, hit);
|
|
1614
|
+
return hit.value;
|
|
1615
|
+
}
|
|
1616
|
+
}
|
|
1617
|
+
const pending = inflight.get(key);
|
|
1618
|
+
if (pending) return pending;
|
|
1619
|
+
const request = load().then(
|
|
1620
|
+
(value) => {
|
|
1621
|
+
inflight.delete(key);
|
|
1622
|
+
store(key, value);
|
|
1623
|
+
return value;
|
|
1624
|
+
},
|
|
1625
|
+
(error2) => {
|
|
1626
|
+
inflight.delete(key);
|
|
1627
|
+
throw error2;
|
|
1628
|
+
}
|
|
1629
|
+
);
|
|
1630
|
+
inflight.set(key, request);
|
|
1631
|
+
return request;
|
|
1632
|
+
},
|
|
1633
|
+
clear() {
|
|
1634
|
+
entries.clear();
|
|
1635
|
+
},
|
|
1636
|
+
get size() {
|
|
1637
|
+
return entries.size;
|
|
1638
|
+
}
|
|
1639
|
+
};
|
|
1640
|
+
}
|
|
1641
|
+
var INTROSPECTION_ACTIVE_TTL_MS = 6e4;
|
|
1642
|
+
var INTROSPECTION_INACTIVE_TTL_MS = 1e4;
|
|
1643
|
+
function introspectionTtlMs(value, now, activeTtlMs = INTROSPECTION_ACTIVE_TTL_MS, inactiveTtlMs = INTROSPECTION_INACTIVE_TTL_MS) {
|
|
1644
|
+
if (!value.active) return inactiveTtlMs;
|
|
1645
|
+
if (value.exp === void 0) return activeTtlMs;
|
|
1646
|
+
if (typeof value.exp !== "number" || !Number.isFinite(value.exp)) return 0;
|
|
1647
|
+
return Math.min(activeTtlMs, value.exp * 1e3 - now);
|
|
1648
|
+
}
|
|
1649
|
+
|
|
1471
1650
|
// src/index.ts
|
|
1472
1651
|
dotenv.config({ debug: false, quiet: true });
|
|
1473
1652
|
var require2 = createRequire2(import.meta.url);
|
|
@@ -1658,7 +1837,40 @@ function credentialMetadata(data) {
|
|
|
1658
1837
|
resource: typeof data.aud === "string" ? data.aud : void 0
|
|
1659
1838
|
};
|
|
1660
1839
|
}
|
|
1661
|
-
|
|
1840
|
+
function introspectionActiveTtlMs() {
|
|
1841
|
+
const raw = normalizeHeader(
|
|
1842
|
+
process.env.FIRECRAWL_OAUTH_INTROSPECT_CACHE_TTL_MS
|
|
1843
|
+
);
|
|
1844
|
+
if (raw === void 0) return INTROSPECTION_ACTIVE_TTL_MS;
|
|
1845
|
+
const parsed = Number(raw);
|
|
1846
|
+
return Number.isFinite(parsed) && parsed >= 0 ? parsed : INTROSPECTION_ACTIVE_TTL_MS;
|
|
1847
|
+
}
|
|
1848
|
+
var introspectionCache = createIntrospectionCache({
|
|
1849
|
+
maxEntries: 5e4,
|
|
1850
|
+
ttlMs: (value, now) => {
|
|
1851
|
+
const activeTtlMs = introspectionActiveTtlMs();
|
|
1852
|
+
if (activeTtlMs === 0) return 0;
|
|
1853
|
+
return introspectionTtlMs(
|
|
1854
|
+
value,
|
|
1855
|
+
now,
|
|
1856
|
+
activeTtlMs,
|
|
1857
|
+
Math.min(activeTtlMs, INTROSPECTION_INACTIVE_TTL_MS)
|
|
1858
|
+
);
|
|
1859
|
+
}
|
|
1860
|
+
});
|
|
1861
|
+
function introspectToken(token, expectedResource) {
|
|
1862
|
+
return introspectionCache.get(
|
|
1863
|
+
[token, expectedResource],
|
|
1864
|
+
() => fetchIntrospection(token, expectedResource)
|
|
1865
|
+
);
|
|
1866
|
+
}
|
|
1867
|
+
var EDGE_MITIGATIONS = /* @__PURE__ */ new Set(["deny", "challenge", "rate_limit"]);
|
|
1868
|
+
function edgeMitigation(response) {
|
|
1869
|
+
const value = response.headers.get("x-vercel-mitigated")?.trim().toLowerCase();
|
|
1870
|
+
if (!value) return void 0;
|
|
1871
|
+
return EDGE_MITIGATIONS.has(value) ? value : "other";
|
|
1872
|
+
}
|
|
1873
|
+
async function fetchIntrospection(token, expectedResource) {
|
|
1662
1874
|
const introspectionSecret = getOAuthIntrospectionSecret();
|
|
1663
1875
|
if (!introspectionSecret) {
|
|
1664
1876
|
throw credentialValidationUnavailable({
|
|
@@ -1697,6 +1909,7 @@ async function introspectToken(token, expectedResource) {
|
|
|
1697
1909
|
}
|
|
1698
1910
|
if (!response.ok) {
|
|
1699
1911
|
throw credentialValidationUnavailable({
|
|
1912
|
+
edgeMitigation: edgeMitigation(response),
|
|
1700
1913
|
elapsedMs: elapsedMs(),
|
|
1701
1914
|
reason: "introspect_http_status",
|
|
1702
1915
|
resource: expectedResource,
|
|
@@ -2010,7 +2223,7 @@ function termsRequiredAction(body) {
|
|
|
2010
2223
|
url: action.url
|
|
2011
2224
|
};
|
|
2012
2225
|
}
|
|
2013
|
-
function termsRequiredError(action, context) {
|
|
2226
|
+
function termsRequiredError(action, context, hints) {
|
|
2014
2227
|
const requestId = context.requestId;
|
|
2015
2228
|
const retryIdentity = requestId ? ` and requestId ${requestId}` : "";
|
|
2016
2229
|
const message = `Alexandria provider terms required. An organization admin must accept the ${action.terms} provider's terms (version ${action.version}) before this request can run.`;
|
|
@@ -2025,6 +2238,7 @@ Do not retry until acceptance is confirmed.`,
|
|
|
2025
2238
|
code: TERMS_REQUIRED_CODE,
|
|
2026
2239
|
status: 403,
|
|
2027
2240
|
message,
|
|
2241
|
+
...hints ? { agent_hints: hints } : {},
|
|
2028
2242
|
...requestId ? { requestId } : {},
|
|
2029
2243
|
requiresAction: action,
|
|
2030
2244
|
nextTool: {
|
|
@@ -2055,7 +2269,8 @@ Do not retry until acceptance is confirmed.`,
|
|
|
2055
2269
|
function throwIfTermsRequired(error2, context) {
|
|
2056
2270
|
const source = error2;
|
|
2057
2271
|
const action = termsRequiredAction(source?.response?.data ?? source?.details);
|
|
2058
|
-
if (action)
|
|
2272
|
+
if (action)
|
|
2273
|
+
throw termsRequiredError(action, context, readErrorAgentHints(error2));
|
|
2059
2274
|
}
|
|
2060
2275
|
async function relayTermsRequired(run, context) {
|
|
2061
2276
|
try {
|
|
@@ -2073,6 +2288,7 @@ async function relayExchangeError(run, context) {
|
|
|
2073
2288
|
const response = error2?.response;
|
|
2074
2289
|
if (!response || response.status === 401) throw error2;
|
|
2075
2290
|
const data = response.data;
|
|
2291
|
+
const hints = readErrorAgentHints(error2);
|
|
2076
2292
|
const message = typeof data?.error === "string" ? data.error : `Alexandria request failed (HTTP ${response.status})`;
|
|
2077
2293
|
const disabledProvider = response.status === 403 ? context.providers?.find((provider) => message === `Access to ${provider} is disabled for this organization.`) : void 0;
|
|
2078
2294
|
if (disabledProvider) {
|
|
@@ -2082,6 +2298,7 @@ async function relayExchangeError(run, context) {
|
|
|
2082
2298
|
code: typeof data?.code === "string" ? data.code : "exchange_error",
|
|
2083
2299
|
status: 403,
|
|
2084
2300
|
message,
|
|
2301
|
+
...hints ? { agent_hints: hints } : {},
|
|
2085
2302
|
nextTool: {
|
|
2086
2303
|
name: "firecrawl_scrape",
|
|
2087
2304
|
arguments: {
|
|
@@ -2095,6 +2312,7 @@ async function relayExchangeError(run, context) {
|
|
|
2095
2312
|
code: typeof data?.code === "string" ? data.code : "exchange_error",
|
|
2096
2313
|
status: response.status,
|
|
2097
2314
|
message,
|
|
2315
|
+
...hints ? { agent_hints: hints } : {},
|
|
2098
2316
|
...typeof data?.chargeId === "string" ? { chargeId: data.chargeId } : {}
|
|
2099
2317
|
});
|
|
2100
2318
|
}
|
|
@@ -2148,7 +2366,7 @@ var ConsoleLogger = class {
|
|
|
2148
2366
|
var openAiAppsChallengeToken = normalizeHeader(
|
|
2149
2367
|
process.env.OPENAI_APPS_CHALLENGE_TOKEN
|
|
2150
2368
|
);
|
|
2151
|
-
var FULL_PROFILE_INSTRUCTIONS = `Firecrawl provides web search, page retrieval, site URL discovery, multi-page collection, structured page data, monitoring, and multi-source research that returns structured data. Match the requested operation to the tool boundary: firecrawl_scrape retrieves one supplied page and can return JSON matching a supplied schema, firecrawl_map enumerates URLs under a site without retrieving their content, and firecrawl_agent runs multi-source research and returns structured data when the URLs are not known or the answer spans several sites (an entity plus its fields, a list, a dataset); its result is read with firecrawl_agent_status. Authenticated firecrawl_search returns web results together with matching Alexandria providers in data.tools. ${ALEXANDRIA_CATALOGUE_SENTENCE} A matching provider can return the same fields across several entities, provenance, exact figures or timestamps, or a large set of records through firecrawl_scrape with its published contract. If web results already answer the question, use them. For the same fields across multiple pages, firecrawl_find_tools offers free provider discovery. Use firecrawl_find_tools to read a contract that was not returned in full or to browse the catalogue by category. ${ALEXANDRIA_SOURCES_OPT_OUT} If no provider fits, continue with web search or firecrawl_agent. For biomedical, life-science, clinical, or arXiv literature, the firecrawl_research_* tools search a paper index of abstracts and full text; firecrawl_search with categories: ["research"] is a website filter over ordinary web results and reaches different sources. For a programming question (code behaviour, a library or framework, an API contract, an error message, or a known bug), firecrawl_developer_search (or firecrawl_search with categories: ["developer"]) searches an index of public repositories, GitHub issues, merged pull requests, READMEs, and code documentation.
|
|
2369
|
+
var FULL_PROFILE_INSTRUCTIONS = `Firecrawl provides web search, page retrieval, site URL discovery, multi-page collection, structured page data, monitoring, and multi-source research that returns structured data. Match the requested operation to the tool boundary: firecrawl_scrape retrieves one supplied page and can return JSON matching a supplied schema, firecrawl_map enumerates URLs under a site without retrieving their content, and firecrawl_agent runs multi-source research and returns structured data when the URLs are not known or the answer spans several sites (an entity plus its fields, a list, a dataset); its result is read with firecrawl_agent_status. Authenticated firecrawl_search returns web results together with matching Alexandria providers in data.tools. ${ALEXANDRIA_CATALOGUE_SENTENCE} A matching provider can return the same fields across several entities, provenance, exact figures or timestamps, or a large set of records through firecrawl_scrape with its published contract. If web results already answer the question, use them. For the same fields across multiple pages, firecrawl_find_tools offers free provider discovery. Use firecrawl_find_tools to read a contract that was not returned in full or to browse the catalogue by category. ${ALEXANDRIA_SOURCES_OPT_OUT} If no provider fits, continue with web search or firecrawl_agent. For biomedical, life-science, clinical, or arXiv literature, the firecrawl_research_* tools search a paper index of abstracts and full text; firecrawl_search with categories: ["research"] is a website filter over ordinary web results and reaches different sources. For a programming question (code behaviour, a library or framework, an API contract, an error message, or a known bug), firecrawl_developer_search (or firecrawl_search with categories: ["developer"]) searches an index of public repositories, GitHub issues, merged pull requests, READMEs, and code documentation. firecrawl_search with sources: [{type: "alexandria"}] returns compact tool summaries in data.tools; toolDetail: "full" includes contracts, firecrawl_find_tools starts with categories, lists providers, then compact tools, and expands the selected full contract, and firecrawl_scrape with alexandria: [{provider, capability, options}] executes up to ten capabilities and returns their results. Alexandria access needs an API key on a team with it enabled. Provide only the required inputs and account for stated network or external side effects.`;
|
|
2152
2370
|
var KEYLESS_PROFILE_INSTRUCTIONS = `Hosted keyless sessions expose firecrawl_search, firecrawl_scrape, and firecrawl_parse with usage limits. firecrawl_search searches the web. For programming questions, firecrawl_search with categories: ["developer"] searches indexed public repositories, GitHub issues, merged pull requests, repository READMEs, and code documentation. For biomedical, life-science, clinical, or arXiv literature, firecrawl_search with categories: ["research"] filters ordinary web results to research-affiliated websites. firecrawl_scrape retrieves one supplied page and can return JSON matching a supplied schema. firecrawl_parse processes supported local files through its two-phase upload flow. An Authorization bearer API key can provide higher usage limits and expose additional tools, subject to plan, deployment, and team policy, including firecrawl_map for site URL discovery, firecrawl_agent and firecrawl_agent_status for multi-source research that returns structured data when the URLs are not known, firecrawl_research_* for paper-index and repository research, and firecrawl_find_tools as the progressive Alexandria catalogue lookup alongside the Alexandria options of firecrawl_search and firecrawl_scrape for catalogued data providers.`;
|
|
2153
2371
|
var SEARCH_PROFILE_INSTRUCTIONS = ALEXANDRIA_SEARCH_INSTRUCTIONS + ` Firecrawl provides web, developer, and research search, and executes catalogued Alexandria data providers. Use firecrawl_search to find relevant results across the web and specialized indexes; authenticated searches also return matching Alexandria providers in data.tools. firecrawl_find_tools provides catalogue browsing and provider contracts; firecrawl_scrape with an alexandria body executes a selected capability; firecrawl_scrape with a url retrieves one supplied page. For a programming question, firecrawl_developer_search searches indexed public repositories, GitHub issues, merged pull requests, READMEs, and code documentation and returns the matched passages, and skills: "only" narrows it to agent-skill files; firecrawl_search with categories: ["developer"] reaches the same index beside ordinary web results, returning the hits in the web group rather than as passages and offering no skills filter. For a biomedical, life-science, clinical, or arXiv literature question, the firecrawl_research_* tools search the paper index, while categories: ["research"] on firecrawl_search filters ordinary web results to research-affiliated websites. Use the firecrawl_research_* tools to search academic and research literature, expand from anchor papers via the citation graph, and read full-text passages from a specific paper. Search and discovery tools are read-only and return ranked results. Billing: web, developer and research search are billed per request; Alexandria discovery (firecrawl_search with sources ["alexandria"] alone, and firecrawl_find_tools) is free; firecrawl_scrape is billed, as a page retrieval in url mode or at each executed capability's listed price in alexandria mode.`;
|
|
2154
2372
|
var SEARCH_PROFILE_TOOLS = /* @__PURE__ */ new Set([
|
|
@@ -2260,16 +2478,36 @@ function isHostedKeylessSession(session) {
|
|
|
2260
2478
|
function isLocalKeylessStartup() {
|
|
2261
2479
|
return process.env.CLOUD_SERVICE !== "true" && !isHttpStreamingTransport() && !resolveCredentialFromEnv() && !normalizeHeader(process.env.FIRECRAWL_API_URL);
|
|
2262
2480
|
}
|
|
2263
|
-
var
|
|
2264
|
-
var
|
|
2481
|
+
var KEYLESS_SIGNUP_FALLBACK_URL = "https://www.firecrawl.dev/signin?utm_source=keyless&utm_medium=mcp&redirect=%2Fapp%2Fapi-keys";
|
|
2482
|
+
var droppedKeylessSignupUrls = /* @__PURE__ */ new Set();
|
|
2483
|
+
function keylessSignupUrlFrom(value) {
|
|
2484
|
+
const check = checkKeylessSignupUrl(value);
|
|
2485
|
+
if (check.ok) return check.url;
|
|
2486
|
+
if (check.firecrawlHost && typeof value === "string" && droppedKeylessSignupUrls.size < 50 && !droppedKeylessSignupUrls.has(value)) {
|
|
2487
|
+
droppedKeylessSignupUrls.add(value);
|
|
2488
|
+
console.warn(
|
|
2489
|
+
"[WARN]",
|
|
2490
|
+
(/* @__PURE__ */ new Date()).toISOString(),
|
|
2491
|
+
"Ignoring an unrecognized Firecrawl keyless signup link from the API; using the fallback",
|
|
2492
|
+
{ signupUrl: value }
|
|
2493
|
+
);
|
|
2494
|
+
}
|
|
2495
|
+
return void 0;
|
|
2496
|
+
}
|
|
2497
|
+
function keylessAccountFix(signupUrl) {
|
|
2498
|
+
return `Fix: Create an API key at ${signupUrl} and then:
|
|
2499
|
+
- Set the header: Authorization: Bearer YOUR_API_KEY on https://mcp.firecrawl.dev/v2/mcp
|
|
2500
|
+
Then start a new session.`;
|
|
2501
|
+
}
|
|
2502
|
+
var keylessQuotaMessage = (signupUrl) => `You've hit Firecrawl's free MCP rate limit. To continue using without limits, create a Firecrawl API key.
|
|
2265
2503
|
|
|
2266
|
-
${
|
|
2267
|
-
var
|
|
2504
|
+
${keylessAccountFix(signupUrl)}`;
|
|
2505
|
+
var keylessToolMessage = (signupUrl) => `This tool needs a Firecrawl account.
|
|
2268
2506
|
|
|
2269
|
-
${
|
|
2270
|
-
var
|
|
2507
|
+
${keylessAccountFix(signupUrl)}`;
|
|
2508
|
+
var keylessAccessMessage = (signupUrl) => `Anonymous keyless access is unavailable for this request.
|
|
2271
2509
|
|
|
2272
|
-
${
|
|
2510
|
+
${keylessAccountFix(signupUrl)}`;
|
|
2273
2511
|
var INVALID_API_KEY_MESSAGE = "The Firecrawl API key is invalid or revoked.\nFix: Replace the key on the existing Firecrawl MCP server, then start a new session. Get an API key at https://www.firecrawl.dev/app/api-keys";
|
|
2274
2512
|
var INVALID_OAUTH_MESSAGE = "This Firecrawl account connection is no longer valid.\nFix: Reconnect the existing Firecrawl server in the client, or set that existing server URL to https://mcp.firecrawl.dev/v2/mcp-oauth, then start a new session.";
|
|
2275
2513
|
function connectionRecoveryPayload(params) {
|
|
@@ -2304,6 +2542,20 @@ async function runWithCredentialRecovery(run, requestId, session) {
|
|
|
2304
2542
|
return await run();
|
|
2305
2543
|
} catch (error2) {
|
|
2306
2544
|
if (session?.authType !== "api-key" || !isCoreCredentialRejection(error2)) {
|
|
2545
|
+
const hints = readErrorAgentHints(error2);
|
|
2546
|
+
if (hints) {
|
|
2547
|
+
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
2548
|
+
throw new UserError2(
|
|
2549
|
+
hints.length ? `${message}
|
|
2550
|
+
|
|
2551
|
+
${agentHintsText(hints)}` : message,
|
|
2552
|
+
{
|
|
2553
|
+
...error2 instanceof UserError2 ? error2.extras : {},
|
|
2554
|
+
error: message,
|
|
2555
|
+
agent_hints: hints
|
|
2556
|
+
}
|
|
2557
|
+
);
|
|
2558
|
+
}
|
|
2307
2559
|
throw error2;
|
|
2308
2560
|
}
|
|
2309
2561
|
const payload = recoveryPayload("CREDENTIAL_INVALID", requestId);
|
|
@@ -2312,6 +2564,7 @@ async function runWithCredentialRecovery(run, requestId, session) {
|
|
|
2312
2564
|
}
|
|
2313
2565
|
function recoveryPayload(code, requestId = randomUUID(), options = {}) {
|
|
2314
2566
|
const retryAfterSeconds = options.retryAfterSeconds;
|
|
2567
|
+
const signupUrl = options.signupUrl ?? KEYLESS_SIGNUP_FALLBACK_URL;
|
|
2315
2568
|
const isQuotaExhausted = code === "KEYLESS_QUOTA_EXHAUSTED" || code === "KEYLESS_LIMIT_REACHED";
|
|
2316
2569
|
const isToolUnavailable = code === "KEYLESS_TOOL_NOT_AVAILABLE";
|
|
2317
2570
|
const isKeylessAccessUnavailable = code === "KEYLESS_ACCESS_NOT_AVAILABLE";
|
|
@@ -2321,7 +2574,7 @@ function recoveryPayload(code, requestId = randomUUID(), options = {}) {
|
|
|
2321
2574
|
code,
|
|
2322
2575
|
request_id: requestId,
|
|
2323
2576
|
auth_mode: code === "CREDENTIAL_INVALID" ? "credential_error" : "keyless",
|
|
2324
|
-
message: code === "CREDENTIAL_INVALID" ? INVALID_API_KEY_MESSAGE : isQuotaExhausted ?
|
|
2577
|
+
message: code === "CREDENTIAL_INVALID" ? INVALID_API_KEY_MESSAGE : isQuotaExhausted ? keylessQuotaMessage(signupUrl) : isToolUnavailable ? keylessToolMessage(signupUrl) : isKeylessAccessUnavailable ? keylessAccessMessage(signupUrl) : isKeylessEligibilityUnavailable ? "The anonymous keyless eligibility check is temporarily unavailable. Retry shortly." : "This tool requires a Firecrawl account or API key.",
|
|
2325
2578
|
// CREDENTIAL_INVALID sessions gate every tool call (including keyless
|
|
2326
2579
|
// tools) on the credentialError check before the keyless branch ever
|
|
2327
2580
|
// runs, so none of KEYLESS_TOOL_NAMES are actually callable here. Listing
|
|
@@ -2330,6 +2583,7 @@ function recoveryPayload(code, requestId = randomUUID(), options = {}) {
|
|
|
2330
2583
|
// tools, and ineligible access omit available_tools and next_actions for
|
|
2331
2584
|
// the same reason: those fields retry tools that cannot clear the error.
|
|
2332
2585
|
...isKeylessConversion || code === "CREDENTIAL_INVALID" ? {} : { available_tools: [...KEYLESS_TOOL_NAMES] },
|
|
2586
|
+
...isKeylessConversion ? { signup_url: signupUrl } : {},
|
|
2333
2587
|
docs_url: MCP_CONNECTION_GUIDE_URL,
|
|
2334
2588
|
...retryAfterSeconds ? { retry_after_seconds: retryAfterSeconds } : {},
|
|
2335
2589
|
...isKeylessEligibilityUnavailable ? { next_actions: [{ kind: "retry_later", after_seconds: 30 }] } : {}
|
|
@@ -2389,6 +2643,26 @@ function emitActionLog(toolName, status, session, error2, requestId = randomUUID
|
|
|
2389
2643
|
signal: AbortSignal.timeout(1500)
|
|
2390
2644
|
}).catch(() => void 0);
|
|
2391
2645
|
}
|
|
2646
|
+
var AGENT_HINT_LOG_MAX_CHARS = 1e3;
|
|
2647
|
+
function emitAgentHintsLog(toolName, status, hints, session, requestId) {
|
|
2648
|
+
if (process.env.CLOUD_SERVICE !== "true" || !hints) return;
|
|
2649
|
+
console.error(
|
|
2650
|
+
"[MCP_AGENT_HINTS]",
|
|
2651
|
+
JSON.stringify({
|
|
2652
|
+
tool_name: toolName,
|
|
2653
|
+
status,
|
|
2654
|
+
request_id: requestId,
|
|
2655
|
+
auth_type: session.authType ?? "none",
|
|
2656
|
+
// The companion search server shares this process and wrapper.
|
|
2657
|
+
profile: session.profile ?? primaryProfile.id,
|
|
2658
|
+
hint_count: hints.length,
|
|
2659
|
+
hints: hints.map((hint) => hint.slice(0, AGENT_HINT_LOG_MAX_CHARS))
|
|
2660
|
+
})
|
|
2661
|
+
);
|
|
2662
|
+
}
|
|
2663
|
+
function resultAgentHints(result) {
|
|
2664
|
+
return result && typeof result === "object" ? readAgentHints(result.structuredContent) : void 0;
|
|
2665
|
+
}
|
|
2392
2666
|
function guardHostedTool(tool, { logActions }) {
|
|
2393
2667
|
const keylessTool = KEYLESS_TOOL_NAMES.has(tool.name);
|
|
2394
2668
|
const execute = tool.execute;
|
|
@@ -2410,7 +2684,9 @@ function guardHostedTool(tool, { logActions }) {
|
|
|
2410
2684
|
const code = session?.credentialError ? "CREDENTIAL_INVALID" : isHostedKeylessSession(session) && !keylessTool ? "KEYLESS_TOOL_NOT_AVAILABLE" : void 0;
|
|
2411
2685
|
if (code) {
|
|
2412
2686
|
const requestId = randomUUID();
|
|
2413
|
-
const payload2 = recoveryPayload(code, requestId
|
|
2687
|
+
const payload2 = recoveryPayload(code, requestId, {
|
|
2688
|
+
signupUrl: code === "KEYLESS_TOOL_NOT_AVAILABLE" ? await hostedKeylessSignupUrl(session) : void 0
|
|
2689
|
+
});
|
|
2414
2690
|
if (logActions) {
|
|
2415
2691
|
emitActionLog(tool.name, "error", session, new UserError2(String(payload2.message), payload2), requestId, code);
|
|
2416
2692
|
}
|
|
@@ -2454,15 +2730,32 @@ function guardHostedTool(tool, { logActions }) {
|
|
|
2454
2730
|
}
|
|
2455
2731
|
if (isHostedKeylessSession(invocationSession) && !keylessTool) {
|
|
2456
2732
|
const code = "KEYLESS_TOOL_NOT_AVAILABLE";
|
|
2457
|
-
const payload = recoveryPayload(code, requestId
|
|
2733
|
+
const payload = recoveryPayload(code, requestId, {
|
|
2734
|
+
signupUrl: await hostedKeylessSignupUrl(invocationSession)
|
|
2735
|
+
});
|
|
2458
2736
|
if (logActions) emitActionLog(tool.name, "error", invocationSession, new UserError2(String(payload.message), payload), requestId, code);
|
|
2459
2737
|
throw new UserError2(String(payload.message), payload);
|
|
2460
2738
|
}
|
|
2461
|
-
const runTool = () =>
|
|
2462
|
-
|
|
2463
|
-
|
|
2464
|
-
|
|
2465
|
-
|
|
2739
|
+
const runTool = async () => {
|
|
2740
|
+
try {
|
|
2741
|
+
const result = await runWithCredentialRecovery(
|
|
2742
|
+
() => execute(args2, invocationContext),
|
|
2743
|
+
requestId,
|
|
2744
|
+
invocationSession
|
|
2745
|
+
);
|
|
2746
|
+
emitAgentHintsLog(
|
|
2747
|
+
tool.name,
|
|
2748
|
+
result?.isError ? "error" : "success",
|
|
2749
|
+
resultAgentHints(result),
|
|
2750
|
+
invocationSession,
|
|
2751
|
+
requestId
|
|
2752
|
+
);
|
|
2753
|
+
return result;
|
|
2754
|
+
} catch (error2) {
|
|
2755
|
+
emitAgentHintsLog(tool.name, "error", readErrorAgentHints(error2), invocationSession, requestId);
|
|
2756
|
+
throw error2;
|
|
2757
|
+
}
|
|
2758
|
+
};
|
|
2466
2759
|
if (!logActions) return runTool();
|
|
2467
2760
|
emitActionLog(tool.name, "started", invocationSession, void 0, requestId);
|
|
2468
2761
|
try {
|
|
@@ -2517,6 +2810,7 @@ server.getApp().get("/ready", (context) => {
|
|
|
2517
2810
|
}
|
|
2518
2811
|
return missing.length ? context.json({ ok: false, missing }, 503) : context.json({ ok: true }, 200);
|
|
2519
2812
|
});
|
|
2813
|
+
var warnedMissingAgentHintsInterceptor = false;
|
|
2520
2814
|
function createClient(apiKey) {
|
|
2521
2815
|
const config = {
|
|
2522
2816
|
...process.env.FIRECRAWL_API_URL && {
|
|
@@ -2526,7 +2820,44 @@ function createClient(apiKey) {
|
|
|
2526
2820
|
if (apiKey) {
|
|
2527
2821
|
config.apiKey = apiKey;
|
|
2528
2822
|
}
|
|
2529
|
-
|
|
2823
|
+
const client = new FirecrawlApp(config);
|
|
2824
|
+
const axiosInstance = client.http?.instance;
|
|
2825
|
+
if (axiosInstance?.interceptors?.request?.use) {
|
|
2826
|
+
axiosInstance.interceptors.request.use((request) => {
|
|
2827
|
+
if (typeof request.headers?.set === "function") {
|
|
2828
|
+
request.headers.set("X-Firecrawl-Agent-Hints", "true");
|
|
2829
|
+
} else {
|
|
2830
|
+
request.headers = { ...request.headers ?? {}, ...AGENT_HINTS_HEADERS };
|
|
2831
|
+
}
|
|
2832
|
+
return request;
|
|
2833
|
+
});
|
|
2834
|
+
} else if (!warnedMissingAgentHintsInterceptor) {
|
|
2835
|
+
warnedMissingAgentHintsInterceptor = true;
|
|
2836
|
+
console.warn(
|
|
2837
|
+
"[firecrawl-mcp] SDK request interceptor unavailable; API agent hints may be absent."
|
|
2838
|
+
);
|
|
2839
|
+
}
|
|
2840
|
+
return client;
|
|
2841
|
+
}
|
|
2842
|
+
async function sdkResultWithAgentHints(client, execute) {
|
|
2843
|
+
const responseInterceptors = client.http?.instance?.interceptors?.response;
|
|
2844
|
+
if (!responseInterceptors?.use) return execute();
|
|
2845
|
+
let hints;
|
|
2846
|
+
const interceptor = responseInterceptors.use((response) => {
|
|
2847
|
+
hints = readAgentHints(response?.data) ?? hints;
|
|
2848
|
+
return response;
|
|
2849
|
+
});
|
|
2850
|
+
try {
|
|
2851
|
+
const result = await execute();
|
|
2852
|
+
return preserveAgentHints(result, { agent_hints: hints });
|
|
2853
|
+
} catch (error2) {
|
|
2854
|
+
if (hints && error2 && typeof error2 === "object" && !readErrorAgentHints(error2)) {
|
|
2855
|
+
error2.agent_hints = hints;
|
|
2856
|
+
}
|
|
2857
|
+
throw error2;
|
|
2858
|
+
} finally {
|
|
2859
|
+
responseInterceptors.eject?.(interceptor);
|
|
2860
|
+
}
|
|
2530
2861
|
}
|
|
2531
2862
|
var SAFE_MODE = process.env.CLOUD_SERVICE === "true";
|
|
2532
2863
|
function getClient(session) {
|
|
@@ -2555,10 +2886,14 @@ function getClient(session) {
|
|
|
2555
2886
|
reason: "delegated_credential_unavailable"
|
|
2556
2887
|
});
|
|
2557
2888
|
}
|
|
2558
|
-
config.headers
|
|
2559
|
-
|
|
2560
|
-
|
|
2561
|
-
|
|
2889
|
+
if (typeof config.headers?.set === "function") {
|
|
2890
|
+
config.headers.set("Authorization", `Bearer ${credential}`);
|
|
2891
|
+
} else {
|
|
2892
|
+
config.headers = {
|
|
2893
|
+
...config.headers ?? {},
|
|
2894
|
+
Authorization: `Bearer ${credential}`
|
|
2895
|
+
};
|
|
2896
|
+
}
|
|
2562
2897
|
return config;
|
|
2563
2898
|
});
|
|
2564
2899
|
return client;
|
|
@@ -2706,10 +3041,10 @@ var scrapeToolParamsSchema = scrapeParamsSchema.extend({
|
|
|
2706
3041
|
url: z8.string().url().optional(),
|
|
2707
3042
|
timeout: z8.number().int().positive().optional().describe("Execution timeout in milliseconds."),
|
|
2708
3043
|
requestId: z8.string().regex(/^[A-Za-z0-9._:-]{1,128}$/).optional().describe(
|
|
2709
|
-
"
|
|
3044
|
+
"Idempotency key bound to one Alexandria execution payload. Generated when omitted and returned with the result."
|
|
2710
3045
|
),
|
|
2711
3046
|
alexandria: z8.union([exchangeCallSchema, exchangeCallsSchema]).optional().describe(
|
|
2712
|
-
"Catalogued Alexandria capability invocation, mutually exclusive with url. One {provider, capability, options} object or an array of 1-10, with contracts available through firecrawl_search or firecrawl_find_tools. Each call may include version to pin a published workflow; omitting it uses latest. Only requestId and timeout are supported alongside alexandria. " + ALEXANDRIA_CONTRACT_GUIDANCE + " Returns per-capability results in data.alexandria with data, records, or an error with a code; individual capabilities can fail even when the outer response succeeds.
|
|
3047
|
+
"Catalogued Alexandria capability invocation, mutually exclusive with url. One {provider, capability, options} object or an array of 1-10, with contracts available through firecrawl_search or firecrawl_find_tools. Each call may include version to pin a published workflow; omitting it uses latest. Only requestId and timeout are supported alongside alexandria. " + ALEXANDRIA_CONTRACT_GUIDANCE + " Returns per-capability results in data.alexandria with data, records, or an error with a code; individual capabilities can fail even when the outer response succeeds. Requires an API key on a team with Alexandria enabled. Some providers require accepted terms; blocked requests return the applicable requirements."
|
|
2713
3048
|
),
|
|
2714
3049
|
toolDetail: z8.enum(["compact", "summary", "full"]).optional().describe("URL mode only: domain discovery detail, summary by default; compact returns provider/capability/description, full includes contracts. Ignored with alexandria."),
|
|
2715
3050
|
domainTools: z8.boolean().optional().describe(
|
|
@@ -2846,6 +3181,7 @@ async function apiPostJson(pathName, body, apiKey, origin) {
|
|
|
2846
3181
|
const response = await fetch(`${resolveApiBaseUrl()}${pathName}`, {
|
|
2847
3182
|
method: "POST",
|
|
2848
3183
|
headers: {
|
|
3184
|
+
...AGENT_HINTS_HEADERS,
|
|
2849
3185
|
"Content-Type": "application/json",
|
|
2850
3186
|
Authorization: `Bearer ${apiKey}`,
|
|
2851
3187
|
...origin ? originHeaders(origin) : {}
|
|
@@ -2862,7 +3198,8 @@ async function apiPostJson(pathName, body, apiKey, origin) {
|
|
|
2862
3198
|
if (!response.ok) {
|
|
2863
3199
|
throw new CoreHttpError(
|
|
2864
3200
|
parsed?.error || parsed?.message || `Firecrawl request failed (HTTP ${response.status})`,
|
|
2865
|
-
response.status
|
|
3201
|
+
response.status,
|
|
3202
|
+
readAgentHints(parsed)
|
|
2866
3203
|
);
|
|
2867
3204
|
}
|
|
2868
3205
|
return parsed;
|
|
@@ -2944,6 +3281,7 @@ async function executeHostedParse(args2, session, log, origin) {
|
|
|
2944
3281
|
session,
|
|
2945
3282
|
origin
|
|
2946
3283
|
);
|
|
3284
|
+
const uploadHints = readAgentHints(uploadJson);
|
|
2947
3285
|
const upload = parseApiData(uploadJson);
|
|
2948
3286
|
if (!upload?.uploadUrl || !upload?.uploadRef) {
|
|
2949
3287
|
throw new Error(
|
|
@@ -2954,6 +3292,7 @@ async function executeHostedParse(args2, session, log, origin) {
|
|
|
2954
3292
|
const uploadForCommand = { ...upload, headers: uploadHeaders };
|
|
2955
3293
|
return asText({
|
|
2956
3294
|
success: true,
|
|
3295
|
+
...uploadHints ? { agent_hints: uploadHints } : {},
|
|
2957
3296
|
mode: "hosted-upload-ref-awaiting-upload",
|
|
2958
3297
|
message: "Hosted MCP cannot read local files. Run the local upload command, then call firecrawl_parse again with uploadRef. No Firecrawl API key is included in this command.",
|
|
2959
3298
|
upload: {
|
|
@@ -3008,7 +3347,7 @@ Firecrawl may serve recently indexed content; set \`maxAge: 0\` for a live fetch
|
|
|
3008
3347
|
|
|
3009
3348
|
On an authenticated session with Alexandria access, \`firecrawl_search\` with \`sources\` unset and \`firecrawl_find_tools\` can discover providers for the same fields across several pages; a matching provider returns typed records in one call. Keyless sessions have no provider matches.
|
|
3010
3349
|
|
|
3011
|
-
Alexandria mode, on an authenticated session with Alexandria access: \`alexandria\` selects catalogued capability execution and is mutually exclusive with \`url
|
|
3350
|
+
Alexandria mode, on an authenticated session with Alexandria access: \`alexandria\` selects catalogued capability execution and is mutually exclusive with \`url\`.
|
|
3012
3351
|
`,
|
|
3013
3352
|
outputSchema: scrapeOutputSchema,
|
|
3014
3353
|
parameters: scrapeToolParamsSchema,
|
|
@@ -3048,14 +3387,17 @@ Alexandria mode, on an authenticated session with Alexandria access: \`alexandri
|
|
|
3048
3387
|
},
|
|
3049
3388
|
session
|
|
3050
3389
|
);
|
|
3051
|
-
return structuredText(json?.data ?? json);
|
|
3390
|
+
return structuredText(preserveAgentHints(json?.data ?? json, json));
|
|
3052
3391
|
}
|
|
3053
3392
|
const client = getClient(session);
|
|
3054
3393
|
const res = await relayTermsRequired(
|
|
3055
|
-
() =>
|
|
3056
|
-
|
|
3057
|
-
|
|
3058
|
-
|
|
3394
|
+
() => sdkResultWithAgentHints(
|
|
3395
|
+
client,
|
|
3396
|
+
() => client.scrape(String(url), {
|
|
3397
|
+
...cleaned,
|
|
3398
|
+
origin
|
|
3399
|
+
})
|
|
3400
|
+
),
|
|
3059
3401
|
{ tool: "firecrawl_scrape" }
|
|
3060
3402
|
);
|
|
3061
3403
|
return structuredText(res);
|
|
@@ -3095,10 +3437,13 @@ Returns matching URLs rather than page bodies. Retrieve one page with \`firecraw
|
|
|
3095
3437
|
const client = getClient(session);
|
|
3096
3438
|
const cleaned = removeEmptyTopLevel(options);
|
|
3097
3439
|
log.info("Mapping URL", { url: String(url) });
|
|
3098
|
-
const res = await
|
|
3099
|
-
|
|
3100
|
-
|
|
3101
|
-
|
|
3440
|
+
const res = await sdkResultWithAgentHints(
|
|
3441
|
+
client,
|
|
3442
|
+
() => client.map(String(url), {
|
|
3443
|
+
...cleaned,
|
|
3444
|
+
origin: requestOrigin(mcpClient, session)
|
|
3445
|
+
})
|
|
3446
|
+
);
|
|
3102
3447
|
return structuredText(res);
|
|
3103
3448
|
}
|
|
3104
3449
|
});
|
|
@@ -3267,7 +3612,7 @@ Scrape one URL and return its content, or execute catalogued Alexandria capabili
|
|
|
3267
3612
|
|
|
3268
3613
|
\`firecrawl_search\` with \`sources\` unset and \`firecrawl_find_tools\` can discover providers for the same fields across several pages; a matching Alexandria provider returns typed records in one call.
|
|
3269
3614
|
|
|
3270
|
-
Alexandria mode: \`alexandria\` selects catalogued capability execution and is mutually exclusive with \`url
|
|
3615
|
+
Alexandria mode: \`alexandria\` selects catalogued capability execution and is mutually exclusive with \`url\`. Alexandria execution is billed at the capability's listed price and needs a team with Alexandria enabled.
|
|
3271
3616
|
`;
|
|
3272
3617
|
var searchSurfaceScrapeTool = {
|
|
3273
3618
|
...scrapeTool,
|
|
@@ -3292,12 +3637,12 @@ function extractClientIp(request) {
|
|
|
3292
3637
|
function keylessQuotaReason(reason) {
|
|
3293
3638
|
return reason === "requests" || reason === "credits";
|
|
3294
3639
|
}
|
|
3295
|
-
async function keylessEligible(clientIp, origin) {
|
|
3640
|
+
async function keylessEligible(clientIp, origin, { signupLink = false } = {}) {
|
|
3296
3641
|
const secret = process.env.KEYLESS_PROXY_SECRET;
|
|
3297
3642
|
if (!secret) return { eligible: false, unavailable: true };
|
|
3298
3643
|
try {
|
|
3299
3644
|
const response = await fetch(
|
|
3300
|
-
`${resolveApiBaseUrl()}/v2/keyless/eligibility`,
|
|
3645
|
+
`${resolveApiBaseUrl()}/v2/keyless/eligibility${signupLink ? "?signup_link=1" : ""}`,
|
|
3301
3646
|
{
|
|
3302
3647
|
headers: {
|
|
3303
3648
|
...originHeaders(origin),
|
|
@@ -3314,12 +3659,22 @@ async function keylessEligible(clientIp, origin) {
|
|
|
3314
3659
|
return {
|
|
3315
3660
|
eligible: json?.eligible === true,
|
|
3316
3661
|
...typeof json?.reason === "string" ? { reason: json.reason } : {},
|
|
3317
|
-
...Number.isFinite(json?.retryAfterSeconds) && json.retryAfterSeconds > 0 ? { retryAfterSeconds: json.retryAfterSeconds } : {}
|
|
3662
|
+
...Number.isFinite(json?.retryAfterSeconds) && json.retryAfterSeconds > 0 ? { retryAfterSeconds: json.retryAfterSeconds } : {},
|
|
3663
|
+
...keylessSignupUrlFrom(json?.signupUrl) ? { signupUrl: json.signupUrl } : {}
|
|
3318
3664
|
};
|
|
3319
3665
|
} catch {
|
|
3320
3666
|
return { eligible: false, unavailable: true };
|
|
3321
3667
|
}
|
|
3322
3668
|
}
|
|
3669
|
+
async function hostedKeylessSignupUrl(session) {
|
|
3670
|
+
if (!session?.keylessClientIp) return void 0;
|
|
3671
|
+
const eligibility = await keylessEligible(
|
|
3672
|
+
session.keylessClientIp,
|
|
3673
|
+
requestOrigin(void 0, session),
|
|
3674
|
+
{ signupLink: true }
|
|
3675
|
+
);
|
|
3676
|
+
return eligibility.signupUrl;
|
|
3677
|
+
}
|
|
3323
3678
|
function isKeylessMode(session) {
|
|
3324
3679
|
if (hasCredential(session) || session?.credentialError) return false;
|
|
3325
3680
|
if (process.env.CLOUD_SERVICE === "true") {
|
|
@@ -3334,12 +3689,14 @@ async function keylessPost(path2, body, session, originOverride) {
|
|
|
3334
3689
|
if (!eligibility.eligible) {
|
|
3335
3690
|
const code = eligibility.unavailable ? "KEYLESS_ELIGIBILITY_UNAVAILABLE" : keylessQuotaReason(eligibility.reason) ? "KEYLESS_QUOTA_EXHAUSTED" : "KEYLESS_ACCESS_NOT_AVAILABLE";
|
|
3336
3691
|
const payload = recoveryPayload(code, session?.requestId, {
|
|
3337
|
-
retryAfterSeconds: eligibility.retryAfterSeconds
|
|
3692
|
+
retryAfterSeconds: eligibility.retryAfterSeconds,
|
|
3693
|
+
signupUrl: eligibility.signupUrl
|
|
3338
3694
|
});
|
|
3339
3695
|
throw new UserError2(String(payload.message), payload);
|
|
3340
3696
|
}
|
|
3341
3697
|
}
|
|
3342
3698
|
const headers = {
|
|
3699
|
+
...AGENT_HINTS_HEADERS,
|
|
3343
3700
|
...originHeaders(origin),
|
|
3344
3701
|
"Content-Type": "application/json"
|
|
3345
3702
|
};
|
|
@@ -3357,12 +3714,17 @@ async function keylessPost(path2, body, session, originOverride) {
|
|
|
3357
3714
|
if (isKeylessMode(session) && response.status === 429) {
|
|
3358
3715
|
const code = keylessQuotaReason(json?.reason) ? "KEYLESS_QUOTA_EXHAUSTED" : "KEYLESS_LIMIT_REACHED";
|
|
3359
3716
|
const payload = recoveryPayload(code, session?.requestId, {
|
|
3360
|
-
retryAfterSeconds: Number.isFinite(json?.retry_after_seconds) && json.retry_after_seconds > 0 ? json.retry_after_seconds : void 0
|
|
3717
|
+
retryAfterSeconds: Number.isFinite(json?.retry_after_seconds) && json.retry_after_seconds > 0 ? json.retry_after_seconds : void 0,
|
|
3718
|
+
signupUrl: keylessSignupUrlFrom(json?.signup_url)
|
|
3361
3719
|
});
|
|
3720
|
+
const hints = readAgentHints(json);
|
|
3721
|
+
if (hints) payload.agent_hints = hints;
|
|
3362
3722
|
throw new UserError2(String(payload.message), payload);
|
|
3363
3723
|
}
|
|
3364
|
-
throw new
|
|
3365
|
-
json?.error || `Firecrawl request failed (HTTP ${response.status})
|
|
3724
|
+
throw new CoreHttpError(
|
|
3725
|
+
json?.error || `Firecrawl request failed (HTTP ${response.status})`,
|
|
3726
|
+
response.status,
|
|
3727
|
+
readAgentHints(json)
|
|
3366
3728
|
);
|
|
3367
3729
|
}
|
|
3368
3730
|
return json;
|
|
@@ -3383,7 +3745,8 @@ async function getCrawlStatusWithOrigin(client, jobId, origin) {
|
|
|
3383
3745
|
creditsUsed: body.creditsUsed,
|
|
3384
3746
|
expiresAt: body.expiresAt,
|
|
3385
3747
|
next: body.next ?? null,
|
|
3386
|
-
data: initialDocs
|
|
3748
|
+
data: initialDocs,
|
|
3749
|
+
...readAgentHints(body) ? { agent_hints: readAgentHints(body) } : {}
|
|
3387
3750
|
};
|
|
3388
3751
|
}
|
|
3389
3752
|
const docs = initialDocs.slice();
|
|
@@ -3407,7 +3770,8 @@ async function getCrawlStatusWithOrigin(client, jobId, origin) {
|
|
|
3407
3770
|
creditsUsed: body.creditsUsed,
|
|
3408
3771
|
expiresAt: body.expiresAt,
|
|
3409
3772
|
next: null,
|
|
3410
|
-
data: docs
|
|
3773
|
+
data: docs,
|
|
3774
|
+
...readAgentHints(body) ? { agent_hints: readAgentHints(body) } : {}
|
|
3411
3775
|
};
|
|
3412
3776
|
}
|
|
3413
3777
|
async function waitForCrawlCompletionWithOrigin(client, jobId, origin, pollInterval = 2, timeout) {
|
|
@@ -3518,6 +3882,7 @@ Eligibility is limited to successful searches within the feedback age window. Th
|
|
|
3518
3882
|
}
|
|
3519
3883
|
if (querySuggestions) body.querySuggestions = querySuggestions;
|
|
3520
3884
|
const headers = {
|
|
3885
|
+
...AGENT_HINTS_HEADERS,
|
|
3521
3886
|
...originHeaders(origin),
|
|
3522
3887
|
"Content-Type": "application/json"
|
|
3523
3888
|
};
|
|
@@ -3556,7 +3921,8 @@ Eligibility is limited to successful searches within the feedback age window. Th
|
|
|
3556
3921
|
status: response.status,
|
|
3557
3922
|
feedbackErrorCode: parsed?.feedbackErrorCode,
|
|
3558
3923
|
error: parsed?.error ?? `HTTP ${response.status}`,
|
|
3559
|
-
retryable: response.status >= 500
|
|
3924
|
+
retryable: response.status >= 500,
|
|
3925
|
+
...readAgentHints(parsed) ? { agent_hints: readAgentHints(parsed) } : {}
|
|
3560
3926
|
});
|
|
3561
3927
|
}
|
|
3562
3928
|
return structuredText(parsed);
|
|
@@ -3638,6 +4004,7 @@ Returns submission status, feedback ID, and accounting fields.
|
|
|
3638
4004
|
} = args2;
|
|
3639
4005
|
const apiBase = resolveApiBaseUrl();
|
|
3640
4006
|
const headers = {
|
|
4007
|
+
...AGENT_HINTS_HEADERS,
|
|
3641
4008
|
...originHeaders(origin),
|
|
3642
4009
|
"Content-Type": "application/json"
|
|
3643
4010
|
};
|
|
@@ -3691,7 +4058,8 @@ Returns submission status, feedback ID, and accounting fields.
|
|
|
3691
4058
|
status: response.status,
|
|
3692
4059
|
feedbackErrorCode: parsed?.feedbackErrorCode,
|
|
3693
4060
|
error: parsed?.error ?? `HTTP ${response.status}`,
|
|
3694
|
-
retryable: response.status >= 500
|
|
4061
|
+
retryable: response.status >= 500,
|
|
4062
|
+
...readAgentHints(parsed) ? { agent_hints: readAgentHints(parsed) } : {}
|
|
3695
4063
|
});
|
|
3696
4064
|
}
|
|
3697
4065
|
return structuredText(parsed);
|
|
@@ -3838,6 +4206,29 @@ Deprecated compatibility entry point. Use firecrawl_scrape once per known URL wi
|
|
|
3838
4206
|
throw new UserError2(payload.message, payload);
|
|
3839
4207
|
}
|
|
3840
4208
|
});
|
|
4209
|
+
var agentExchangeSchema = z8.strictObject({
|
|
4210
|
+
enabled: z8.boolean().optional().describe("Let the agent call Alexandria providers. On by default."),
|
|
4211
|
+
toolkits: z8.array(z8.string()).max(5).optional().describe("Pin up to 5 providers the agent may use, by slug. Omitted means the whole catalog."),
|
|
4212
|
+
maxCalls: z8.number().int().min(1).max(30).optional().describe("Most provider calls the agent may make in this turn."),
|
|
4213
|
+
requireApproval: z8.boolean().optional().describe(
|
|
4214
|
+
'End the turn with a paid-call pendingApproval before any paid provider call. Requires mode "chat" on the same request, even on a follow-up.'
|
|
4215
|
+
),
|
|
4216
|
+
approve: z8.strictObject({
|
|
4217
|
+
approvalId: z8.string().uuid(),
|
|
4218
|
+
callIds: z8.array(z8.string()).optional(),
|
|
4219
|
+
always: z8.boolean().optional()
|
|
4220
|
+
}).optional().describe(
|
|
4221
|
+
"Answer yes to the pendingApproval the previous turn of this thread ended on (its id, also exchange.requiresAction.approvalId). Needs threadId. For a terms offer, send it only after the user explicitly agreed and terms/accept succeeded; callIds and always are ignored on terms offers. For paid calls, callIds picks a subset (default all) and always stops asking for the rest of the thread."
|
|
4222
|
+
),
|
|
4223
|
+
decline: z8.strictObject({ approvalId: z8.string().uuid() }).optional().describe(
|
|
4224
|
+
"Answer no to that pendingApproval. Needs threadId. A declined terms offer keeps those providers out of the rest of the thread."
|
|
4225
|
+
),
|
|
4226
|
+
onTermsRequired: z8.enum(["skip", "ask"]).optional().describe(
|
|
4227
|
+
`What to do when a provider the agent would use needs data terms the team has not accepted. Gated providers are never called. "skip" (default): answer with accepted providers and list the rest in exchange.skippedProviders. "ask": the same, plus a terms pendingApproval and exchange.requiresAction with the terms/show and terms/accept calls. Each provider digest is string | null and always present; when null, terms/show returns it. There is no auto-accept. Omitted on a follow-up keeps the previous turn's value.`
|
|
4228
|
+
)
|
|
4229
|
+
}).describe(
|
|
4230
|
+
"Alexandria provider settings for this turn, forwarded as the request's exchange object."
|
|
4231
|
+
);
|
|
3841
4232
|
server.addTool({
|
|
3842
4233
|
name: "firecrawl_agent",
|
|
3843
4234
|
annotations: {
|
|
@@ -3850,27 +4241,74 @@ server.addTool({
|
|
|
3850
4241
|
// Gathers information only; does not delete external data or user resources.
|
|
3851
4242
|
},
|
|
3852
4243
|
description: `
|
|
3853
|
-
Run web research that returns structured data when the URLs are not known or the answer spans several sites. Describe the fields you need in \`prompt\`, optionally pass a JSON \`schema\` and seed \`urls\`, and the research agent searches, navigates, reads pages, and returns JSON assembled across sources. Use it to research an entity plus its fields (founders, pricing, contact details), to build lists and datasets (companies, people, products, jobs, papers), and for pages that need navigation or interaction to reach the data.
|
|
4244
|
+
Run web research that returns structured data when the URLs are not known or the answer spans several sites. Describe the fields you need in \`prompt\`, optionally pass a JSON \`schema\` and seed \`urls\`, and the research agent searches, navigates, reads pages, and returns JSON assembled across sources. Use it to research an entity plus its fields (founders, pricing, contact details), to build lists and datasets (companies, people, products, jobs, papers), and for pages that need navigation or interaction to reach the data. Optional \`effort\` sets the reasoning budget, \`maxCredits\` caps spend, and \`strictConstrainToURLs\` keeps the agent to the supplied \`urls\`.
|
|
3854
4245
|
|
|
3855
4246
|
This call returns only a job ID, not the research result. Read the job with \`firecrawl_agent_status\` until it reaches \`completed\` or \`failed\`; a typical research run takes one to three minutes. For one known URL use \`firecrawl_scrape\` (with formats: ["json"] for structured output); for a plain lookup that a results page answers, use \`firecrawl_search\`.
|
|
4247
|
+
|
|
4248
|
+
The job also returns a \`threadId\`. To continue that thread, pass it with a follow-up \`prompt\`; omitted \`mode\`, \`urls\`, \`schema\` and exchange settings carry over from the previous turn.
|
|
4249
|
+
|
|
4250
|
+
The agent only calls Alexandria providers whose data terms the team has accepted. The status result's \`exchange.skippedProviders\` lists gated providers that would have helped. With \`exchange.onTermsRequired\` "ask", a terms offer ends the turn: \`pendingApproval\` (kind "terms") and \`exchange.requiresAction\` carry the \`approvalId\` and the exact terms/show and terms/accept calls. Show the user the terms, get their EXPLICIT consent, run terms/accept through \`firecrawl_scrape\`, then call \`firecrawl_agent\` with the same \`threadId\` and \`exchange.approve: {approvalId}\`. If they decline, send \`exchange.decline: {approvalId}\` instead. Never call terms/accept without that consent; a data request is not consent.
|
|
3856
4251
|
`,
|
|
3857
4252
|
outputSchema: agentOutputSchema,
|
|
3858
4253
|
parameters: z8.object({
|
|
3859
4254
|
prompt: z8.string().min(1).max(1e4),
|
|
3860
4255
|
urls: z8.array(z8.string().url()).optional(),
|
|
3861
|
-
schema: z8.record(z8.string(), z8.any()).optional()
|
|
3862
|
-
|
|
4256
|
+
schema: z8.record(z8.string(), z8.any()).optional(),
|
|
4257
|
+
effort: z8.enum(["low", "medium", "high"]).optional().describe("Reasoning budget for the agent task."),
|
|
4258
|
+
maxCredits: z8.number().int().positive().optional().describe(
|
|
4259
|
+
"Spending limit in credits for this run. Defaults to 2500 on the API."
|
|
4260
|
+
),
|
|
4261
|
+
strictConstrainToURLs: z8.boolean().optional().describe(
|
|
4262
|
+
"If true, agent will only visit URLs provided in the urls array."
|
|
4263
|
+
),
|
|
4264
|
+
threadId: z8.string().uuid().optional().describe(
|
|
4265
|
+
"Continue this thread: the threadId from an earlier firecrawl_agent or firecrawl_agent_status result. Omit to start a new thread."
|
|
4266
|
+
),
|
|
4267
|
+
mode: z8.enum(["extract", "chat"]).optional().describe(
|
|
4268
|
+
`"extract" (default) returns the complete structured result every turn. "chat" lets a follow-up that asks no new data get a short reply in message instead of a re-run; required for exchange.requireApproval. Omitted on a follow-up keeps the previous turn's mode.`
|
|
4269
|
+
),
|
|
4270
|
+
exchange: agentExchangeSchema.optional()
|
|
4271
|
+
}).refine(
|
|
4272
|
+
(data) => !(data.exchange?.approve && data.exchange?.decline),
|
|
4273
|
+
{
|
|
4274
|
+
message: "Send exchange.approve or exchange.decline, not both: each answers the pending approval one way.",
|
|
4275
|
+
path: ["exchange"]
|
|
4276
|
+
}
|
|
4277
|
+
).refine(
|
|
4278
|
+
(data) => !(data.exchange?.approve || data.exchange?.decline) || Boolean(data.threadId),
|
|
4279
|
+
{
|
|
4280
|
+
message: "exchange.approve and exchange.decline answer a pending approval on an existing thread: pass that thread's threadId.",
|
|
4281
|
+
path: ["threadId"]
|
|
4282
|
+
}
|
|
4283
|
+
).refine(
|
|
4284
|
+
(data) => !data.exchange?.requireApproval || data.mode === "chat",
|
|
4285
|
+
{
|
|
4286
|
+
// The agent service checks the mode sent on this request, not the
|
|
4287
|
+
// thread's inherited one, and the gateway turns its 400 into a 500.
|
|
4288
|
+
message: 'exchange.requireApproval needs mode: "chat" on the same request, including on a follow-up.',
|
|
4289
|
+
path: ["mode"]
|
|
4290
|
+
}
|
|
4291
|
+
),
|
|
3863
4292
|
execute: async (args2, { session, log, client: mcpClient }) => {
|
|
3864
4293
|
const client = getClient(session);
|
|
3865
4294
|
const a = args2;
|
|
3866
4295
|
log.info("Starting agent", {
|
|
3867
4296
|
prompt: a.prompt.substring(0, 100),
|
|
3868
|
-
urlCount: Array.isArray(a.urls) ? a.urls.length : 0
|
|
4297
|
+
urlCount: Array.isArray(a.urls) ? a.urls.length : 0,
|
|
4298
|
+
threadId: a.threadId ?? null
|
|
3869
4299
|
});
|
|
3870
4300
|
const agentBody = removeEmptyTopLevel({
|
|
3871
4301
|
prompt: a.prompt,
|
|
3872
4302
|
urls: a.urls,
|
|
3873
|
-
schema: a.schema || void 0
|
|
4303
|
+
schema: a.schema || void 0,
|
|
4304
|
+
effort: a.effort,
|
|
4305
|
+
maxCredits: a.maxCredits,
|
|
4306
|
+
strictConstrainToURLs: a.strictConstrainToURLs,
|
|
4307
|
+
threadId: a.threadId,
|
|
4308
|
+
mode: a.mode,
|
|
4309
|
+
// Forwarded verbatim: the agent service owns the defaults and the
|
|
4310
|
+
// per-thread inheritance.
|
|
4311
|
+
exchange: a.exchange
|
|
3874
4312
|
});
|
|
3875
4313
|
const res = await client.startAgent({
|
|
3876
4314
|
...agentBody,
|
|
@@ -4076,7 +4514,10 @@ Set \`redactPII\` to request redaction of personally identifiable information in
|
|
|
4076
4514
|
});
|
|
4077
4515
|
form.append("file", blob, filename);
|
|
4078
4516
|
form.append("options", JSON.stringify(optionsPayload));
|
|
4079
|
-
const headers = {
|
|
4517
|
+
const headers = {
|
|
4518
|
+
...AGENT_HINTS_HEADERS,
|
|
4519
|
+
...originHeaders(origin)
|
|
4520
|
+
};
|
|
4080
4521
|
const credential = credentialForOutboundRequest(session);
|
|
4081
4522
|
if (credential) {
|
|
4082
4523
|
headers["Authorization"] = `Bearer ${credential}`;
|
|
@@ -4094,8 +4535,15 @@ Set \`redactPII\` to request redaction of personally identifiable information in
|
|
|
4094
4535
|
});
|
|
4095
4536
|
const responseText = await response.text();
|
|
4096
4537
|
if (!response.ok) {
|
|
4097
|
-
|
|
4098
|
-
|
|
4538
|
+
let parsed;
|
|
4539
|
+
try {
|
|
4540
|
+
parsed = JSON.parse(responseText);
|
|
4541
|
+
} catch {
|
|
4542
|
+
}
|
|
4543
|
+
throw new CoreHttpError(
|
|
4544
|
+
`Parse request failed with status ${response.status}: ${responseText}`,
|
|
4545
|
+
response.status,
|
|
4546
|
+
readAgentHints(parsed)
|
|
4099
4547
|
);
|
|
4100
4548
|
}
|
|
4101
4549
|
try {
|