firecrawl-mcp 3.31.1 → 3.33.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -1
- package/dist/index.js +94 -13
- package/dist/integration.js +19 -0
- package/dist/origin.js +46 -5
- package/dist/server.d.ts +11 -0
- package/dist/server.js +91 -12
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
|
|
9
9
|
# Firecrawl MCP Server
|
|
10
10
|
|
|
11
|
-
A Model Context Protocol (MCP) server that brings [Firecrawl](https://github.com/firecrawl/firecrawl) to MCP-compatible
|
|
11
|
+
A Model Context Protocol (MCP) server that brings [Firecrawl](https://github.com/firecrawl/firecrawl), the context API for AI agents, to MCP-compatible clients: search, scrape, and interact with the live web and get back clean Markdown or structured data.
|
|
12
12
|
|
|
13
13
|
> Big thanks to [@vrknetha](https://github.com/vrknetha), [@knacklabs](https://www.knacklabs.ai) for the initial implementation!
|
|
14
14
|
|
|
@@ -273,6 +273,9 @@ Optionally, you can add it to a file called `.vscode/mcp.json` in your workspace
|
|
|
273
273
|
- `FIRECRAWL_API_URL` (Optional): Custom API endpoint for self-hosted instances
|
|
274
274
|
- Example: `https://firecrawl.your-domain.com`
|
|
275
275
|
- If not provided, the cloud API will be used (requires API key)
|
|
276
|
+
- `FIRECRAWL_INTEGRATION` (Optional): Integration tag for apps that launch this server, such as an editor extension
|
|
277
|
+
- Sent as the `integration` field of scrape, batch scrape, crawl, map, search, extract, agent, parse, interact and feedback requests
|
|
278
|
+
- Use a value that starts with `_`, for example `_zed`
|
|
276
279
|
|
|
277
280
|
#### MCP OAuth (Bearer access tokens)
|
|
278
281
|
|
package/dist/index.js
CHANGED
|
@@ -2637,25 +2637,64 @@ function initializeClientSlug(name2) {
|
|
|
2637
2637
|
const slug = slugOf(name2);
|
|
2638
2638
|
return isOwnSlug(slug) || GENERIC_CLIENT_NAMES.has(slug) ? "" : slug;
|
|
2639
2639
|
}
|
|
2640
|
-
function
|
|
2641
|
-
let
|
|
2640
|
+
function publicHost(url) {
|
|
2641
|
+
let parsed;
|
|
2642
2642
|
try {
|
|
2643
|
-
|
|
2643
|
+
parsed = new URL(url ?? "");
|
|
2644
2644
|
} catch {
|
|
2645
2645
|
return "";
|
|
2646
2646
|
}
|
|
2647
|
+
const host = parsed.hostname.toLowerCase().replace(/\.$/, "");
|
|
2647
2648
|
if (!host.includes(".") || host.endsWith(".localhost") || host.endsWith(".local") || /^[\d.]+$/.test(host) || host.startsWith("[")) {
|
|
2648
2649
|
return "";
|
|
2649
2650
|
}
|
|
2650
|
-
return
|
|
2651
|
+
return host.replace(/^www\./, "");
|
|
2652
|
+
}
|
|
2653
|
+
function referrerSlug(referrer) {
|
|
2654
|
+
return slugOf(publicHost(referrer));
|
|
2655
|
+
}
|
|
2656
|
+
var SHARED_STATIC_HOSTS = [
|
|
2657
|
+
"github.io",
|
|
2658
|
+
"gitlab.io",
|
|
2659
|
+
"pages.dev",
|
|
2660
|
+
"netlify.app",
|
|
2661
|
+
"vercel.app"
|
|
2662
|
+
];
|
|
2663
|
+
function isHttpUrl(value) {
|
|
2664
|
+
try {
|
|
2665
|
+
return /^https?:$/.test(new URL(value ?? "").protocol);
|
|
2666
|
+
} catch {
|
|
2667
|
+
return false;
|
|
2668
|
+
}
|
|
2669
|
+
}
|
|
2670
|
+
function oauthClientSlug(clientId, clientName) {
|
|
2671
|
+
if (!isHttpUrl(clientId)) return initializeClientSlug(clientName);
|
|
2672
|
+
const host = publicHost(clientId);
|
|
2673
|
+
if (!host || host === "firecrawl.dev" || host.endsWith(".firecrawl.dev")) {
|
|
2674
|
+
return "";
|
|
2675
|
+
}
|
|
2676
|
+
if (!SHARED_STATIC_HOSTS.some(
|
|
2677
|
+
(shared) => host === shared || host.endsWith(`.${shared}`)
|
|
2678
|
+
)) {
|
|
2679
|
+
return slugOf(host);
|
|
2680
|
+
}
|
|
2681
|
+
const [first = ""] = new URL(clientId ?? "").pathname.split("/").filter(Boolean);
|
|
2682
|
+
const segment = slugOf(first).slice(0, NAME_MAX / 2);
|
|
2683
|
+
if (!segment) return slugOf(host);
|
|
2684
|
+
const room = NAME_MAX - segment.length - 1;
|
|
2685
|
+
return slugOf(`${slugOf(host).slice(0, room)}-${segment}`);
|
|
2651
2686
|
}
|
|
2652
2687
|
function originForClient(client, serverVersion2, userAgent, signals = {}) {
|
|
2653
2688
|
const declared = slugOf(signals.declaredOrigin);
|
|
2654
2689
|
const slug = slugOf(client?.name);
|
|
2655
2690
|
const product = userAgentProduct(userAgent);
|
|
2656
2691
|
const remembered = initializeClientSlug(signals.initializeClient?.name);
|
|
2692
|
+
const oauthClient = oauthClientSlug(
|
|
2693
|
+
signals.oauthClientId,
|
|
2694
|
+
signals.oauthClientName
|
|
2695
|
+
);
|
|
2657
2696
|
const referrer = referrerSlug(signals.referrer);
|
|
2658
|
-
const base = !isOwnSlug(declared) ? `mcp-${declared}` : !isOwnSlug(slug) ? `mcp-${slug}` : product && !RUNTIME_PRODUCTS.has(product) ? `mcp-ua-${product}` : remembered ? `mcp-${remembered}` : referrer ? `mcp-ref-${referrer}` : product ? `mcp-ua-${product}` : ORIGIN_SERVER;
|
|
2697
|
+
const base = !isOwnSlug(declared) ? `mcp-${declared}` : !isOwnSlug(slug) ? `mcp-${slug}` : product && !RUNTIME_PRODUCTS.has(product) ? `mcp-ua-${product}` : remembered ? `mcp-${remembered}` : oauthClient ? `mcp-oauth-${oauthClient}` : referrer ? `mcp-ref-${referrer}` : product ? `mcp-ua-${product}` : ORIGIN_SERVER;
|
|
2659
2698
|
return serverVersion2 ? `${base}@${serverVersion2}` : base;
|
|
2660
2699
|
}
|
|
2661
2700
|
function originHeaders(origin) {
|
|
@@ -2672,6 +2711,8 @@ function requestOrigin(client, session) {
|
|
|
2672
2711
|
{
|
|
2673
2712
|
declaredOrigin: session?.clientOrigin,
|
|
2674
2713
|
initializeClient: session?.initializeClient,
|
|
2714
|
+
oauthClientId: session?.oauthClientId,
|
|
2715
|
+
oauthClientName: session?.oauthClientName,
|
|
2675
2716
|
referrer: session?.clientReferrer
|
|
2676
2717
|
}
|
|
2677
2718
|
);
|
|
@@ -3559,6 +3600,22 @@ The historical view returns periods sorted by start date with \`startDate\`, \`e
|
|
|
3559
3600
|
});
|
|
3560
3601
|
}
|
|
3561
3602
|
|
|
3603
|
+
// src/integration.ts
|
|
3604
|
+
var TAGGED_PATH = /^\/v2\/(scrape|batch\/scrape|crawl|map|search|extract|agent|parse|feedback|scrape\/[^/]+\/interact|search\/[^/]+\/feedback)$/;
|
|
3605
|
+
function integrationFrom(value) {
|
|
3606
|
+
const tag = (value ?? "").trim();
|
|
3607
|
+
return tag === "" ? void 0 : tag;
|
|
3608
|
+
}
|
|
3609
|
+
function pathOf(url) {
|
|
3610
|
+
const path2 = url.replace(/^[a-z][a-z0-9+.-]*:\/\/[^/]+/i, "").split(/[?#]/)[0];
|
|
3611
|
+
return path2.replace(/\/+$/, "");
|
|
3612
|
+
}
|
|
3613
|
+
function withIntegration(url, body, integration) {
|
|
3614
|
+
if (!integration || body.integration != null) return body;
|
|
3615
|
+
if (!TAGGED_PATH.test(pathOf(url))) return body;
|
|
3616
|
+
return { ...body, integration };
|
|
3617
|
+
}
|
|
3618
|
+
|
|
3562
3619
|
// src/headers.ts
|
|
3563
3620
|
function normalizeHeader(value) {
|
|
3564
3621
|
if (value == null) return void 0;
|
|
@@ -3999,7 +4056,8 @@ function createFirecrawlMcpServer(options2 = {}) {
|
|
|
3999
4056
|
fileAccess: options2.fileAccess ?? "local",
|
|
4000
4057
|
requireCredential: options2.requireCredential ?? false,
|
|
4001
4058
|
searchFeedback: options2.searchFeedback ?? true,
|
|
4002
|
-
endpointFeedback: options2.endpointFeedback ?? true
|
|
4059
|
+
endpointFeedback: options2.endpointFeedback ?? true,
|
|
4060
|
+
integration: integrationFrom(options2.integration)
|
|
4003
4061
|
};
|
|
4004
4062
|
const logging = options2.logging ?? config.transport === "httpStream";
|
|
4005
4063
|
const instructionDefaults = {
|
|
@@ -4127,7 +4185,22 @@ function createFirecrawlMcpServer(options2 = {}) {
|
|
|
4127
4185
|
if (apiKey2) {
|
|
4128
4186
|
clientConfig.apiKey = apiKey2;
|
|
4129
4187
|
}
|
|
4130
|
-
|
|
4188
|
+
const client = new FirecrawlApp(clientConfig);
|
|
4189
|
+
if (config.integration) {
|
|
4190
|
+
client.http?.instance?.interceptors?.request?.use((request) => {
|
|
4191
|
+
const isPost = String(request.method ?? "").toLowerCase() === "post";
|
|
4192
|
+
const isPlainObject = request.data != null && typeof request.data === "object" && !Array.isArray(request.data) && !(request.data instanceof FormData);
|
|
4193
|
+
if (isPost && isPlainObject) {
|
|
4194
|
+
request.data = withIntegration(
|
|
4195
|
+
request.url ?? "",
|
|
4196
|
+
request.data,
|
|
4197
|
+
config.integration
|
|
4198
|
+
);
|
|
4199
|
+
}
|
|
4200
|
+
return request;
|
|
4201
|
+
});
|
|
4202
|
+
}
|
|
4203
|
+
return client;
|
|
4131
4204
|
}
|
|
4132
4205
|
const SAFE_MODE = config.safeMode;
|
|
4133
4206
|
function getClient(session) {
|
|
@@ -4461,7 +4534,7 @@ function createFirecrawlMcpServer(options2 = {}) {
|
|
|
4461
4534
|
function buildParseOptionsPayload(options3, origin) {
|
|
4462
4535
|
const transformed = transformScrapeParams(options3);
|
|
4463
4536
|
const cleaned = removeEmptyTopLevel(transformed);
|
|
4464
|
-
return { origin, ...cleaned };
|
|
4537
|
+
return withIntegration("/v2/parse", { origin, ...cleaned }, config.integration);
|
|
4465
4538
|
}
|
|
4466
4539
|
function buildContinuationArguments(uploadRef, options3) {
|
|
4467
4540
|
return {
|
|
@@ -4485,7 +4558,7 @@ function createFirecrawlMcpServer(options2 = {}) {
|
|
|
4485
4558
|
Authorization: `Bearer ${apiKey2}`,
|
|
4486
4559
|
...origin ? originHeaders(origin) : {}
|
|
4487
4560
|
},
|
|
4488
|
-
body: JSON.stringify(body)
|
|
4561
|
+
body: JSON.stringify(withIntegration(pathName, body, config.integration))
|
|
4489
4562
|
});
|
|
4490
4563
|
const responseText = await response.text();
|
|
4491
4564
|
let parsed;
|
|
@@ -4928,7 +5001,7 @@ For a programming question, add \`categories: ["developer"]\`; its hits return i
|
|
|
4928
5001
|
const response = await fetch(`${resolveApiBaseUrl()}${path2}`, {
|
|
4929
5002
|
method: "POST",
|
|
4930
5003
|
headers,
|
|
4931
|
-
body: JSON.stringify(body)
|
|
5004
|
+
body: JSON.stringify(withIntegration(path2, body, config.integration))
|
|
4932
5005
|
});
|
|
4933
5006
|
const json = await response.json().catch(() => ({}));
|
|
4934
5007
|
if (!response.ok) {
|
|
@@ -5098,7 +5171,13 @@ Eligibility is limited to successful searches within the feedback age window. Th
|
|
|
5098
5171
|
const response = await fetch(endpoint, {
|
|
5099
5172
|
method: "POST",
|
|
5100
5173
|
headers,
|
|
5101
|
-
body: JSON.stringify(
|
|
5174
|
+
body: JSON.stringify(
|
|
5175
|
+
withIntegration(
|
|
5176
|
+
`/v2/search/${encodeURIComponent(searchId)}/feedback`,
|
|
5177
|
+
body,
|
|
5178
|
+
config.integration
|
|
5179
|
+
)
|
|
5180
|
+
)
|
|
5102
5181
|
});
|
|
5103
5182
|
const responseText = await response.text();
|
|
5104
5183
|
let parsed;
|
|
@@ -5237,7 +5316,7 @@ Returns submission status, feedback ID, and accounting fields.
|
|
|
5237
5316
|
const response = await fetch(`${apiBase}/v2/feedback`, {
|
|
5238
5317
|
method: "POST",
|
|
5239
5318
|
headers,
|
|
5240
|
-
body: JSON.stringify(body)
|
|
5319
|
+
body: JSON.stringify(withIntegration("/v2/feedback", body, config.integration))
|
|
5241
5320
|
});
|
|
5242
5321
|
const responseText = await response.text();
|
|
5243
5322
|
let parsed;
|
|
@@ -6184,6 +6263,7 @@ function credentialMetadata(data) {
|
|
|
6184
6263
|
userId: typeof data.sub === "string" ? data.sub : void 0,
|
|
6185
6264
|
apiKeyId: typeof data.api_key_id === "string" ? data.api_key_id : void 0,
|
|
6186
6265
|
oauthClientId: typeof data.client_id === "string" ? data.client_id : void 0,
|
|
6266
|
+
oauthClientName: typeof data.client_name === "string" ? data.client_name : void 0,
|
|
6187
6267
|
resource: typeof data.aud === "string" ? data.aud : void 0
|
|
6188
6268
|
};
|
|
6189
6269
|
}
|
|
@@ -6718,7 +6798,8 @@ var options = {
|
|
|
6718
6798
|
fileAccess: hosted ? "upload" : "local",
|
|
6719
6799
|
requireCredential: hosted,
|
|
6720
6800
|
searchFeedback: !searchFeedbackDisabled,
|
|
6721
|
-
endpointFeedback: !endpointFeedbackDisabled
|
|
6801
|
+
endpointFeedback: !endpointFeedbackDisabled,
|
|
6802
|
+
integration: process.env.FIRECRAWL_INTEGRATION
|
|
6722
6803
|
};
|
|
6723
6804
|
var serviceSettings = { hosted, transport, apiKey, apiUrl };
|
|
6724
6805
|
var primaryHooks = createServiceHooks(primaryProfile, serviceSettings);
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
// src/integration.ts
|
|
2
|
+
var TAGGED_PATH = /^\/v2\/(scrape|batch\/scrape|crawl|map|search|extract|agent|parse|feedback|scrape\/[^/]+\/interact|search\/[^/]+\/feedback)$/;
|
|
3
|
+
function integrationFrom(value) {
|
|
4
|
+
const tag = (value ?? "").trim();
|
|
5
|
+
return tag === "" ? void 0 : tag;
|
|
6
|
+
}
|
|
7
|
+
function pathOf(url) {
|
|
8
|
+
const path = url.replace(/^[a-z][a-z0-9+.-]*:\/\/[^/]+/i, "").split(/[?#]/)[0];
|
|
9
|
+
return path.replace(/\/+$/, "");
|
|
10
|
+
}
|
|
11
|
+
function withIntegration(url, body, integration) {
|
|
12
|
+
if (!integration || body.integration != null) return body;
|
|
13
|
+
if (!TAGGED_PATH.test(pathOf(url))) return body;
|
|
14
|
+
return { ...body, integration };
|
|
15
|
+
}
|
|
16
|
+
export {
|
|
17
|
+
integrationFrom,
|
|
18
|
+
withIntegration
|
|
19
|
+
};
|
package/dist/origin.js
CHANGED
|
@@ -59,25 +59,64 @@ function initializeClientSlug(name) {
|
|
|
59
59
|
const slug = slugOf(name);
|
|
60
60
|
return isOwnSlug(slug) || GENERIC_CLIENT_NAMES.has(slug) ? "" : slug;
|
|
61
61
|
}
|
|
62
|
-
function
|
|
63
|
-
let
|
|
62
|
+
function publicHost(url) {
|
|
63
|
+
let parsed;
|
|
64
64
|
try {
|
|
65
|
-
|
|
65
|
+
parsed = new URL(url ?? "");
|
|
66
66
|
} catch {
|
|
67
67
|
return "";
|
|
68
68
|
}
|
|
69
|
+
const host = parsed.hostname.toLowerCase().replace(/\.$/, "");
|
|
69
70
|
if (!host.includes(".") || host.endsWith(".localhost") || host.endsWith(".local") || /^[\d.]+$/.test(host) || host.startsWith("[")) {
|
|
70
71
|
return "";
|
|
71
72
|
}
|
|
72
|
-
return
|
|
73
|
+
return host.replace(/^www\./, "");
|
|
74
|
+
}
|
|
75
|
+
function referrerSlug(referrer) {
|
|
76
|
+
return slugOf(publicHost(referrer));
|
|
77
|
+
}
|
|
78
|
+
var SHARED_STATIC_HOSTS = [
|
|
79
|
+
"github.io",
|
|
80
|
+
"gitlab.io",
|
|
81
|
+
"pages.dev",
|
|
82
|
+
"netlify.app",
|
|
83
|
+
"vercel.app"
|
|
84
|
+
];
|
|
85
|
+
function isHttpUrl(value) {
|
|
86
|
+
try {
|
|
87
|
+
return /^https?:$/.test(new URL(value ?? "").protocol);
|
|
88
|
+
} catch {
|
|
89
|
+
return false;
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
function oauthClientSlug(clientId, clientName) {
|
|
93
|
+
if (!isHttpUrl(clientId)) return initializeClientSlug(clientName);
|
|
94
|
+
const host = publicHost(clientId);
|
|
95
|
+
if (!host || host === "firecrawl.dev" || host.endsWith(".firecrawl.dev")) {
|
|
96
|
+
return "";
|
|
97
|
+
}
|
|
98
|
+
if (!SHARED_STATIC_HOSTS.some(
|
|
99
|
+
(shared) => host === shared || host.endsWith(`.${shared}`)
|
|
100
|
+
)) {
|
|
101
|
+
return slugOf(host);
|
|
102
|
+
}
|
|
103
|
+
const [first = ""] = new URL(clientId ?? "").pathname.split("/").filter(Boolean);
|
|
104
|
+
const segment = slugOf(first).slice(0, NAME_MAX / 2);
|
|
105
|
+
if (!segment) return slugOf(host);
|
|
106
|
+
const room = NAME_MAX - segment.length - 1;
|
|
107
|
+
return slugOf(`${slugOf(host).slice(0, room)}-${segment}`);
|
|
73
108
|
}
|
|
74
109
|
function originForClient(client, serverVersion2, userAgent, signals = {}) {
|
|
75
110
|
const declared = slugOf(signals.declaredOrigin);
|
|
76
111
|
const slug = slugOf(client?.name);
|
|
77
112
|
const product = userAgentProduct(userAgent);
|
|
78
113
|
const remembered = initializeClientSlug(signals.initializeClient?.name);
|
|
114
|
+
const oauthClient = oauthClientSlug(
|
|
115
|
+
signals.oauthClientId,
|
|
116
|
+
signals.oauthClientName
|
|
117
|
+
);
|
|
79
118
|
const referrer = referrerSlug(signals.referrer);
|
|
80
|
-
const base = !isOwnSlug(declared) ? `mcp-${declared}` : !isOwnSlug(slug) ? `mcp-${slug}` : product && !RUNTIME_PRODUCTS.has(product) ? `mcp-ua-${product}` : remembered ? `mcp-${remembered}` : referrer ? `mcp-ref-${referrer}` : product ? `mcp-ua-${product}` : ORIGIN_SERVER;
|
|
119
|
+
const base = !isOwnSlug(declared) ? `mcp-${declared}` : !isOwnSlug(slug) ? `mcp-${slug}` : product && !RUNTIME_PRODUCTS.has(product) ? `mcp-ua-${product}` : remembered ? `mcp-${remembered}` : oauthClient ? `mcp-oauth-${oauthClient}` : referrer ? `mcp-ref-${referrer}` : product ? `mcp-ua-${product}` : ORIGIN_SERVER;
|
|
81
120
|
return serverVersion2 ? `${base}@${serverVersion2}` : base;
|
|
82
121
|
}
|
|
83
122
|
function originHeaders(origin) {
|
|
@@ -94,6 +133,8 @@ function requestOrigin(client, session) {
|
|
|
94
133
|
{
|
|
95
134
|
declaredOrigin: session?.clientOrigin,
|
|
96
135
|
initializeClient: session?.initializeClient,
|
|
136
|
+
oauthClientId: session?.oauthClientId,
|
|
137
|
+
oauthClientName: session?.oauthClientName,
|
|
97
138
|
referrer: session?.clientReferrer
|
|
98
139
|
}
|
|
99
140
|
);
|
package/dist/server.d.ts
CHANGED
|
@@ -35,6 +35,8 @@ type OriginSession = {
|
|
|
35
35
|
clientOrigin?: string;
|
|
36
36
|
clientReferrer?: string;
|
|
37
37
|
initializeClient?: ClientImplementation;
|
|
38
|
+
oauthClientId?: string;
|
|
39
|
+
oauthClientName?: string;
|
|
38
40
|
};
|
|
39
41
|
/**
|
|
40
42
|
* The origin of every request a tool call makes: the call's client, the
|
|
@@ -241,6 +243,13 @@ interface SessionData {
|
|
|
241
243
|
name?: string;
|
|
242
244
|
version?: string;
|
|
243
245
|
};
|
|
246
|
+
/**
|
|
247
|
+
* The OAuth client id the access token was issued to. A metadata URL id
|
|
248
|
+
* names the product; read only when the User-Agent names a runtime.
|
|
249
|
+
*/
|
|
250
|
+
oauthClientId?: string;
|
|
251
|
+
/** The registered name of that client; read when the id is not a URL. */
|
|
252
|
+
oauthClientName?: string;
|
|
244
253
|
authType?: 'api-key' | 'oauth' | 'env' | 'keyless' | 'none';
|
|
245
254
|
/** Admitted, but the supplied credential cannot be used. */
|
|
246
255
|
credentialError?: 'CREDENTIAL_INVALID';
|
|
@@ -420,6 +429,8 @@ interface FirecrawlMcpServerOptions {
|
|
|
420
429
|
searchFeedback?: boolean;
|
|
421
430
|
/** Register firecrawl_feedback. Defaults to true. */
|
|
422
431
|
endpointFeedback?: boolean;
|
|
432
|
+
/** Tag sent as the `integration` field of API requests that accept one, such as `_zed`. */
|
|
433
|
+
integration?: string;
|
|
423
434
|
/** Extension hooks. Unstable: may change in any release. */
|
|
424
435
|
unstable_hooks?: FirecrawlMcpServerHooks;
|
|
425
436
|
}
|
package/dist/server.js
CHANGED
|
@@ -2636,25 +2636,64 @@ function initializeClientSlug(name2) {
|
|
|
2636
2636
|
const slug = slugOf(name2);
|
|
2637
2637
|
return isOwnSlug(slug) || GENERIC_CLIENT_NAMES.has(slug) ? "" : slug;
|
|
2638
2638
|
}
|
|
2639
|
-
function
|
|
2640
|
-
let
|
|
2639
|
+
function publicHost(url) {
|
|
2640
|
+
let parsed;
|
|
2641
2641
|
try {
|
|
2642
|
-
|
|
2642
|
+
parsed = new URL(url ?? "");
|
|
2643
2643
|
} catch {
|
|
2644
2644
|
return "";
|
|
2645
2645
|
}
|
|
2646
|
+
const host = parsed.hostname.toLowerCase().replace(/\.$/, "");
|
|
2646
2647
|
if (!host.includes(".") || host.endsWith(".localhost") || host.endsWith(".local") || /^[\d.]+$/.test(host) || host.startsWith("[")) {
|
|
2647
2648
|
return "";
|
|
2648
2649
|
}
|
|
2649
|
-
return
|
|
2650
|
+
return host.replace(/^www\./, "");
|
|
2651
|
+
}
|
|
2652
|
+
function referrerSlug(referrer) {
|
|
2653
|
+
return slugOf(publicHost(referrer));
|
|
2654
|
+
}
|
|
2655
|
+
var SHARED_STATIC_HOSTS = [
|
|
2656
|
+
"github.io",
|
|
2657
|
+
"gitlab.io",
|
|
2658
|
+
"pages.dev",
|
|
2659
|
+
"netlify.app",
|
|
2660
|
+
"vercel.app"
|
|
2661
|
+
];
|
|
2662
|
+
function isHttpUrl(value) {
|
|
2663
|
+
try {
|
|
2664
|
+
return /^https?:$/.test(new URL(value ?? "").protocol);
|
|
2665
|
+
} catch {
|
|
2666
|
+
return false;
|
|
2667
|
+
}
|
|
2668
|
+
}
|
|
2669
|
+
function oauthClientSlug(clientId, clientName) {
|
|
2670
|
+
if (!isHttpUrl(clientId)) return initializeClientSlug(clientName);
|
|
2671
|
+
const host = publicHost(clientId);
|
|
2672
|
+
if (!host || host === "firecrawl.dev" || host.endsWith(".firecrawl.dev")) {
|
|
2673
|
+
return "";
|
|
2674
|
+
}
|
|
2675
|
+
if (!SHARED_STATIC_HOSTS.some(
|
|
2676
|
+
(shared) => host === shared || host.endsWith(`.${shared}`)
|
|
2677
|
+
)) {
|
|
2678
|
+
return slugOf(host);
|
|
2679
|
+
}
|
|
2680
|
+
const [first = ""] = new URL(clientId ?? "").pathname.split("/").filter(Boolean);
|
|
2681
|
+
const segment = slugOf(first).slice(0, NAME_MAX / 2);
|
|
2682
|
+
if (!segment) return slugOf(host);
|
|
2683
|
+
const room = NAME_MAX - segment.length - 1;
|
|
2684
|
+
return slugOf(`${slugOf(host).slice(0, room)}-${segment}`);
|
|
2650
2685
|
}
|
|
2651
2686
|
function originForClient(client, serverVersion2, userAgent, signals = {}) {
|
|
2652
2687
|
const declared = slugOf(signals.declaredOrigin);
|
|
2653
2688
|
const slug = slugOf(client?.name);
|
|
2654
2689
|
const product = userAgentProduct(userAgent);
|
|
2655
2690
|
const remembered = initializeClientSlug(signals.initializeClient?.name);
|
|
2691
|
+
const oauthClient = oauthClientSlug(
|
|
2692
|
+
signals.oauthClientId,
|
|
2693
|
+
signals.oauthClientName
|
|
2694
|
+
);
|
|
2656
2695
|
const referrer = referrerSlug(signals.referrer);
|
|
2657
|
-
const base = !isOwnSlug(declared) ? `mcp-${declared}` : !isOwnSlug(slug) ? `mcp-${slug}` : product && !RUNTIME_PRODUCTS.has(product) ? `mcp-ua-${product}` : remembered ? `mcp-${remembered}` : referrer ? `mcp-ref-${referrer}` : product ? `mcp-ua-${product}` : ORIGIN_SERVER;
|
|
2696
|
+
const base = !isOwnSlug(declared) ? `mcp-${declared}` : !isOwnSlug(slug) ? `mcp-${slug}` : product && !RUNTIME_PRODUCTS.has(product) ? `mcp-ua-${product}` : remembered ? `mcp-${remembered}` : oauthClient ? `mcp-oauth-${oauthClient}` : referrer ? `mcp-ref-${referrer}` : product ? `mcp-ua-${product}` : ORIGIN_SERVER;
|
|
2658
2697
|
return serverVersion2 ? `${base}@${serverVersion2}` : base;
|
|
2659
2698
|
}
|
|
2660
2699
|
function originHeaders(origin) {
|
|
@@ -2671,6 +2710,8 @@ function requestOrigin(client, session) {
|
|
|
2671
2710
|
{
|
|
2672
2711
|
declaredOrigin: session?.clientOrigin,
|
|
2673
2712
|
initializeClient: session?.initializeClient,
|
|
2713
|
+
oauthClientId: session?.oauthClientId,
|
|
2714
|
+
oauthClientName: session?.oauthClientName,
|
|
2674
2715
|
referrer: session?.clientReferrer
|
|
2675
2716
|
}
|
|
2676
2717
|
);
|
|
@@ -3558,6 +3599,22 @@ The historical view returns periods sorted by start date with \`startDate\`, \`e
|
|
|
3558
3599
|
});
|
|
3559
3600
|
}
|
|
3560
3601
|
|
|
3602
|
+
// src/integration.ts
|
|
3603
|
+
var TAGGED_PATH = /^\/v2\/(scrape|batch\/scrape|crawl|map|search|extract|agent|parse|feedback|scrape\/[^/]+\/interact|search\/[^/]+\/feedback)$/;
|
|
3604
|
+
function integrationFrom(value) {
|
|
3605
|
+
const tag = (value ?? "").trim();
|
|
3606
|
+
return tag === "" ? void 0 : tag;
|
|
3607
|
+
}
|
|
3608
|
+
function pathOf(url) {
|
|
3609
|
+
const path2 = url.replace(/^[a-z][a-z0-9+.-]*:\/\/[^/]+/i, "").split(/[?#]/)[0];
|
|
3610
|
+
return path2.replace(/\/+$/, "");
|
|
3611
|
+
}
|
|
3612
|
+
function withIntegration(url, body, integration) {
|
|
3613
|
+
if (!integration || body.integration != null) return body;
|
|
3614
|
+
if (!TAGGED_PATH.test(pathOf(url))) return body;
|
|
3615
|
+
return { ...body, integration };
|
|
3616
|
+
}
|
|
3617
|
+
|
|
3561
3618
|
// src/headers.ts
|
|
3562
3619
|
function normalizeHeader(value) {
|
|
3563
3620
|
if (value == null) return void 0;
|
|
@@ -3998,7 +4055,8 @@ function createFirecrawlMcpServer(options = {}) {
|
|
|
3998
4055
|
fileAccess: options.fileAccess ?? "local",
|
|
3999
4056
|
requireCredential: options.requireCredential ?? false,
|
|
4000
4057
|
searchFeedback: options.searchFeedback ?? true,
|
|
4001
|
-
endpointFeedback: options.endpointFeedback ?? true
|
|
4058
|
+
endpointFeedback: options.endpointFeedback ?? true,
|
|
4059
|
+
integration: integrationFrom(options.integration)
|
|
4002
4060
|
};
|
|
4003
4061
|
const logging = options.logging ?? config.transport === "httpStream";
|
|
4004
4062
|
const instructionDefaults = {
|
|
@@ -4126,7 +4184,22 @@ function createFirecrawlMcpServer(options = {}) {
|
|
|
4126
4184
|
if (apiKey) {
|
|
4127
4185
|
clientConfig.apiKey = apiKey;
|
|
4128
4186
|
}
|
|
4129
|
-
|
|
4187
|
+
const client = new FirecrawlApp(clientConfig);
|
|
4188
|
+
if (config.integration) {
|
|
4189
|
+
client.http?.instance?.interceptors?.request?.use((request) => {
|
|
4190
|
+
const isPost = String(request.method ?? "").toLowerCase() === "post";
|
|
4191
|
+
const isPlainObject = request.data != null && typeof request.data === "object" && !Array.isArray(request.data) && !(request.data instanceof FormData);
|
|
4192
|
+
if (isPost && isPlainObject) {
|
|
4193
|
+
request.data = withIntegration(
|
|
4194
|
+
request.url ?? "",
|
|
4195
|
+
request.data,
|
|
4196
|
+
config.integration
|
|
4197
|
+
);
|
|
4198
|
+
}
|
|
4199
|
+
return request;
|
|
4200
|
+
});
|
|
4201
|
+
}
|
|
4202
|
+
return client;
|
|
4130
4203
|
}
|
|
4131
4204
|
const SAFE_MODE = config.safeMode;
|
|
4132
4205
|
function getClient(session) {
|
|
@@ -4460,7 +4533,7 @@ function createFirecrawlMcpServer(options = {}) {
|
|
|
4460
4533
|
function buildParseOptionsPayload(options2, origin) {
|
|
4461
4534
|
const transformed = transformScrapeParams(options2);
|
|
4462
4535
|
const cleaned = removeEmptyTopLevel(transformed);
|
|
4463
|
-
return { origin, ...cleaned };
|
|
4536
|
+
return withIntegration("/v2/parse", { origin, ...cleaned }, config.integration);
|
|
4464
4537
|
}
|
|
4465
4538
|
function buildContinuationArguments(uploadRef, options2) {
|
|
4466
4539
|
return {
|
|
@@ -4484,7 +4557,7 @@ function createFirecrawlMcpServer(options = {}) {
|
|
|
4484
4557
|
Authorization: `Bearer ${apiKey}`,
|
|
4485
4558
|
...origin ? originHeaders(origin) : {}
|
|
4486
4559
|
},
|
|
4487
|
-
body: JSON.stringify(body)
|
|
4560
|
+
body: JSON.stringify(withIntegration(pathName, body, config.integration))
|
|
4488
4561
|
});
|
|
4489
4562
|
const responseText = await response.text();
|
|
4490
4563
|
let parsed;
|
|
@@ -4927,7 +5000,7 @@ For a programming question, add \`categories: ["developer"]\`; its hits return i
|
|
|
4927
5000
|
const response = await fetch(`${resolveApiBaseUrl()}${path2}`, {
|
|
4928
5001
|
method: "POST",
|
|
4929
5002
|
headers,
|
|
4930
|
-
body: JSON.stringify(body)
|
|
5003
|
+
body: JSON.stringify(withIntegration(path2, body, config.integration))
|
|
4931
5004
|
});
|
|
4932
5005
|
const json = await response.json().catch(() => ({}));
|
|
4933
5006
|
if (!response.ok) {
|
|
@@ -5097,7 +5170,13 @@ Eligibility is limited to successful searches within the feedback age window. Th
|
|
|
5097
5170
|
const response = await fetch(endpoint, {
|
|
5098
5171
|
method: "POST",
|
|
5099
5172
|
headers,
|
|
5100
|
-
body: JSON.stringify(
|
|
5173
|
+
body: JSON.stringify(
|
|
5174
|
+
withIntegration(
|
|
5175
|
+
`/v2/search/${encodeURIComponent(searchId)}/feedback`,
|
|
5176
|
+
body,
|
|
5177
|
+
config.integration
|
|
5178
|
+
)
|
|
5179
|
+
)
|
|
5101
5180
|
});
|
|
5102
5181
|
const responseText = await response.text();
|
|
5103
5182
|
let parsed;
|
|
@@ -5236,7 +5315,7 @@ Returns submission status, feedback ID, and accounting fields.
|
|
|
5236
5315
|
const response = await fetch(`${apiBase}/v2/feedback`, {
|
|
5237
5316
|
method: "POST",
|
|
5238
5317
|
headers,
|
|
5239
|
-
body: JSON.stringify(body)
|
|
5318
|
+
body: JSON.stringify(withIntegration("/v2/feedback", body, config.integration))
|
|
5240
5319
|
});
|
|
5241
5320
|
const responseText = await response.text();
|
|
5242
5321
|
let parsed;
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "firecrawl-mcp",
|
|
3
|
-
"version": "3.
|
|
4
|
-
"description": "MCP server for
|
|
3
|
+
"version": "3.33.0",
|
|
4
|
+
"description": "Firecrawl MCP server, the context API for AI agents: search, scrape, parse, crawl, map, and interact with the web, run Agent and Monitor jobs, and search scientific papers (PubMed, bioRxiv, medRxiv, arXiv). Works with Firecrawl cloud or a self-hosted instance.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"mcpName": "io.github.firecrawl/firecrawl-mcp-server",
|
|
7
7
|
"bin": {
|