firecrawl-mcp 3.30.3 → 3.31.0

package/README.md

@@ -73,6 +73,10 @@ Authorization: Bearer <FIRECRAWL_API_KEY>
 
 Never put an API key in the server URL. Never put an API key in an agent chat. Configure it directly in the client or secret manager. See the [hosted MCP setup guide](https://docs.firecrawl.dev/mcp-server) and the [agent onboarding guide](https://www.firecrawl.dev/agent-onboarding/SKILL.md) for client-specific instructions.
 
+#### Naming your integration
+
+An integration that connects over HTTP can name itself with an `X-Firecrawl-Origin` header, for example `X-Firecrawl-Origin: my-agent`. The server reports it to Firecrawl as the request origin `mcp-my-agent@<server version>`, in place of the client name or User-Agent it would otherwise report. Use letters, digits and dashes: each run of other characters becomes one dash, leading and trailing dashes are dropped, and the name is cut at 48 characters. Tool results are unaffected; only the origin sent to the Firecrawl API (and passed to an embedder's `beforeKeylessRequest`) changes.
+
 #### Search-only endpoint
 
 A fixed-scope search surface is also hosted at:
@@ -1246,7 +1250,7 @@ Options mirror the CLI's environment variables: `apiUrl`, `apiKey`, `transport`,
 
 FastMCP is bundled inside the package, so use the types exported from `firecrawl-mcp/server` (`Session`, `ToolDefinition`, `ToolRegistrar`, `FirecrawlMcpServerHooks` and others) rather than importing `fastmcp` yourself.
 
-The entry point also exports the helpers the built-in tools use, such as `recoveryPayload`, `normalizeHeader`, `requestOrigin`, `searchToolBaseFields`, `searchOutputSchema` and `structuredCompact`, so extra tools and hooks can match the built-in behaviour. They are unstable as well.
+The entry point also exports the helpers the built-in tools use, such as `recoveryPayload`, `normalizeHeader`, `requestOrigin`, `isRuntimeUserAgent`, `initializeClientSlug`, `searchToolBaseFields`, `searchOutputSchema` and `structuredCompact`, so extra tools and hooks can match the built-in behaviour. They are unstable as well.
 
 ## Logging System
 

package/dist/index.js

@@ -2583,19 +2583,79 @@ import { z as z6 } from "zod";
 // src/origin.ts
 import { createRequire } from "module";
 var ORIGIN_SERVER = "mcp-fastmcp";
+var DECLARED_ORIGIN_HEADER = "x-firecrawl-origin";
 var NAME_MAX = 48;
+var RUNTIME_PRODUCTS = /* @__PURE__ */ new Set([
+  "apache-httpasyncclient",
+  "apache-httpclient",
+  "aiohttp",
+  "axios",
+  "bun",
+  "curl",
+  "dart",
+  "deno",
+  "faraday",
+  "go-http-client",
+  "got",
+  "guzzlehttp",
+  "java-http-client",
+  "ktor-client",
+  "libcurl-agent",
+  "mozilla",
+  "node",
+  "node-fetch",
+  "okhttp",
+  "python",
+  "python-httpx",
+  "python-httpx2",
+  "python-requests",
+  "python-urllib",
+  "reqwest",
+  "ruby",
+  "undici",
+  "wget"
+]);
+var GENERIC_CLIENT_NAMES = /* @__PURE__ */ new Set([
+  "client",
+  "example-client",
+  "mcp",
+  "mcp-client",
+  "test",
+  "test-client"
+]);
 function slugOf(name2) {
   return (name2 ?? "").toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, NAME_MAX).replace(/-+$/, "");
 }
+function isOwnSlug(slug) {
+  return slug === "" || slug === "fastmcp" || slug === "firecrawl-fastmcp";
+}
 function userAgentProduct(userAgent) {
   const first = (userAgent ?? "").trim().split(/\s+/)[0] ?? "";
   return slugOf(first.split("/")[0]);
 }
-function originForClient(client, serverVersion2, userAgent) {
+function initializeClientSlug(name2) {
+  const slug = slugOf(name2);
+  return isOwnSlug(slug) || GENERIC_CLIENT_NAMES.has(slug) ? "" : slug;
+}
+function referrerSlug(referrer) {
+  let host;
+  try {
+    host = new URL(referrer ?? "").hostname.toLowerCase().replace(/\.$/, "");
+  } catch {
+    return "";
+  }
+  if (!host.includes(".") || host.endsWith(".localhost") || host.endsWith(".local") || /^[\d.]+$/.test(host) || host.startsWith("[")) {
+    return "";
+  }
+  return slugOf(host.replace(/^www\./, ""));
+}
+function originForClient(client, serverVersion2, userAgent, signals = {}) {
+  const declared = slugOf(signals.declaredOrigin);
   const slug = slugOf(client?.name);
-  const own = slug === "" || slug === "fastmcp" || slug === "firecrawl-fastmcp";
-  const product = own ? userAgentProduct(userAgent) : "";
-  const base = !own ? `mcp-${slug}` : product ? `mcp-ua-${product}` : ORIGIN_SERVER;
+  const product = userAgentProduct(userAgent);
+  const remembered = initializeClientSlug(signals.initializeClient?.name);
+  const referrer = referrerSlug(signals.referrer);
+  const base = !isOwnSlug(declared) ? `mcp-${declared}` : !isOwnSlug(slug) ? `mcp-${slug}` : product && !RUNTIME_PRODUCTS.has(product) ? `mcp-ua-${product}` : remembered ? `mcp-${remembered}` : referrer ? `mcp-ref-${referrer}` : product ? `mcp-ua-${product}` : ORIGIN_SERVER;
   return serverVersion2 ? `${base}@${serverVersion2}` : base;
 }
 function originHeaders(origin) {
@@ -2608,7 +2668,12 @@ function requestOrigin(client, session) {
   return originForClient(
     client?.version ?? null,
     serverVersion,
-    session?.clientUserAgent
+    session?.clientUserAgent,
+    {
+      declaredOrigin: session?.clientOrigin,
+      initializeClient: session?.initializeClient,
+      referrer: session?.clientReferrer
+    }
   );
 }
 
@@ -3961,10 +4026,18 @@ function createFirecrawlMcpServer(options2 = {}) {
     const cached = request?.[authResultByRequest];
     if (cached) return cached;
     const authResult = authenticateSession(request).then((session) => {
-      const userAgent = request?.headers?.["user-agent"];
+      const headers = request?.headers;
+      const userAgent = headers?.["user-agent"];
       if (typeof userAgent === "string" && userAgent !== "") {
         session.clientUserAgent = userAgent;
       }
+      const declaredOrigin = normalizeHeader(headers?.[DECLARED_ORIGIN_HEADER]);
+      if (declaredOrigin) session.clientOrigin = declaredOrigin;
+      const referrer = [
+        normalizeHeader(headers?.["origin"]),
+        normalizeHeader(headers?.["referer"])
+      ].find((value) => value && value !== "null");
+      if (referrer) session.clientReferrer = referrer;
       return session;
     });
     if (request) request[authResultByRequest] = authResult;

package/dist/origin.js

@@ -1,19 +1,83 @@
 // src/origin.ts
 import { createRequire } from "module";
 var ORIGIN_SERVER = "mcp-fastmcp";
+var DECLARED_ORIGIN_HEADER = "x-firecrawl-origin";
 var NAME_MAX = 48;
+var RUNTIME_PRODUCTS = /* @__PURE__ */ new Set([
+  "apache-httpasyncclient",
+  "apache-httpclient",
+  "aiohttp",
+  "axios",
+  "bun",
+  "curl",
+  "dart",
+  "deno",
+  "faraday",
+  "go-http-client",
+  "got",
+  "guzzlehttp",
+  "java-http-client",
+  "ktor-client",
+  "libcurl-agent",
+  "mozilla",
+  "node",
+  "node-fetch",
+  "okhttp",
+  "python",
+  "python-httpx",
+  "python-httpx2",
+  "python-requests",
+  "python-urllib",
+  "reqwest",
+  "ruby",
+  "undici",
+  "wget"
+]);
+var GENERIC_CLIENT_NAMES = /* @__PURE__ */ new Set([
+  "client",
+  "example-client",
+  "mcp",
+  "mcp-client",
+  "test",
+  "test-client"
+]);
 function slugOf(name) {
   return (name ?? "").toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, NAME_MAX).replace(/-+$/, "");
 }
+function isOwnSlug(slug) {
+  return slug === "" || slug === "fastmcp" || slug === "firecrawl-fastmcp";
+}
 function userAgentProduct(userAgent) {
   const first = (userAgent ?? "").trim().split(/\s+/)[0] ?? "";
   return slugOf(first.split("/")[0]);
 }
-function originForClient(client, serverVersion2, userAgent) {
+function isRuntimeUserAgent(userAgent) {
+  const product = userAgentProduct(userAgent);
+  return product === "" || RUNTIME_PRODUCTS.has(product);
+}
+function initializeClientSlug(name) {
+  const slug = slugOf(name);
+  return isOwnSlug(slug) || GENERIC_CLIENT_NAMES.has(slug) ? "" : slug;
+}
+function referrerSlug(referrer) {
+  let host;
+  try {
+    host = new URL(referrer ?? "").hostname.toLowerCase().replace(/\.$/, "");
+  } catch {
+    return "";
+  }
+  if (!host.includes(".") || host.endsWith(".localhost") || host.endsWith(".local") || /^[\d.]+$/.test(host) || host.startsWith("[")) {
+    return "";
+  }
+  return slugOf(host.replace(/^www\./, ""));
+}
+function originForClient(client, serverVersion2, userAgent, signals = {}) {
+  const declared = slugOf(signals.declaredOrigin);
   const slug = slugOf(client?.name);
-  const own = slug === "" || slug === "fastmcp" || slug === "firecrawl-fastmcp";
-  const product = own ? userAgentProduct(userAgent) : "";
-  const base = !own ? `mcp-${slug}` : product ? `mcp-ua-${product}` : ORIGIN_SERVER;
+  const product = userAgentProduct(userAgent);
+  const remembered = initializeClientSlug(signals.initializeClient?.name);
+  const referrer = referrerSlug(signals.referrer);
+  const base = !isOwnSlug(declared) ? `mcp-${declared}` : !isOwnSlug(slug) ? `mcp-${slug}` : product && !RUNTIME_PRODUCTS.has(product) ? `mcp-ua-${product}` : remembered ? `mcp-${remembered}` : referrer ? `mcp-ref-${referrer}` : product ? `mcp-ua-${product}` : ORIGIN_SERVER;
   return serverVersion2 ? `${base}@${serverVersion2}` : base;
 }
 function originHeaders(origin) {
@@ -26,11 +90,19 @@ function requestOrigin(client, session) {
   return originForClient(
     client?.version ?? null,
     serverVersion,
-    session?.clientUserAgent
+    session?.clientUserAgent,
+    {
+      declaredOrigin: session?.clientOrigin,
+      initializeClient: session?.initializeClient,
+      referrer: session?.clientReferrer
+    }
   );
 }
 export {
+  DECLARED_ORIGIN_HEADER,
   ORIGIN_SERVER,
+  initializeClientSlug,
+  isRuntimeUserAgent,
   originForClient,
   originHeaders,
   requestOrigin,

package/dist/server.d.ts

@@ -6,10 +6,20 @@ import { z } from 'zod';
 declare function normalizeHeader(value: string | string[] | undefined): string | undefined;
 declare function withoutTrailingSlash(value: string): string;
 
+/** The header a connection sets to name the integration it comes from. */
+declare const DECLARED_ORIGIN_HEADER = "x-firecrawl-origin";
 type ClientImplementation = {
     name?: string;
     version?: string;
 };
+/** Whether a User-Agent is absent or names an HTTP runtime rather than an MCP client. */
+declare function isRuntimeUserAgent(userAgent: string | null | undefined): boolean;
+/**
+ * The token an initialize client name stamps when it reaches a call only
+ * through `ClientSignals.initializeClient`; empty for the server's own names
+ * and the defaults SDKs send when the integration sets none.
+ */
+declare function initializeClientSlug(name: string | null | undefined): string;
 /** The header form of an origin, for the requests that reach the API without a body. */
 declare function originHeaders(origin: string): Record<string, string>;
 /**
@@ -19,14 +29,19 @@ declare function originHeaders(origin: string): Record<string, string>;
 type McpClient = {
     version?: ClientImplementation | undefined;
 } | undefined;
+/** The client signals the authenticated session carries (see SessionData). */
+type OriginSession = {
+    clientUserAgent?: string;
+    clientOrigin?: string;
+    clientReferrer?: string;
+    initializeClient?: ClientImplementation;
+};
 /**
  * The origin of every request a tool call makes: the call's client, the
- * User-Agent the session was authenticated with (stateless HTTP calls carry
- * no client name), and this server's version.
+ * client signals of the request the session was authenticated with
+ * (stateless HTTP calls carry no client name), and this server's version.
  */
-declare function requestOrigin(client: McpClient, session?: {
-    clientUserAgent?: string;
-} | undefined): string;
+declare function requestOrigin(client: McpClient, session?: OriginSession | undefined): string;
 
 /** An API-issued keyless signup link, or undefined for anything else. */
 declare function keylessSignupUrlFrom(value: unknown): string | undefined;
@@ -209,10 +224,23 @@ interface SessionData {
      */
     firecrawlApiKey?: string;
     /**
-     * The User-Agent of the HTTP request that opened the session, the only
-     * client signal a stateless HTTP tool call carries (see src/origin.ts).
+     * The User-Agent of the HTTP request that opened the session, one of the
+     * client signals a stateless HTTP tool call carries (see src/origin.ts).
      */
     clientUserAgent?: string;
+    /** The `X-Firecrawl-Origin` header: the integration the connection names. */
+    clientOrigin?: string;
+    /** The `Origin` or `Referer` header of that request. */
+    clientReferrer?: string;
+    /**
+     * The client an earlier initialize named, for stateless HTTP calls that do
+     * not carry it. Set by an embedder that can recover it; read only when the
+     * User-Agent names a runtime rather than a client.
+     */
+    initializeClient?: {
+        name?: string;
+        version?: string;
+    };
     authType?: 'api-key' | 'oauth' | 'env' | 'keyless' | 'none';
     /** Admitted, but the supplied credential cannot be used. */
     credentialError?: 'CREDENTIAL_INVALID';
@@ -428,4 +456,4 @@ declare const structuredCompactResult: (data: unknown) => ToolResult;
 
 declare function createFirecrawlMcpServer(options?: FirecrawlMcpServerOptions): FirecrawlMcpServer;
 
-export { ALEXANDRIA_CATALOGUE_VERTICALS, ALEXANDRIA_SEARCH_INSTRUCTIONS, type AuthenticationRequest, type ExtraToolsContext, type FirecrawlMcpServer, type FirecrawlMcpServerHooks, type FirecrawlMcpServerOptions, type FirecrawlMcpServerStartArgs, type HttpApp, type InstructionDefaults, keylessToolNames as KEYLESS_TOOL_NAMES, type OutboundRequest, type ProtectedResourceMetadata, type ResourceContent, type ResourceDefinition, type ResourceRegistrar, SEARCH_DOMAINS_CONFLICT_MESSAGE, type Session, type SessionData, type ToolCallEvent, type ToolContext, type ToolDefinition, type ToolIcon, type ToolLogger, type ToolRegistrar, type ToolResult, UserError, assertExchangeCredential, createFirecrawlMcpServer, defaultDomainTools, enterpriseAllowsAlexandria, hasAlexandria, invalidApiKeyRecoveryPayload, invalidOAuthRecoveryPayload, keylessQuotaReason, keylessSignupUrlFrom, normalizeHeader, normalizeSearchSources, originHeaders, postSearchWithFallback, recoveryPayload, relayExchangeError, relayTermsRequired, removeEmptyTopLevel, requestOrigin, searchDomainsAreExclusive, searchOutputSchema, searchQueryIsValid, searchToolBaseFields, structuredCompactResult as structuredCompact, withAlexandriaSource, withoutTrailingSlash };
+export { ALEXANDRIA_CATALOGUE_VERTICALS, ALEXANDRIA_SEARCH_INSTRUCTIONS, type AuthenticationRequest, DECLARED_ORIGIN_HEADER, type ExtraToolsContext, type FirecrawlMcpServer, type FirecrawlMcpServerHooks, type FirecrawlMcpServerOptions, type FirecrawlMcpServerStartArgs, type HttpApp, type InstructionDefaults, keylessToolNames as KEYLESS_TOOL_NAMES, type OutboundRequest, type ProtectedResourceMetadata, type ResourceContent, type ResourceDefinition, type ResourceRegistrar, SEARCH_DOMAINS_CONFLICT_MESSAGE, type Session, type SessionData, type ToolCallEvent, type ToolContext, type ToolDefinition, type ToolIcon, type ToolLogger, type ToolRegistrar, type ToolResult, UserError, assertExchangeCredential, createFirecrawlMcpServer, defaultDomainTools, enterpriseAllowsAlexandria, hasAlexandria, initializeClientSlug, invalidApiKeyRecoveryPayload, invalidOAuthRecoveryPayload, isRuntimeUserAgent, keylessQuotaReason, keylessSignupUrlFrom, normalizeHeader, normalizeSearchSources, originHeaders, postSearchWithFallback, recoveryPayload, relayExchangeError, relayTermsRequired, removeEmptyTopLevel, requestOrigin, searchDomainsAreExclusive, searchOutputSchema, searchQueryIsValid, searchToolBaseFields, structuredCompactResult as structuredCompact, withAlexandriaSource, withoutTrailingSlash };

package/dist/server.js

@@ -2578,19 +2578,83 @@ import { z as z6 } from "zod";
 // src/origin.ts
 import { createRequire } from "module";
 var ORIGIN_SERVER = "mcp-fastmcp";
+var DECLARED_ORIGIN_HEADER = "x-firecrawl-origin";
 var NAME_MAX = 48;
+var RUNTIME_PRODUCTS = /* @__PURE__ */ new Set([
+  "apache-httpasyncclient",
+  "apache-httpclient",
+  "aiohttp",
+  "axios",
+  "bun",
+  "curl",
+  "dart",
+  "deno",
+  "faraday",
+  "go-http-client",
+  "got",
+  "guzzlehttp",
+  "java-http-client",
+  "ktor-client",
+  "libcurl-agent",
+  "mozilla",
+  "node",
+  "node-fetch",
+  "okhttp",
+  "python",
+  "python-httpx",
+  "python-httpx2",
+  "python-requests",
+  "python-urllib",
+  "reqwest",
+  "ruby",
+  "undici",
+  "wget"
+]);
+var GENERIC_CLIENT_NAMES = /* @__PURE__ */ new Set([
+  "client",
+  "example-client",
+  "mcp",
+  "mcp-client",
+  "test",
+  "test-client"
+]);
 function slugOf(name2) {
   return (name2 ?? "").toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, NAME_MAX).replace(/-+$/, "");
 }
+function isOwnSlug(slug) {
+  return slug === "" || slug === "fastmcp" || slug === "firecrawl-fastmcp";
+}
 function userAgentProduct(userAgent) {
   const first = (userAgent ?? "").trim().split(/\s+/)[0] ?? "";
   return slugOf(first.split("/")[0]);
 }
-function originForClient(client, serverVersion2, userAgent) {
+function isRuntimeUserAgent(userAgent) {
+  const product = userAgentProduct(userAgent);
+  return product === "" || RUNTIME_PRODUCTS.has(product);
+}
+function initializeClientSlug(name2) {
+  const slug = slugOf(name2);
+  return isOwnSlug(slug) || GENERIC_CLIENT_NAMES.has(slug) ? "" : slug;
+}
+function referrerSlug(referrer) {
+  let host;
+  try {
+    host = new URL(referrer ?? "").hostname.toLowerCase().replace(/\.$/, "");
+  } catch {
+    return "";
+  }
+  if (!host.includes(".") || host.endsWith(".localhost") || host.endsWith(".local") || /^[\d.]+$/.test(host) || host.startsWith("[")) {
+    return "";
+  }
+  return slugOf(host.replace(/^www\./, ""));
+}
+function originForClient(client, serverVersion2, userAgent, signals = {}) {
+  const declared = slugOf(signals.declaredOrigin);
   const slug = slugOf(client?.name);
-  const own = slug === "" || slug === "fastmcp" || slug === "firecrawl-fastmcp";
-  const product = own ? userAgentProduct(userAgent) : "";
-  const base = !own ? `mcp-${slug}` : product ? `mcp-ua-${product}` : ORIGIN_SERVER;
+  const product = userAgentProduct(userAgent);
+  const remembered = initializeClientSlug(signals.initializeClient?.name);
+  const referrer = referrerSlug(signals.referrer);
+  const base = !isOwnSlug(declared) ? `mcp-${declared}` : !isOwnSlug(slug) ? `mcp-${slug}` : product && !RUNTIME_PRODUCTS.has(product) ? `mcp-ua-${product}` : remembered ? `mcp-${remembered}` : referrer ? `mcp-ref-${referrer}` : product ? `mcp-ua-${product}` : ORIGIN_SERVER;
   return serverVersion2 ? `${base}@${serverVersion2}` : base;
 }
 function originHeaders(origin) {
@@ -2603,7 +2667,12 @@ function requestOrigin(client, session) {
   return originForClient(
     client?.version ?? null,
     serverVersion,
-    session?.clientUserAgent
+    session?.clientUserAgent,
+    {
+      declaredOrigin: session?.clientOrigin,
+      initializeClient: session?.initializeClient,
+      referrer: session?.clientReferrer
+    }
   );
 }
 
@@ -3956,10 +4025,18 @@ function createFirecrawlMcpServer(options = {}) {
     const cached = request?.[authResultByRequest];
     if (cached) return cached;
     const authResult = authenticateSession(request).then((session) => {
-      const userAgent = request?.headers?.["user-agent"];
+      const headers = request?.headers;
+      const userAgent = headers?.["user-agent"];
       if (typeof userAgent === "string" && userAgent !== "") {
         session.clientUserAgent = userAgent;
       }
+      const declaredOrigin = normalizeHeader(headers?.[DECLARED_ORIGIN_HEADER]);
+      if (declaredOrigin) session.clientOrigin = declaredOrigin;
+      const referrer = [
+        normalizeHeader(headers?.["origin"]),
+        normalizeHeader(headers?.["referer"])
+      ].find((value) => value && value !== "null");
+      if (referrer) session.clientReferrer = referrer;
       return session;
     });
     if (request) request[authResultByRequest] = authResult;
@@ -5734,6 +5811,7 @@ Set \`redactPII\` to request redaction of personally identifiable information in
 export {
   ALEXANDRIA_CATALOGUE_VERTICALS,
   ALEXANDRIA_SEARCH_INSTRUCTIONS,
+  DECLARED_ORIGIN_HEADER,
   keylessToolNames as KEYLESS_TOOL_NAMES,
   SEARCH_DOMAINS_CONFLICT_MESSAGE,
   UserError2 as UserError,
@@ -5742,8 +5820,10 @@ export {
   defaultDomainTools,
   enterpriseAllowsAlexandria,
   hasAlexandria,
+  initializeClientSlug,
   invalidApiKeyRecoveryPayload,
   invalidOAuthRecoveryPayload,
+  isRuntimeUserAgent,
   keylessQuotaReason,
   keylessSignupUrlFrom,
   normalizeHeader,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "firecrawl-mcp",
-  "version": "3.30.3",
+  "version": "3.31.0",
   "description": "MCP server for Firecrawl — search, scrape, and interact with the web, and search scientific papers. Supports both cloud and self-hosted instances. Features include web search, scraping, page interaction, batch processing, LLM-powered content analysis, and research paper search over biomedical and arXiv literature (PubMed, bioRxiv, medRxiv, arXiv) with citation-graph expansion and full-text reading.",
   "type": "module",
   "mcpName": "io.github.firecrawl/firecrawl-mcp-server",