firecrawl-mcp 3.23.3 → 3.23.6

package/README.md

@@ -35,15 +35,29 @@ Connect to the remote hosted server with no setup:
 https://mcp.firecrawl.dev/v2/mcp
 ```
 
-On the keyless free tier, `scrape`, `search`, and `interact` work without an API key (rate-limited). Other tools such as `crawl`, `map`, `agent`, and `extract` still need a key.
+On the keyless free tier, `scrape`, `search`, and `interact` work without an API key (rate-limited). Other tools such as `crawl`, `map`, and `agent` still need a key.
 
-Prefer an API key or OAuth whenever the human can sign up. It unlocks the full tool set and higher limits. With a key, use:
+Prefer OAuth or an API key whenever the human can sign up. It unlocks the full tool set and higher limits.
+
+For an interactive account connection, use:
+
+```
+https://mcp.firecrawl.dev/v2/mcp-oauth
+```
+
+For an API-key connection (for example, an unattended integration), keep the server URL as:
+
+```
+https://mcp.firecrawl.dev/v2/mcp
+```
+
+Then configure the client's secure header or secret setting with:
 
 ```
-https://mcp.firecrawl.dev/{FIRECRAWL_API_KEY}/v2/mcp
+Authorization: Bearer <FIRECRAWL_API_KEY>
 ```
 
-See the [MCP server docs](https://docs.firecrawl.dev/mcp-server) and the [agent onboarding guide](https://www.firecrawl.dev/agent-onboarding/SKILL.md) for setup details.
+Never put an API key in the server URL. See the [hosted MCP setup guide](https://docs.firecrawl.dev/developer-guides/mcp-setup-guides/oauth) and the [agent onboarding guide](https://www.firecrawl.dev/agent-onboarding/SKILL.md) for client-specific instructions.
 
 #### Search-only endpoint
 
@@ -290,7 +304,6 @@ Use this guide to select the right tool for your task:
 | map          | Discovering URLs on a site                     | URL[]                          |
 | crawl        | Multi-page extraction (with limits)            | final crawl status/data after internal polling |
 | parse        | Files and hosted upload refs                   | markdown, JSON, or document output |
-| extract      | Structured extraction from URLs                | JSON structured data           |
 | search       | Web search for info                            | results[]                      |
 | developer    | Programming questions over developer sources   | results[] with passages        |
 | agent        | Complex multi-source research                  | JSON (structured data)         |
@@ -643,79 +656,34 @@ Parse local files or hosted upload references with Firecrawl's `/v2/parse` endpo
 
 **Returns:** Parsed document content or hosted upload instructions with a `nextToolCall`.
 
-### 7. Extract Tool (`firecrawl_extract`)
-
-Extract structured information from web pages using LLM capabilities. Supports both cloud AI and self-hosted LLM extraction.
-
-**Best for:**
-
-- Extracting specific structured data like prices, names, details.
-
-**Not recommended for:**
-
-- When you need the full content of a page (use scrape)
-- When you're not looking for specific structured data
-
-**Arguments:**
-
-- `urls`: Array of URLs to extract information from
-- `prompt`: Custom prompt for the LLM extraction
-- `systemPrompt`: System prompt to guide the LLM
-- `schema`: JSON schema for structured data extraction
-- `allowExternalLinks`: Allow extraction from external links
-- `enableWebSearch`: Enable web search for additional context
-- `includeSubdomains`: Include subdomains in extraction
-
-When using a self-hosted instance, the extraction will use your configured LLM. For cloud API, it uses Firecrawl's managed LLM service.
-**Prompt Example:**
-
-> "Extract the product name, price, and description from these product pages."
+### 7. Structured data with Scrape JSON
 
-**Usage Example:**
+For structured data from a known page, call `firecrawl_scrape` once per URL with `formats: ["json"]`. Put the extraction prompt and JSON schema in `jsonOptions`.
 
 ```json
 {
-  "name": "firecrawl_extract",
+  "name": "firecrawl_scrape",
   "arguments": {
-    "urls": ["https://example.com/page1", "https://example.com/page2"],
-    "prompt": "Extract product information including name, price, and description",
-    "systemPrompt": "You are a helpful assistant that extracts product information",
-    "schema": {
-      "type": "object",
-      "properties": {
-        "name": { "type": "string" },
-        "price": { "type": "number" },
-        "description": { "type": "string" }
-      },
-      "required": ["name", "price"]
-    },
-    "allowExternalLinks": false,
-    "enableWebSearch": false,
-    "includeSubdomains": false
-  }
-}
-```
-
-**Returns:**
-
-- Extracted structured data as defined by your schema
-
-```json
-{
-  "content": [
-    {
-      "type": "text",
-      "text": {
-        "name": "Example Product",
-        "price": 99.99,
-        "description": "This is an example product description"
+    "url": "https://example.com/product",
+    "formats": ["json"],
+    "jsonOptions": {
+      "prompt": "Extract the product name, price, and description.",
+      "schema": {
+        "type": "object",
+        "properties": {
+          "name": { "type": "string" },
+          "price": { "type": "number" },
+          "description": { "type": "string" }
+        },
+        "required": ["name", "price"]
       }
     }
-  ],
-  "isError": false
+  }
 }
 ```
 
+For unknown URLs or multi-source research, use `firecrawl_search` or `firecrawl_agent` before Scrape.
+
 ### 8. Agent Tool (`firecrawl_agent`)
 
 Autonomous web research agent. This is a separate AI agent layer that independently browses the internet, searches for information, navigates through pages, and extracts structured data based on your query.

package/dist/index.js

@@ -1059,15 +1059,7 @@ async function authenticateRequest(request, profile) {
   if (process.env.CLOUD_SERVICE === "true") {
     if (!headerCred && !managedCred) {
       if (resolved?.invalid) {
-        if (!profile.acceptApiKeys) {
-          throw new Error(
-            `OAuth access token required for the Firecrawl MCP resource ${profile.endpoint}`
-          );
-        }
-        if (isLegacyKeyPathRequest(request)) {
-          throw new InvalidFirecrawlCredentialError();
-        }
-        return { authType: "none", credentialError: "CREDENTIAL_INVALID" };
+        throw new InvalidFirecrawlCredentialError();
       }
       if (profile.allowKeyless) {
         return {
@@ -1281,8 +1273,8 @@ var ConsoleLogger = class {
 var openAiAppsChallengeToken = normalizeHeader(
   process.env.OPENAI_APPS_CHALLENGE_TOKEN
 );
-var FULL_PROFILE_INSTRUCTIONS = `Firecrawl provides web search, page retrieval, site URL discovery, multi-page collection, structured extraction, monitoring, and asynchronous research. Match the requested operation to the tool boundary: firecrawl_scrape retrieves one supplied page, firecrawl_map enumerates URLs under a site without retrieving their content, and firecrawl_agent starts multi-source research whose result is read with firecrawl_agent_status. Provide only the required inputs and account for stated network or external side effects.`;
-var KEYLESS_PROFILE_INSTRUCTIONS = `Without authentication, this endpoint exposes Search, Scrape, and Parse with usage limits. An OAuth connection or Authorization bearer API key exposes account tools; unavailable tools return connection guidance. Firecrawl provides web search, page retrieval, site URL discovery, multi-page collection, structured extraction, monitoring, and asynchronous research. Match the requested operation to the tool boundary: firecrawl_scrape retrieves one supplied page, firecrawl_map enumerates URLs under a site without retrieving their content, and firecrawl_agent starts multi-source research whose result is read with firecrawl_agent_status. Provide only the required inputs.`;
+var FULL_PROFILE_INSTRUCTIONS = `Firecrawl provides web search, page retrieval, site URL discovery, multi-page collection, structured page data, monitoring, and asynchronous research. Match the requested operation to the tool boundary: firecrawl_scrape retrieves one supplied page and can return JSON matching a supplied schema, firecrawl_map enumerates URLs under a site without retrieving their content, and firecrawl_agent starts multi-source research whose result is read with firecrawl_agent_status. Provide only the required inputs and account for stated network or external side effects.`;
+var KEYLESS_PROFILE_INSTRUCTIONS = `Without authentication, this endpoint exposes Search, Scrape, and Parse with usage limits. An OAuth connection or Authorization bearer API key exposes account tools; unavailable tools return connection guidance. Firecrawl provides web search, page retrieval, site URL discovery, multi-page collection, structured page data, monitoring, and asynchronous research. Match the requested operation to the tool boundary: firecrawl_scrape retrieves one supplied page and can return JSON matching a supplied schema, firecrawl_map enumerates URLs under a site without retrieving their content, and firecrawl_agent starts multi-source research whose result is read with firecrawl_agent_status. Provide only the required inputs.`;
 var SEARCH_PROFILE_INSTRUCTIONS = `Firecrawl provides web, developer, and research search. Use firecrawl_search to find relevant results across the web and specialized indexes. For a programming question, use firecrawl_search with categories: ["developer"] to search indexed GitHub issues, merged pull requests, READMEs, and documentation. Use the firecrawl_research_* tools to search academic and research literature, expand from anchor papers via the citation graph, read full-text passages from a specific paper, and search public code repositories. All tools are read-only and return ranked results.`;
 var SEARCH_PROFILE_TOOLS = /* @__PURE__ */ new Set([
   "firecrawl_search",
@@ -1388,7 +1380,7 @@ function recoveryPayload(code, requestId = randomUUID(), options = {}) {
     code,
     request_id: requestId,
     auth_mode: code === "CREDENTIAL_INVALID" ? "credential_error" : "keyless",
-    message: code === "CREDENTIAL_INVALID" ? "The supplied Firecrawl credential is invalid or revoked. Replace it or reconnect the account, then retry." : isQuotaExhausted ? `The free daily limit for this network has been reached${retryAfterSeconds ? `; try again in about ${retryAfterSeconds} seconds` : ""}. To continue now, connect a Firecrawl account via OAuth or configure an API key.` : isToolUnavailable ? "The free tier includes Search, Scrape, and Parse. This tool needs a connected account; Search, Scrape, and Parse still work, so no action is required." : isKeylessAccessUnavailable ? "Anonymous keyless access is unavailable for this request. To continue, connect a Firecrawl account via OAuth or configure an API key." : isKeylessEligibilityUnavailable ? "The anonymous keyless eligibility check is temporarily unavailable. Retry shortly." : "This tool requires a Firecrawl account or API key. Connect an account or configure Authorization: Bearer <FIRECRAWL_API_KEY>, then retry.",
+    message: code === "CREDENTIAL_INVALID" ? `The supplied Firecrawl credential is invalid or revoked. Reconnect the account via OAuth (https://mcp.firecrawl.dev/v2/mcp-oauth) or create a new API key at https://www.firecrawl.dev/signin and send it as Authorization: Bearer <FIRECRAWL_API_KEY>, then retry.` : isQuotaExhausted ? `The free daily limit for this network has been reached${retryAfterSeconds ? `; try again in about ${retryAfterSeconds} seconds` : ""}. To continue now, connect a Firecrawl account via OAuth (https://mcp.firecrawl.dev/v2/mcp-oauth) or create a free API key at https://www.firecrawl.dev/signin and send it as Authorization: Bearer <FIRECRAWL_API_KEY>.` : isToolUnavailable ? "The free tier includes Search, Scrape, and Parse. This tool needs a connected account; Search, Scrape, and Parse still work, so no action is required." : isKeylessAccessUnavailable ? `Anonymous keyless access is unavailable for this request. To continue, connect a Firecrawl account via OAuth (https://mcp.firecrawl.dev/v2/mcp-oauth) or create a free API key at https://www.firecrawl.dev/signin and send it as Authorization: Bearer <FIRECRAWL_API_KEY>.` : isKeylessEligibilityUnavailable ? "The anonymous keyless eligibility check is temporarily unavailable. Retry shortly." : `This tool requires a Firecrawl account or API key. Connect an account via OAuth (https://mcp.firecrawl.dev/v2/mcp-oauth) or create a free API key at https://www.firecrawl.dev/signin and send it as Authorization: Bearer <FIRECRAWL_API_KEY>, then retry.`,
     ...isKeylessAccessUnavailable ? {} : { available_tools: [...KEYLESS_TOOL_NAMES] },
     docs_url: "https://docs.firecrawl.dev/mcp-server",
     ...retryAfterSeconds ? { retry_after_seconds: retryAfterSeconds } : {},
@@ -1401,6 +1393,28 @@ function recoveryPayload(code, requestId = randomUUID(), options = {}) {
     ]
   };
 }
+function deprecatedExtractPayload() {
+  return {
+    code: "DEPRECATED_TOOL",
+    message: 'firecrawl_extract is deprecated and unavailable through MCP. For structured data from a known page, call firecrawl_scrape once per URL with formats: ["json"] and jsonOptions containing the prompt and schema. For unknown URLs or multi-source research, use firecrawl_search or firecrawl_agent first.',
+    replacement: {
+      name: "firecrawl_scrape",
+      instructions: 'Call once per known URL. Set formats to ["json"] and pass the extraction prompt and JSON schema in jsonOptions.',
+      example_arguments: {
+        url: "https://example.com/page",
+        formats: ["json"],
+        jsonOptions: {
+          prompt: "Extract the requested fields from this page.",
+          schema: {
+            type: "object",
+            properties: {}
+          }
+        }
+      }
+    },
+    docs_url: "https://docs.firecrawl.dev/developer-guides/usage-guides/choosing-the-data-extractor"
+  };
+}
 function emitActionLog(toolName, status, session, error, requestId = randomUUID(), code) {
   if (process.env.CLOUD_SERVICE !== "true") return;
   const payload = {
@@ -1436,22 +1450,39 @@ function emitActionLog(toolName, status, session, error, requestId = randomUUID(
 function guardHostedTool(tool, { logActions }) {
   const keylessTool = KEYLESS_TOOL_NAMES.has(tool.name);
   const execute = tool.execute;
+  const canList = tool.canList;
+  const beforeValidate = tool.beforeValidate;
   return {
     ...tool,
-    canList: (session) => !session?.credentialError && (!isHostedKeylessSession(session) || keylessTool),
-    beforeValidate: (_args, session) => {
+    canList: (session) => !session?.credentialError && (!isHostedKeylessSession(session) || keylessTool) && (canList?.(session) ?? true),
+    beforeValidate: async (args2, session) => {
       const code = session?.credentialError ? "CREDENTIAL_INVALID" : isHostedKeylessSession(session) && !keylessTool ? "KEYLESS_TOOL_NOT_AVAILABLE" : void 0;
-      if (!code) return void 0;
-      const requestId = randomUUID();
-      const payload = recoveryPayload(code, requestId);
-      if (logActions) {
-        emitActionLog(tool.name, "error", session, new UserError(String(payload.message), payload), requestId, code);
+      if (code) {
+        const requestId = randomUUID();
+        const payload2 = recoveryPayload(code, requestId);
+        if (logActions) {
+          emitActionLog(tool.name, "error", session, new UserError(String(payload2.message), payload2), requestId, code);
+        }
+        return {
+          content: [{ type: "text", text: String(payload2.message) }],
+          isError: true,
+          structuredContent: payload2
+        };
       }
-      return {
-        content: [{ type: "text", text: String(payload.message) }],
-        isError: true,
-        structuredContent: payload
-      };
+      const earlyResult = await beforeValidate?.(args2, session);
+      const payload = earlyResult?.structuredContent;
+      const recoveryCode = payload && typeof payload === "object" && "code" in payload && typeof payload.code === "string" ? payload.code : void 0;
+      if (logActions && earlyResult?.isError && recoveryCode) {
+        emitActionLog(
+          tool.name,
+          "error",
+          session,
+          new UserError(`Tool validation failed: ${recoveryCode}`, payload),
+          randomUUID(),
+          recoveryCode
+        );
+      }
+      return earlyResult;
     },
     execute: async (args2, context) => {
       const requestId = randomUUID();
@@ -1742,7 +1773,7 @@ var parseOptionParamsSchema = z4.object({
   skipTlsVerification: z4.boolean().optional(),
   storeInCache: z4.boolean().optional(),
   zeroDataRetention: z4.boolean().optional(),
-  maxAge: z4.number().optional(),
+  maxAge: z4.number().optional().describe("Ignored: parse never reuses or stores indexed content."),
   proxy: z4.enum(["basic", "auto"]).optional()
 });
 var localParseParamsSchema = parseOptionParamsSchema.extend({
@@ -1980,6 +2011,8 @@ Retrieve and extract content from one supplied URL through Firecrawl. Use this w
 
 This tool operates on a known page. For a set of pages use \`firecrawl_crawl\`, and to discover page URLs use \`firecrawl_map\` or \`firecrawl_search\`. Options include JavaScript render delay, cache age, main-content filtering, PII redaction, and lockdown cache-only retrieval. Browser actions may change the live page when interactive actions are enabled.
 
+Firecrawl may reuse recently indexed content instead of refetching the page, and the reuse window varies by domain. Set \`maxAge: 0\` to force a live fetch, or a smaller \`maxAge\` to bound how stale reused content may be. A successful response does not by itself confirm that the state it describes is still current.
+
 Returns the selected content formats and page metadata.
 `,
   parameters: scrapeParamsSchema,
@@ -2066,7 +2099,7 @@ Search web, news, or image sources and return ranked results. Operators include
 
 For a programming question, add \`categories: ["developer"]\`. It searches an index of GitHub issues, merged pull requests, repository READMEs, and curated documentation sites, and returns the hits in \`data.developer\` beside the web results.
 
-\`scrapeOptions\` can attach extracted page content. Returns source-type result groups and usage metadata. Authenticated responses can include an \`id\` for optional search feedback.
+\`scrapeOptions\` can attach extracted page content; pages fetched this way use a fixed reuse window and ignore \`maxAge\`, so use \`firecrawl_scrape\` when a live fetch is required. Returns source-type result groups and usage metadata. Authenticated responses can include an \`id\` for optional search feedback.
 `,
   parameters: z4.object({
     ...searchToolBaseFields,
@@ -2584,18 +2617,13 @@ Retrieve the current status, progress, and available results for an existing cra
 server.addTool({
   name: "firecrawl_extract",
   annotations: {
-    title: "Extract structured data",
+    title: "Deprecated: use Scrape JSON",
     readOnlyHint: true,
-    // Uses LLM extraction to pull structured data from URLs without modifying those sites.
     openWorldHint: true,
-    // Accepts arbitrary user-supplied URLs on the public web.
     destructiveHint: false
-    // Read-only extraction; no destructive changes to external content.
   },
   description: `
-Extract structured information from one or more URLs with an optional natural-language prompt and JSON schema. It can include subdomains, follow external links, or use web search when those options are enabled.
-
-Use this for a defined structured result rather than full page content. Returns data shaped by the supplied schema or prompt.
+Deprecated compatibility entry point. Use firecrawl_scrape once per known URL with formats: ["json"] and jsonOptions containing the prompt and schema. Use firecrawl_search or firecrawl_agent before Scrape when URLs are not known.
 `,
   parameters: z4.object({
     urls: z4.array(z4.string()),
@@ -2605,23 +2633,18 @@ Use this for a defined structured result rather than full page content. Returns
     enableWebSearch: z4.boolean().optional(),
     includeSubdomains: z4.boolean().optional()
   }),
-  execute: async (args2, { session, log }) => {
-    const client = getClient(session);
-    const a = args2;
-    log.info("Extracting from URLs", {
-      count: Array.isArray(a.urls) ? a.urls.length : 0
-    });
-    const extractBody = removeEmptyTopLevel({
-      urls: a.urls,
-      prompt: a.prompt,
-      schema: a.schema || void 0,
-      allowExternalLinks: a.allowExternalLinks,
-      enableWebSearch: a.enableWebSearch,
-      includeSubdomains: a.includeSubdomains,
-      origin: ORIGIN
-    });
-    const res = await client.extract(extractBody);
-    return asText2(res);
+  canList: () => false,
+  beforeValidate: () => {
+    const payload = deprecatedExtractPayload();
+    return {
+      content: [{ type: "text", text: payload.message }],
+      isError: true,
+      structuredContent: payload
+    };
+  },
+  execute: async () => {
+    const payload = deprecatedExtractPayload();
+    throw new UserError(payload.message, payload);
   }
 });
 server.addTool({

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "firecrawl-mcp",
-  "version": "3.23.3",
+  "version": "3.23.6",
   "description": "MCP server for Firecrawl — search, scrape, and interact with the web. Supports both cloud and self-hosted instances. Features include web search, scraping, page interaction, batch processing, and LLM-powered content analysis.",
   "type": "module",
   "mcpName": "io.github.firecrawl/firecrawl-mcp-server",