firecrawl-mcp 3.21.2 → 3.21.3

package/dist/index.js

@@ -2880,6 +2880,61 @@ async function keylessPost(path2, body, session) {
   }
   return json;
 }
+async function getCrawlStatusWithOrigin(client, jobId) {
+  const res = await client.http.get(
+    `/v2/crawl/${encodeURIComponent(jobId)}`,
+    ORIGIN_HEADERS2
+  );
+  const body = res?.data ?? {};
+  const initialDocs = Array.isArray(body.data) ? body.data : [];
+  if (!body.next) {
+    return {
+      id: jobId,
+      status: body.status,
+      completed: body.completed ?? 0,
+      total: body.total ?? 0,
+      creditsUsed: body.creditsUsed,
+      expiresAt: body.expiresAt,
+      next: body.next ?? null,
+      data: initialDocs
+    };
+  }
+  const docs = initialDocs.slice();
+  let current = body.next;
+  while (current) {
+    const pageRes = await client.http.get(current, ORIGIN_HEADERS2);
+    const payload = pageRes?.data ?? {};
+    if (!payload.success) break;
+    const pageData = Array.isArray(payload.data) ? payload.data : payload.data?.pages || [];
+    docs.push(...pageData);
+    current = payload.next ?? (Array.isArray(payload.data) ? null : payload.data?.next) ?? null;
+  }
+  return {
+    id: jobId,
+    status: body.status,
+    completed: body.completed ?? 0,
+    total: body.total ?? 0,
+    creditsUsed: body.creditsUsed,
+    expiresAt: body.expiresAt,
+    next: null,
+    data: docs
+  };
+}
+async function waitForCrawlCompletionWithOrigin(client, jobId, pollInterval = 2, timeout) {
+  const startedAt = Date.now();
+  while (true) {
+    const status = await getCrawlStatusWithOrigin(client, jobId);
+    if (["completed", "failed", "cancelled"].includes(String(status.status ?? ""))) {
+      return status;
+    }
+    if (timeout != null && Date.now() - startedAt > timeout * 1e3) {
+      throw new Error(`Crawl job ${jobId} did not complete within ${timeout}s`);
+    }
+    await new Promise(
+      (resolve) => setTimeout(resolve, Math.max(1e3, pollInterval * 1e3))
+    );
+  }
+}
 var feedbackIssueSchema = z4.string().trim().min(1).max(80).regex(
   /^[a-z0-9][a-z0-9_-]*$/,
   "Issue codes must use lowercase letters, numbers, underscores, or hyphens"
@@ -3255,11 +3310,26 @@ server.addTool({
     if (webhook) opts.webhook = webhook;
     delete opts.webhookHeaders;
     const cleaned = removeEmptyTopLevel(opts);
+    const pollInterval = typeof cleaned.pollInterval === "number" ? cleaned.pollInterval : 2;
+    const timeout = typeof cleaned.timeout === "number" ? cleaned.timeout : void 0;
+    delete cleaned.pollInterval;
+    delete cleaned.timeout;
     log.info("Starting crawl", { url: String(url) });
-    const res = await client.crawl(String(url), {
+    const started = await client.http.post("/v2/crawl", {
+      url: String(url),
       ...cleaned,
       origin: ORIGIN
     });
+    const crawlId = started?.data?.id;
+    if (!crawlId) {
+      return asText2(started?.data ?? {});
+    }
+    const res = await waitForCrawlCompletionWithOrigin(
+      client,
+      crawlId,
+      pollInterval,
+      timeout
+    );
     return asText2(res);
   }
 });
@@ -3292,11 +3362,8 @@ Check the status of a crawl job.
   execute: async (args2, { session }) => {
     const client = getClient(session);
     const id = args2.id;
-    const res = await client.http.get(
-      `/v2/crawl/${encodeURIComponent(id)}`,
-      ORIGIN_HEADERS2
-    );
-    return asText2(res?.data ?? {});
+    const res = await getCrawlStatusWithOrigin(client, id);
+    return asText2(res);
   }
 });
 server.addTool({

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "firecrawl-mcp",
-  "version": "3.21.2",
+  "version": "3.21.3",
   "description": "MCP server for Firecrawl — search, scrape, and interact with the web. Supports both cloud and self-hosted instances. Features include web search, scraping, page interaction, batch processing, and LLM-powered content analysis.",
   "type": "module",
   "mcpName": "io.github.firecrawl/firecrawl-mcp-server",