firecrawl 4.38.0 → 4.39.0

package/src/__tests__/e2e/v2/search.test.ts

@@ -5,6 +5,7 @@ import Firecrawl from "../../../index";
 import type { Document, SearchResultWeb, SearchResultNews, SearchResultImages } from "../../../index";
 import { config } from "dotenv";
 import { getIdentity, getApiUrl } from "./utils/idmux";
+import { testTimeoutMs, withRateLimitRetry } from "./utils/rateLimit";
 import { describe, test, expect, beforeAll } from "@jest/globals";
 
 config();
@@ -14,7 +15,7 @@ let client: Firecrawl;
 
 beforeAll(async () => {
   const { apiKey } = await getIdentity({ name: "js-e2e-search" });
-  client = new Firecrawl({ apiKey, apiUrl: API_URL });
+  client = withRateLimitRetry(new Firecrawl({ apiKey, apiUrl: API_URL }));
 });
 
 function collectTexts(entries: any[] | undefined): string[] {
@@ -72,7 +73,7 @@ describe("v2.search e2e", () => {
 
     expect(results.news == null).toBe(true);
     expect(results.images == null).toBe(true);
-  }, 90_000);
+  }, testTimeoutMs(90_000));
 
   test("with sources web+news and limit", async () => {
     if (!client) throw new Error();
@@ -93,7 +94,7 @@ describe("v2.search e2e", () => {
       .map(r => (r.description || "").toString().toLowerCase());
     const allWebText = (webTitles.concat(webDescriptions)).join(" ");
     expect(allWebText.includes("firecrawl")).toBe(true);
-  }, 90_000);
+  }, testTimeoutMs(90_000));
 
   test("result structure", async () => {
     if (!client) throw new Error();
@@ -108,7 +109,7 @@ describe("v2.search e2e", () => {
       expect(typeof result.description === "string" || result.description == null).toBe(true);
       expect(result.url.startsWith("http")).toBe(true);
     }
-  }, 90_000);
+  }, testTimeoutMs(90_000));
 
   test("all parameters (comprehensive)", async () => {
     if (!client) throw new Error();
@@ -196,7 +197,7 @@ describe("v2.search e2e", () => {
         expect(result.url?.startsWith("http")).toBe(true);
       }
     }
-  }, 120_000);
+  }, testTimeoutMs(120_000));
 
   test("formats flexibility: list vs object", async () => {
     if (!client) throw new Error();
@@ -212,7 +213,7 @@ describe("v2.search e2e", () => {
     expect(results2).toBeTruthy();
     expect(results1.web).toBeTruthy();
     expect(results2.web).toBeTruthy();
-  }, 90_000);
+  }, testTimeoutMs(151_000));
 
   test("with json format object", async () => {
     if (!client) throw new Error();
@@ -229,7 +230,7 @@ describe("v2.search e2e", () => {
     });
     expect(results).toBeTruthy();
     expect(Array.isArray(results.web) || results.web == null).toBe(true);
-  }, 90_000);
+  }, testTimeoutMs(90_000));
 
   test("with summary format, documents include summary when present", async () => {
     if (!client) throw new Error();
@@ -242,6 +243,6 @@ describe("v2.search e2e", () => {
       expect(typeof docs[0].summary).toBe("string");
       expect((docs[0].summary || "").length).toBeGreaterThan(5);
     }
-  }, 90_000);
+  }, testTimeoutMs(90_000));
 });
 

package/src/__tests__/e2e/v2/usage.test.ts

@@ -4,6 +4,7 @@
 import Firecrawl from "../../../index";
 import { config } from "dotenv";
 import { getIdentity, getApiUrl } from "./utils/idmux";
+import { testTimeoutMs, withRateLimitRetry } from "./utils/rateLimit";
 import { describe, test, expect, beforeAll } from "@jest/globals";
 
 config();
@@ -13,7 +14,7 @@ let client: Firecrawl;
 
 beforeAll(async () => {
   const { apiKey } = await getIdentity({ name: "js-e2e-usage" });
-  client = new Firecrawl({ apiKey, apiUrl: API_URL });
+  client = withRateLimitRetry(new Firecrawl({ apiKey, apiUrl: API_URL }));
 });
 
 describe("v2.usage e2e", () => {
@@ -21,17 +22,17 @@ describe("v2.usage e2e", () => {
     const resp = await client.getConcurrency();
     expect(typeof resp.concurrency).toBe("number");
     expect(typeof resp.maxConcurrency).toBe("number");
-  }, 60_000);
+  }, testTimeoutMs(60_000));
 
   test("get_credit_usage", async () => {
     const resp = await client.getCreditUsage();
     expect(typeof resp.remainingCredits).toBe("number");
-  }, 60_000);
+  }, testTimeoutMs(60_000));
 
   test("get_token_usage", async () => {
     const resp = await client.getTokenUsage();
     expect(typeof resp.remainingTokens).toBe("number");
-  }, 60_000);
+  }, testTimeoutMs(60_000));
 
   test("get_queue_status", async () => {
     const resp = await client.getQueueStatus();
@@ -39,6 +40,6 @@ describe("v2.usage e2e", () => {
     expect(typeof resp.activeJobsInQueue).toBe("number");
     expect(typeof resp.waitingJobsInQueue).toBe("number");
     expect(typeof resp.maxConcurrency).toBe("number");
-  }, 60_000);
+  }, testTimeoutMs(60_000));
 });
 

package/src/__tests__/e2e/v2/utils/rateLimit.ts

@@ -0,0 +1,257 @@
+/**
+ * Rate-limit retry for the v2 e2e suites.
+ *
+ * The e2e suites run against a live API. Each suite mints a CI identity through
+ * idmux, and that identity gets the base per-minute limits: 2/min for crawl and
+ * extract, 10/min for scrape, search and map, 500/min for job status. Several
+ * suites send more calls than that in one minute, so the API answers 429 and the
+ * suite fails.
+ *
+ * This helper wraps a client so every call waits for the limit window to reset
+ * and then runs again. It is test-only code. The shipped SDK keeps its own retry
+ * behaviour.
+ *
+ * Batch scrape creation counts against the scrape limit, not a limit of its
+ * own. Job status reads use the 500/min limit, so they almost never wait.
+ *
+ * The wrapper only runs a call again when the refused request created nothing.
+ * A single request is safe: the API rejected it, so no job exists. A composite
+ * method is not safe. It starts a job, then polls that job. The poll can hit
+ * the limit after the job exists, and a second run of the method would start a
+ * second job. The wrapper passes these methods straight through, so they get no
+ * retry. See COMPOSITE_METHODS.
+ *
+ * A suite that needs a job therefore does the two steps itself: it calls the
+ * start method, then waitForJob. Both calls go through the wrapper, so both
+ * retry, and the job starts once.
+ */
+
+/** Attempts per call, including the first one. */
+const MAX_ATTEMPTS = 3;
+
+/** Wait used when the error carries no reset time. */
+const FALLBACK_WAIT_MS = 30_000;
+
+/** Upper bound for one wait. The API window is 60 seconds. */
+const MAX_WAIT_MS = 75_000;
+
+/** Added to the reset time so the retry lands after the window, not on it. */
+const WAIT_BUFFER_MS = 1_000;
+
+/**
+ * Worst-case time the retry adds to one call.
+ *
+ * The first attempt does not wait. Every later attempt waits first, and one
+ * wait is never longer than MAX_WAIT_MS. The value comes from the two bounds
+ * above, so it cannot drift from them.
+ */
+export const RETRY_BUDGET_MS = (MAX_ATTEMPTS - 1) * MAX_WAIT_MS;
+
+/**
+ * Time waitForJob allows when the caller passes no timeout.
+ *
+ * Every caller now passes its own timeout, sized to the work it waits for.
+ * This value is only a backstop for a new caller that forgets one. Without a
+ * bound the poll loop runs until the jest timeout, which hides the reason.
+ *
+ * The value stays short because it cannot know the work. It fits the smallest
+ * base any suite passes to testTimeoutMs, which is 60_000 ms:
+ * 45_000 + 2_000 + RETRY_BUDGET_MS = 197_000 ms, inside the 210_000 ms of
+ * testTimeoutMs(60_000). So a forgotten timeout still reports "job did not
+ * finish" in every test the suites have today.
+ *
+ * A caller that waits for real work must pass a timeout. Size it to the work,
+ * not to the budget, then set the budget from the sum of the calls the test
+ * makes. The unit test "every wrapped call in a test fits its budget" holds
+ * that rule: it counts this bound, one poll interval, and the retry budget of
+ * every wrapped call in the test, and requires the sum to fit.
+ */
+export const DEFAULT_JOB_TIMEOUT_MS = 45_000;
+
+/**
+ * Jest timeout for a test that calls a wrapped client.
+ *
+ * Pass the base: the time the test needs when the API answers at once. The
+ * result adds one retry budget on top, so a test whose single call hits the
+ * limit still finishes inside its own timeout instead of failing on it.
+ *
+ * One retry budget covers one call. A test that makes several wrapped calls
+ * can spend the budget once per call, so the base must carry the rest. Do not
+ * pick the base by hand: take the number the unit test "every wrapped call in
+ * a test fits its budget" computes, which sums every call in the test.
+ */
+export function testTimeoutMs(baseMs: number): number {
+  return baseMs + RETRY_BUDGET_MS;
+}
+
+/**
+ * Client methods that start a job and then poll it.
+ *
+ * These never run again after a rate-limit error. A retry would call the whole
+ * method, which starts a second job. The single-request methods that the job
+ * waiters use inside, such as startCrawl and getCrawlStatus, keep their retry
+ * when a test calls them directly.
+ */
+export const COMPOSITE_METHODS: ReadonlySet<string> = new Set([
+  "crawl",
+  "crawlUrl",
+  "batchScrape",
+  "batchScrapeUrls",
+  "extract",
+  "agent",
+]);
+
+function sleep(ms: number): Promise<void> {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+/** States a job stops in. Matches the waiters in src/v2/methods. */
+const TERMINAL_STATES = ["completed", "failed", "cancelled"];
+
+/** The part of a job snapshot this module reads. */
+interface JobSnapshot {
+  status?: string;
+}
+
+/**
+ * Polls a job that a test already started.
+ *
+ * The composite client methods, such as crawl, start a job and then poll it in
+ * one call, so the wrapper cannot run them again. A test uses this instead: it
+ * starts the job with the start method, then polls with the status method. Both
+ * calls go through the wrapper, so both keep their retry, and the job starts
+ * once. Compare waitForCrawlCompletion in src/v2/methods/crawl.ts, which this
+ * mirrors.
+ *
+ * @param getStatus Reads the job state. Call the wrapped client here.
+ * @param opts.pollInterval Seconds between reads. Least value is 1.
+ * @param opts.timeout Seconds to allow. Throws when the job runs longer.
+ *   Defaults to DEFAULT_JOB_TIMEOUT_MS, so the loop always has a bound.
+ */
+export async function waitForJob<T extends JobSnapshot>(
+  getStatus: () => Promise<T>,
+  opts: { pollInterval?: number; timeout?: number } = {},
+): Promise<T> {
+  const pollIntervalMs = Math.max(1000, (opts.pollInterval ?? 2) * 1000);
+  const timeoutMs =
+    opts.timeout != null ? opts.timeout * 1000 : DEFAULT_JOB_TIMEOUT_MS;
+  const startedAt = Date.now();
+
+  let snapshot = await getStatus();
+  while (!TERMINAL_STATES.includes(snapshot.status ?? "")) {
+    if (Date.now() - startedAt > timeoutMs) {
+      throw new Error(
+        `job did not finish in ${timeoutMs / 1000}s. Last status: ${
+          snapshot.status
+        }`,
+      );
+    }
+    await sleep(pollIntervalMs);
+    snapshot = await getStatus();
+  }
+  return snapshot;
+}
+
+function isPromiseLike(value: unknown): value is Promise<unknown> {
+  return (
+    typeof value === "object" &&
+    value !== null &&
+    typeof (value as { then?: unknown }).then === "function"
+  );
+}
+
+/** True when the API refused the call because the per-minute limit is spent. */
+export function isRateLimitError(err: unknown): boolean {
+  if (typeof err !== "object" || err === null) return false;
+  const candidate = err as { status?: unknown; message?: unknown };
+  if (candidate.status === 429) return true;
+  return (
+    typeof candidate.message === "string" &&
+    /rate limit exceeded/i.test(candidate.message)
+  );
+}
+
+/** Milliseconds to wait before the next attempt. */
+export function rateLimitWaitMs(err: unknown): number {
+  let seconds: number | undefined;
+
+  const details = (err as { details?: Record<string, unknown> })?.details;
+  for (const key of ["retryAfter", "retryAfterSeconds"]) {
+    const value = details?.[key];
+    if (typeof value === "number" && Number.isFinite(value) && value > 0) {
+      seconds = value;
+      break;
+    }
+  }
+
+  if (seconds === undefined) {
+    const message = (err as { message?: unknown })?.message;
+    const match =
+      typeof message === "string"
+        ? message.match(/retry after (\d+)\s*s/i)
+        : null;
+    if (match) seconds = Number(match[1]);
+  }
+
+  const waitMs =
+    seconds !== undefined && seconds > 0
+      ? seconds * 1000 + WAIT_BUFFER_MS
+      : FALLBACK_WAIT_MS;
+  return Math.min(waitMs, MAX_WAIT_MS);
+}
+
+async function runWithRetry<T>(
+  first: Promise<T>,
+  again: () => Promise<T>,
+  label: string,
+): Promise<T> {
+  let pending = first;
+
+  for (let attempt = 1; ; attempt++) {
+    try {
+      return await pending;
+    } catch (err) {
+      if (attempt >= MAX_ATTEMPTS || !isRateLimitError(err)) throw err;
+      const waitMs = rateLimitWaitMs(err);
+      console.warn(
+        `[e2e] ${label} hit the API rate limit. Waiting ${Math.round(
+          waitMs / 1000,
+        )}s, then attempt ${attempt + 1} of ${MAX_ATTEMPTS}.`,
+      );
+      await sleep(waitMs);
+      pending = again();
+    }
+  }
+}
+
+/**
+ * Wraps a client so each method call retries after a rate-limit error.
+ *
+ * The wrapper only covers the calls the test makes. Methods that return a value
+ * other than a promise pass through unchanged, and so do the composite methods
+ * listed in COMPOSITE_METHODS.
+ */
+export function withRateLimitRetry<T extends object>(client: T): T {
+  return new Proxy(client, {
+    get(target, prop) {
+      const value = (target as Record<string | symbol, unknown>)[prop];
+      if (typeof value !== "function") return value;
+
+      const name = String(prop);
+
+      return (...args: unknown[]) => {
+        const call = () =>
+          (value as (...a: unknown[]) => unknown).apply(target, args);
+        const result = call();
+        // A composite method already created a job, so never run it again.
+        if (COMPOSITE_METHODS.has(name)) return result;
+        if (!isPromiseLike(result)) return result;
+        return runWithRetry(
+          result,
+          call as () => Promise<unknown>,
+          name,
+        );
+      };
+    },
+  });
+}

package/src/__tests__/e2e/v2/watcher.test.ts

@@ -2,6 +2,7 @@ import Firecrawl from "../../../index";
 import { config } from "dotenv";
 import { describe, test, expect, beforeAll } from "@jest/globals";
 import { getIdentity } from "./utils/idmux";
+import { testTimeoutMs, withRateLimitRetry } from "./utils/rateLimit";
 
 config();
 
@@ -10,7 +11,7 @@ let client: Firecrawl;
 
 beforeAll(async () => {
   const { apiKey } = await getIdentity({ name: "js-e2e-watcher" });
-  client = new Firecrawl({ apiKey, apiUrl: API_URL });
+  client = withRateLimitRetry(new Firecrawl({ apiKey, apiUrl: API_URL }));
 });
 
 describe("v2.watcher e2e", () => {
@@ -52,7 +53,7 @@ describe("v2.watcher e2e", () => {
     expect(snapshots).toBeGreaterThanOrEqual(1);
     expect(documents).toBeGreaterThanOrEqual(0);
     watcher.close();
-  }, 240_000);
+  }, testTimeoutMs(240_000));
 
   test("batch watcher with options (kind, pollInterval, timeout)", async () => {
     // client is initialized in beforeAll
@@ -91,6 +92,6 @@ describe("v2.watcher e2e", () => {
     expect(snapshots).toBeGreaterThanOrEqual(1);
     expect(gotCompleted || final.status !== "completed").toBe(true);
     watcher.close();
-  }, 300_000);
+  }, testTimeoutMs(300_000));
 });
 

package/src/__tests__/unit/v2/agent-effort-trace.unit.test.ts

@@ -1,4 +1,4 @@
-import { describe, expect, test } from "@jest/globals";
+import { describe, expect, jest, test } from "@jest/globals";
 import {
   startAgent,
   getAgentTrace,

package/src/__tests__/unit/v2/agent-thread.unit.test.ts

@@ -0,0 +1,164 @@
+import { describe, expect, jest, test } from "@jest/globals";
+import {
+  getAgentStatus,
+  getAgentThread,
+  startAgent,
+} from "../../../v2/methods/agent";
+
+const okPost = () =>
+  jest.fn().mockResolvedValue({
+    status: 200,
+    data: { success: true, id: "agent-job", threadId: "thread-1", threadTurn: 2 },
+  });
+
+describe("v2.agent threads unit", () => {
+  test("startAgent forwards the thread fields when set", async () => {
+    const post = okPost();
+
+    await startAgent({ post } as any, {
+      prompt: "Which tier has SSO?",
+      threadId: "thread-1",
+      mode: "chat",
+      exchange: { enabled: true, toolkits: ["a", "b"] },
+    });
+
+    expect(post).toHaveBeenCalledWith("/v2/agent", {
+      prompt: "Which tier has SSO?",
+      threadId: "thread-1",
+      mode: "chat",
+      exchange: { enabled: true, toolkits: ["a", "b"] },
+    });
+  });
+
+  test("startAgent omits the thread fields when unset", async () => {
+    const post = okPost();
+
+    await startAgent({ post } as any, { prompt: "List the pricing tiers" });
+
+    expect(post).toHaveBeenCalledWith("/v2/agent", {
+      prompt: "List the pricing tiers",
+    });
+  });
+
+  test("startAgent returns the thread the run belongs to", async () => {
+    const started = await startAgent({ post: okPost() } as any, {
+      prompt: "Which tier has SSO?",
+      threadId: "thread-1",
+    });
+
+    expect(started.threadId).toBe("thread-1");
+    expect(started.threadTurn).toBe(2);
+  });
+
+  test("getAgentStatus parses a chat-mode status payload", async () => {
+    const get = jest.fn().mockResolvedValue({
+      status: 200,
+      data: {
+        success: true,
+        status: "completed",
+        data: null,
+        model: "spark-2",
+        expiresAt: "2026-09-02T00:00:00.000Z",
+        threadId: "thread-1",
+        threadTurn: 2,
+        mode: "chat",
+        message: "Only the Enterprise tier lists SSO.",
+        suggestions: [{ label: "Seat caps?", prompt: "Does Team cap seats?" }],
+        pendingApproval: {
+          id: "approval-1",
+          reason: "One paid call answers this.",
+          calls: [
+            {
+              id: "call-1",
+              provider: "provider-1",
+              capability: "capability-1",
+              input: { query: "sso" },
+              creditsEstimate: 5,
+            },
+          ],
+          resolution: null,
+        },
+        exchange: { enabled: true, paidCalls: 0, creditsUsed: null },
+      },
+    });
+
+    const status = await getAgentStatus({ get } as any, "agent-job");
+
+    expect(get).toHaveBeenCalledWith("/v2/agent/agent-job");
+    expect(status.data).toBeNull();
+    expect(status.message).toBe("Only the Enterprise tier lists SSO.");
+    expect(status.mode).toBe("chat");
+    expect(status.threadTurn).toBe(2);
+    expect(status.suggestions).toHaveLength(1);
+    expect(status.pendingApproval!.calls[0].creditsEstimate).toBe(5);
+    expect(status.exchange!.paidCalls).toBe(0);
+  });
+
+  test("getAgentStatus parses a status payload without the thread fields", async () => {
+    const get = jest.fn().mockResolvedValue({
+      status: 200,
+      data: {
+        success: true,
+        status: "completed",
+        data: { price: 42 },
+        model: "spark-2",
+        expiresAt: "2026-09-02T00:00:00.000Z",
+        creditsUsed: 12,
+      },
+    });
+
+    const status = await getAgentStatus({ get } as any, "agent-job");
+
+    expect(status.data).toEqual({ price: 42 });
+    expect(status.threadId).toBeUndefined();
+    expect(status.message).toBeUndefined();
+    expect(status.pendingApproval).toBeUndefined();
+  });
+
+  test("getAgentThread hits the thread endpoint", async () => {
+    const get = jest.fn().mockResolvedValue({
+      status: 200,
+      data: {
+        success: true,
+        thread: {
+          id: "thread-1",
+          createdAt: "2026-09-01T00:00:00.000Z",
+          updatedAt: "2026-09-01T00:01:00.000Z",
+          status: "idle",
+          runs: [
+            {
+              id: "agent-job",
+              turn: 1,
+              mode: "chat",
+              prompt: "List the pricing tiers",
+              status: "succeeded",
+              createdAt: "2026-09-01T00:00:00.000Z",
+              finishedAt: "2026-09-01T00:00:30.000Z",
+              creditsUsed: 212,
+              message: null,
+            },
+          ],
+        },
+      },
+    });
+
+    const thread = await getAgentThread({ get } as any, "thread-1");
+
+    expect(get).toHaveBeenCalledWith("/v2/agent/threads/thread-1");
+    expect(thread.thread!.runs).toHaveLength(1);
+    expect(thread.thread!.runs[0].turn).toBe(1);
+  });
+
+  test("getAgentThread appends includeData when requested", async () => {
+    const get = jest.fn().mockResolvedValue({
+      status: 200,
+      data: { success: true, thread: { id: "thread-1", runs: [] } },
+    });
+
+    await getAgentThread({ get } as any, "thread-1", { includeData: true });
+
+    expect(get).toHaveBeenCalledWith(
+      "/v2/agent/threads/thread-1?includeData=true",
+    );
+  });
+});