@mendable/firecrawl-js 4.35.0 → 4.38.0

package/src/__tests__/unit/v2/agent-list.unit.test.ts

@@ -0,0 +1,68 @@
+import { describe, expect, jest, test } from "@jest/globals";
+import { listAgents } from "../../../v2/methods/agent";
+
+describe("v2.agent listAgents unit", () => {
+  const sampleAgent = {
+    id: "018f3c5e-0000-7000-8000-000000000000",
+    createdAt: "2026-08-31T12:00:00.000Z",
+    targetHint: "https://example.com",
+    origin: "api",
+    settings: { hidden: false, starred: false },
+    status: "completed",
+    options: {
+      urls: ["https://example.com"],
+      prompt: "Find the pricing",
+      model: "spark-1-pro",
+    },
+  };
+
+  test("hits GET /v2/agent without query by default", async () => {
+    const get = jest.fn().mockResolvedValue({
+      status: 200,
+      data: { success: true, agents: [sampleAgent] },
+    });
+
+    const res = await listAgents({ get } as any);
+
+    expect(get).toHaveBeenCalledWith("/v2/agent");
+    expect(res.success).toBe(true);
+    expect(res.agents).toHaveLength(1);
+    expect(res.agents![0].id).toBe(sampleAgent.id);
+    expect(res.agents![0].settings.starred).toBe(false);
+    expect(res.next).toBeUndefined();
+  });
+
+  test("passes before as a query param", async () => {
+    const get = jest.fn().mockResolvedValue({
+      status: 200,
+      data: {
+        success: true,
+        agents: [sampleAgent],
+        next: "https://api.firecrawl.dev/v2/agent?before=1756600000000",
+      },
+    });
+
+    const res = await listAgents({ get } as any, { before: 1756600000000 });
+
+    expect(get).toHaveBeenCalledWith("/v2/agent?before=1756600000000");
+    expect(res.next).toBe(
+      "https://api.firecrawl.dev/v2/agent?before=1756600000000",
+    );
+  });
+
+  test("does not auto-paginate when next is present", async () => {
+    const get = jest.fn().mockResolvedValue({
+      status: 200,
+      data: {
+        success: true,
+        agents: [sampleAgent],
+        next: "https://api.firecrawl.dev/v2/agent?before=1756600000000",
+      },
+    });
+
+    const res = await listAgents({ get } as any);
+
+    expect(get).toHaveBeenCalledTimes(1);
+    expect(res.agents).toHaveLength(1);
+  });
+});

package/src/__tests__/unit/v2/scrapeAutoResume.test.ts

@@ -0,0 +1,145 @@
+import { describe, expect, test } from "@jest/globals";
+import {
+  processingContinuesDelayMs,
+  scrape,
+} from "../../../v2/methods/scrape";
+
+const DOC = { markdown: "# big doc" };
+
+function processingContinues408(retryAfterSeconds = 10) {
+  return {
+    isAxiosError: true,
+    response: {
+      status: 408,
+      headers: { "retry-after": String(retryAfterSeconds) },
+      data: {
+        success: false,
+        code: "SCRAPE_TIMEOUT",
+        error: "Request timed out, but this 700-page PDF is still being processed on our side.",
+        details: {
+          state: "processing_continues",
+          documentPages: 700,
+          jobStatus: "running",
+          estimatedRemainingSeconds: retryAfterSeconds,
+          retryAfterSeconds,
+        },
+      },
+    },
+  };
+}
+
+describe("v2.scrape auto-resume", () => {
+  test("resumes after processing_continues and returns the finished document", async () => {
+    const post = jest
+      .fn()
+      .mockRejectedValueOnce(processingContinues408(10))
+      .mockResolvedValueOnce({ status: 200, data: { success: true, data: DOC } });
+    const sleeps: number[] = [];
+    const sleepImpl = async (ms: number) => {
+      sleeps.push(ms);
+    };
+
+    const doc = await scrape({ post } as any, "https://example.com/big.pdf", undefined, {
+      sleepImpl,
+    });
+
+    expect(doc).toEqual(DOC);
+    expect(post).toHaveBeenCalledTimes(2);
+    expect(sleeps).toEqual([10_000]);
+  });
+
+  test("autoResume: false surfaces the timeout immediately and never sleeps", async () => {
+    const post = jest.fn().mockRejectedValueOnce(processingContinues408());
+    const sleepImpl = jest.fn();
+
+    await expect(
+      scrape({ post } as any, "https://example.com/big.pdf", { autoResume: false }, {
+        sleepImpl: sleepImpl as any,
+      }),
+    ).rejects.toBeDefined();
+    expect(post).toHaveBeenCalledTimes(1);
+    expect(sleepImpl).not.toHaveBeenCalled();
+  });
+
+  test("autoResume never enters the wire payload", async () => {
+    const post = jest
+      .fn()
+      .mockResolvedValueOnce({ status: 200, data: { success: true, data: DOC } });
+    await scrape({ post } as any, "https://example.com/x", { autoResume: true });
+    const payload = post.mock.calls[0][1];
+    expect(payload).toEqual({ url: "https://example.com/x" });
+  });
+
+  test("gives up after the attempt bound and surfaces the error", async () => {
+    const post = jest.fn().mockRejectedValue(processingContinues408(10));
+    const sleepImpl = jest.fn().mockResolvedValue(undefined);
+
+    await expect(
+      scrape({ post } as any, "https://example.com/big.pdf", undefined, {
+        sleepImpl: sleepImpl as any,
+      }),
+    ).rejects.toBeDefined();
+    // 1 initial + 5 resumes.
+    expect(post).toHaveBeenCalledTimes(6);
+    expect(sleepImpl).toHaveBeenCalledTimes(5);
+  });
+
+  test("plain timeouts without the processing signal do not resume", async () => {
+    const post = jest.fn().mockRejectedValueOnce({
+      isAxiosError: true,
+      response: {
+        status: 408,
+        headers: {},
+        data: { success: false, code: "SCRAPE_TIMEOUT", error: "Request timed out" },
+      },
+    });
+    const sleepImpl = jest.fn();
+
+    await expect(
+      scrape({ post } as any, "https://example.com/slow", undefined, {
+        sleepImpl: sleepImpl as any,
+      }),
+    ).rejects.toBeDefined();
+    expect(post).toHaveBeenCalledTimes(1);
+    expect(sleepImpl).not.toHaveBeenCalled();
+  });
+
+  test("request timeout: opt-out keeps legacy behavior, resume adds the cushion", async () => {
+    const ok = { status: 200, data: { success: true, data: DOC } };
+    // Opt-out, no explicit timeout → no per-request override at all.
+    const post1 = jest.fn().mockResolvedValueOnce(ok);
+    await scrape({ post: post1 } as any, "https://example.com/x", { autoResume: false });
+    expect(post1.mock.calls[0][2]).toEqual({});
+    // Opt-out with explicit timeout → the pre-existing +5s.
+    const post2 = jest.fn().mockResolvedValueOnce(ok);
+    await scrape({ post: post2 } as any, "https://example.com/x", {
+      autoResume: false,
+      timeout: 10_000,
+    });
+    expect(post2.mock.calls[0][2]).toEqual({ timeoutMs: 15_000 });
+    // Resume enabled → cushion over the server wall, respecting a larger
+    // configured client default as the floor.
+    const post3 = jest.fn().mockResolvedValueOnce(ok);
+    await scrape(
+      { post: post3, getTimeoutMs: () => 600_000 } as any,
+      "https://example.com/x",
+    );
+    expect(post3.mock.calls[0][2]).toEqual({ timeoutMs: 630_000 });
+    // Resume enabled, no client getter → server wall default + cushion.
+    const post4 = jest.fn().mockResolvedValueOnce(ok);
+    await scrape({ post: post4 } as any, "https://example.com/x", { timeout: 900_000 });
+    expect(post4.mock.calls[0][2]).toEqual({ timeoutMs: 930_000 });
+  });
+
+  test("processingContinuesDelayMs clamps and falls back sensibly", () => {
+    expect(processingContinuesDelayMs(processingContinues408(1))).toBe(5_000);
+    expect(processingContinuesDelayMs(processingContinues408(3_600))).toBe(600_000);
+    // No payload field → Retry-After header.
+    const headerOnly = processingContinues408(0);
+    delete (headerOnly.response.data.details as any).retryAfterSeconds;
+    headerOnly.response.headers["retry-after"] = "90";
+    expect(processingContinuesDelayMs(headerOnly)).toBe(90_000);
+    expect(processingContinuesDelayMs(new Error("net"))).toBeUndefined();
+    expect(processingContinuesDelayMs(undefined)).toBeUndefined();
+  });
+});

package/src/v1/index.ts

@@ -181,6 +181,7 @@ export interface ScrapeParams<LLMSchema extends zt.ZodSchema = any, ActionsSchem
     prompt?: string;
     schema?: LLMSchema;
     systemPrompt?: string;
+    checkPromptInjection?: boolean;
   }
   changeTrackingOptions?: {
     prompt?: string;

package/src/v2/client.ts

@@ -1,6 +1,6 @@
 import { HttpClient } from "./utils/httpClient";
 import {
-  scrape,
+  scrape, type ScrapeCallOptions,
   interact as interactMethod,
   stopInteraction as stopInteractionMethod,
 } from "./methods/scrape";
@@ -26,7 +26,7 @@ import {
   batchScrape as batchWaiter,
 } from "./methods/batch";
 import { startExtract, getExtractStatus, extract as extractWaiter } from "./methods/extract";
-import { startAgent, getAgentStatus, cancelAgent, agent as agentWaiter } from "./methods/agent";
+import { startAgent, getAgentStatus, getAgentTrace, getAgentSnapshot, cancelAgent, listAgents, agent as agentWaiter } from "./methods/agent";
 import {
   browser as browserMethod,
   browserExecute,
@@ -68,6 +68,10 @@ import type {
   ExtractResponse,
   AgentResponse,
   AgentStatusResponse,
+  AgentTraceResponse,
+  AgentSnapshotResponse,
+  AgentListOptions,
+  AgentListResponse,
   CrawlOptions,
   BatchScrapeOptions,
   PaginationConfig,
@@ -168,8 +172,8 @@ export class FirecrawlClient {
     url: string,
     options: Opts
   ): Promise<Omit<Document, "json"> & { json?: InferredJsonFromOptions<Opts> }>;
-  async scrape(url: string, options?: ScrapeOptions): Promise<Document>;
-  async scrape(url: string, options?: ScrapeOptions): Promise<Document> {
+  async scrape(url: string, options?: ScrapeCallOptions): Promise<Document>;
+  async scrape(url: string, options?: ScrapeCallOptions): Promise<Document> {
     return scrape(this.http, url, options);
   }
   /**
@@ -519,6 +523,17 @@ export class FirecrawlClient {
   async getAgentStatus(jobId: string): Promise<AgentStatusResponse> {
     return getAgentStatus(this.http, jobId);
   }
+  /**
+   * List agent runs, most recent first.
+   *
+   * Pages are fixed at 20 runs. To fetch the next page, pass the `before`
+   * value from the previous page's `next` URL. This method does not
+   * auto-paginate.
+   * @param options.before Only return runs created before this unix ms timestamp.
+   */
+  async listAgents(options?: AgentListOptions): Promise<AgentListResponse> {
+    return listAgents(this.http, options);
+  }
   /**
    * Convenience waiter: start an agent and poll until it finishes.
    * @param args Agent request plus waiter controls (pollInterval, timeout seconds).
@@ -535,6 +550,22 @@ export class FirecrawlClient {
   async cancelAgent(jobId: string): Promise<boolean> {
     return cancelAgent(this.http, jobId);
   }
+  /**
+   * Get the execution trace of an agent job (spark-2 runs only).
+   * @param jobId Agent job id.
+   * @param options.liveView Also include currently active browser sessions with live view URLs.
+   */
+  async getAgentTrace(jobId: string, options?: { liveView?: boolean }): Promise<AgentTraceResponse> {
+    return getAgentTrace(this.http, jobId, options);
+  }
+  /**
+   * Get the full content of an artifact snapshot referenced by a trace event.
+   * @param jobId Agent job id.
+   * @param snapshotId Snapshot id from an artifact.updated trace event.
+   */
+  async getAgentSnapshot(jobId: string, snapshotId: string): Promise<AgentSnapshotResponse> {
+    return getAgentSnapshot(this.http, jobId, snapshotId);
+  }
 
   // Browser
   /**

package/src/v2/methods/agent.ts

@@ -1,4 +1,4 @@
-import { type AgentResponse, type AgentStatusResponse, type AgentWebhookConfig, type AuditMetadata, type ThreatProtectionOptions } from "../types";
+import { type AgentEffort, type AgentListOptions, type AgentListResponse, type AgentResponse, type AgentSnapshotResponse, type AgentStatusResponse, type AgentTraceResponse, type AgentWebhookConfig, type AuditMetadata, type ThreatProtectionOptions } from "../types";
 import { HttpClient } from "../utils/httpClient";
 import { normalizeAxiosError, throwForBadResponse } from "../utils/errorHandler";
 import { isZodSchema, zodSchemaToJsonSchema } from "../../utils/zodSchemaToJson";
@@ -13,6 +13,7 @@ function prepareAgentPayload(args: {
   maxCredits?: number;
   strictConstrainToURLs?: boolean;
   model?: "spark-1-pro" | "spark-1-mini" | "spark-2";
+  effort?: AgentEffort;
   webhook?: string | AgentWebhookConfig;
   threatProtection?: ThreatProtectionOptions;
   auditMetadata?: AuditMetadata;
@@ -28,6 +29,7 @@ function prepareAgentPayload(args: {
   if (args.maxCredits !== null && args.maxCredits !== undefined) body.maxCredits = args.maxCredits;
   if (args.strictConstrainToURLs !== null && args.strictConstrainToURLs !== undefined) body.strictConstrainToURLs = args.strictConstrainToURLs;
   if (args.model !== null && args.model !== undefined) body.model = args.model;
+  if (args.effort !== null && args.effort !== undefined) body.effort = args.effort;
   if (args.webhook != null) body.webhook = args.webhook;
   if (args.threatProtection != null)
     body.threatProtection = args.threatProtection;
@@ -59,6 +61,18 @@ export async function getAgentStatus(http: HttpClient, jobId: string): Promise<A
   }
 }
 
+export async function listAgents(http: HttpClient, options?: AgentListOptions): Promise<AgentListResponse> {
+  try {
+    const query = options?.before !== undefined ? `?before=${options.before}` : "";
+    const res = await http.get<AgentListResponse>(`/v2/agent${query}`);
+    if (res.status !== 200) throwForBadResponse(res, "list agents");
+    return res.data;
+  } catch (err: any) {
+    if (err?.isAxiosError) return normalizeAxiosError(err, "list agents");
+    throw err;
+  }
+}
+
 export async function waitAgent(
   http: HttpClient,
   jobId: string,
@@ -84,6 +98,37 @@ export async function agent(
   return waitAgent(http, jobId, args.pollInterval ?? 2, args.timeout);
 }
 
+export async function getAgentTrace(
+  http: HttpClient,
+  jobId: string,
+  options?: { liveView?: boolean }
+): Promise<AgentTraceResponse> {
+  try {
+    const query = options?.liveView ? "?liveView=true" : "";
+    const res = await http.get<AgentTraceResponse>(`/v2/agent/${jobId}/trace${query}`);
+    if (res.status !== 200) throwForBadResponse(res, "agent trace");
+    return res.data;
+  } catch (err: any) {
+    if (err?.isAxiosError) return normalizeAxiosError(err, "agent trace");
+    throw err;
+  }
+}
+
+export async function getAgentSnapshot(
+  http: HttpClient,
+  jobId: string,
+  snapshotId: string
+): Promise<AgentSnapshotResponse> {
+  try {
+    const res = await http.get<AgentSnapshotResponse>(`/v2/agent/${jobId}/snapshots/${snapshotId}`);
+    if (res.status !== 200) throwForBadResponse(res, "agent snapshot");
+    return res.data;
+  } catch (err: any) {
+    if (err?.isAxiosError) return normalizeAxiosError(err, "agent snapshot");
+    throw err;
+  }
+}
+
 export async function cancelAgent(http: HttpClient, jobId: string): Promise<boolean> {
   try {
     const res = await http.delete<{ success: boolean }>(`/v2/agent/${jobId}`);

package/src/v2/methods/scrape.ts

@@ -12,38 +12,142 @@ import {
   normalizeAxiosError,
 } from "../utils/errorHandler";
 
+/** Auto-resume bounds: a resume only ever follows the server's explicit
+ * "processing continues" signal, and stops after this many resumes or
+ * this much total sleeping — whichever comes first. */
+const RESUME_MAX_ATTEMPTS = 5;
+const RESUME_MAX_TOTAL_WAIT_MS = 20 * 60 * 1000;
+const RESUME_MIN_DELAY_MS = 5 * 1000;
+const RESUME_MAX_DELAY_MS = 10 * 60 * 1000;
+
+/**
+ * When a request failed because the document is still processing
+ * server-side (large PDFs outlive their request window by design), the
+ * error carries `details.state === "processing_continues"` plus a retry
+ * delay. Returns that delay in ms, or undefined for every other error.
+ */
+export function processingContinuesDelayMs(err: unknown): number | undefined {
+  const response = (err as { isAxiosError?: boolean; response?: any })
+    ?.isAxiosError
+    ? (err as { response?: any }).response
+    : undefined;
+  if (!response || response.status !== 408) return undefined;
+  const body = response.data as
+    | { code?: string; details?: { state?: string; retryAfterSeconds?: number } }
+    | undefined;
+  if (
+    body?.code !== "SCRAPE_TIMEOUT" ||
+    body?.details?.state !== "processing_continues"
+  ) {
+    return undefined;
+  }
+  const headerSeconds = Number(response.headers?.["retry-after"]);
+  const seconds =
+    typeof body.details.retryAfterSeconds === "number" &&
+    Number.isFinite(body.details.retryAfterSeconds)
+      ? body.details.retryAfterSeconds
+      : Number.isFinite(headerSeconds)
+        ? headerSeconds
+        : 60;
+  return Math.min(RESUME_MAX_DELAY_MS, Math.max(RESUME_MIN_DELAY_MS, seconds * 1000));
+}
+
+/**
+ * Options for a direct scrape call. `autoResume` lives here — NOT on the
+ * shared ScrapeOptions — because that type is reused verbatim inside
+ * batch/crawl/search/extract payloads, where an SDK-only key would leak
+ * onto the wire and fail strict endpoints.
+ */
+export type ScrapeCallOptions = ScrapeOptions & {
+  /**
+   * SDK-only (never sent to the API). Large documents (big PDFs) that
+   * outlive the request window keep processing server-side; the API's
+   * timeout error then carries `details.state === "processing_continues"`
+   * and a Retry-After. When this is not `false`, the SDK sleeps that
+   * long and re-issues the same request — the retry attaches to the
+   * in-flight job and returns the finished result, so a large document
+   * behaves like one slow successful call. Bounded (at most 5 resumes /
+   * 20 minutes total wait); set `false` to surface the timeout error
+   * immediately instead.
+   */
+  autoResume?: boolean;
+};
+
 export async function scrape(
   http: HttpClient,
   url: string,
-  options?: ScrapeOptions,
+  options?: ScrapeCallOptions,
+  deps: { sleepImpl?: (ms: number) => Promise<void> } = {},
 ): Promise<Document> {
+  const sleep =
+    deps.sleepImpl ??
+    ((ms: number) => new Promise<void>(resolve => setTimeout(resolve, ms)));
   if (!url || !url.trim()) {
     throw new Error("URL cannot be empty");
   }
   if (options) ensureValidScrapeOptions(options);
 
+  // autoResume is SDK behavior, never part of the wire payload.
+  const { autoResume, ...requestOptions } = options ?? {};
   const payload: Record<string, unknown> = { url: url.trim() };
-  if (options) Object.assign(payload, options);
+  Object.assign(payload, requestOptions);
 
-  try {
-    const res = await http.post<{
-      success: boolean;
-      data?: Document;
-      error?: string;
-    }>(
-      "/v2/scrape",
-      payload,
-      typeof options?.timeout === "number"
+  // Per-request timeout. With auto-resume disabled this is exactly the
+  // pre-existing behavior (explicit timeout +5s, else the client's own
+  // default). With auto-resume on, the client-side timeout must outlast
+  // the SERVER's wall — options.timeout, or the API's 5-minute default —
+  // otherwise axios can abort in a photo-finish with the server's
+  // processing_continues response and the resume signal is lost exactly
+  // when it matters. The caller's configured client timeout is
+  // respected as a floor, never overridden downward.
+  const requestTimeout: { timeoutMs?: number } =
+    autoResume === false
+      ? typeof options?.timeout === "number"
         ? { timeoutMs: options.timeout + 5000 }
-        : {},
-    );
-    if (res.status !== 200 || !res.data?.success) {
-      throwForBadResponse(res, "scrape");
+        : {}
+      : {
+          timeoutMs:
+            Math.max(
+              typeof options?.timeout === "number" ? options.timeout : 300_000,
+              typeof http.getTimeoutMs === "function" ? http.getTimeoutMs() : 0,
+            ) + 30_000,
+        };
+
+  let resumes = 0;
+  let resumeWaitedMs = 0;
+  while (true) {
+    try {
+      const res = await http.post<{
+        success: boolean;
+        data?: Document;
+        error?: string;
+      }>(
+        "/v2/scrape",
+        payload,
+        requestTimeout,
+      );
+      if (res.status !== 200 || !res.data?.success) {
+        throwForBadResponse(res, "scrape");
+      }
+      return (res.data.data || {}) as Document;
+    } catch (err: any) {
+      const delayMs = autoResume !== false ? processingContinuesDelayMs(err) : undefined;
+      if (
+        delayMs !== undefined &&
+        resumes < RESUME_MAX_ATTEMPTS &&
+        resumeWaitedMs + delayMs <= RESUME_MAX_TOTAL_WAIT_MS
+      ) {
+        // The document keeps processing server-side; the retry attaches
+        // to the same in-flight job (content adoption) and returns the
+        // finished result instead of restarting the work.
+        resumes += 1;
+        resumeWaitedMs += delayMs;
+        await sleep(delayMs);
+        continue;
+      }
+      if (err?.isAxiosError) return normalizeAxiosError(err, "scrape");
+      throw err;
     }
-    return (res.data.data || {}) as Document;
-  } catch (err: any) {
-    if (err?.isAxiosError) return normalizeAxiosError(err, "scrape");
-    throw err;
   }
 }