firecrawl 4.35.0 → 4.37.0

package/src/v2/methods/scrape.ts

@@ -12,38 +12,142 @@ import {
   normalizeAxiosError,
 } from "../utils/errorHandler";
 
+/** Auto-resume bounds: a resume only ever follows the server's explicit
+ * "processing continues" signal, and stops after this many resumes or
+ * this much total sleeping — whichever comes first. */
+const RESUME_MAX_ATTEMPTS = 5;
+const RESUME_MAX_TOTAL_WAIT_MS = 20 * 60 * 1000;
+const RESUME_MIN_DELAY_MS = 5 * 1000;
+const RESUME_MAX_DELAY_MS = 10 * 60 * 1000;
+
+/**
+ * When a request failed because the document is still processing
+ * server-side (large PDFs outlive their request window by design), the
+ * error carries `details.state === "processing_continues"` plus a retry
+ * delay. Returns that delay in ms, or undefined for every other error.
+ */
+export function processingContinuesDelayMs(err: unknown): number | undefined {
+  const response = (err as { isAxiosError?: boolean; response?: any })
+    ?.isAxiosError
+    ? (err as { response?: any }).response
+    : undefined;
+  if (!response || response.status !== 408) return undefined;
+  const body = response.data as
+    | { code?: string; details?: { state?: string; retryAfterSeconds?: number } }
+    | undefined;
+  if (
+    body?.code !== "SCRAPE_TIMEOUT" ||
+    body?.details?.state !== "processing_continues"
+  ) {
+    return undefined;
+  }
+  const headerSeconds = Number(response.headers?.["retry-after"]);
+  const seconds =
+    typeof body.details.retryAfterSeconds === "number" &&
+    Number.isFinite(body.details.retryAfterSeconds)
+      ? body.details.retryAfterSeconds
+      : Number.isFinite(headerSeconds)
+        ? headerSeconds
+        : 60;
+  return Math.min(RESUME_MAX_DELAY_MS, Math.max(RESUME_MIN_DELAY_MS, seconds * 1000));
+}
+
+/**
+ * Options for a direct scrape call. `autoResume` lives here — NOT on the
+ * shared ScrapeOptions — because that type is reused verbatim inside
+ * batch/crawl/search/extract payloads, where an SDK-only key would leak
+ * onto the wire and fail strict endpoints.
+ */
+export type ScrapeCallOptions = ScrapeOptions & {
+  /**
+   * SDK-only (never sent to the API). Large documents (big PDFs) that
+   * outlive the request window keep processing server-side; the API's
+   * timeout error then carries `details.state === "processing_continues"`
+   * and a Retry-After. When this is not `false`, the SDK sleeps that
+   * long and re-issues the same request — the retry attaches to the
+   * in-flight job and returns the finished result, so a large document
+   * behaves like one slow successful call. Bounded (at most 5 resumes /
+   * 20 minutes total wait); set `false` to surface the timeout error
+   * immediately instead.
+   */
+  autoResume?: boolean;
+};
+
 export async function scrape(
   http: HttpClient,
   url: string,
-  options?: ScrapeOptions,
+  options?: ScrapeCallOptions,
+  deps: { sleepImpl?: (ms: number) => Promise<void> } = {},
 ): Promise<Document> {
+  const sleep =
+    deps.sleepImpl ??
+    ((ms: number) => new Promise<void>(resolve => setTimeout(resolve, ms)));
   if (!url || !url.trim()) {
     throw new Error("URL cannot be empty");
   }
   if (options) ensureValidScrapeOptions(options);
 
+  // autoResume is SDK behavior, never part of the wire payload.
+  const { autoResume, ...requestOptions } = options ?? {};
   const payload: Record<string, unknown> = { url: url.trim() };
-  if (options) Object.assign(payload, options);
+  Object.assign(payload, requestOptions);
 
-  try {
-    const res = await http.post<{
-      success: boolean;
-      data?: Document;
-      error?: string;
-    }>(
-      "/v2/scrape",
-      payload,
-      typeof options?.timeout === "number"
+  // Per-request timeout. With auto-resume disabled this is exactly the
+  // pre-existing behavior (explicit timeout +5s, else the client's own
+  // default). With auto-resume on, the client-side timeout must outlast
+  // the SERVER's wall — options.timeout, or the API's 5-minute default —
+  // otherwise axios can abort in a photo-finish with the server's
+  // processing_continues response and the resume signal is lost exactly
+  // when it matters. The caller's configured client timeout is
+  // respected as a floor, never overridden downward.
+  const requestTimeout: { timeoutMs?: number } =
+    autoResume === false
+      ? typeof options?.timeout === "number"
         ? { timeoutMs: options.timeout + 5000 }
-        : {},
-    );
-    if (res.status !== 200 || !res.data?.success) {
-      throwForBadResponse(res, "scrape");
+        : {}
+      : {
+          timeoutMs:
+            Math.max(
+              typeof options?.timeout === "number" ? options.timeout : 300_000,
+              typeof http.getTimeoutMs === "function" ? http.getTimeoutMs() : 0,
+            ) + 30_000,
+        };
+
+  let resumes = 0;
+  let resumeWaitedMs = 0;
+  while (true) {
+    try {
+      const res = await http.post<{
+        success: boolean;
+        data?: Document;
+        error?: string;
+      }>(
+        "/v2/scrape",
+        payload,
+        requestTimeout,
+      );
+      if (res.status !== 200 || !res.data?.success) {
+        throwForBadResponse(res, "scrape");
+      }
+      return (res.data.data || {}) as Document;
+    } catch (err: any) {
+      const delayMs = autoResume !== false ? processingContinuesDelayMs(err) : undefined;
+      if (
+        delayMs !== undefined &&
+        resumes < RESUME_MAX_ATTEMPTS &&
+        resumeWaitedMs + delayMs <= RESUME_MAX_TOTAL_WAIT_MS
+      ) {
+        // The document keeps processing server-side; the retry attaches
+        // to the same in-flight job (content adoption) and returns the
+        // finished result instead of restarting the work.
+        resumes += 1;
+        resumeWaitedMs += delayMs;
+        await sleep(delayMs);
+        continue;
+      }
+      if (err?.isAxiosError) return normalizeAxiosError(err, "scrape");
+      throw err;
     }
-    return (res.data.data || {}) as Document;
-  } catch (err: any) {
-    if (err?.isAxiosError) return normalizeAxiosError(err, "scrape");
-    throw err;
   }
 }
 

package/src/v2/types.ts

@@ -31,6 +31,7 @@ export interface JsonFormat extends Format {
   type: "json";
   prompt?: string;
   schema?: Record<string, unknown> | ZodTypeAny;
+  checkPromptInjection?: boolean;
 }
 
 export interface ScreenshotFormat {
@@ -1327,10 +1328,174 @@ export interface AgentStatusResponse {
    * names makes every future model a type error at the call site.
    */
   model?: "spark-1-pro" | "spark-1-mini" | "spark-2" | (string & {});
+  /**
+   * Reasoning effort the job ran with. Only set for runs that specified it;
+   * older rows and default runs have no effort.
+   */
+  effort?: "low" | "medium" | "high";
   expiresAt: string;
   creditsUsed?: number;
 }
 
+/** Reasoning effort for agent jobs. Every level runs spark-2. */
+export type AgentEffort = "low" | "medium" | "high";
+
+export type AgentTraceAgentRole = "orchestrator" | "subagent" | "browser" | "system";
+
+export interface AgentTraceAgentIdentity {
+  id: string;
+  role: AgentTraceAgentRole;
+  name: string;
+  parentId?: string;
+}
+
+export interface AgentTraceError {
+  code: "cancelled" | "credit_limit_reached" | "parent_finished" | "refused" | "internal";
+  source: "agent" | "tool" | "billing" | "system";
+  retryable: boolean;
+  message: string;
+}
+
+export interface AgentTraceArtifactChange {
+  kind: "json" | "markdown" | "html" | "screenshot" | "text";
+  artifactId: string;
+  path?: string;
+  /** Fetch the full content via getAgentSnapshot(jobId, snapshotId). */
+  snapshotId: string;
+  change: "init" | "partial" | "append" | "modify" | "update";
+  changedFields?: string[];
+  itemCount?: number;
+  sourceToolCallId?: string;
+}
+
+interface AgentTraceEventBase {
+  schemaVersion: 1;
+  eventId: string;
+  runId: string;
+  occurredAt: string;
+  producerSequence: number;
+  agent: AgentTraceAgentIdentity;
+}
+
+export interface AgentTraceRunStartedEvent extends AgentTraceEventBase {
+  type: "run.started";
+}
+
+export interface AgentTraceRunCancelRequestedEvent extends AgentTraceEventBase {
+  type: "run.cancel_requested";
+  reason: "user";
+}
+
+export interface AgentTraceRunFinishedEvent extends AgentTraceEventBase {
+  type: "run.finished";
+  outcome: "succeeded" | "failed" | "cancelled" | "refused" | "credit_limit_reached";
+  /** The canonical schema always writes this key (nullable), but older rows may omit it. */
+  error?: AgentTraceError | null;
+}
+
+export interface AgentTraceAgentStartedEvent extends AgentTraceEventBase {
+  type: "agent.started";
+}
+
+export interface AgentTraceAgentFinishedEvent extends AgentTraceEventBase {
+  type: "agent.finished";
+  outcome: "succeeded" | "failed" | "cancelled" | "refused";
+  durationMs: number;
+  /** The canonical schema always writes this key (nullable), but older rows may omit it. */
+  error?: AgentTraceError | null;
+}
+
+export interface AgentTraceBrowserSessionStartedEvent extends AgentTraceEventBase {
+  type: "browser.session.started";
+  sessionId: string;
+}
+
+export interface AgentTraceBrowserSessionFinishedEvent extends AgentTraceEventBase {
+  type: "browser.session.finished";
+  sessionId: string;
+  durationMs: number;
+}
+
+export interface AgentTraceProgressReportedEvent extends AgentTraceEventBase {
+  type: "progress.reported";
+  phase: "planning" | "working" | "finalizing";
+  message: string;
+}
+
+export interface AgentTraceReasoningSummaryEvent extends AgentTraceEventBase {
+  type: "reasoning.summary";
+  text: string;
+}
+
+export interface AgentTraceToolCallStartedEvent extends AgentTraceEventBase {
+  type: "tool_call.started";
+  toolCallId: string;
+  toolName: string;
+  parameters: unknown;
+}
+
+export interface AgentTraceToolCallFinishedEvent extends AgentTraceEventBase {
+  type: "tool_call.finished";
+  toolCallId: string;
+  toolName: string;
+  result: unknown;
+}
+
+export interface AgentTraceArtifactUpdatedEvent extends AgentTraceEventBase {
+  type: "artifact.updated";
+  artifact: AgentTraceArtifactChange;
+}
+
+export interface AgentTraceErrorOccurredEvent extends AgentTraceEventBase {
+  type: "error.occurred";
+  error: AgentTraceError;
+}
+
+/**
+ * One event in an agent job's trace. Mirrors the canonical event schema of
+ * the agent service (schemaVersion 1); `usage.recorded` events are withheld
+ * server-side and `agent.started` carries no model name.
+ */
+export type AgentTraceEvent =
+  | AgentTraceRunStartedEvent
+  | AgentTraceRunCancelRequestedEvent
+  | AgentTraceRunFinishedEvent
+  | AgentTraceAgentStartedEvent
+  | AgentTraceAgentFinishedEvent
+  | AgentTraceBrowserSessionStartedEvent
+  | AgentTraceBrowserSessionFinishedEvent
+  | AgentTraceProgressReportedEvent
+  | AgentTraceReasoningSummaryEvent
+  | AgentTraceToolCallStartedEvent
+  | AgentTraceToolCallFinishedEvent
+  | AgentTraceArtifactUpdatedEvent
+  | AgentTraceErrorOccurredEvent;
+
+export interface AgentTraceActiveBrowserSession {
+  id: string;
+  liveViewUrl: string;
+  viewport: { width: number; height: number };
+}
+
+export interface AgentTraceResponse {
+  success: boolean;
+  id?: string;
+  events?: AgentTraceEvent[];
+  creditsUsed?: number;
+  /** Present only when the trace was requested with liveView: true. */
+  activeBrowserSessions?: AgentTraceActiveBrowserSession[];
+  error?: string;
+}
+
+export interface AgentSnapshotResponse {
+  success: boolean;
+  id?: string;
+  snapshotId?: string;
+  /** Full artifact content as a JSON/string blob. */
+  snapshot?: string;
+  error?: string;
+}
+
 export interface AgentOptions {
   model: "FIRE-1" | "v3-beta";
 }

package/src/v2/utils/httpClient.ts

@@ -51,6 +51,12 @@ export class HttpClient {
     return this.apiKey;
   }
 
+  /** The client-wide default request timeout, for callers that need to
+   * extend a specific request beyond it (e.g. scrape auto-resume). */
+  getTimeoutMs(): number {
+    return (this.instance.defaults.timeout as number | undefined) ?? 300000;
+  }
+
   private async request<T = any>(
     config: AxiosRequestConfig,
   ): Promise<AxiosResponse<T>> {