firecrawl 1.29.3 → 3.0.2

package/src/v2/methods/scrape.ts

@@ -0,0 +1,26 @@
+import { type Document, type ScrapeOptions } from "../types";
+import { HttpClient } from "../utils/httpClient";
+import { ensureValidScrapeOptions } from "../utils/validation";
+import { throwForBadResponse, normalizeAxiosError } from "../utils/errorHandler";
+
+export async function scrape(http: HttpClient, url: string, options?: ScrapeOptions): Promise<Document> {
+  if (!url || !url.trim()) {
+    throw new Error("URL cannot be empty");
+  }
+  if (options) ensureValidScrapeOptions(options);
+
+  const payload: Record<string, unknown> = { url: url.trim() };
+  if (options) Object.assign(payload, options);
+
+  try {
+    const res = await http.post<{ success: boolean; data?: Document; error?: string }>("/v2/scrape", payload);
+    if (res.status !== 200 || !res.data?.success) {
+      throwForBadResponse(res, "scrape");
+    }
+    return (res.data.data || {}) as Document;
+  } catch (err: any) {
+    if (err?.isAxiosError) return normalizeAxiosError(err, "scrape");
+    throw err;
+  }
+}
+

package/src/v2/methods/search.ts

@@ -0,0 +1,69 @@
+import { type Document, type SearchData, type SearchRequest, type SearchResult, type ScrapeOptions } from "../types";
+import { HttpClient } from "../utils/httpClient";
+import { ensureValidScrapeOptions } from "../utils/validation";
+import { throwForBadResponse, normalizeAxiosError } from "../utils/errorHandler";
+
+function prepareSearchPayload(req: SearchRequest): Record<string, unknown> {
+  if (!req.query || !req.query.trim()) throw new Error("Query cannot be empty");
+  if (req.limit != null && req.limit <= 0) throw new Error("limit must be positive");
+  if (req.timeout != null && req.timeout <= 0) throw new Error("timeout must be positive");
+  const payload: Record<string, unknown> = {
+    query: req.query,
+  };
+  if (req.sources) payload.sources = req.sources;
+  if (req.limit != null) payload.limit = req.limit;
+  if (req.tbs != null) payload.tbs = req.tbs;
+  if (req.location != null) payload.location = req.location;
+  if (req.ignoreInvalidURLs != null) payload.ignoreInvalidURLs = req.ignoreInvalidURLs;
+  if (req.timeout != null) payload.timeout = req.timeout;
+  if (req.scrapeOptions) {
+    ensureValidScrapeOptions(req.scrapeOptions as ScrapeOptions);
+    payload.scrapeOptions = req.scrapeOptions;
+  }
+  return payload;
+}
+
+export async function search(http: HttpClient, request: SearchRequest): Promise<SearchData> {
+  const payload = prepareSearchPayload(request);
+  try {
+    const res = await http.post<{ success: boolean; data?: Record<string, unknown>; error?: string }>("/v2/search", payload);
+    if (res.status !== 200 || !res.data?.success) {
+      throwForBadResponse(res, "search");
+    }
+    const data = (res.data.data || {}) as Record<string, any>;
+    const out: SearchData = {};
+    for (const key of Object.keys(data)) {
+      const arr = data[key];
+      if (Array.isArray(arr)) {
+        const results: Array<SearchResult | Document> = [] as any;
+        for (const item of arr) {
+          if (item && typeof item === "object") {
+            // If scraped page fields present, treat as Document; otherwise SearchResult
+            if (
+              "markdown" in item ||
+              "html" in item ||
+              "rawHtml" in item ||
+              "links" in item ||
+              "screenshot" in item ||
+              "changeTracking" in item ||
+              "summary" in item ||
+              "json" in item
+            ) {
+              results.push(item as Document);
+            } else {
+              results.push({ url: item.url, title: item.title, description: item.description } as SearchResult);
+            }
+          } else if (typeof item === "string") {
+            results.push({ url: item } as SearchResult);
+          }
+        }
+        (out as any)[key] = results;
+      }
+    }
+    return out;
+  } catch (err: any) {
+    if (err?.isAxiosError) return normalizeAxiosError(err, "search");
+    throw err;
+  }
+}
+

package/src/v2/methods/usage.ts

@@ -0,0 +1,39 @@
+import type { ConcurrencyCheck, CreditUsage, TokenUsage } from "../types";
+import { HttpClient } from "../utils/httpClient";
+import { normalizeAxiosError, throwForBadResponse } from "../utils/errorHandler";
+
+export async function getConcurrency(http: HttpClient): Promise<ConcurrencyCheck> {
+  try {
+    const res = await http.get<{ success: boolean; data?: { concurrency: number; maxConcurrency: number } }>("/v2/concurrency-check");
+    if (res.status !== 200 || !res.data?.success) throwForBadResponse(res, "get concurrency");
+    const d = res.data.data || (res.data as any);
+    return { concurrency: d.concurrency, maxConcurrency: d.maxConcurrency ?? d.max_concurrency };
+  } catch (err: any) {
+    if (err?.isAxiosError) return normalizeAxiosError(err, "get concurrency");
+    throw err;
+  }
+}
+
+export async function getCreditUsage(http: HttpClient): Promise<CreditUsage> {
+  try {
+    const res = await http.get<{ success: boolean; data?: { remainingCredits?: number; remaining_credits?: number } }>("/v2/team/credit-usage");
+    if (res.status !== 200 || !res.data?.success) throwForBadResponse(res, "get credit usage");
+    const d = res.data.data || (res.data as any);
+    return { remainingCredits: d.remainingCredits ?? d.remaining_credits ?? 0 };
+  } catch (err: any) {
+    if (err?.isAxiosError) return normalizeAxiosError(err, "get credit usage");
+    throw err;
+  }
+}
+
+export async function getTokenUsage(http: HttpClient): Promise<TokenUsage> {
+  try {
+    const res = await http.get<{ success: boolean; data?: TokenUsage }>("/v2/team/token-usage");
+    if (res.status !== 200 || !res.data?.success) throwForBadResponse(res, "get token usage");
+    return (res.data.data || (res.data as any)) as TokenUsage;
+  } catch (err: any) {
+    if (err?.isAxiosError) return normalizeAxiosError(err, "get token usage");
+    throw err;
+  }
+}
+

package/src/v2/types.ts

@@ -0,0 +1,308 @@
+import type { ZodTypeAny } from "zod";
+// Public types for Firecrawl JS/TS SDK v2 (camelCase only)
+
+export type FormatString =
+  | "markdown"
+  | "html"
+  | "rawHtml"
+  | "links"
+  | "screenshot"
+  | "summary"
+  | "changeTracking"
+  | "json";
+
+export interface Viewport {
+  width: number;
+  height: number;
+}
+
+export interface Format {
+  type: FormatString;
+}
+
+export interface JsonFormat extends Format {
+  type: "json";
+  prompt?: string;
+  schema?: Record<string, unknown> | ZodTypeAny;
+}
+
+export interface ScreenshotFormat {
+  type: "screenshot";
+  fullPage?: boolean;
+  quality?: number;
+  viewport?: Viewport | { width: number; height: number };
+}
+
+export interface ChangeTrackingFormat extends Format {
+  type: "changeTracking";
+  modes: ("git-diff" | "json")[];
+  schema?: Record<string, unknown>;
+  prompt?: string;
+  tag?: string;
+}
+
+export type FormatOption =
+  | FormatString
+  | Format
+  | JsonFormat
+  | ChangeTrackingFormat
+  | ScreenshotFormat;
+
+export interface LocationConfig {
+  country?: string;
+  languages?: string[];
+}
+
+export interface WaitAction {
+  type: "wait";
+  milliseconds?: number;
+  selector?: string;
+}
+
+export interface ScreenshotAction {
+  type: "screenshot";
+  fullPage?: boolean;
+  quality?: number;
+  viewport?: Viewport | { width: number; height: number };
+}
+
+export interface ClickAction {
+  type: "click";
+  selector: string;
+}
+
+export interface WriteAction {
+  type: "write";
+  text: string;
+}
+
+export interface PressAction {
+  type: "press";
+  key: string;
+}
+
+export interface ScrollAction {
+  type: "scroll";
+  direction: "up" | "down";
+  selector?: string;
+}
+
+export interface ScrapeAction {
+  type: "scrape";
+}
+
+export interface ExecuteJavascriptAction {
+  type: "executeJavascript";
+  script: string;
+}
+
+export interface PDFAction {
+  type: "pdf";
+  format?: "A0" | "A1" | "A2" | "A3" | "A4" | "A5" | "A6" | "Letter" | "Legal" | "Tabloid" | "Ledger";
+  landscape?: boolean;
+  scale?: number;
+}
+
+export type ActionOption =
+  | WaitAction
+  | ScreenshotAction
+  | ClickAction
+  | WriteAction
+  | PressAction
+  | ScrollAction
+  | ScrapeAction
+  | ExecuteJavascriptAction
+  | PDFAction;
+
+export interface ScrapeOptions {
+  formats?: FormatOption[];
+  headers?: Record<string, string>;
+  includeTags?: string[];
+  excludeTags?: string[];
+  onlyMainContent?: boolean;
+  timeout?: number;
+  waitFor?: number;
+  mobile?: boolean;
+  parsers?: string[];
+  actions?: ActionOption[];
+  location?: LocationConfig;
+  skipTlsVerification?: boolean;
+  removeBase64Images?: boolean;
+  fastMode?: boolean;
+  useMock?: string;
+  blockAds?: boolean;
+  proxy?: "basic" | "stealth" | "auto" | string;
+  maxAge?: number;
+  storeInCache?: boolean;
+}
+
+export interface WebhookConfig {
+  url: string;
+  headers?: Record<string, string>;
+  metadata?: Record<string, string>;
+  events?: Array<"completed" | "failed" | "page" | "started">;
+}
+
+export interface DocumentMetadata {
+  title?: string;
+  description?: string;
+  language?: string;
+  keywords?: string | string[];
+  robots?: string;
+  ogTitle?: string;
+  ogDescription?: string;
+  ogUrl?: string;
+  ogImage?: string;
+  sourceURL?: string;
+  statusCode?: number;
+  error?: string;
+  [key: string]: unknown;
+}
+
+export interface Document {
+  markdown?: string;
+  html?: string;
+  rawHtml?: string;
+  json?: unknown;
+  summary?: string;
+  metadata?: DocumentMetadata;
+  links?: string[];
+  screenshot?: string;
+  actions?: Record<string, unknown>;
+  warning?: string;
+  changeTracking?: Record<string, unknown>;
+}
+
+export interface SearchResult {
+  url: string;
+  title?: string;
+  description?: string;
+}
+
+export interface SearchData {
+  web?: Array<SearchResult | Document>;
+  news?: Array<SearchResult | Document>;
+  images?: Array<SearchResult | Document>;
+}
+
+export interface SearchRequest {
+  query: string;
+  sources?: Array<"web" | "news" | "images" | { type: "web" | "news" | "images" }>;
+  limit?: number;
+  tbs?: string;
+  location?: string;
+  ignoreInvalidURLs?: boolean;
+  timeout?: number; // ms
+  scrapeOptions?: ScrapeOptions;
+}
+
+export interface CrawlResponse {
+  id: string;
+  url: string;
+}
+
+export interface CrawlJob {
+  status: "scraping" | "completed" | "failed" | "cancelled";
+  total: number;
+  completed: number;
+  creditsUsed?: number;
+  expiresAt?: string;
+  next?: string | null;
+  data: Document[];
+}
+
+export interface BatchScrapeResponse {
+  id: string;
+  url: string;
+  invalidURLs?: string[];
+}
+
+export interface BatchScrapeJob {
+  status: "scraping" | "completed" | "failed" | "cancelled";
+  completed: number;
+  total: number;
+  creditsUsed?: number;
+  expiresAt?: string;
+  next?: string | null;
+  data: Document[];
+}
+
+export interface MapData {
+  links: SearchResult[];
+}
+
+export interface MapOptions {
+  search?: string;
+  sitemap?: "only" | "include" | "skip";
+  includeSubdomains?: boolean;
+  limit?: number;
+  timeout?: number;
+}
+
+export interface ExtractResponse {
+  success?: boolean;
+  id?: string;
+  status?: "processing" | "completed" | "failed" | "cancelled";
+  data?: unknown;
+  error?: string;
+  warning?: string;
+  sources?: Record<string, unknown>;
+  expiresAt?: string;
+}
+
+export interface ConcurrencyCheck {
+  concurrency: number;
+  maxConcurrency: number;
+}
+
+export interface CreditUsage {
+  remainingCredits: number;
+}
+
+export interface TokenUsage {
+  remainingTokens: number;
+}
+
+export interface CrawlErrorsResponse {
+  errors: {
+    id: string;
+    timestamp?: string;
+    url: string;
+    code?: string;
+    error: string;
+  }[];
+  robotsBlocked: string[];
+}
+
+export interface ActiveCrawl {
+  id: string;
+  teamId: string;
+  url: string;
+  options?: Record<string, unknown> | null;
+}
+
+export interface ActiveCrawlsResponse {
+  success: boolean;
+  crawls: ActiveCrawl[];
+}
+
+export interface ErrorDetails {
+  code?: string;
+  message: string;
+  details?: Record<string, unknown>;
+  status?: number;
+}
+
+export class SdkError extends Error {
+  status?: number;
+  code?: string;
+  details?: unknown;
+  constructor(message: string, status?: number, code?: string, details?: unknown) {
+    super(message);
+    this.name = "FirecrawlSdkError";
+    this.status = status;
+    this.code = code;
+    this.details = details;
+  }
+}
+

package/src/v2/utils/errorHandler.ts

@@ -0,0 +1,18 @@
+import { type AxiosError, type AxiosResponse } from "axios";
+import { SdkError } from "../types";
+
+export function throwForBadResponse(resp: AxiosResponse, action: string): never {
+  const status = resp.status;
+  const body = resp.data || {};
+  const msg = body?.error || body?.message || `Request failed (${status}) while trying to ${action}`;
+  throw new SdkError(msg, status, undefined, body?.details);
+}
+
+export function normalizeAxiosError(err: AxiosError, action: string): never {
+  const status = err.response?.status;
+  const body: any = err.response?.data;
+  const message = body?.error || err.message || `Request failed${status ? ` (${status})` : ""} while trying to ${action}`;
+  const code = (body?.code as string) || err.code;
+  throw new SdkError(message, status, code, body?.details ?? body);
+}
+

package/src/v2/utils/getVersion.ts

@@ -0,0 +1,14 @@
+export function getVersion(): string {
+  try {
+    if (typeof process !== "undefined" && process.env && process.env.npm_package_version) {
+      return process.env.npm_package_version as string;
+    }
+
+    // eslint-disable-next-line @typescript-eslint/no-var-requires
+    const pkg = require("../../../package.json");
+    return (pkg?.version as string) || "3.x.x";
+  } catch {
+    return "3.x.x";
+  }
+}
+

package/src/v2/utils/httpClient.ts

@@ -0,0 +1,99 @@
+import axios, { type AxiosInstance, type AxiosRequestConfig, type AxiosResponse } from "axios";
+import { getVersion } from "./getVersion";
+
+export interface HttpClientOptions {
+  apiKey: string;
+  apiUrl: string;
+  timeoutMs?: number;
+  maxRetries?: number;
+  backoffFactor?: number; // seconds factor for 0.5, 1, 2...
+}
+
+export class HttpClient {
+  private instance: AxiosInstance;
+  private readonly apiKey: string;
+  private readonly apiUrl: string;
+  private readonly maxRetries: number;
+  private readonly backoffFactor: number;
+
+  constructor(options: HttpClientOptions) {
+    this.apiKey = options.apiKey;
+    this.apiUrl = options.apiUrl.replace(/\/$/, "");
+    this.maxRetries = options.maxRetries ?? 3;
+    this.backoffFactor = options.backoffFactor ?? 0.5;
+    this.instance = axios.create({
+      baseURL: this.apiUrl,
+      timeout: options.timeoutMs ?? 60000,
+      headers: {
+        "Content-Type": "application/json",
+        Authorization: `Bearer ${this.apiKey}`,
+      },
+      transitional: { clarifyTimeoutError: true },
+    });
+  }
+
+  getApiUrl(): string {
+    return this.apiUrl;
+  }
+
+  getApiKey(): string {
+    return this.apiKey;
+  }
+
+  private async request<T = any>(config: AxiosRequestConfig): Promise<AxiosResponse<T>> {
+    const version = getVersion();
+    config.headers = {
+      ...(config.headers || {}),
+    };
+
+    let lastError: any;
+    for (let attempt = 0; attempt < this.maxRetries; attempt++) {
+      try {
+        const cfg: AxiosRequestConfig = { ...config };
+        // For POST/PUT, ensure origin is present in JSON body too
+        if (cfg.method && ["post", "put", "patch"].includes(cfg.method.toLowerCase())) {
+          const data = (cfg.data ?? {}) as Record<string, unknown>;
+          cfg.data = { ...data, origin: `js-sdk@${version}` };
+        }
+        const res = await this.instance.request<T>(cfg);
+        if (res.status === 502 && attempt < this.maxRetries - 1) {
+          await this.sleep(this.backoffFactor * Math.pow(2, attempt));
+          continue;
+        }
+        return res;
+      } catch (err: any) {
+        lastError = err;
+        const status = err?.response?.status;
+        if (status === 502 && attempt < this.maxRetries - 1) {
+          await this.sleep(this.backoffFactor * Math.pow(2, attempt));
+          continue;
+        }
+        throw err;
+      }
+    }
+    throw lastError ?? new Error("Unexpected HTTP client error");
+  }
+
+  private sleep(seconds: number): Promise<void> {
+    return new Promise((r) => setTimeout(r, seconds * 1000));
+  }
+
+  post<T = any>(endpoint: string, body: Record<string, unknown>, headers?: Record<string, string>) {
+    return this.request<T>({ method: "post", url: endpoint, data: body, headers });
+  }
+
+  get<T = any>(endpoint: string, headers?: Record<string, string>) {
+    return this.request<T>({ method: "get", url: endpoint, headers });
+  }
+
+  delete<T = any>(endpoint: string, headers?: Record<string, string>) {
+    return this.request<T>({ method: "delete", url: endpoint, headers });
+  }
+
+  prepareHeaders(idempotencyKey?: string): Record<string, string> {
+    const headers: Record<string, string> = {};
+    if (idempotencyKey) headers["x-idempotency-key"] = idempotencyKey;
+    return headers;
+  }
+}
+

package/src/v2/utils/validation.ts

@@ -0,0 +1,50 @@
+import { type FormatOption, type JsonFormat, type ScrapeOptions, type ScreenshotFormat } from "../types";
+import zodToJsonSchema from "zod-to-json-schema";
+
+export function ensureValidFormats(formats?: FormatOption[]): void {
+  if (!formats) return;
+  for (const fmt of formats) {
+    if (typeof fmt === "string") {
+      if (fmt === "json") {
+        throw new Error("json format must be an object with { type: 'json', prompt, schema }");
+      }
+      continue;
+    }
+    if ((fmt as JsonFormat).type === "json") {
+      const j = fmt as JsonFormat;
+      if (!j.prompt && !j.schema) {
+        throw new Error("json format requires either 'prompt' or 'schema' (or both)");
+      }
+      // Flexibility: allow passing a Zod schema. Convert to JSON schema internally.
+      const maybeSchema: any = j.schema as any;
+      const isZod = !!maybeSchema && (typeof maybeSchema.safeParse === "function" || typeof maybeSchema.parse === "function") && !!maybeSchema._def;
+      if (isZod) {
+        try {
+          (j as any).schema = zodToJsonSchema(maybeSchema);
+        } catch {
+          // If conversion fails, leave as-is; server-side may still handle, or request will fail explicitly
+        }
+      }
+      continue;
+    }
+    if ((fmt as ScreenshotFormat).type === "screenshot") {
+      // no-op; already camelCase; validate numeric fields if present
+      const s = fmt as ScreenshotFormat;
+      if (s.quality != null && (typeof s.quality !== "number" || s.quality < 0)) {
+        throw new Error("screenshot.quality must be a non-negative number");
+      }
+    }
+  }
+}
+
+export function ensureValidScrapeOptions(options?: ScrapeOptions): void {
+  if (!options) return;
+  if (options.timeout != null && options.timeout <= 0) {
+    throw new Error("timeout must be positive");
+  }
+  if (options.waitFor != null && options.waitFor < 0) {
+    throw new Error("waitFor must be non-negative");
+  }
+  ensureValidFormats(options.formats);
+}
+