firecrawl 1.29.3 → 3.0.2

package/src/v2/client.ts

@@ -0,0 +1,281 @@
+import { HttpClient } from "./utils/httpClient";
+import { scrape } from "./methods/scrape";
+import { search } from "./methods/search";
+import { map as mapMethod } from "./methods/map";
+import {
+  startCrawl,
+  getCrawlStatus,
+  cancelCrawl,
+  crawl as crawlWaiter,
+  getCrawlErrors,
+  getActiveCrawls,
+  crawlParamsPreview,
+} from "./methods/crawl";
+import {
+  startBatchScrape,
+  getBatchScrapeStatus,
+  getBatchScrapeErrors,
+  cancelBatchScrape,
+  batchScrape as batchWaiter,
+} from "./methods/batch";
+import { startExtract, getExtractStatus, extract as extractWaiter } from "./methods/extract";
+import { getConcurrency, getCreditUsage, getTokenUsage } from "./methods/usage";
+import type {
+  Document,
+  ScrapeOptions,
+  SearchData,
+  SearchRequest,
+  MapData,
+  MapOptions,
+  CrawlResponse,
+  CrawlJob,
+  CrawlErrorsResponse,
+  ActiveCrawlsResponse,
+  BatchScrapeResponse,
+  BatchScrapeJob,
+  ExtractResponse,
+} from "./types";
+import { Watcher } from "./watcher";
+import type { WatcherOptions } from "./watcher";
+import type { ZodTypeAny, infer as ZodInfer } from "zod";
+
+// Helper types to infer the `json` field from a Zod schema included in `formats`
+type ExtractJsonSchemaFromFormats<Formats> = Formats extends readonly any[]
+  ? Extract<Formats[number], { type: "json"; schema?: unknown }>["schema"]
+  : never;
+
+type InferredJsonFromOptions<Opts> = Opts extends { formats?: infer Fmts }
+  ? ExtractJsonSchemaFromFormats<Fmts> extends ZodTypeAny
+    ? ZodInfer<ExtractJsonSchemaFromFormats<Fmts>>
+    : unknown
+  : unknown;
+
+/**
+ * Configuration for the v2 client transport.
+ */
+export interface FirecrawlClientOptions {
+  /** API key (falls back to FIRECRAWL_API_KEY). */
+  apiKey?: string | null;
+  /** API base URL (falls back to FIRECRAWL_API_URL or https://api.firecrawl.dev). */
+  apiUrl?: string | null;
+  /** Per-request timeout in milliseconds (optional). */
+  timeoutMs?: number;
+  /** Max automatic retries for transient failures (optional). */
+  maxRetries?: number;
+  /** Exponential backoff factor for retries (optional). */
+  backoffFactor?: number;
+}
+
+/**
+ * Firecrawl v2 client. Provides typed access to all v2 endpoints and utilities.
+ */
+export class FirecrawlClient {
+  private readonly http: HttpClient;
+
+  /**
+   * Create a v2 client.
+   * @param options Transport configuration (API key, base URL, timeouts, retries).
+   */
+  constructor(options: FirecrawlClientOptions = {}) {
+    const apiKey = options.apiKey ?? process.env.FIRECRAWL_API_KEY ?? "";
+    const apiUrl = (options.apiUrl ?? process.env.FIRECRAWL_API_URL ?? "https://api.firecrawl.dev").replace(/\/$/, "");
+    if (!apiKey) {
+      throw new Error("API key is required. Set FIRECRAWL_API_KEY env or pass apiKey.");
+    }
+    this.http = new HttpClient({
+      apiKey,
+      apiUrl,
+      timeoutMs: options.timeoutMs,
+      maxRetries: options.maxRetries,
+      backoffFactor: options.backoffFactor,
+    });
+  }
+
+  // Scrape
+  /**
+   * Scrape a single URL.
+   * @param url Target URL.
+   * @param options Optional scrape options (formats, headers, etc.).
+   * @returns Resolved document with requested formats.
+   */
+  async scrape<Opts extends ScrapeOptions>(
+    url: string,
+    options: Opts
+  ): Promise<Omit<Document, "json"> & { json?: InferredJsonFromOptions<Opts> }>;
+  async scrape(url: string, options?: ScrapeOptions): Promise<Document>;
+  async scrape(url: string, options?: ScrapeOptions): Promise<Document> {
+    return scrape(this.http, url, options);
+  }
+
+  // Search
+  /**
+   * Search the web and optionally scrape each result.
+   * @param query Search query string.
+   * @param req Additional search options (sources, limit, scrapeOptions, etc.).
+   * @returns Structured search results.
+   */
+  async search(query: string, req: Omit<SearchRequest, "query"> = {}): Promise<SearchData> {
+    return search(this.http, { query, ...req });
+  }
+
+  // Map
+  /**
+   * Map a site to discover URLs (sitemap-aware).
+   * @param url Root URL to map.
+   * @param options Mapping options (sitemap mode, includeSubdomains, limit, timeout).
+   * @returns Discovered links.
+   */
+  async map(url: string, options?: MapOptions): Promise<MapData> {
+    return mapMethod(this.http, url, options);
+  }
+
+  // Crawl
+  /**
+   * Start a crawl job (async).
+   * @param url Root URL to crawl.
+   * @param req Crawl configuration (paths, limits, scrapeOptions, webhook, etc.).
+   * @returns Job id and url.
+   */
+  async startCrawl(url: string, req: Omit<Parameters<typeof startCrawl>[1], "url"> = {}): Promise<CrawlResponse> {
+    return startCrawl(this.http, { url, ...(req as any) });
+  }
+  /**
+   * Get the status and partial data of a crawl job.
+   * @param jobId Crawl job id.
+   */
+  async getCrawlStatus(jobId: string): Promise<CrawlJob> {
+    return getCrawlStatus(this.http, jobId);
+  }
+  /**
+   * Cancel a crawl job.
+   * @param jobId Crawl job id.
+   * @returns True if cancelled.
+   */
+  async cancelCrawl(jobId: string): Promise<boolean> {
+    return cancelCrawl(this.http, jobId);
+  }
+  /**
+   * Convenience waiter: start a crawl and poll until it finishes.
+   * @param url Root URL to crawl.
+   * @param req Crawl configuration plus waiter controls (pollInterval, timeout seconds).
+   * @returns Final job snapshot.
+   */
+  async crawl(url: string, req: Omit<Parameters<typeof startCrawl>[1], "url"> & { pollInterval?: number; timeout?: number } = {}): Promise<CrawlJob> {
+    return crawlWaiter(this.http, { url, ...(req as any) }, req.pollInterval, req.timeout);
+  }
+  /**
+   * Retrieve crawl errors and robots.txt blocks.
+   * @param crawlId Crawl job id.
+   */
+  async getCrawlErrors(crawlId: string): Promise<CrawlErrorsResponse> {
+    return getCrawlErrors(this.http, crawlId);
+  }
+  /**
+   * List active crawls for the authenticated team.
+   */
+  async getActiveCrawls(): Promise<ActiveCrawlsResponse> {
+    return getActiveCrawls(this.http);
+  }
+  /**
+   * Preview normalized crawl parameters produced by a natural-language prompt.
+   * @param url Root URL.
+   * @param prompt Natural-language instruction.
+   */
+  async crawlParamsPreview(url: string, prompt: string): Promise<Record<string, unknown>> {
+    return crawlParamsPreview(this.http, url, prompt);
+  }
+
+  // Batch
+  /**
+   * Start a batch scrape job for multiple URLs (async).
+   * @param urls URLs to scrape.
+   * @param opts Batch options (scrape options, webhook, concurrency, idempotency key, etc.).
+   * @returns Job id and url.
+   */
+  async startBatchScrape(urls: string[], opts?: Parameters<typeof startBatchScrape>[2]): Promise<BatchScrapeResponse> {
+    return startBatchScrape(this.http, urls, opts);
+  }
+  /**
+   * Get the status and partial data of a batch scrape job.
+   * @param jobId Batch job id.
+   */
+  async getBatchScrapeStatus(jobId: string): Promise<BatchScrapeJob> {
+    return getBatchScrapeStatus(this.http, jobId);
+  }
+  /**
+   * Retrieve batch scrape errors and robots.txt blocks.
+   * @param jobId Batch job id.
+   */
+  async getBatchScrapeErrors(jobId: string): Promise<CrawlErrorsResponse> {
+    return getBatchScrapeErrors(this.http, jobId);
+  }
+  /**
+   * Cancel a batch scrape job.
+   * @param jobId Batch job id.
+   * @returns True if cancelled.
+   */
+  async cancelBatchScrape(jobId: string): Promise<boolean> {
+    return cancelBatchScrape(this.http, jobId);
+  }
+  /**
+   * Convenience waiter: start a batch scrape and poll until it finishes.
+   * @param urls URLs to scrape.
+   * @param opts Batch options plus waiter controls (pollInterval, timeout seconds).
+   * @returns Final job snapshot.
+   */
+  async batchScrape(urls: string[], opts?: Parameters<typeof startBatchScrape>[2] & { pollInterval?: number; timeout?: number }): Promise<BatchScrapeJob> {
+    return batchWaiter(this.http, urls, opts);
+  }
+
+  // Extract
+  /**
+   * Start an extract job (async).
+   * @param args Extraction request (urls, schema or prompt, flags).
+   * @returns Job id or processing state.
+   */
+  async startExtract(args: Parameters<typeof startExtract>[1]): Promise<ExtractResponse> {
+    return startExtract(this.http, args);
+  }
+  /**
+   * Get extract job status/data.
+   * @param jobId Extract job id.
+   */
+  async getExtractStatus(jobId: string): Promise<ExtractResponse> {
+    return getExtractStatus(this.http, jobId);
+  }
+  /**
+   * Convenience waiter: start an extract and poll until it finishes.
+   * @param args Extraction request plus waiter controls (pollInterval, timeout seconds).
+   * @returns Final extract response.
+   */
+  async extract(args: Parameters<typeof startExtract>[1] & { pollInterval?: number; timeout?: number }): Promise<ExtractResponse> {
+    return extractWaiter(this.http, args);
+  }
+
+  // Usage
+  /** Current concurrency usage. */
+  async getConcurrency() {
+    return getConcurrency(this.http);
+  }
+  /** Current credit usage. */
+  async getCreditUsage() {
+    return getCreditUsage(this.http);
+  }
+  /** Recent token usage. */
+  async getTokenUsage() {
+    return getTokenUsage(this.http);
+  }
+
+  // Watcher
+  /**
+   * Create a watcher for a crawl or batch job. Emits: `document`, `snapshot`, `done`, `error`.
+   * @param jobId Job id.
+   * @param opts Watcher options (kind, pollInterval, timeout seconds).
+   */
+  watcher(jobId: string, opts: WatcherOptions = {}): Watcher {
+    return new Watcher(this.http, jobId, opts);
+  }
+}
+
+export default FirecrawlClient;
+

package/src/v2/methods/batch.ts

@@ -0,0 +1,131 @@
+import {
+  type BatchScrapeJob,
+  type BatchScrapeResponse,
+  type CrawlErrorsResponse,
+  type Document,
+  type ScrapeOptions,
+  type WebhookConfig,
+} from "../types";
+import { HttpClient } from "../utils/httpClient";
+import { ensureValidScrapeOptions } from "../utils/validation";
+import { normalizeAxiosError, throwForBadResponse } from "../utils/errorHandler";
+
+export interface StartBatchOptions {
+  options?: ScrapeOptions;
+  webhook?: string | WebhookConfig;
+  appendToId?: string;
+  ignoreInvalidURLs?: boolean;
+  maxConcurrency?: number;
+  zeroDataRetention?: boolean;
+  integration?: string;
+  idempotencyKey?: string;
+}
+
+export async function startBatchScrape(
+  http: HttpClient,
+  urls: string[],
+  {
+    options,
+    webhook,
+    appendToId,
+    ignoreInvalidURLs,
+    maxConcurrency,
+    zeroDataRetention,
+    integration,
+    idempotencyKey,
+  }: StartBatchOptions = {}
+): Promise<BatchScrapeResponse> {
+  if (!Array.isArray(urls) || urls.length === 0) throw new Error("URLs list cannot be empty");
+  const payload: Record<string, unknown> = { urls };
+  if (options) {
+    ensureValidScrapeOptions(options);
+    Object.assign(payload, options);
+  }
+  if (webhook != null) payload.webhook = webhook;
+  if (appendToId != null) payload.appendToId = appendToId;
+  if (ignoreInvalidURLs != null) payload.ignoreInvalidURLs = ignoreInvalidURLs;
+  if (maxConcurrency != null) payload.maxConcurrency = maxConcurrency;
+  if (zeroDataRetention != null) payload.zeroDataRetention = zeroDataRetention;
+  if (integration != null) payload.integration = integration;
+
+  try {
+    const headers = http.prepareHeaders(idempotencyKey);
+    const res = await http.post<{ success: boolean; id: string; url: string; invalidURLs?: string[]; error?: string }>("/v2/batch/scrape", payload, headers);
+    if (res.status !== 200 || !res.data?.success) throwForBadResponse(res, "start batch scrape");
+    return { id: res.data.id, url: res.data.url, invalidURLs: res.data.invalidURLs || undefined };
+  } catch (err: any) {
+    if (err?.isAxiosError) return normalizeAxiosError(err, "start batch scrape");
+    throw err;
+  }
+}
+
+export async function getBatchScrapeStatus(http: HttpClient, jobId: string): Promise<BatchScrapeJob> {
+  try {
+    const res = await http.get<{ success: boolean; status: BatchScrapeJob["status"]; completed?: number; total?: number; creditsUsed?: number; expiresAt?: string; next?: string | null; data?: Document[] }>(`/v2/batch/scrape/${jobId}`);
+    if (res.status !== 200 || !res.data?.success) throwForBadResponse(res, "get batch scrape status");
+    const body = res.data;
+    return {
+      status: body.status,
+      completed: body.completed ?? 0,
+      total: body.total ?? 0,
+      creditsUsed: body.creditsUsed,
+      expiresAt: body.expiresAt,
+      next: body.next ?? null,
+      data: (body.data || []) as Document[],
+    };
+  } catch (err: any) {
+    if (err?.isAxiosError) return normalizeAxiosError(err, "get batch scrape status");
+    throw err;
+  }
+}
+
+export async function cancelBatchScrape(http: HttpClient, jobId: string): Promise<boolean> {
+  try {
+    const res = await http.delete<{ status: string }>(`/v2/batch/scrape/${jobId}`);
+    if (res.status !== 200) throwForBadResponse(res, "cancel batch scrape");
+    return res.data?.status === "cancelled";
+  } catch (err: any) {
+    if (err?.isAxiosError) return normalizeAxiosError(err, "cancel batch scrape");
+    throw err;
+  }
+}
+
+export async function getBatchScrapeErrors(http: HttpClient, jobId: string): Promise<CrawlErrorsResponse> {
+  try {
+    const res = await http.get<{ success?: boolean; data?: { errors: Array<Record<string, string>>; robotsBlocked: string[] } }>(`/v2/batch/scrape/${jobId}/errors`);
+    if (res.status !== 200) throwForBadResponse(res, "get batch scrape errors");
+    const payload = res.data?.data ?? (res.data as any);
+    return { errors: payload.errors || [], robotsBlocked: payload.robotsBlocked || [] };
+  } catch (err: any) {
+    if (err?.isAxiosError) return normalizeAxiosError(err, "get batch scrape errors");
+    throw err;
+  }
+}
+
+export async function waitForBatchCompletion(http: HttpClient, jobId: string, pollInterval = 2, timeout?: number): Promise<BatchScrapeJob> {
+  const start = Date.now();
+  while (true) {
+    const status = await getBatchScrapeStatus(http, jobId);
+    if (["completed", "failed", "cancelled"].includes(status.status)) return status;
+    if (timeout != null && Date.now() - start > timeout * 1000) {
+      throw new Error(`Batch scrape job ${jobId} did not complete within ${timeout} seconds`);
+    }
+    await new Promise((r) => setTimeout(r, Math.max(1000, pollInterval * 1000)));
+  }
+}
+
+export async function batchScrape(
+  http: HttpClient,
+  urls: string[],
+  opts: StartBatchOptions & { pollInterval?: number; timeout?: number } = {}
+): Promise<BatchScrapeJob> {
+  const start = await startBatchScrape(http, urls, opts);
+  return waitForBatchCompletion(http, start.id, opts.pollInterval ?? 2, opts.timeout);
+}
+
+export function chunkUrls(urls: string[], chunkSize = 100): string[][] {
+  const chunks: string[][] = [];
+  for (let i = 0; i < urls.length; i += chunkSize) chunks.push(urls.slice(i, i + chunkSize));
+  return chunks;
+}
+

package/src/v2/methods/crawl.ts

@@ -0,0 +1,160 @@
+import {
+  type ActiveCrawlsResponse,
+  type CrawlErrorsResponse,
+  type CrawlJob,
+  type CrawlResponse,
+  type Document,
+  type ScrapeOptions,
+  type WebhookConfig,
+} from "../types";
+import { HttpClient } from "../utils/httpClient";
+import { ensureValidScrapeOptions } from "../utils/validation";
+import { normalizeAxiosError, throwForBadResponse } from "../utils/errorHandler";
+
+export interface CrawlRequest {
+  url: string;
+  prompt?: string | null;
+  excludePaths?: string[] | null;
+  includePaths?: string[] | null;
+  maxDiscoveryDepth?: number | null;
+  sitemap?: "skip" | "include";
+  ignoreQueryParameters?: boolean;
+  limit?: number | null;
+  crawlEntireDomain?: boolean;
+  allowExternalLinks?: boolean;
+  allowSubdomains?: boolean;
+  delay?: number | null;
+  maxConcurrency?: number | null;
+  webhook?: string | WebhookConfig | null;
+  scrapeOptions?: ScrapeOptions | null;
+  zeroDataRetention?: boolean;
+}
+
+function prepareCrawlPayload(request: CrawlRequest): Record<string, unknown> {
+  if (!request.url || !request.url.trim()) throw new Error("URL cannot be empty");
+  const data: Record<string, unknown> = { url: request.url.trim() };
+  if (request.prompt) data.prompt = request.prompt;
+  if (request.excludePaths) data.excludePaths = request.excludePaths;
+  if (request.includePaths) data.includePaths = request.includePaths;
+  if (request.maxDiscoveryDepth != null) data.maxDiscoveryDepth = request.maxDiscoveryDepth;
+  if (request.sitemap != null) data.sitemap = request.sitemap;
+  if (request.ignoreQueryParameters != null) data.ignoreQueryParameters = request.ignoreQueryParameters;
+  if (request.limit != null) data.limit = request.limit;
+  if (request.crawlEntireDomain != null) data.crawlEntireDomain = request.crawlEntireDomain;
+  if (request.allowExternalLinks != null) data.allowExternalLinks = request.allowExternalLinks;
+  if (request.allowSubdomains != null) data.allowSubdomains = request.allowSubdomains;
+  if (request.delay != null) data.delay = request.delay;
+  if (request.maxConcurrency != null) data.maxConcurrency = request.maxConcurrency;
+  if (request.webhook != null) data.webhook = request.webhook;
+  if (request.scrapeOptions) {
+    ensureValidScrapeOptions(request.scrapeOptions);
+    data.scrapeOptions = request.scrapeOptions;
+  }
+  if (request.zeroDataRetention != null) data.zeroDataRetention = request.zeroDataRetention;
+  return data;
+}
+
+export async function startCrawl(http: HttpClient, request: CrawlRequest): Promise<CrawlResponse> {
+  const payload = prepareCrawlPayload(request);
+  try {
+    const res = await http.post<{ success: boolean; id: string; url: string; error?: string }>("/v2/crawl", payload);
+    if (res.status !== 200 || !res.data?.success) {
+      throwForBadResponse(res, "start crawl");
+    }
+    return { id: res.data.id, url: res.data.url };
+  } catch (err: any) {
+    if (err?.isAxiosError) return normalizeAxiosError(err, "start crawl");
+    throw err;
+  }
+}
+
+export async function getCrawlStatus(http: HttpClient, jobId: string): Promise<CrawlJob> {
+  try {
+    const res = await http.get<{ success: boolean; status: CrawlJob["status"]; completed?: number; total?: number; creditsUsed?: number; expiresAt?: string; next?: string | null; data?: Document[] }>(`/v2/crawl/${jobId}`);
+    if (res.status !== 200 || !res.data?.success) {
+      throwForBadResponse(res, "get crawl status");
+    }
+    const body = res.data;
+    return {
+      status: body.status,
+      completed: body.completed ?? 0,
+      total: body.total ?? 0,
+      creditsUsed: body.creditsUsed,
+      expiresAt: body.expiresAt,
+      next: body.next ?? null,
+      data: (body.data || []) as Document[],
+    };
+  } catch (err: any) {
+    if (err?.isAxiosError) return normalizeAxiosError(err, "get crawl status");
+    throw err;
+  }
+}
+
+export async function cancelCrawl(http: HttpClient, jobId: string): Promise<boolean> {
+  try {
+    const res = await http.delete<{ status: string }>(`/v2/crawl/${jobId}`);
+    if (res.status !== 200) throwForBadResponse(res, "cancel crawl");
+    return res.data?.status === "cancelled";
+  } catch (err: any) {
+    if (err?.isAxiosError) return normalizeAxiosError(err, "cancel crawl");
+    throw err;
+  }
+}
+
+export async function waitForCrawlCompletion(http: HttpClient, jobId: string, pollInterval = 2, timeout?: number): Promise<CrawlJob> {
+  const start = Date.now();
+  while (true) {
+    const status = await getCrawlStatus(http, jobId);
+    if (["completed", "failed", "cancelled"].includes(status.status)) return status;
+    if (timeout != null && Date.now() - start > timeout * 1000) {
+      throw new Error(`Crawl job ${jobId} did not complete within ${timeout} seconds`);
+    }
+    await new Promise((r) => setTimeout(r, Math.max(1000, pollInterval * 1000)));
+  }
+}
+
+export async function crawl(http: HttpClient, request: CrawlRequest, pollInterval = 2, timeout?: number): Promise<CrawlJob> {
+  const started = await startCrawl(http, request);
+  return waitForCrawlCompletion(http, started.id, pollInterval, timeout);
+}
+
+export async function getCrawlErrors(http: HttpClient, crawlId: string): Promise<CrawlErrorsResponse> {
+  try {
+    const res = await http.get<{ success?: boolean; data?: { errors: Array<Record<string, string>>; robotsBlocked: string[] } }>(`/v2/crawl/${crawlId}/errors`);
+    if (res.status !== 200) throwForBadResponse(res, "get crawl errors");
+    const payload = res.data?.data ?? (res.data as any);
+    return { errors: payload.errors || [], robotsBlocked: payload.robotsBlocked || [] };
+  } catch (err: any) {
+    if (err?.isAxiosError) return normalizeAxiosError(err, "get crawl errors");
+    throw err;
+  }
+}
+
+export async function getActiveCrawls(http: HttpClient): Promise<ActiveCrawlsResponse> {
+  try {
+    const res = await http.get<{ success: boolean; crawls: Array<{ id: string; teamId?: string; team_id?: string; url: string; options?: any }> }>(`/v2/crawl/active`);
+    if (res.status !== 200 || !res.data?.success) throwForBadResponse(res, "get active crawls");
+    const crawlsIn = res.data?.crawls || [];
+    const crawls = crawlsIn.map((c) => ({ id: c.id, teamId: (c as any).teamId ?? (c as any).team_id, url: c.url, options: c.options ?? null }));
+    return { success: true, crawls };
+  } catch (err: any) {
+    if (err?.isAxiosError) return normalizeAxiosError(err, "get active crawls");
+    throw err;
+  }
+}
+
+export async function crawlParamsPreview(http: HttpClient, url: string, prompt: string): Promise<Record<string, unknown>> {
+  if (!url || !url.trim()) throw new Error("URL cannot be empty");
+  if (!prompt || !prompt.trim()) throw new Error("Prompt cannot be empty");
+  try {
+    const res = await http.post<{ success: boolean; data?: Record<string, unknown>; warning?: string }>("/v2/crawl/params-preview", { url: url.trim(), prompt });
+    if (res.status !== 200 || !res.data?.success) throwForBadResponse(res, "crawl params preview");
+    const data = res.data.data || {};
+    if (res.data.warning) (data as any).warning = res.data.warning;
+    return data;
+  } catch (err: any) {
+    if (err?.isAxiosError) return normalizeAxiosError(err, "crawl params preview");
+    throw err;
+  }
+}
+

package/src/v2/methods/extract.ts

@@ -0,0 +1,86 @@
+import { type ExtractResponse, type ScrapeOptions } from "../types";
+import { HttpClient } from "../utils/httpClient";
+import { ensureValidScrapeOptions } from "../utils/validation";
+import { normalizeAxiosError, throwForBadResponse } from "../utils/errorHandler";
+import { zodToJsonSchema } from "zod-to-json-schema";
+import type { ZodTypeAny } from "zod";
+
+function prepareExtractPayload(args: {
+  urls?: string[];
+  prompt?: string;
+  schema?: Record<string, unknown> | ZodTypeAny;
+  systemPrompt?: string;
+  allowExternalLinks?: boolean;
+  enableWebSearch?: boolean;
+  showSources?: boolean;
+  scrapeOptions?: ScrapeOptions;
+  ignoreInvalidURLs?: boolean;
+}): Record<string, unknown> {
+  const body: Record<string, unknown> = {};
+  if (args.urls) body.urls = args.urls;
+  if (args.prompt != null) body.prompt = args.prompt;
+  if (args.schema != null) {
+    const s: any = args.schema;
+    const isZod = s && (typeof s.safeParse === "function" || typeof s.parse === "function") && s._def;
+    body.schema = isZod ? zodToJsonSchema(s) : args.schema;
+  }
+  if (args.systemPrompt != null) body.systemPrompt = args.systemPrompt;
+  if (args.allowExternalLinks != null) body.allowExternalLinks = args.allowExternalLinks;
+  if (args.enableWebSearch != null) body.enableWebSearch = args.enableWebSearch;
+  if (args.showSources != null) body.showSources = args.showSources;
+  if (args.ignoreInvalidURLs != null) body.ignoreInvalidURLs = args.ignoreInvalidURLs;
+  if (args.scrapeOptions) {
+    ensureValidScrapeOptions(args.scrapeOptions);
+    body.scrapeOptions = args.scrapeOptions;
+  }
+  return body;
+}
+
+export async function startExtract(http: HttpClient, args: Parameters<typeof prepareExtractPayload>[0]): Promise<ExtractResponse> {
+  const payload = prepareExtractPayload(args);
+  try {
+    const res = await http.post<ExtractResponse>("/v2/extract", payload);
+    if (res.status !== 200) throwForBadResponse(res, "extract");
+    return res.data;
+  } catch (err: any) {
+    if (err?.isAxiosError) return normalizeAxiosError(err, "extract");
+    throw err;
+  }
+}
+
+export async function getExtractStatus(http: HttpClient, jobId: string): Promise<ExtractResponse> {
+  try {
+    const res = await http.get<ExtractResponse>(`/v2/extract/${jobId}`);
+    if (res.status !== 200) throwForBadResponse(res, "extract status");
+    return res.data;
+  } catch (err: any) {
+    if (err?.isAxiosError) return normalizeAxiosError(err, "extract status");
+    throw err;
+  }
+}
+
+export async function waitExtract(
+  http: HttpClient,
+  jobId: string,
+  pollInterval = 2,
+  timeout?: number
+): Promise<ExtractResponse> {
+  const start = Date.now();
+  while (true) {
+    const status = await getExtractStatus(http, jobId);
+    if (["completed", "failed", "cancelled"].includes(status.status || "")) return status;
+    if (timeout != null && Date.now() - start > timeout * 1000) return status;
+    await new Promise((r) => setTimeout(r, Math.max(1000, pollInterval * 1000)));
+  }
+}
+
+export async function extract(
+  http: HttpClient,
+  args: Parameters<typeof prepareExtractPayload>[0] & { pollInterval?: number; timeout?: number }
+): Promise<ExtractResponse> {
+  const started = await startExtract(http, args);
+  const jobId = started.id;
+  if (!jobId) return started;
+  return waitExtract(http, jobId, args.pollInterval ?? 2, args.timeout);
+}
+

package/src/v2/methods/map.ts

@@ -0,0 +1,37 @@
+import { type MapData, type MapOptions, type SearchResult } from "../types";
+import { HttpClient } from "../utils/httpClient";
+import { throwForBadResponse, normalizeAxiosError } from "../utils/errorHandler";
+
+function prepareMapPayload(url: string, options?: MapOptions): Record<string, unknown> {
+  if (!url || !url.trim()) throw new Error("URL cannot be empty");
+  const payload: Record<string, unknown> = { url: url.trim() };
+  if (options) {
+    if (options.sitemap != null) payload.sitemap = options.sitemap;
+    if (options.search != null) payload.search = options.search;
+    if (options.includeSubdomains != null) payload.includeSubdomains = options.includeSubdomains;
+    if (options.limit != null) payload.limit = options.limit;
+    if (options.timeout != null) payload.timeout = options.timeout;
+  }
+  return payload;
+}
+
+export async function map(http: HttpClient, url: string, options?: MapOptions): Promise<MapData> {
+  const payload = prepareMapPayload(url, options);
+  try {
+    const res = await http.post<{ success: boolean; error?: string; links?: Array<string | SearchResult> }>("/v2/map", payload);
+    if (res.status !== 200 || !res.data?.success) {
+      throwForBadResponse(res, "map");
+    }
+    const linksIn = res.data.links || [];
+    const links: SearchResult[] = [];
+    for (const item of linksIn) {
+      if (typeof item === "string") links.push({ url: item });
+      else if (item && typeof item === "object") links.push({ url: item.url, title: (item as any).title, description: (item as any).description });
+    }
+    return { links };
+  } catch (err: any) {
+    if (err?.isAxiosError) return normalizeAxiosError(err, "map");
+    throw err;
+  }
+}
+