firecrawl 1.2.0 → 1.3.0

package/dist/index.cjs

@@ -0,0 +1,410 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/index.ts
+var src_exports = {};
+__export(src_exports, {
+  CrawlWatcher: () => CrawlWatcher,
+  default: () => FirecrawlApp
+});
+module.exports = __toCommonJS(src_exports);
+var import_axios = __toESM(require("axios"), 1);
+var import_zod_to_json_schema = require("zod-to-json-schema");
+var import_isows = require("isows");
+var import_typescript_event_target = require("typescript-event-target");
+var FirecrawlApp = class {
+  apiKey;
+  apiUrl;
+  /**
+   * Initializes a new instance of the FirecrawlApp class.
+   * @param config - Configuration options for the FirecrawlApp instance.
+   */
+  constructor({ apiKey = null, apiUrl = null }) {
+    if (typeof apiKey !== "string") {
+      throw new Error("No API key provided");
+    }
+    this.apiKey = apiKey;
+    this.apiUrl = apiUrl || "https://api.firecrawl.dev";
+  }
+  /**
+   * Scrapes a URL using the Firecrawl API.
+   * @param url - The URL to scrape.
+   * @param params - Additional parameters for the scrape request.
+   * @returns The response from the scrape operation.
+   */
+  async scrapeUrl(url, params) {
+    const headers = {
+      "Content-Type": "application/json",
+      Authorization: `Bearer ${this.apiKey}`
+    };
+    let jsonData = { url, ...params };
+    if (jsonData?.extract?.schema) {
+      let schema = jsonData.extract.schema;
+      try {
+        schema = (0, import_zod_to_json_schema.zodToJsonSchema)(schema);
+      } catch (error) {
+      }
+      jsonData = {
+        ...jsonData,
+        extract: {
+          ...jsonData.extract,
+          schema
+        }
+      };
+    }
+    try {
+      const response = await import_axios.default.post(
+        this.apiUrl + `/v1/scrape`,
+        jsonData,
+        { headers }
+      );
+      if (response.status === 200) {
+        const responseData = response.data;
+        if (responseData.success) {
+          return {
+            success: true,
+            warning: responseData.warning,
+            error: responseData.error,
+            ...responseData.data
+          };
+        } else {
+          throw new Error(`Failed to scrape URL. Error: ${responseData.error}`);
+        }
+      } else {
+        this.handleError(response, "scrape URL");
+      }
+    } catch (error) {
+      throw new Error(error.message);
+    }
+    return { success: false, error: "Internal server error." };
+  }
+  /**
+   * This method is intended to search for a query using the Firecrawl API. However, it is not supported in version 1 of the API.
+   * @param query - The search query string.
+   * @param params - Additional parameters for the search.
+   * @returns Throws an error advising to use version 0 of the API.
+   */
+  async search(query, params) {
+    throw new Error("Search is not supported in v1, please update FirecrawlApp() initialization to use v0.");
+  }
+  /**
+   * Initiates a crawl job for a URL using the Firecrawl API.
+   * @param url - The URL to crawl.
+   * @param params - Additional parameters for the crawl request.
+   * @param pollInterval - Time in seconds for job status checks.
+   * @param idempotencyKey - Optional idempotency key for the request.
+   * @returns The response from the crawl operation.
+   */
+  async crawlUrl(url, params, pollInterval = 2, idempotencyKey) {
+    const headers = this.prepareHeaders(idempotencyKey);
+    let jsonData = { url, ...params };
+    try {
+      const response = await this.postRequest(
+        this.apiUrl + `/v1/crawl`,
+        jsonData,
+        headers
+      );
+      if (response.status === 200) {
+        const id = response.data.id;
+        return this.monitorJobStatus(id, headers, pollInterval);
+      } else {
+        this.handleError(response, "start crawl job");
+      }
+    } catch (error) {
+      if (error.response?.data?.error) {
+        throw new Error(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ""}`);
+      } else {
+        throw new Error(error.message);
+      }
+    }
+    return { success: false, error: "Internal server error." };
+  }
+  async asyncCrawlUrl(url, params, idempotencyKey) {
+    const headers = this.prepareHeaders(idempotencyKey);
+    let jsonData = { url, ...params };
+    try {
+      const response = await this.postRequest(
+        this.apiUrl + `/v1/crawl`,
+        jsonData,
+        headers
+      );
+      if (response.status === 200) {
+        return response.data;
+      } else {
+        this.handleError(response, "start crawl job");
+      }
+    } catch (error) {
+      if (error.response?.data?.error) {
+        throw new Error(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ""}`);
+      } else {
+        throw new Error(error.message);
+      }
+    }
+    return { success: false, error: "Internal server error." };
+  }
+  /**
+   * Checks the status of a crawl job using the Firecrawl API.
+   * @param id - The ID of the crawl operation.
+   * @param getAllData - Paginate through all the pages of documents, returning the full list of all documents. (default: `false`)
+   * @returns The response containing the job status.
+   */
+  async checkCrawlStatus(id, getAllData = false) {
+    if (!id) {
+      throw new Error("No crawl ID provided");
+    }
+    const headers = this.prepareHeaders();
+    try {
+      const response = await this.getRequest(
+        `${this.apiUrl}/v1/crawl/${id}`,
+        headers
+      );
+      if (response.status === 200) {
+        let allData = response.data.data;
+        if (getAllData && response.data.status === "completed") {
+          let statusData = response.data;
+          if ("data" in statusData) {
+            let data = statusData.data;
+            while ("next" in statusData) {
+              statusData = (await this.getRequest(statusData.next, headers)).data;
+              data = data.concat(statusData.data);
+            }
+            allData = data;
+          }
+        }
+        return {
+          success: response.data.success,
+          status: response.data.status,
+          total: response.data.total,
+          completed: response.data.completed,
+          creditsUsed: response.data.creditsUsed,
+          expiresAt: new Date(response.data.expiresAt),
+          next: response.data.next,
+          data: allData,
+          error: response.data.error
+        };
+      } else {
+        this.handleError(response, "check crawl status");
+      }
+    } catch (error) {
+      throw new Error(error.message);
+    }
+    return { success: false, error: "Internal server error." };
+  }
+  async crawlUrlAndWatch(url, params, idempotencyKey) {
+    const crawl = await this.asyncCrawlUrl(url, params, idempotencyKey);
+    if (crawl.success && crawl.id) {
+      const id = crawl.id;
+      return new CrawlWatcher(id, this);
+    }
+    throw new Error("Crawl job failed to start");
+  }
+  async mapUrl(url, params) {
+    const headers = this.prepareHeaders();
+    let jsonData = { url, ...params };
+    try {
+      const response = await this.postRequest(
+        this.apiUrl + `/v1/map`,
+        jsonData,
+        headers
+      );
+      if (response.status === 200) {
+        return response.data;
+      } else {
+        this.handleError(response, "map");
+      }
+    } catch (error) {
+      throw new Error(error.message);
+    }
+    return { success: false, error: "Internal server error." };
+  }
+  /**
+   * Prepares the headers for an API request.
+   * @param idempotencyKey - Optional key to ensure idempotency.
+   * @returns The prepared headers.
+   */
+  prepareHeaders(idempotencyKey) {
+    return {
+      "Content-Type": "application/json",
+      Authorization: `Bearer ${this.apiKey}`,
+      ...idempotencyKey ? { "x-idempotency-key": idempotencyKey } : {}
+    };
+  }
+  /**
+   * Sends a POST request to the specified URL.
+   * @param url - The URL to send the request to.
+   * @param data - The data to send in the request.
+   * @param headers - The headers for the request.
+   * @returns The response from the POST request.
+   */
+  postRequest(url, data, headers) {
+    return import_axios.default.post(url, data, { headers });
+  }
+  /**
+   * Sends a GET request to the specified URL.
+   * @param url - The URL to send the request to.
+   * @param headers - The headers for the request.
+   * @returns The response from the GET request.
+   */
+  getRequest(url, headers) {
+    return import_axios.default.get(url, { headers });
+  }
+  /**
+   * Monitors the status of a crawl job until completion or failure.
+   * @param id - The ID of the crawl operation.
+   * @param headers - The headers for the request.
+   * @param checkInterval - Interval in seconds for job status checks.
+   * @param checkUrl - Optional URL to check the status (used for v1 API)
+   * @returns The final job status or data.
+   */
+  async monitorJobStatus(id, headers, checkInterval) {
+    while (true) {
+      let statusResponse = await this.getRequest(
+        `${this.apiUrl}/v1/crawl/${id}`,
+        headers
+      );
+      if (statusResponse.status === 200) {
+        let statusData = statusResponse.data;
+        if (statusData.status === "completed") {
+          if ("data" in statusData) {
+            let data = statusData.data;
+            while ("next" in statusData) {
+              statusResponse = await this.getRequest(statusData.next, headers);
+              statusData = statusResponse.data;
+              data = data.concat(statusData.data);
+            }
+            statusData.data = data;
+            return statusData;
+          } else {
+            throw new Error("Crawl job completed but no data was returned");
+          }
+        } else if (["active", "paused", "pending", "queued", "waiting", "scraping"].includes(statusData.status)) {
+          checkInterval = Math.max(checkInterval, 2);
+          await new Promise(
+            (resolve) => setTimeout(resolve, checkInterval * 1e3)
+          );
+        } else {
+          throw new Error(
+            `Crawl job failed or was stopped. Status: ${statusData.status}`
+          );
+        }
+      } else {
+        this.handleError(statusResponse, "check crawl status");
+      }
+    }
+  }
+  /**
+   * Handles errors from API responses.
+   * @param {AxiosResponse} response - The response from the API.
+   * @param {string} action - The action being performed when the error occurred.
+   */
+  handleError(response, action) {
+    if ([402, 408, 409, 500].includes(response.status)) {
+      const errorMessage = response.data.error || "Unknown error occurred";
+      throw new Error(
+        `Failed to ${action}. Status code: ${response.status}. Error: ${errorMessage}`
+      );
+    } else {
+      throw new Error(
+        `Unexpected error occurred while trying to ${action}. Status code: ${response.status}`
+      );
+    }
+  }
+};
+var CrawlWatcher = class extends import_typescript_event_target.TypedEventTarget {
+  ws;
+  data;
+  status;
+  constructor(id, app) {
+    super();
+    this.ws = new import_isows.WebSocket(`${app.apiUrl}/v1/crawl/${id}`, app.apiKey);
+    this.status = "scraping";
+    this.data = [];
+    const messageHandler = (msg) => {
+      if (msg.type === "done") {
+        this.status = "completed";
+        this.dispatchTypedEvent("done", new CustomEvent("done", {
+          detail: {
+            status: this.status,
+            data: this.data
+          }
+        }));
+      } else if (msg.type === "error") {
+        this.status = "failed";
+        this.dispatchTypedEvent("error", new CustomEvent("error", {
+          detail: {
+            status: this.status,
+            data: this.data,
+            error: msg.error
+          }
+        }));
+      } else if (msg.type === "catchup") {
+        this.status = msg.data.status;
+        this.data.push(...msg.data.data ?? []);
+        for (const doc of this.data) {
+          this.dispatchTypedEvent("document", new CustomEvent("document", {
+            detail: doc
+          }));
+        }
+      } else if (msg.type === "document") {
+        this.dispatchTypedEvent("document", new CustomEvent("document", {
+          detail: msg.data
+        }));
+      }
+    };
+    this.ws.onmessage = ((ev) => {
+      if (typeof ev.data !== "string") {
+        this.ws.close();
+        return;
+      }
+      const msg = JSON.parse(ev.data);
+      messageHandler(msg);
+    }).bind(this);
+    this.ws.onclose = ((ev) => {
+      const msg = JSON.parse(ev.reason);
+      messageHandler(msg);
+    }).bind(this);
+    this.ws.onerror = ((_) => {
+      this.status = "failed";
+      this.dispatchTypedEvent("error", new CustomEvent("error", {
+        detail: {
+          status: this.status,
+          data: this.data,
+          error: "WebSocket error"
+        }
+      }));
+    }).bind(this);
+  }
+  close() {
+    this.ws.close();
+  }
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  CrawlWatcher
+});

package/dist/index.d.cts

@@ -0,0 +1,262 @@
+import { AxiosRequestHeaders, AxiosResponse } from 'axios';
+import { ZodSchema } from 'zod';
+import { TypedEventTarget } from 'typescript-event-target';
+
+/**
+ * Configuration interface for FirecrawlApp.
+ * @param apiKey - Optional API key for authentication.
+ * @param apiUrl - Optional base URL of the API; defaults to 'https://api.firecrawl.dev'.
+ */
+interface FirecrawlAppConfig {
+    apiKey?: string | null;
+    apiUrl?: string | null;
+}
+/**
+ * Metadata for a Firecrawl document.
+ * Includes various optional properties for document metadata.
+ */
+interface FirecrawlDocumentMetadata {
+    title?: string;
+    description?: string;
+    language?: string;
+    keywords?: string;
+    robots?: string;
+    ogTitle?: string;
+    ogDescription?: string;
+    ogUrl?: string;
+    ogImage?: string;
+    ogAudio?: string;
+    ogDeterminer?: string;
+    ogLocale?: string;
+    ogLocaleAlternate?: string[];
+    ogSiteName?: string;
+    ogVideo?: string;
+    dctermsCreated?: string;
+    dcDateCreated?: string;
+    dcDate?: string;
+    dctermsType?: string;
+    dcType?: string;
+    dctermsAudience?: string;
+    dctermsSubject?: string;
+    dcSubject?: string;
+    dcDescription?: string;
+    dctermsKeywords?: string;
+    modifiedTime?: string;
+    publishedTime?: string;
+    articleTag?: string;
+    articleSection?: string;
+    sourceURL?: string;
+    statusCode?: number;
+    error?: string;
+    [key: string]: any;
+}
+/**
+ * Document interface for Firecrawl.
+ * Represents a document retrieved or processed by Firecrawl.
+ */
+interface FirecrawlDocument {
+    url?: string;
+    markdown?: string;
+    html?: string;
+    rawHtml?: string;
+    links?: string[];
+    extract?: Record<any, any>;
+    screenshot?: string;
+    metadata?: FirecrawlDocumentMetadata;
+}
+/**
+ * Parameters for scraping operations.
+ * Defines the options and configurations available for scraping web content.
+ */
+interface ScrapeParams {
+    formats: ("markdown" | "html" | "rawHtml" | "content" | "links" | "screenshot" | "extract" | "full@scrennshot")[];
+    headers?: Record<string, string>;
+    includeTags?: string[];
+    excludeTags?: string[];
+    onlyMainContent?: boolean;
+    extract?: {
+        prompt?: string;
+        schema?: ZodSchema | any;
+        systemPrompt?: string;
+    };
+    waitFor?: number;
+    timeout?: number;
+}
+/**
+ * Response interface for scraping operations.
+ * Defines the structure of the response received after a scraping operation.
+ */
+interface ScrapeResponse extends FirecrawlDocument {
+    success: true;
+    warning?: string;
+    error?: string;
+}
+/**
+ * Parameters for crawling operations.
+ * Includes options for both scraping and mapping during a crawl.
+ */
+interface CrawlParams {
+    includePaths?: string[];
+    excludePaths?: string[];
+    maxDepth?: number;
+    limit?: number;
+    allowBackwardLinks?: boolean;
+    allowExternalLinks?: boolean;
+    ignoreSitemap?: boolean;
+    scrapeOptions?: ScrapeParams;
+    webhook?: string;
+}
+/**
+ * Response interface for crawling operations.
+ * Defines the structure of the response received after initiating a crawl.
+ */
+interface CrawlResponse {
+    id?: string;
+    url?: string;
+    success: true;
+    error?: string;
+}
+/**
+ * Response interface for job status checks.
+ * Provides detailed status of a crawl job including progress and results.
+ */
+interface CrawlStatusResponse {
+    success: true;
+    status: "scraping" | "completed" | "failed" | "cancelled";
+    completed: number;
+    total: number;
+    creditsUsed: number;
+    expiresAt: Date;
+    next?: string;
+    data: FirecrawlDocument[];
+}
+/**
+ * Parameters for mapping operations.
+ * Defines options for mapping URLs during a crawl.
+ */
+interface MapParams {
+    search?: string;
+    ignoreSitemap?: boolean;
+    includeSubdomains?: boolean;
+    limit?: number;
+}
+/**
+ * Response interface for mapping operations.
+ * Defines the structure of the response received after a mapping operation.
+ */
+interface MapResponse {
+    success: true;
+    links?: string[];
+    error?: string;
+}
+/**
+ * Error response interface.
+ * Defines the structure of the response received when an error occurs.
+ */
+interface ErrorResponse {
+    success: false;
+    error: string;
+}
+/**
+ * Main class for interacting with the Firecrawl API.
+ * Provides methods for scraping, searching, crawling, and mapping web content.
+ */
+declare class FirecrawlApp {
+    apiKey: string;
+    apiUrl: string;
+    /**
+     * Initializes a new instance of the FirecrawlApp class.
+     * @param config - Configuration options for the FirecrawlApp instance.
+     */
+    constructor({ apiKey, apiUrl }: FirecrawlAppConfig);
+    /**
+     * Scrapes a URL using the Firecrawl API.
+     * @param url - The URL to scrape.
+     * @param params - Additional parameters for the scrape request.
+     * @returns The response from the scrape operation.
+     */
+    scrapeUrl(url: string, params?: ScrapeParams): Promise<ScrapeResponse | ErrorResponse>;
+    /**
+     * This method is intended to search for a query using the Firecrawl API. However, it is not supported in version 1 of the API.
+     * @param query - The search query string.
+     * @param params - Additional parameters for the search.
+     * @returns Throws an error advising to use version 0 of the API.
+     */
+    search(query: string, params?: any): Promise<any>;
+    /**
+     * Initiates a crawl job for a URL using the Firecrawl API.
+     * @param url - The URL to crawl.
+     * @param params - Additional parameters for the crawl request.
+     * @param pollInterval - Time in seconds for job status checks.
+     * @param idempotencyKey - Optional idempotency key for the request.
+     * @returns The response from the crawl operation.
+     */
+    crawlUrl(url: string, params?: CrawlParams, pollInterval?: number, idempotencyKey?: string): Promise<CrawlStatusResponse | ErrorResponse>;
+    asyncCrawlUrl(url: string, params?: CrawlParams, idempotencyKey?: string): Promise<CrawlResponse | ErrorResponse>;
+    /**
+     * Checks the status of a crawl job using the Firecrawl API.
+     * @param id - The ID of the crawl operation.
+     * @param getAllData - Paginate through all the pages of documents, returning the full list of all documents. (default: `false`)
+     * @returns The response containing the job status.
+     */
+    checkCrawlStatus(id?: string, getAllData?: boolean): Promise<CrawlStatusResponse | ErrorResponse>;
+    crawlUrlAndWatch(url: string, params?: CrawlParams, idempotencyKey?: string): Promise<CrawlWatcher>;
+    mapUrl(url: string, params?: MapParams): Promise<MapResponse | ErrorResponse>;
+    /**
+     * Prepares the headers for an API request.
+     * @param idempotencyKey - Optional key to ensure idempotency.
+     * @returns The prepared headers.
+     */
+    prepareHeaders(idempotencyKey?: string): AxiosRequestHeaders;
+    /**
+     * Sends a POST request to the specified URL.
+     * @param url - The URL to send the request to.
+     * @param data - The data to send in the request.
+     * @param headers - The headers for the request.
+     * @returns The response from the POST request.
+     */
+    postRequest(url: string, data: any, headers: AxiosRequestHeaders): Promise<AxiosResponse>;
+    /**
+     * Sends a GET request to the specified URL.
+     * @param url - The URL to send the request to.
+     * @param headers - The headers for the request.
+     * @returns The response from the GET request.
+     */
+    getRequest(url: string, headers: AxiosRequestHeaders): Promise<AxiosResponse>;
+    /**
+     * Monitors the status of a crawl job until completion or failure.
+     * @param id - The ID of the crawl operation.
+     * @param headers - The headers for the request.
+     * @param checkInterval - Interval in seconds for job status checks.
+     * @param checkUrl - Optional URL to check the status (used for v1 API)
+     * @returns The final job status or data.
+     */
+    monitorJobStatus(id: string, headers: AxiosRequestHeaders, checkInterval: number): Promise<CrawlStatusResponse | ErrorResponse>;
+    /**
+     * Handles errors from API responses.
+     * @param {AxiosResponse} response - The response from the API.
+     * @param {string} action - The action being performed when the error occurred.
+     */
+    handleError(response: AxiosResponse, action: string): void;
+}
+interface CrawlWatcherEvents {
+    document: CustomEvent<FirecrawlDocument>;
+    done: CustomEvent<{
+        status: CrawlStatusResponse["status"];
+        data: FirecrawlDocument[];
+    }>;
+    error: CustomEvent<{
+        status: CrawlStatusResponse["status"];
+        data: FirecrawlDocument[];
+        error: string;
+    }>;
+}
+declare class CrawlWatcher extends TypedEventTarget<CrawlWatcherEvents> {
+    private ws;
+    data: FirecrawlDocument[];
+    status: CrawlStatusResponse["status"];
+    constructor(id: string, app: FirecrawlApp);
+    close(): void;
+}
+
+export { type CrawlParams, type CrawlResponse, type CrawlStatusResponse, CrawlWatcher, type ErrorResponse, type FirecrawlAppConfig, type FirecrawlDocument, type FirecrawlDocumentMetadata, type MapParams, type MapResponse, type ScrapeParams, type ScrapeResponse, FirecrawlApp as default };