firecrawl 1.1.0 → 1.3.0

package/{types → dist}/index.d.ts

@@ -1,11 +1,13 @@
-import { AxiosResponse, AxiosRequestHeaders } from "axios";
-import { TypedEventTarget } from "typescript-event-target";
+import { AxiosRequestHeaders, AxiosResponse } from 'axios';
+import { ZodSchema } from 'zod';
+import { TypedEventTarget } from 'typescript-event-target';
+
 /**
  * Configuration interface for FirecrawlApp.
  * @param apiKey - Optional API key for authentication.
  * @param apiUrl - Optional base URL of the API; defaults to 'https://api.firecrawl.dev'.
  */
-export interface FirecrawlAppConfig {
+interface FirecrawlAppConfig {
     apiKey?: string | null;
     apiUrl?: string | null;
 }
@@ -13,7 +15,7 @@ export interface FirecrawlAppConfig {
  * Metadata for a Firecrawl document.
  * Includes various optional properties for document metadata.
  */
-export interface FirecrawlDocumentMetadata {
+interface FirecrawlDocumentMetadata {
     title?: string;
     description?: string;
     language?: string;
@@ -52,12 +54,13 @@ export interface FirecrawlDocumentMetadata {
  * Document interface for Firecrawl.
  * Represents a document retrieved or processed by Firecrawl.
  */
-export interface FirecrawlDocument {
+interface FirecrawlDocument {
     url?: string;
     markdown?: string;
     html?: string;
     rawHtml?: string;
     links?: string[];
+    extract?: Record<any, any>;
     screenshot?: string;
     metadata?: FirecrawlDocumentMetadata;
 }
@@ -65,12 +68,17 @@ export interface FirecrawlDocument {
  * Parameters for scraping operations.
  * Defines the options and configurations available for scraping web content.
  */
-export interface ScrapeParams {
-    formats: ("markdown" | "html" | "rawHtml" | "content" | "links" | "screenshot" | "full@scrennshot")[];
+interface ScrapeParams {
+    formats: ("markdown" | "html" | "rawHtml" | "content" | "links" | "screenshot" | "extract" | "full@scrennshot")[];
     headers?: Record<string, string>;
     includeTags?: string[];
     excludeTags?: string[];
     onlyMainContent?: boolean;
+    extract?: {
+        prompt?: string;
+        schema?: ZodSchema | any;
+        systemPrompt?: string;
+    };
     waitFor?: number;
     timeout?: number;
 }
@@ -78,7 +86,7 @@ export interface ScrapeParams {
  * Response interface for scraping operations.
  * Defines the structure of the response received after a scraping operation.
  */
-export interface ScrapeResponse extends FirecrawlDocument {
+interface ScrapeResponse extends FirecrawlDocument {
     success: true;
     warning?: string;
     error?: string;
@@ -87,7 +95,7 @@ export interface ScrapeResponse extends FirecrawlDocument {
  * Parameters for crawling operations.
  * Includes options for both scraping and mapping during a crawl.
  */
-export interface CrawlParams {
+interface CrawlParams {
     includePaths?: string[];
     excludePaths?: string[];
     maxDepth?: number;
@@ -96,12 +104,13 @@ export interface CrawlParams {
     allowExternalLinks?: boolean;
     ignoreSitemap?: boolean;
     scrapeOptions?: ScrapeParams;
+    webhook?: string;
 }
 /**
  * Response interface for crawling operations.
  * Defines the structure of the response received after initiating a crawl.
  */
-export interface CrawlResponse {
+interface CrawlResponse {
     id?: string;
     url?: string;
     success: true;
@@ -111,22 +120,21 @@ export interface CrawlResponse {
  * Response interface for job status checks.
  * Provides detailed status of a crawl job including progress and results.
  */
-export interface CrawlStatusResponse {
+interface CrawlStatusResponse {
     success: true;
-    total: number;
+    status: "scraping" | "completed" | "failed" | "cancelled";
     completed: number;
+    total: number;
     creditsUsed: number;
     expiresAt: Date;
-    status: "scraping" | "completed" | "failed";
-    next: string;
-    data?: FirecrawlDocument[];
-    error?: string;
+    next?: string;
+    data: FirecrawlDocument[];
 }
 /**
  * Parameters for mapping operations.
  * Defines options for mapping URLs during a crawl.
  */
-export interface MapParams {
+interface MapParams {
     search?: string;
     ignoreSitemap?: boolean;
     includeSubdomains?: boolean;
@@ -136,7 +144,7 @@ export interface MapParams {
  * Response interface for mapping operations.
  * Defines the structure of the response received after a mapping operation.
  */
-export interface MapResponse {
+interface MapResponse {
     success: true;
     links?: string[];
     error?: string;
@@ -145,7 +153,7 @@ export interface MapResponse {
  * Error response interface.
  * Defines the structure of the response received when an error occurs.
  */
-export interface ErrorResponse {
+interface ErrorResponse {
     success: false;
     error: string;
 }
@@ -153,7 +161,7 @@ export interface ErrorResponse {
  * Main class for interacting with the Firecrawl API.
  * Provides methods for scraping, searching, crawling, and mapping web content.
  */
-export default class FirecrawlApp {
+declare class FirecrawlApp {
     apiKey: string;
     apiUrl: string;
     /**
@@ -188,9 +196,10 @@ export default class FirecrawlApp {
     /**
      * Checks the status of a crawl job using the Firecrawl API.
      * @param id - The ID of the crawl operation.
+     * @param getAllData - Paginate through all the pages of documents, returning the full list of all documents. (default: `false`)
      * @returns The response containing the job status.
      */
-    checkCrawlStatus(id?: string): Promise<CrawlStatusResponse | ErrorResponse>;
+    checkCrawlStatus(id?: string, getAllData?: boolean): Promise<CrawlStatusResponse | ErrorResponse>;
     crawlUrlAndWatch(url: string, params?: CrawlParams, idempotencyKey?: string): Promise<CrawlWatcher>;
     mapUrl(url: string, params?: MapParams): Promise<MapResponse | ErrorResponse>;
     /**
@@ -222,7 +231,7 @@ export default class FirecrawlApp {
      * @param checkUrl - Optional URL to check the status (used for v1 API)
      * @returns The final job status or data.
      */
-    monitorJobStatus(id: string, headers: AxiosRequestHeaders, checkInterval: number): Promise<CrawlStatusResponse>;
+    monitorJobStatus(id: string, headers: AxiosRequestHeaders, checkInterval: number): Promise<CrawlStatusResponse | ErrorResponse>;
     /**
      * Handles errors from API responses.
      * @param {AxiosResponse} response - The response from the API.
@@ -242,11 +251,12 @@ interface CrawlWatcherEvents {
         error: string;
     }>;
 }
-export declare class CrawlWatcher extends TypedEventTarget<CrawlWatcherEvents> {
+declare class CrawlWatcher extends TypedEventTarget<CrawlWatcherEvents> {
     private ws;
     data: FirecrawlDocument[];
     status: CrawlStatusResponse["status"];
     constructor(id: string, app: FirecrawlApp);
     close(): void;
 }
-export {};
+
+export { type CrawlParams, type CrawlResponse, type CrawlStatusResponse, CrawlWatcher, type ErrorResponse, type FirecrawlAppConfig, type FirecrawlDocument, type FirecrawlDocumentMetadata, type MapParams, type MapResponse, type ScrapeParams, type ScrapeResponse, FirecrawlApp as default };

package/dist/index.js

@@ -0,0 +1,375 @@
+// src/index.ts
+import axios from "axios";
+import { zodToJsonSchema } from "zod-to-json-schema";
+import { WebSocket } from "isows";
+import { TypedEventTarget } from "typescript-event-target";
+var FirecrawlApp = class {
+  apiKey;
+  apiUrl;
+  /**
+   * Initializes a new instance of the FirecrawlApp class.
+   * @param config - Configuration options for the FirecrawlApp instance.
+   */
+  constructor({ apiKey = null, apiUrl = null }) {
+    if (typeof apiKey !== "string") {
+      throw new Error("No API key provided");
+    }
+    this.apiKey = apiKey;
+    this.apiUrl = apiUrl || "https://api.firecrawl.dev";
+  }
+  /**
+   * Scrapes a URL using the Firecrawl API.
+   * @param url - The URL to scrape.
+   * @param params - Additional parameters for the scrape request.
+   * @returns The response from the scrape operation.
+   */
+  async scrapeUrl(url, params) {
+    const headers = {
+      "Content-Type": "application/json",
+      Authorization: `Bearer ${this.apiKey}`
+    };
+    let jsonData = { url, ...params };
+    if (jsonData?.extract?.schema) {
+      let schema = jsonData.extract.schema;
+      try {
+        schema = zodToJsonSchema(schema);
+      } catch (error) {
+      }
+      jsonData = {
+        ...jsonData,
+        extract: {
+          ...jsonData.extract,
+          schema
+        }
+      };
+    }
+    try {
+      const response = await axios.post(
+        this.apiUrl + `/v1/scrape`,
+        jsonData,
+        { headers }
+      );
+      if (response.status === 200) {
+        const responseData = response.data;
+        if (responseData.success) {
+          return {
+            success: true,
+            warning: responseData.warning,
+            error: responseData.error,
+            ...responseData.data
+          };
+        } else {
+          throw new Error(`Failed to scrape URL. Error: ${responseData.error}`);
+        }
+      } else {
+        this.handleError(response, "scrape URL");
+      }
+    } catch (error) {
+      throw new Error(error.message);
+    }
+    return { success: false, error: "Internal server error." };
+  }
+  /**
+   * This method is intended to search for a query using the Firecrawl API. However, it is not supported in version 1 of the API.
+   * @param query - The search query string.
+   * @param params - Additional parameters for the search.
+   * @returns Throws an error advising to use version 0 of the API.
+   */
+  async search(query, params) {
+    throw new Error("Search is not supported in v1, please update FirecrawlApp() initialization to use v0.");
+  }
+  /**
+   * Initiates a crawl job for a URL using the Firecrawl API.
+   * @param url - The URL to crawl.
+   * @param params - Additional parameters for the crawl request.
+   * @param pollInterval - Time in seconds for job status checks.
+   * @param idempotencyKey - Optional idempotency key for the request.
+   * @returns The response from the crawl operation.
+   */
+  async crawlUrl(url, params, pollInterval = 2, idempotencyKey) {
+    const headers = this.prepareHeaders(idempotencyKey);
+    let jsonData = { url, ...params };
+    try {
+      const response = await this.postRequest(
+        this.apiUrl + `/v1/crawl`,
+        jsonData,
+        headers
+      );
+      if (response.status === 200) {
+        const id = response.data.id;
+        return this.monitorJobStatus(id, headers, pollInterval);
+      } else {
+        this.handleError(response, "start crawl job");
+      }
+    } catch (error) {
+      if (error.response?.data?.error) {
+        throw new Error(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ""}`);
+      } else {
+        throw new Error(error.message);
+      }
+    }
+    return { success: false, error: "Internal server error." };
+  }
+  async asyncCrawlUrl(url, params, idempotencyKey) {
+    const headers = this.prepareHeaders(idempotencyKey);
+    let jsonData = { url, ...params };
+    try {
+      const response = await this.postRequest(
+        this.apiUrl + `/v1/crawl`,
+        jsonData,
+        headers
+      );
+      if (response.status === 200) {
+        return response.data;
+      } else {
+        this.handleError(response, "start crawl job");
+      }
+    } catch (error) {
+      if (error.response?.data?.error) {
+        throw new Error(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ""}`);
+      } else {
+        throw new Error(error.message);
+      }
+    }
+    return { success: false, error: "Internal server error." };
+  }
+  /**
+   * Checks the status of a crawl job using the Firecrawl API.
+   * @param id - The ID of the crawl operation.
+   * @param getAllData - Paginate through all the pages of documents, returning the full list of all documents. (default: `false`)
+   * @returns The response containing the job status.
+   */
+  async checkCrawlStatus(id, getAllData = false) {
+    if (!id) {
+      throw new Error("No crawl ID provided");
+    }
+    const headers = this.prepareHeaders();
+    try {
+      const response = await this.getRequest(
+        `${this.apiUrl}/v1/crawl/${id}`,
+        headers
+      );
+      if (response.status === 200) {
+        let allData = response.data.data;
+        if (getAllData && response.data.status === "completed") {
+          let statusData = response.data;
+          if ("data" in statusData) {
+            let data = statusData.data;
+            while ("next" in statusData) {
+              statusData = (await this.getRequest(statusData.next, headers)).data;
+              data = data.concat(statusData.data);
+            }
+            allData = data;
+          }
+        }
+        return {
+          success: response.data.success,
+          status: response.data.status,
+          total: response.data.total,
+          completed: response.data.completed,
+          creditsUsed: response.data.creditsUsed,
+          expiresAt: new Date(response.data.expiresAt),
+          next: response.data.next,
+          data: allData,
+          error: response.data.error
+        };
+      } else {
+        this.handleError(response, "check crawl status");
+      }
+    } catch (error) {
+      throw new Error(error.message);
+    }
+    return { success: false, error: "Internal server error." };
+  }
+  async crawlUrlAndWatch(url, params, idempotencyKey) {
+    const crawl = await this.asyncCrawlUrl(url, params, idempotencyKey);
+    if (crawl.success && crawl.id) {
+      const id = crawl.id;
+      return new CrawlWatcher(id, this);
+    }
+    throw new Error("Crawl job failed to start");
+  }
+  async mapUrl(url, params) {
+    const headers = this.prepareHeaders();
+    let jsonData = { url, ...params };
+    try {
+      const response = await this.postRequest(
+        this.apiUrl + `/v1/map`,
+        jsonData,
+        headers
+      );
+      if (response.status === 200) {
+        return response.data;
+      } else {
+        this.handleError(response, "map");
+      }
+    } catch (error) {
+      throw new Error(error.message);
+    }
+    return { success: false, error: "Internal server error." };
+  }
+  /**
+   * Prepares the headers for an API request.
+   * @param idempotencyKey - Optional key to ensure idempotency.
+   * @returns The prepared headers.
+   */
+  prepareHeaders(idempotencyKey) {
+    return {
+      "Content-Type": "application/json",
+      Authorization: `Bearer ${this.apiKey}`,
+      ...idempotencyKey ? { "x-idempotency-key": idempotencyKey } : {}
+    };
+  }
+  /**
+   * Sends a POST request to the specified URL.
+   * @param url - The URL to send the request to.
+   * @param data - The data to send in the request.
+   * @param headers - The headers for the request.
+   * @returns The response from the POST request.
+   */
+  postRequest(url, data, headers) {
+    return axios.post(url, data, { headers });
+  }
+  /**
+   * Sends a GET request to the specified URL.
+   * @param url - The URL to send the request to.
+   * @param headers - The headers for the request.
+   * @returns The response from the GET request.
+   */
+  getRequest(url, headers) {
+    return axios.get(url, { headers });
+  }
+  /**
+   * Monitors the status of a crawl job until completion or failure.
+   * @param id - The ID of the crawl operation.
+   * @param headers - The headers for the request.
+   * @param checkInterval - Interval in seconds for job status checks.
+   * @param checkUrl - Optional URL to check the status (used for v1 API)
+   * @returns The final job status or data.
+   */
+  async monitorJobStatus(id, headers, checkInterval) {
+    while (true) {
+      let statusResponse = await this.getRequest(
+        `${this.apiUrl}/v1/crawl/${id}`,
+        headers
+      );
+      if (statusResponse.status === 200) {
+        let statusData = statusResponse.data;
+        if (statusData.status === "completed") {
+          if ("data" in statusData) {
+            let data = statusData.data;
+            while ("next" in statusData) {
+              statusResponse = await this.getRequest(statusData.next, headers);
+              statusData = statusResponse.data;
+              data = data.concat(statusData.data);
+            }
+            statusData.data = data;
+            return statusData;
+          } else {
+            throw new Error("Crawl job completed but no data was returned");
+          }
+        } else if (["active", "paused", "pending", "queued", "waiting", "scraping"].includes(statusData.status)) {
+          checkInterval = Math.max(checkInterval, 2);
+          await new Promise(
+            (resolve) => setTimeout(resolve, checkInterval * 1e3)
+          );
+        } else {
+          throw new Error(
+            `Crawl job failed or was stopped. Status: ${statusData.status}`
+          );
+        }
+      } else {
+        this.handleError(statusResponse, "check crawl status");
+      }
+    }
+  }
+  /**
+   * Handles errors from API responses.
+   * @param {AxiosResponse} response - The response from the API.
+   * @param {string} action - The action being performed when the error occurred.
+   */
+  handleError(response, action) {
+    if ([402, 408, 409, 500].includes(response.status)) {
+      const errorMessage = response.data.error || "Unknown error occurred";
+      throw new Error(
+        `Failed to ${action}. Status code: ${response.status}. Error: ${errorMessage}`
+      );
+    } else {
+      throw new Error(
+        `Unexpected error occurred while trying to ${action}. Status code: ${response.status}`
+      );
+    }
+  }
+};
+var CrawlWatcher = class extends TypedEventTarget {
+  ws;
+  data;
+  status;
+  constructor(id, app) {
+    super();
+    this.ws = new WebSocket(`${app.apiUrl}/v1/crawl/${id}`, app.apiKey);
+    this.status = "scraping";
+    this.data = [];
+    const messageHandler = (msg) => {
+      if (msg.type === "done") {
+        this.status = "completed";
+        this.dispatchTypedEvent("done", new CustomEvent("done", {
+          detail: {
+            status: this.status,
+            data: this.data
+          }
+        }));
+      } else if (msg.type === "error") {
+        this.status = "failed";
+        this.dispatchTypedEvent("error", new CustomEvent("error", {
+          detail: {
+            status: this.status,
+            data: this.data,
+            error: msg.error
+          }
+        }));
+      } else if (msg.type === "catchup") {
+        this.status = msg.data.status;
+        this.data.push(...msg.data.data ?? []);
+        for (const doc of this.data) {
+          this.dispatchTypedEvent("document", new CustomEvent("document", {
+            detail: doc
+          }));
+        }
+      } else if (msg.type === "document") {
+        this.dispatchTypedEvent("document", new CustomEvent("document", {
+          detail: msg.data
+        }));
+      }
+    };
+    this.ws.onmessage = ((ev) => {
+      if (typeof ev.data !== "string") {
+        this.ws.close();
+        return;
+      }
+      const msg = JSON.parse(ev.data);
+      messageHandler(msg);
+    }).bind(this);
+    this.ws.onclose = ((ev) => {
+      const msg = JSON.parse(ev.reason);
+      messageHandler(msg);
+    }).bind(this);
+    this.ws.onerror = ((_) => {
+      this.status = "failed";
+      this.dispatchTypedEvent("error", new CustomEvent("error", {
+        detail: {
+          status: this.status,
+          data: this.data,
+          error: "WebSocket error"
+        }
+      }));
+    }).bind(this);
+  }
+  close() {
+    this.ws.close();
+  }
+};
+export {
+  CrawlWatcher,
+  FirecrawlApp as default
+};

package/package.json

@@ -1,22 +1,19 @@
 {
   "name": "firecrawl",
-  "version": "1.1.0",
+  "version": "1.3.0",
   "description": "JavaScript SDK for Firecrawl API",
-  "main": "build/cjs/index.js",
-  "types": "types/index.d.ts",
-  "type": "module",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
   "exports": {
-    "require": {
-      "types": "./types/index.d.ts",
-      "default": "./build/cjs/index.js"
-    },
-    "import": {
-      "types": "./types/index.d.ts",
-      "default": "./build/esm/index.js"
+    "./package.json": "./package.json",
+    ".": {
+      "import": "./dist/index.js",
+      "default": "./dist/index.cjs"
     }
   },
+  "type": "module",
   "scripts": {
-    "build": "tsc --module commonjs --moduleResolution node10 --outDir build/cjs/ && echo '{\"type\": \"commonjs\"}' > build/cjs/package.json && npx tsc --module NodeNext --moduleResolution NodeNext --outDir build/esm/ && echo '{\"type\": \"module\"}' > build/esm/package.json",
+    "build": "tsup",
     "build-and-publish": "npm run build && npm publish --access public",
     "publish-beta": "npm run build && npm publish --access public --tag beta",
     "test": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/v1/**/*.test.ts"
@@ -29,10 +26,8 @@
   "license": "MIT",
   "dependencies": {
     "axios": "^1.6.8",
-    "dotenv": "^16.4.5",
     "isows": "^1.0.4",
     "typescript-event-target": "^1.1.1",
-    "uuid": "^9.0.1",
     "zod": "^3.23.8",
     "zod-to-json-schema": "^3.23.0"
   },
@@ -41,6 +36,8 @@
   },
   "homepage": "https://github.com/mendableai/firecrawl#readme",
   "devDependencies": {
+    "uuid": "^9.0.1",
+    "dotenv": "^16.4.5",
     "@jest/globals": "^29.7.0",
     "@types/axios": "^0.14.0",
     "@types/dotenv": "^8.2.0",
@@ -50,6 +47,7 @@
     "@types/uuid": "^9.0.8",
     "jest": "^29.7.0",
     "ts-jest": "^29.2.2",
+    "tsup": "^8.2.4",
     "typescript": "^5.4.5"
   },
   "keywords": [