firecrawl 0.0.29 → 1.1.0

package/src/index.ts

@@ -1,8 +1,13 @@
 import axios, { AxiosResponse, AxiosRequestHeaders } from "axios";
 import { z } from "zod";
 import { zodToJsonSchema } from "zod-to-json-schema";
+import { WebSocket } from "isows";
+import { TypedEventTarget } from "typescript-event-target";
+
 /**
  * Configuration interface for FirecrawlApp.
+ * @param apiKey - Optional API key for authentication.
+ * @param apiUrl - Optional base URL of the API; defaults to 'https://api.firecrawl.dev'.
  */
 export interface FirecrawlAppConfig {
   apiKey?: string | null;
@@ -11,6 +16,7 @@ export interface FirecrawlAppConfig {
 
 /**
  * Metadata for a Firecrawl document.
+ * Includes various optional properties for document metadata.
  */
 export interface FirecrawlDocumentMetadata {
   title?: string;
@@ -43,115 +49,155 @@ export interface FirecrawlDocumentMetadata {
   articleTag?: string;
   articleSection?: string;
   sourceURL?: string;
-  pageStatusCode?: number;
-  pageError?: string;
-  [key: string]: any;
+  statusCode?: number;
+  error?: string;
+  [key: string]: any; // Allows for additional metadata properties not explicitly defined.
 }
 
 /**
  * Document interface for Firecrawl.
+ * Represents a document retrieved or processed by Firecrawl.
  */
 export interface FirecrawlDocument {
-  id?: string;
   url?: string;
-  content: string;
   markdown?: string;
   html?: string;
-  llm_extraction?: Record<string, any>;
-  createdAt?: Date;
-  updatedAt?: Date;
-  type?: string;
-  metadata: FirecrawlDocumentMetadata;
-  childrenLinks?: string[];
-  provider?: string;
-  warning?: string;
+  rawHtml?: string;
+  links?: string[];
+  screenshot?: string;
+  metadata?: FirecrawlDocumentMetadata;
+}
 
-  index?: number;
+/**
+ * Parameters for scraping operations.
+ * Defines the options and configurations available for scraping web content.
+ */
+export interface ScrapeParams {
+  formats: ("markdown" | "html" | "rawHtml" | "content" | "links" | "screenshot" | "full@scrennshot")[];
+  headers?: Record<string, string>;
+  includeTags?: string[];
+  excludeTags?: string[];
+  onlyMainContent?: boolean;
+    waitFor?: number;
+  timeout?: number;
 }
 
 /**
  * Response interface for scraping operations.
+ * Defines the structure of the response received after a scraping operation.
  */
-export interface ScrapeResponse {
-  success: boolean;
-  data?: FirecrawlDocument;
+export interface ScrapeResponse extends FirecrawlDocument {
+  success: true;
+  warning?: string;
   error?: string;
 }
+
 /**
- * Response interface for searching operations.
+ * Parameters for crawling operations.
+ * Includes options for both scraping and mapping during a crawl.
  */
-export interface SearchResponse {
-  success: boolean;
-  data?: FirecrawlDocument[];
-  error?: string;
+export interface CrawlParams {
+  includePaths?: string[];
+  excludePaths?: string[];
+  maxDepth?: number;
+  limit?: number;
+  allowBackwardLinks?: boolean;
+  allowExternalLinks?: boolean;
+  ignoreSitemap?: boolean;
+  scrapeOptions?: ScrapeParams;
 }
+
 /**
  * Response interface for crawling operations.
+ * Defines the structure of the response received after initiating a crawl.
  */
 export interface CrawlResponse {
-  success: boolean;
-  jobId?: string;
-  data?: FirecrawlDocument[];
+  id?: string;
+  url?: string;
+  success: true;
   error?: string;
 }
+
 /**
  * Response interface for job status checks.
+ * Provides detailed status of a crawl job including progress and results.
  */
-export interface JobStatusResponse {
-  success: boolean;
-  status: string;
-  jobId?: string;
+export interface CrawlStatusResponse {
+  success: true;
+  total: number;
+  completed: number;
+  creditsUsed: number;
+  expiresAt: Date;
+  status: "scraping" | "completed" | "failed";
+  next: string;
   data?: FirecrawlDocument[];
-  partial_data?: FirecrawlDocument[];
   error?: string;
 }
+
+/**
+ * Parameters for mapping operations.
+ * Defines options for mapping URLs during a crawl.
+ */
+export interface MapParams {
+  search?: string;
+  ignoreSitemap?: boolean;
+  includeSubdomains?: boolean;
+  limit?: number;
+}
+
 /**
- * Generic parameter interface.
+ * Response interface for mapping operations.
+ * Defines the structure of the response received after a mapping operation.
  */
-export interface Params {
-  [key: string]: any;
-  extractorOptions?: {
-    extractionSchema: z.ZodSchema | any;
-    mode?: "llm-extraction";
-    extractionPrompt?: string;
-  };
+export interface MapResponse {
+  success: true;
+  links?: string[];
+  error?: string;
 }
+
+/**
+ * Error response interface.
+ * Defines the structure of the response received when an error occurs.
+ */
+export interface ErrorResponse {
+  success: false;
+  error: string;
+}
+
 /**
  * Main class for interacting with the Firecrawl API.
+ * Provides methods for scraping, searching, crawling, and mapping web content.
  */
 export default class FirecrawlApp {
-  private apiKey: string;
-  private apiUrl: string;
+  public apiKey: string;
+  public apiUrl: string;
 
   /**
    * Initializes a new instance of the FirecrawlApp class.
-   * @param {FirecrawlAppConfig} config - Configuration options for the FirecrawlApp instance.
+   * @param config - Configuration options for the FirecrawlApp instance.
    */
   constructor({ apiKey = null, apiUrl = null }: FirecrawlAppConfig) {
     this.apiKey = apiKey || "";
     this.apiUrl = apiUrl || "https://api.firecrawl.dev";
-    if (!this.apiKey) {
-      throw new Error("No API key provided");
-    }
   }
 
   /**
    * Scrapes a URL using the Firecrawl API.
-   * @param {string} url - The URL to scrape.
-   * @param {Params | null} params - Additional parameters for the scrape request.
-   * @returns {Promise<ScrapeResponse>} The response from the scrape operation.
+   * @param url - The URL to scrape.
+   * @param params - Additional parameters for the scrape request.
+   * @returns The response from the scrape operation.
    */
   async scrapeUrl(
     url: string,
-    params: Params | null = null
-  ): Promise<ScrapeResponse> {
+    params?: ScrapeParams
+  ): Promise<ScrapeResponse | ErrorResponse> {
     const headers: AxiosRequestHeaders = {
       "Content-Type": "application/json",
       Authorization: `Bearer ${this.apiKey}`,
     } as AxiosRequestHeaders;
-    let jsonData: Params = { url, ...params };
-    if (params?.extractorOptions?.extractionSchema) {
-      let schema = params.extractorOptions.extractionSchema;
+    let jsonData: any = { url, ...params };
+    if (jsonData?.extractorOptions?.extractionSchema) {
+      let schema = jsonData.extractorOptions.extractionSchema;
       // Check if schema is an instance of ZodSchema to correctly identify Zod schemas
       if (schema instanceof z.ZodSchema) {
         schema = zodToJsonSchema(schema);
@@ -159,22 +205,27 @@ export default class FirecrawlApp {
       jsonData = {
         ...jsonData,
         extractorOptions: {
-          ...params.extractorOptions,
+          ...jsonData.extractorOptions,
           extractionSchema: schema,
-          mode: params.extractorOptions.mode || "llm-extraction",
+          mode: jsonData.extractorOptions.mode || "llm-extraction",
         },
       };
     }
     try {
       const response: AxiosResponse = await axios.post(
-        this.apiUrl + "/v0/scrape",
+        this.apiUrl + `/v1/scrape`,
         jsonData,
         { headers }
       );
       if (response.status === 200) {
         const responseData = response.data;
         if (responseData.success) {
-          return responseData;
+          return {
+            success: true,
+            warning: responseData.warning,
+            error: responseData.error,
+            ...responseData.data
+          };
         } else {
           throw new Error(`Failed to scrape URL. Error: ${responseData.error}`);
         }
@@ -188,126 +239,161 @@ export default class FirecrawlApp {
   }
 
   /**
-   * Searches for a query using the Firecrawl API.
-   * @param {string} query - The query to search for.
-   * @param {Params | null} params - Additional parameters for the search request.
-   * @returns {Promise<SearchResponse>} The response from the search operation.
+   * This method is intended to search for a query using the Firecrawl API. However, it is not supported in version 1 of the API.
+   * @param query - The search query string.
+   * @param params - Additional parameters for the search.
+   * @returns Throws an error advising to use version 0 of the API.
    */
   async search(
     query: string,
-    params: Params | null = null
-  ): Promise<SearchResponse> {
-    const headers: AxiosRequestHeaders = {
-      "Content-Type": "application/json",
-      Authorization: `Bearer ${this.apiKey}`,
-    } as AxiosRequestHeaders;
-    let jsonData: Params = { query };
-    if (params) {
-      jsonData = { ...jsonData, ...params };
-    }
+    params?: any
+  ): Promise<any> {
+    throw new Error("Search is not supported in v1, please update FirecrawlApp() initialization to use v0.");
+  }
+
+  /**
+   * Initiates a crawl job for a URL using the Firecrawl API.
+   * @param url - The URL to crawl.
+   * @param params - Additional parameters for the crawl request.
+   * @param pollInterval - Time in seconds for job status checks.
+   * @param idempotencyKey - Optional idempotency key for the request.
+   * @returns The response from the crawl operation.
+   */
+  async crawlUrl(
+    url: string,
+    params?: CrawlParams,
+    pollInterval: number = 2,
+    idempotencyKey?: string
+  ): Promise<CrawlStatusResponse | ErrorResponse> {
+    const headers = this.prepareHeaders(idempotencyKey);
+    let jsonData: any = { url, ...params };
     try {
-      const response: AxiosResponse = await axios.post(
-        this.apiUrl + "/v0/search",
+      const response: AxiosResponse = await this.postRequest(
+        this.apiUrl + `/v1/crawl`,
         jsonData,
-        { headers }
+        headers
       );
       if (response.status === 200) {
-        const responseData = response.data;
-        if (responseData.success) {
-          return responseData;
-        } else {
-          throw new Error(`Failed to search. Error: ${responseData.error}`);
-        }
+        const id: string = response.data.id;
+        return this.monitorJobStatus(id, headers, pollInterval);
       } else {
-        this.handleError(response, "search");
+        this.handleError(response, "start crawl job");
       }
     } catch (error: any) {
-      throw new Error(error.message);
+      if (error.response?.data?.error) {
+        throw new Error(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`);
+      } else {
+        throw new Error(error.message);
+      }
     }
     return { success: false, error: "Internal server error." };
   }
 
-  /**
-   * Initiates a crawl job for a URL using the Firecrawl API.
-   * @param {string} url - The URL to crawl.
-   * @param {Params | null} params - Additional parameters for the crawl request.
-   * @param {boolean} waitUntilDone - Whether to wait for the crawl job to complete.
-   * @param {number} pollInterval - Time in seconds for job status checks.
-   * @param {string} idempotencyKey - Optional idempotency key for the request.
-   * @returns {Promise<CrawlResponse | any>} The response from the crawl operation.
-   */
-  async crawlUrl(
+  async asyncCrawlUrl(
     url: string,
-    params: Params | null = null,
-    waitUntilDone: boolean = true,
-    pollInterval: number = 2,
+    params?: CrawlParams,
     idempotencyKey?: string
-  ): Promise<CrawlResponse | any> {
+  ): Promise<CrawlResponse | ErrorResponse> {
     const headers = this.prepareHeaders(idempotencyKey);
-    let jsonData: Params = { url };
-    if (params) {
-      jsonData = { ...jsonData, ...params };
-    }
+    let jsonData: any = { url, ...params };
     try {
       const response: AxiosResponse = await this.postRequest(
-        this.apiUrl + "/v0/crawl",
+        this.apiUrl + `/v1/crawl`,
         jsonData,
         headers
       );
       if (response.status === 200) {
-        const jobId: string = response.data.jobId;
-        if (waitUntilDone) {
-          return this.monitorJobStatus(jobId, headers, pollInterval);
-        } else {
-          return { success: true, jobId };
-        }
+        return response.data;
       } else {
         this.handleError(response, "start crawl job");
       }
     } catch (error: any) {
-      console.log(error);
-      throw new Error(error.message);
+      if (error.response?.data?.error) {
+        throw new Error(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`);
+      } else {
+        throw new Error(error.message);
+      }
     }
     return { success: false, error: "Internal server error." };
   }
 
   /**
    * Checks the status of a crawl job using the Firecrawl API.
-   * @param {string} jobId - The job ID of the crawl operation.
-   * @returns {Promise<JobStatusResponse>} The response containing the job status.
+   * @param id - The ID of the crawl operation.
+   * @returns The response containing the job status.
    */
-  async checkCrawlStatus(jobId: string): Promise<JobStatusResponse> {
+  async checkCrawlStatus(id?: string): Promise<CrawlStatusResponse | ErrorResponse> {
+    if (!id) {
+      throw new Error("No crawl ID provided");
+    }
+
     const headers: AxiosRequestHeaders = this.prepareHeaders();
     try {
       const response: AxiosResponse = await this.getRequest(
-        this.apiUrl + `/v0/crawl/status/${jobId}`,
+        `${this.apiUrl}/v1/crawl/${id}`,
         headers
       );
       if (response.status === 200) {
-        return {
+        return ({
           success: true,
           status: response.data.status,
+          total: response.data.total,
+          completed: response.data.completed,
+          creditsUsed: response.data.creditsUsed,
+          expiresAt: new Date(response.data.expiresAt),
+          next: response.data.next,
           data: response.data.data,
-          partial_data: !response.data.data
-            ? response.data.partial_data
-            : undefined,
-        };
+          error: response.data.error
+        })
       } else {
         this.handleError(response, "check crawl status");
       }
     } catch (error: any) {
       throw new Error(error.message);
     }
-    return {
-      success: false,
-      status: "unknown",
-      error: "Internal server error.",
-    };
+    return { success: false, error: "Internal server error." };
+  }
+
+  async crawlUrlAndWatch(
+    url: string,
+    params?: CrawlParams,
+    idempotencyKey?: string,
+  ) {
+    const crawl = await this.asyncCrawlUrl(url, params, idempotencyKey);
+
+    if (crawl.success && crawl.id) {
+      const id = crawl.id;
+      return new CrawlWatcher(id, this);
+    }
+
+    throw new Error("Crawl job failed to start");
+  }
+
+  async mapUrl(url: string, params?: MapParams): Promise<MapResponse | ErrorResponse> {
+    const headers = this.prepareHeaders();
+    let jsonData: { url: string } & MapParams = { url, ...params };
+
+    try {
+      const response: AxiosResponse = await this.postRequest(
+        this.apiUrl + `/v1/map`,
+        jsonData,
+        headers
+      );
+      if (response.status === 200) {
+        return response.data as MapResponse;
+      } else {
+        this.handleError(response, "map");
+      }
+    } catch (error: any) {
+      throw new Error(error.message);
+    }
+    return { success: false, error: "Internal server error." };
   }
 
   /**
    * Prepares the headers for an API request.
-   * @returns {AxiosRequestHeaders} The prepared headers.
+   * @param idempotencyKey - Optional key to ensure idempotency.
+   * @returns The prepared headers.
    */
   prepareHeaders(idempotencyKey?: string): AxiosRequestHeaders {
     return {
@@ -319,14 +405,14 @@ export default class FirecrawlApp {
 
   /**
    * Sends a POST request to the specified URL.
-   * @param {string} url - The URL to send the request to.
-   * @param {Params} data - The data to send in the request.
-   * @param {AxiosRequestHeaders} headers - The headers for the request.
-   * @returns {Promise<AxiosResponse>} The response from the POST request.
+   * @param url - The URL to send the request to.
+   * @param data - The data to send in the request.
+   * @param headers - The headers for the request.
+   * @returns The response from the POST request.
    */
   postRequest(
     url: string,
-    data: Params,
+    data: any,
     headers: AxiosRequestHeaders
   ): Promise<AxiosResponse> {
     return axios.post(url, data, { headers });
@@ -334,9 +420,9 @@ export default class FirecrawlApp {
 
   /**
    * Sends a GET request to the specified URL.
-   * @param {string} url - The URL to send the request to.
-   * @param {AxiosRequestHeaders} headers - The headers for the request.
-   * @returns {Promise<AxiosResponse>} The response from the GET request.
+   * @param url - The URL to send the request to.
+   * @param headers - The headers for the request.
+   * @returns The response from the GET request.
    */
   getRequest(
     url: string,
@@ -347,38 +433,37 @@ export default class FirecrawlApp {
 
   /**
    * Monitors the status of a crawl job until completion or failure.
-   * @param {string} jobId - The job ID of the crawl operation.
-   * @param {AxiosRequestHeaders} headers - The headers for the request.
-   * @param {number} timeout - Timeout in seconds for job status checks.
-   * @returns {Promise<any>} The final job status or data.
+   * @param id - The ID of the crawl operation.
+   * @param headers - The headers for the request.
+   * @param checkInterval - Interval in seconds for job status checks.
+   * @param checkUrl - Optional URL to check the status (used for v1 API)
+   * @returns The final job status or data.
    */
   async monitorJobStatus(
-    jobId: string,
+    id: string,
     headers: AxiosRequestHeaders,
     checkInterval: number
-  ): Promise<any> {
+  ): Promise<CrawlStatusResponse> {
     while (true) {
       const statusResponse: AxiosResponse = await this.getRequest(
-        this.apiUrl + `/v0/crawl/status/${jobId}`,
+        `${this.apiUrl}/v1/crawl/${id}`,
         headers
       );
       if (statusResponse.status === 200) {
         const statusData = statusResponse.data;
         if (statusData.status === "completed") {
           if ("data" in statusData) {
-            return statusData.data;
+            return statusData;
           } else {
             throw new Error("Crawl job completed but no data was returned");
           }
         } else if (
-          ["active", "paused", "pending", "queued"].includes(statusData.status)
+          ["active", "paused", "pending", "queued", "scraping"].includes(statusData.status)
         ) {
-          if (checkInterval < 2) {
-            checkInterval = 2;
-          }
+          checkInterval = Math.max(checkInterval, 2);
           await new Promise((resolve) =>
             setTimeout(resolve, checkInterval * 1000)
-          ); // Wait for the specified timeout before checking again
+          );
         } else {
           throw new Error(
             `Crawl job failed or was stopped. Status: ${statusData.status}`
@@ -409,3 +494,111 @@ export default class FirecrawlApp {
     }
   }
 }
+
+interface CrawlWatcherEvents {
+  document: CustomEvent<FirecrawlDocument>,
+  done: CustomEvent<{
+    status: CrawlStatusResponse["status"];
+    data: FirecrawlDocument[];
+  }>,
+  error: CustomEvent<{
+    status: CrawlStatusResponse["status"],
+    data: FirecrawlDocument[],
+    error: string,
+  }>,
+}
+
+export class CrawlWatcher extends TypedEventTarget<CrawlWatcherEvents> {
+  private ws: WebSocket;
+  public data: FirecrawlDocument[];
+  public status: CrawlStatusResponse["status"];
+
+  constructor(id: string, app: FirecrawlApp) {
+    super();
+    this.ws = new WebSocket(`${app.apiUrl}/v1/crawl/${id}`, app.apiKey);
+    this.status = "scraping";
+    this.data = [];
+
+    type ErrorMessage = {
+      type: "error",
+      error: string,
+    }
+    
+    type CatchupMessage = {
+      type: "catchup",
+      data: CrawlStatusResponse,
+    }
+    
+    type DocumentMessage = {
+      type: "document",
+      data: FirecrawlDocument,
+    }
+    
+    type DoneMessage = { type: "done" }
+    
+    type Message = ErrorMessage | CatchupMessage | DoneMessage | DocumentMessage;
+
+    const messageHandler = (msg: Message) => {
+      if (msg.type === "done") {
+        this.status = "completed";
+        this.dispatchTypedEvent("done", new CustomEvent("done", {
+          detail: {
+            status: this.status,
+            data: this.data,
+          },
+        }));
+      } else if (msg.type === "error") {
+        this.status = "failed";
+        this.dispatchTypedEvent("error", new CustomEvent("error", {
+          detail: {
+            status: this.status,
+            data: this.data,
+            error: msg.error,
+          },
+        }));
+      } else if (msg.type === "catchup") {
+        this.status = msg.data.status;
+        this.data.push(...(msg.data.data ?? []));
+        for (const doc of this.data) {
+          this.dispatchTypedEvent("document", new CustomEvent("document", {
+            detail: doc,
+          }));
+        }
+      } else if (msg.type === "document") {
+        this.dispatchTypedEvent("document", new CustomEvent("document", {
+          detail: msg.data,
+        }));
+      }
+    }
+
+    this.ws.onmessage = ((ev: MessageEvent) => {
+      if (typeof ev.data !== "string") {
+        this.ws.close();
+        return;
+      }
+
+      const msg = JSON.parse(ev.data) as Message;
+      messageHandler(msg);
+    }).bind(this);
+
+    this.ws.onclose = ((ev: CloseEvent) => {
+      const msg = JSON.parse(ev.reason) as Message;
+      messageHandler(msg);
+    }).bind(this);
+
+    this.ws.onerror = ((_: Event) => {
+      this.status = "failed"
+      this.dispatchTypedEvent("error", new CustomEvent("error", {
+        detail: {
+          status: this.status,
+          data: this.data,
+          error: "WebSocket error",
+        },
+      }));
+    }).bind(this);
+  }
+
+  close() {
+    this.ws.close();
+  }
+}