@mendable/firecrawl-js 1.0.3 → 1.1.0

package/src/index.ts

@@ -1,17 +1,17 @@
 import axios, { AxiosResponse, AxiosRequestHeaders } from "axios";
 import { z } from "zod";
 import { zodToJsonSchema } from "zod-to-json-schema";
+import { WebSocket } from "isows";
+import { TypedEventTarget } from "typescript-event-target";
 
 /**
  * Configuration interface for FirecrawlApp.
  * @param apiKey - Optional API key for authentication.
  * @param apiUrl - Optional base URL of the API; defaults to 'https://api.firecrawl.dev'.
- * @param version - API version, either 'v0' or 'v1'.
  */
 export interface FirecrawlAppConfig {
   apiKey?: string | null;
   apiUrl?: string | null;
-  version?: "v0" | "v1";
 }
 
 /**
@@ -54,17 +54,6 @@ export interface FirecrawlDocumentMetadata {
   [key: string]: any; // Allows for additional metadata properties not explicitly defined.
 }
 
-/**
- * Metadata for a Firecrawl document on v0.
- * Similar to FirecrawlDocumentMetadata but includes properties specific to API version v0.
- */
-export interface FirecrawlDocumentMetadataV0 {
-  // Similar properties as FirecrawlDocumentMetadata with additional v0 specific adjustments
-  pageStatusCode?: number;
-  pageError?: string;
-  [key: string]: any;
-}
-
 /**
  * Document interface for Firecrawl.
  * Represents a document retrieved or processed by Firecrawl.
@@ -76,28 +65,7 @@ export interface FirecrawlDocument {
   rawHtml?: string;
   links?: string[];
   screenshot?: string;
-  metadata: FirecrawlDocumentMetadata;
-}
-
-/**
- * Document interface for Firecrawl on v0.
- * Represents a document specifically for API version v0 with additional properties.
- */
-export interface FirecrawlDocumentV0 {
-  id?: string;
-  url?: string;
-  content: string;
-  markdown?: string;
-  html?: string;
-  llm_extraction?: Record<string, any>;
-  createdAt?: Date;
-  updatedAt?: Date;
-  type?: string;
-  metadata: FirecrawlDocumentMetadataV0;
-  childrenLinks?: string[];
-  provider?: string;
-  warning?: string;
-  index?: number;
+  metadata?: FirecrawlDocumentMetadata;
 }
 
 /**
@@ -105,38 +73,12 @@ export interface FirecrawlDocumentV0 {
  * Defines the options and configurations available for scraping web content.
  */
 export interface ScrapeParams {
-  formats: ("markdown" | "html" | "rawHtml" | "content" | "links" | "screenshot")[];
+  formats: ("markdown" | "html" | "rawHtml" | "content" | "links" | "screenshot" | "full@scrennshot")[];
   headers?: Record<string, string>;
   includeTags?: string[];
   excludeTags?: string[];
   onlyMainContent?: boolean;
-  screenshotMode?: "desktop" | "full-desktop" | "mobile" | "full-mobile";
-  waitFor?: number;
-  timeout?: number;
-}
-
-/**
- * Parameters for scraping operations on v0.
- * Includes page and extractor options specific to API version v0.
- */
-export interface ScrapeParamsV0 {
-  pageOptions?: {
-    headers?: Record<string, string>;
-    includeHtml?: boolean;
-    includeRawHtml?: boolean;
-    onlyIncludeTags?: string[];
-    onlyMainContent?: boolean;
-    removeTags?: string[];
-    replaceAllPathsWithAbsolutePaths?: boolean;
-    screenshot?: boolean;
-    fullPageScreenshot?: boolean;
     waitFor?: number;
-  };
-  extractorOptions?: {
-    mode?: "markdown" | "llm-extraction" | "llm-extraction-from-raw-html" | "llm-extraction-from-markdown";
-    extractionPrompt?: string;
-    extractionSchema?: Record<string, any> | z.ZodSchema | any;
-  };
   timeout?: number;
 }
 
@@ -145,21 +87,11 @@ export interface ScrapeParamsV0 {
  * Defines the structure of the response received after a scraping operation.
  */
 export interface ScrapeResponse extends FirecrawlDocument {
-  success: boolean;
+  success: true;
   warning?: string;
   error?: string;
 }
 
-/**
- * Response interface for scraping operations on v0.
- * Similar to ScrapeResponse but tailored for responses from API version v0.
- */
-export interface ScrapeResponseV0 {
-  success: boolean;
-  data?: FirecrawlDocumentV0;
-  error?: string;
-}
-
 /**
  * Parameters for crawling operations.
  * Includes options for both scraping and mapping during a crawl.
@@ -175,37 +107,6 @@ export interface CrawlParams {
   scrapeOptions?: ScrapeParams;
 }
 
-/**
- * Parameters for crawling operations on v0.
- * Tailored for API version v0, includes specific options for crawling.
- */
-export interface CrawlParamsV0 {
-  crawlerOptions?: {
-    includes?: string[];
-    excludes?: string[];
-    generateImgAltText?: boolean;
-    returnOnlyUrls?: boolean;
-    maxDepth?: number;
-    mode?: "default" | "fast";
-    ignoreSitemap?: boolean;
-    limit?: number;
-    allowBackwardCrawling?: boolean;
-    allowExternalContentLinks?: boolean;
-  };
-  pageOptions?: {
-    headers?: Record<string, string>;
-    includeHtml?: boolean;
-    includeRawHtml?: boolean;
-    onlyIncludeTags?: string[];
-    onlyMainContent?: boolean;
-    removeTags?: string[];
-    replaceAllPathsWithAbsolutePaths?: boolean;
-    screenshot?: boolean;
-    fullPageScreenshot?: boolean;
-    waitFor?: number;
-  };
-}
-
 /**
  * Response interface for crawling operations.
  * Defines the structure of the response received after initiating a crawl.
@@ -213,17 +114,7 @@ export interface CrawlParamsV0 {
 export interface CrawlResponse {
   id?: string;
   url?: string;
-  success: boolean;
-  error?: string;
-}
-
-/**
- * Response interface for crawling operations on v0.
- * Similar to CrawlResponse but tailored for responses from API version v0.
- */
-export interface CrawlResponseV0 {
-  jobId?: string;
-  success: boolean;
+  success: true;
   error?: string;
 }
 
@@ -232,7 +123,7 @@ export interface CrawlResponseV0 {
  * Provides detailed status of a crawl job including progress and results.
  */
 export interface CrawlStatusResponse {
-  success: boolean;
+  success: true;
   total: number;
   completed: number;
   creditsUsed: number;
@@ -243,35 +134,15 @@ export interface CrawlStatusResponse {
   error?: string;
 }
 
-/**
- * Response interface for job status checks on v0.
- * Tailored for API version v0, provides status and partial data of a crawl job.
- */
-export interface CrawlStatusResponseV0 {
-  success: boolean;
-  status: string;
-  current?: number;
-  current_url?: string;
-  current_step?: string;
-  total?: number;
-  data?: FirecrawlDocumentV0[];
-  partial_data?: FirecrawlDocumentV0[];
-  error?: string;
-}
-
-
 /**
  * Parameters for mapping operations.
  * Defines options for mapping URLs during a crawl.
  */
 export interface MapParams {
-  includePaths?: string[]
-  excludePaths?: string[]
-  maxDepth?: number
-  limit?: number
-  allowBackwardLinks?: boolean
-  allowExternalLinks?: boolean
-  ignoreSitemap?: boolean
+  search?: string;
+  ignoreSitemap?: boolean;
+  includeSubdomains?: boolean;
+  limit?: number;
 }
 
 /**
@@ -279,57 +150,35 @@ export interface MapParams {
  * Defines the structure of the response received after a mapping operation.
  */
 export interface MapResponse {
-  success: boolean;
+  success: true;
   links?: string[];
   error?: string;
 }
 
 /**
- * Parameters for searching operations on v0.
- * Tailored for API version v0, includes specific options for searching content.
+ * Error response interface.
+ * Defines the structure of the response received when an error occurs.
  */
-export interface SearchParamsV0 {
-  pageOptions?: {
-    onlyMainContent?: boolean;
-    fetchPageContent?: boolean;
-    includeHtml?: boolean;
-    includeRawHtml?: boolean;
-  };
-  searchOptions?: {
-    limit?: number;
-  };
-}
-
-/**
- * Response interface for searching operations on v0.
- * Defines the structure of the response received after a search operation on v0.
- */
-export interface SearchResponseV0 {
-  success: boolean;
-  data?: FirecrawlDocumentV0[];
-  error?: string;
+export interface ErrorResponse {
+  success: false;
+  error: string;
 }
 
 /**
  * Main class for interacting with the Firecrawl API.
  * Provides methods for scraping, searching, crawling, and mapping web content.
  */
-export default class FirecrawlApp<T extends "v0" | "v1"> {
-  private apiKey: string;
-  private apiUrl: string;
-  public version: T;
+export default class FirecrawlApp {
+  public apiKey: string;
+  public apiUrl: string;
 
   /**
    * Initializes a new instance of the FirecrawlApp class.
    * @param config - Configuration options for the FirecrawlApp instance.
    */
-  constructor({ apiKey = null, apiUrl = null, version = "v1" }: FirecrawlAppConfig) {
+  constructor({ apiKey = null, apiUrl = null }: FirecrawlAppConfig) {
     this.apiKey = apiKey || "";
     this.apiUrl = apiUrl || "https://api.firecrawl.dev";
-    this.version = version as T;
-    if (!this.apiKey) {
-      throw new Error("No API key provided");
-    }
   }
 
   /**
@@ -340,8 +189,8 @@ export default class FirecrawlApp<T extends "v0" | "v1"> {
    */
   async scrapeUrl(
     url: string,
-    params?: ScrapeParams | ScrapeParamsV0
-  ): Promise<this['version'] extends 'v0' ? ScrapeResponseV0 : ScrapeResponse> {
+    params?: ScrapeParams
+  ): Promise<ScrapeResponse | ErrorResponse> {
     const headers: AxiosRequestHeaders = {
       "Content-Type": "application/json",
       Authorization: `Bearer ${this.apiKey}`,
@@ -364,19 +213,19 @@ export default class FirecrawlApp<T extends "v0" | "v1"> {
     }
     try {
       const response: AxiosResponse = await axios.post(
-        this.apiUrl + `/${this.version}/scrape`,
+        this.apiUrl + `/v1/scrape`,
         jsonData,
         { headers }
       );
       if (response.status === 200) {
         const responseData = response.data;
         if (responseData.success) {
-          return (this.version === 'v0' ? responseData as ScrapeResponseV0 : {
+          return {
             success: true,
             warning: responseData.warning,
             error: responseData.error,
             ...responseData.data
-          }) as ScrapeResponse;
+          };
         } else {
           throw new Error(`Failed to scrape URL. Error: ${responseData.error}`);
         }
@@ -386,100 +235,47 @@ export default class FirecrawlApp<T extends "v0" | "v1"> {
     } catch (error: any) {
       throw new Error(error.message);
     }
-    return { success: false, error: "Internal server error." } as this['version'] extends 'v0' ? ScrapeResponseV0 : ScrapeResponse;
+    return { success: false, error: "Internal server error." };
   }
 
   /**
-   * Searches for a query using the Firecrawl API.
-   * @param query - The query to search for.
-   * @param params - Additional parameters for the search request.
-   * @returns The response from the search operation.
+   * This method is intended to search for a query using the Firecrawl API. However, it is not supported in version 1 of the API.
+   * @param query - The search query string.
+   * @param params - Additional parameters for the search.
+   * @returns Throws an error advising to use version 0 of the API.
    */
   async search(
     query: string,
-    params?: SearchParamsV0
-  ): Promise<SearchResponseV0> {
-    if (this.version === "v1") {
-      throw new Error("Search is not supported in v1, please update FirecrawlApp() initialization to use v0.");
-    }
-
-    const headers: AxiosRequestHeaders = {
-      "Content-Type": "application/json",
-      Authorization: `Bearer ${this.apiKey}`,
-    } as AxiosRequestHeaders;
-    let jsonData: any = { query };
-    if (params) {
-      jsonData = { ...jsonData, ...params };
-    }
-    try {
-      const response: AxiosResponse = await axios.post(
-        this.apiUrl + "/v0/search",
-        jsonData,
-        { headers }
-      );
-      if (response.status === 200) {
-        const responseData = response.data;
-        if (responseData.success) {
-          return responseData;
-        } else {
-          throw new Error(`Failed to search. Error: ${responseData.error}`);
-        }
-      } else {
-        this.handleError(response, "search");
-      }
-    } catch (error: any) {
-      throw new Error(error.message);
-    }
-    return { success: false, error: "Internal server error." };
+    params?: any
+  ): Promise<any> {
+    throw new Error("Search is not supported in v1, please update FirecrawlApp() initialization to use v0.");
   }
 
   /**
    * Initiates a crawl job for a URL using the Firecrawl API.
    * @param url - The URL to crawl.
    * @param params - Additional parameters for the crawl request.
-   * @param waitUntilDone - Whether to wait for the crawl job to complete.
    * @param pollInterval - Time in seconds for job status checks.
    * @param idempotencyKey - Optional idempotency key for the request.
    * @returns The response from the crawl operation.
    */
   async crawlUrl(
     url: string,
-    params?: this['version'] extends 'v0' ? CrawlParamsV0 : CrawlParams,
-    waitUntilDone: boolean = true,
+    params?: CrawlParams,
     pollInterval: number = 2,
     idempotencyKey?: string
-  ): Promise<
-    this['version'] extends 'v0'
-      ? CrawlResponseV0 | CrawlStatusResponseV0 | FirecrawlDocumentV0[]
-      : CrawlResponse | CrawlStatusResponse
-  > {
+  ): Promise<CrawlStatusResponse | ErrorResponse> {
     const headers = this.prepareHeaders(idempotencyKey);
     let jsonData: any = { url, ...params };
     try {
       const response: AxiosResponse = await this.postRequest(
-        this.apiUrl + `/${this.version}/crawl`,
+        this.apiUrl + `/v1/crawl`,
         jsonData,
         headers
       );
       if (response.status === 200) {
-        const id: string = this.version === 'v0' ? response.data.jobId : response.data.id;
-        let checkUrl: string | undefined = undefined;
-        if (waitUntilDone) {
-          if (this.version === 'v1') { checkUrl = response.data.url }
-          return this.monitorJobStatus(id, headers, pollInterval, checkUrl);
-        } else {
-          if (this.version === 'v0') {
-            return {
-              success: true,
-              jobId: id
-            } as CrawlResponseV0;
-          } else {
-            return {
-              success: true,
-              id: id
-            } as CrawlResponse;
-          }
-        }
+        const id: string = response.data.id;
+        return this.monitorJobStatus(id, headers, pollInterval);
       } else {
         this.handleError(response, "start crawl job");
       }
@@ -490,7 +286,35 @@ export default class FirecrawlApp<T extends "v0" | "v1"> {
         throw new Error(error.message);
       }
     }
-    return { success: false, error: "Internal server error." } as this['version'] extends 'v0' ? CrawlResponseV0 : CrawlResponse;
+    return { success: false, error: "Internal server error." };
+  }
+
+  async asyncCrawlUrl(
+    url: string,
+    params?: CrawlParams,
+    idempotencyKey?: string
+  ): Promise<CrawlResponse | ErrorResponse> {
+    const headers = this.prepareHeaders(idempotencyKey);
+    let jsonData: any = { url, ...params };
+    try {
+      const response: AxiosResponse = await this.postRequest(
+        this.apiUrl + `/v1/crawl`,
+        jsonData,
+        headers
+      );
+      if (response.status === 200) {
+        return response.data;
+      } else {
+        this.handleError(response, "start crawl job");
+      }
+    } catch (error: any) {
+      if (error.response?.data?.error) {
+        throw new Error(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`);
+      } else {
+        throw new Error(error.message);
+      }
+    }
+    return { success: false, error: "Internal server error." };
   }
 
   /**
@@ -498,7 +322,7 @@ export default class FirecrawlApp<T extends "v0" | "v1"> {
    * @param id - The ID of the crawl operation.
    * @returns The response containing the job status.
    */
-  async checkCrawlStatus(id?: string): Promise<this['version'] extends 'v0' ? CrawlStatusResponseV0 : CrawlStatusResponse> {
+  async checkCrawlStatus(id?: string): Promise<CrawlStatusResponse | ErrorResponse> {
     if (!id) {
       throw new Error("No crawl ID provided");
     }
@@ -506,71 +330,52 @@ export default class FirecrawlApp<T extends "v0" | "v1"> {
     const headers: AxiosRequestHeaders = this.prepareHeaders();
     try {
       const response: AxiosResponse = await this.getRequest(
-        this.version === 'v1' ?
-          `${this.apiUrl}/${this.version}/crawl/${id}` :
-          `${this.apiUrl}/${this.version}/crawl/status/${id}`,
+        `${this.apiUrl}/v1/crawl/${id}`,
         headers
       );
       if (response.status === 200) {
-        if (this.version === 'v0') {
-          return ({
-            success: true,
-            status: response.data.status,
-            current: response.data.current,
-            current_url: response.data.current_url,
-            current_step: response.data.current_step,
-            total: response.data.total,
-            data: response.data.data,
-            partial_data: !response.data.data
-              ? response.data.partial_data
-              : undefined,
-          } as CrawlStatusResponseV0) as this['version'] extends 'v0' ? CrawlStatusResponseV0 : CrawlStatusResponse;
-        } else {
-          return ({
-            success: true,
-            status: response.data.status,
-            total: response.data.total,
-            completed: response.data.completed,
-            creditsUsed: response.data.creditsUsed,
-            expiresAt: new Date(response.data.expiresAt),
-            next: response.data.next,
-            data: response.data.data,
-            error: response.data.error
-          } as CrawlStatusResponse) as this['version'] extends 'v0' ? CrawlStatusResponseV0 : CrawlStatusResponse;
-        }
+        return ({
+          success: true,
+          status: response.data.status,
+          total: response.data.total,
+          completed: response.data.completed,
+          creditsUsed: response.data.creditsUsed,
+          expiresAt: new Date(response.data.expiresAt),
+          next: response.data.next,
+          data: response.data.data,
+          error: response.data.error
+        })
       } else {
         this.handleError(response, "check crawl status");
       }
     } catch (error: any) {
       throw new Error(error.message);
     }
-
-    return this.version === 'v0' ?
-      ({
-        success: false,
-        status: "unknown",
-        current: 0,
-        current_url: "",
-        current_step: "",
-        total: 0,
-        error: "Internal server error.",
-      } as this['version'] extends 'v0' ? CrawlStatusResponseV0 : CrawlStatusResponse) :
-      ({
-        success: false,
-        error: "Internal server error.",
-      } as this['version'] extends 'v0' ? CrawlStatusResponseV0 : CrawlStatusResponse);
+    return { success: false, error: "Internal server error." };
   }
 
-  async mapUrl(url: string, params?: MapParams): Promise<MapResponse> {
-    if (this.version == 'v0') {
-      throw new Error("Map is not supported in v0");
+  async crawlUrlAndWatch(
+    url: string,
+    params?: CrawlParams,
+    idempotencyKey?: string,
+  ) {
+    const crawl = await this.asyncCrawlUrl(url, params, idempotencyKey);
+
+    if (crawl.success && crawl.id) {
+      const id = crawl.id;
+      return new CrawlWatcher(id, this);
     }
+
+    throw new Error("Crawl job failed to start");
+  }
+
+  async mapUrl(url: string, params?: MapParams): Promise<MapResponse | ErrorResponse> {
     const headers = this.prepareHeaders();
     let jsonData: { url: string } & MapParams = { url, ...params };
 
     try {
       const response: AxiosResponse = await this.postRequest(
-        this.apiUrl + `/${this.version}/map`,
+        this.apiUrl + `/v1/map`,
         jsonData,
         headers
       );
@@ -582,7 +387,7 @@ export default class FirecrawlApp<T extends "v0" | "v1"> {
     } catch (error: any) {
       throw new Error(error.message);
     }
-    return { success: false, error: "Internal server error." } as MapResponse;
+    return { success: false, error: "Internal server error." };
   }
 
   /**
@@ -637,25 +442,18 @@ export default class FirecrawlApp<T extends "v0" | "v1"> {
   async monitorJobStatus(
     id: string,
     headers: AxiosRequestHeaders,
-    checkInterval: number,
-    checkUrl?: string
-  ): Promise<this['version'] extends 'v0' ? CrawlStatusResponseV0 | FirecrawlDocumentV0[] : CrawlStatusResponse> {
-    let apiUrl: string = '';
+    checkInterval: number
+  ): Promise<CrawlStatusResponse> {
     while (true) {
-      if (this.version === 'v1') {
-        apiUrl = checkUrl ?? `${this.apiUrl}/v1/crawl/${id}`;
-      } else if (this.version === 'v0') {
-        apiUrl = `${this.apiUrl}/v0/crawl/status/${id}`;
-      }
       const statusResponse: AxiosResponse = await this.getRequest(
-        apiUrl,
+        `${this.apiUrl}/v1/crawl/${id}`,
         headers
       );
       if (statusResponse.status === 200) {
         const statusData = statusResponse.data;
         if (statusData.status === "completed") {
           if ("data" in statusData) {
-            return this.version === 'v0' ? statusData.data : statusData;
+            return statusData;
           } else {
             throw new Error("Crawl job completed but no data was returned");
           }
@@ -696,3 +494,111 @@ export default class FirecrawlApp<T extends "v0" | "v1"> {
     }
   }
 }
+
+interface CrawlWatcherEvents {
+  document: CustomEvent<FirecrawlDocument>,
+  done: CustomEvent<{
+    status: CrawlStatusResponse["status"];
+    data: FirecrawlDocument[];
+  }>,
+  error: CustomEvent<{
+    status: CrawlStatusResponse["status"],
+    data: FirecrawlDocument[],
+    error: string,
+  }>,
+}
+
+export class CrawlWatcher extends TypedEventTarget<CrawlWatcherEvents> {
+  private ws: WebSocket;
+  public data: FirecrawlDocument[];
+  public status: CrawlStatusResponse["status"];
+
+  constructor(id: string, app: FirecrawlApp) {
+    super();
+    this.ws = new WebSocket(`${app.apiUrl}/v1/crawl/${id}`, app.apiKey);
+    this.status = "scraping";
+    this.data = [];
+
+    type ErrorMessage = {
+      type: "error",
+      error: string,
+    }
+    
+    type CatchupMessage = {
+      type: "catchup",
+      data: CrawlStatusResponse,
+    }
+    
+    type DocumentMessage = {
+      type: "document",
+      data: FirecrawlDocument,
+    }
+    
+    type DoneMessage = { type: "done" }
+    
+    type Message = ErrorMessage | CatchupMessage | DoneMessage | DocumentMessage;
+
+    const messageHandler = (msg: Message) => {
+      if (msg.type === "done") {
+        this.status = "completed";
+        this.dispatchTypedEvent("done", new CustomEvent("done", {
+          detail: {
+            status: this.status,
+            data: this.data,
+          },
+        }));
+      } else if (msg.type === "error") {
+        this.status = "failed";
+        this.dispatchTypedEvent("error", new CustomEvent("error", {
+          detail: {
+            status: this.status,
+            data: this.data,
+            error: msg.error,
+          },
+        }));
+      } else if (msg.type === "catchup") {
+        this.status = msg.data.status;
+        this.data.push(...(msg.data.data ?? []));
+        for (const doc of this.data) {
+          this.dispatchTypedEvent("document", new CustomEvent("document", {
+            detail: doc,
+          }));
+        }
+      } else if (msg.type === "document") {
+        this.dispatchTypedEvent("document", new CustomEvent("document", {
+          detail: msg.data,
+        }));
+      }
+    }
+
+    this.ws.onmessage = ((ev: MessageEvent) => {
+      if (typeof ev.data !== "string") {
+        this.ws.close();
+        return;
+      }
+
+      const msg = JSON.parse(ev.data) as Message;
+      messageHandler(msg);
+    }).bind(this);
+
+    this.ws.onclose = ((ev: CloseEvent) => {
+      const msg = JSON.parse(ev.reason) as Message;
+      messageHandler(msg);
+    }).bind(this);
+
+    this.ws.onerror = ((_: Event) => {
+      this.status = "failed"
+      this.dispatchTypedEvent("error", new CustomEvent("error", {
+        detail: {
+          status: this.status,
+          data: this.data,
+          error: "WebSocket error",
+        },
+      }));
+    }).bind(this);
+  }
+
+  close() {
+    this.ws.close();
+  }
+}