firecrawl 1.11.2 → 1.13.0

package/dist/index.cjs

@@ -264,23 +264,35 @@ var FirecrawlApp = class {
           if ("data" in statusData) {
             let data = statusData.data;
             while (typeof statusData === "object" && "next" in statusData) {
+              if (data.length === 0) {
+                break;
+              }
               statusData = (await this.getRequest(statusData.next, headers)).data;
               data = data.concat(statusData.data);
             }
             allData = data;
           }
         }
-        return {
+        let resp = {
           success: response.data.success,
           status: response.data.status,
           total: response.data.total,
           completed: response.data.completed,
           creditsUsed: response.data.creditsUsed,
           expiresAt: new Date(response.data.expiresAt),
-          next: response.data.next,
-          data: allData,
-          error: response.data.error
+          data: allData
         };
+        if (!response.data.success && response.data.error) {
+          resp = {
+            ...resp,
+            success: false,
+            error: response.data.error
+          };
+        }
+        if (response.data.next) {
+          resp.next = response.data.next;
+        }
+        return resp;
       } else {
         this.handleError(response, "check crawl status");
       }
@@ -459,23 +471,35 @@ var FirecrawlApp = class {
           if ("data" in statusData) {
             let data = statusData.data;
             while (typeof statusData === "object" && "next" in statusData) {
+              if (data.length === 0) {
+                break;
+              }
               statusData = (await this.getRequest(statusData.next, headers)).data;
               data = data.concat(statusData.data);
             }
             allData = data;
           }
         }
-        return {
+        let resp = {
           success: response.data.success,
           status: response.data.status,
           total: response.data.total,
           completed: response.data.completed,
           creditsUsed: response.data.creditsUsed,
           expiresAt: new Date(response.data.expiresAt),
-          next: response.data.next,
-          data: allData,
-          error: response.data.error
+          data: allData
         };
+        if (!response.data.success && response.data.error) {
+          resp = {
+            ...resp,
+            success: false,
+            error: response.data.error
+          };
+        }
+        if (response.data.next) {
+          resp.next = response.data.next;
+        }
+        return resp;
       } else {
         this.handleError(response, "check batch scrape status");
       }
@@ -509,21 +533,34 @@ var FirecrawlApp = class {
     try {
       const response = await this.postRequest(
         this.apiUrl + `/v1/extract`,
-        { ...jsonData, schema: jsonSchema },
+        { ...jsonData, schema: jsonSchema, origin: "api-sdk" },
         headers
       );
       if (response.status === 200) {
-        const responseData = response.data;
-        if (responseData.success) {
-          return {
-            success: true,
-            data: responseData.data,
-            warning: responseData.warning,
-            error: responseData.error
-          };
-        } else {
-          throw new FirecrawlError(`Failed to scrape URL. Error: ${responseData.error}`, response.status);
-        }
+        const jobId = response.data.id;
+        let extractStatus;
+        do {
+          const statusResponse = await this.getRequest(
+            `${this.apiUrl}/v1/extract/${jobId}`,
+            headers
+          );
+          extractStatus = statusResponse.data;
+          if (extractStatus.status === "completed") {
+            if (extractStatus.success) {
+              return {
+                success: true,
+                data: extractStatus.data,
+                warning: extractStatus.warning,
+                error: extractStatus.error
+              };
+            } else {
+              throw new FirecrawlError(`Failed to extract data. Error: ${extractStatus.error}`, statusResponse.status);
+            }
+          } else if (extractStatus.status === "failed" || extractStatus.status === "cancelled") {
+            throw new FirecrawlError(`Extract job ${extractStatus.status}. Error: ${extractStatus.error}`, statusResponse.status);
+          }
+          await new Promise((resolve) => setTimeout(resolve, 1e3));
+        } while (extractStatus.status !== "completed");
       } else {
         this.handleError(response, "extract");
       }
@@ -532,6 +569,62 @@ var FirecrawlApp = class {
     }
     return { success: false, error: "Internal server error." };
   }
+  /**
+   * Initiates an asynchronous extract job for a URL using the Firecrawl API.
+   * @param url - The URL to extract data from.
+   * @param params - Additional parameters for the extract request.
+   * @param idempotencyKey - Optional idempotency key for the request.
+   * @returns The response from the extract operation.
+   */
+  async asyncExtract(urls, params, idempotencyKey) {
+    const headers = this.prepareHeaders(idempotencyKey);
+    let jsonData = { urls, ...params };
+    let jsonSchema;
+    try {
+      if (params?.schema instanceof zt.ZodType) {
+        jsonSchema = (0, import_zod_to_json_schema.zodToJsonSchema)(params.schema);
+      } else {
+        jsonSchema = params?.schema;
+      }
+    } catch (error) {
+      throw new FirecrawlError("Invalid schema. Schema must be either a valid Zod schema or JSON schema object.", 400);
+    }
+    try {
+      const response = await this.postRequest(
+        this.apiUrl + `/v1/extract`,
+        { ...jsonData, schema: jsonSchema },
+        headers
+      );
+      if (response.status === 200) {
+        return response.data;
+      } else {
+        this.handleError(response, "start extract job");
+      }
+    } catch (error) {
+      throw new FirecrawlError(error.message, 500);
+    }
+    return { success: false, error: "Internal server error." };
+  }
+  /**
+   * Retrieves the status of an extract job.
+   * @param jobId - The ID of the extract job.
+   * @returns The status of the extract job.
+   */
+  async getExtractStatus(jobId) {
+    try {
+      const response = await this.getRequest(
+        `${this.apiUrl}/v1/extract/${jobId}`,
+        this.prepareHeaders()
+      );
+      if (response.status === 200) {
+        return response.data;
+      } else {
+        this.handleError(response, "get extract status");
+      }
+    } catch (error) {
+      throw new FirecrawlError(error.message, 500);
+    }
+  }
   /**
    * Prepares the headers for an API request.
    * @param idempotencyKey - Optional key to ensure idempotency.
@@ -609,6 +702,9 @@ var FirecrawlApp = class {
             if ("data" in statusData) {
               let data = statusData.data;
               while (typeof statusData === "object" && "next" in statusData) {
+                if (data.length === 0) {
+                  break;
+                }
                 statusResponse = await this.getRequest(statusData.next, headers);
                 statusData = statusResponse.data;
                 data = data.concat(statusData.data);

package/dist/index.d.cts

@@ -384,6 +384,20 @@ declare class FirecrawlApp {
      * @returns The response from the extract operation.
      */
     extract<T extends zt.ZodSchema = any>(urls: string[], params?: ExtractParams<T>): Promise<ExtractResponse<zt.infer<T>> | ErrorResponse>;
+    /**
+     * Initiates an asynchronous extract job for a URL using the Firecrawl API.
+     * @param url - The URL to extract data from.
+     * @param params - Additional parameters for the extract request.
+     * @param idempotencyKey - Optional idempotency key for the request.
+     * @returns The response from the extract operation.
+     */
+    asyncExtract(urls: string[], params?: ExtractParams, idempotencyKey?: string): Promise<ExtractResponse | ErrorResponse>;
+    /**
+     * Retrieves the status of an extract job.
+     * @param jobId - The ID of the extract job.
+     * @returns The status of the extract job.
+     */
+    getExtractStatus(jobId: string): Promise<any>;
     /**
      * Prepares the headers for an API request.
      * @param idempotencyKey - Optional key to ensure idempotency.

package/dist/index.d.ts

@@ -384,6 +384,20 @@ declare class FirecrawlApp {
      * @returns The response from the extract operation.
      */
     extract<T extends zt.ZodSchema = any>(urls: string[], params?: ExtractParams<T>): Promise<ExtractResponse<zt.infer<T>> | ErrorResponse>;
+    /**
+     * Initiates an asynchronous extract job for a URL using the Firecrawl API.
+     * @param url - The URL to extract data from.
+     * @param params - Additional parameters for the extract request.
+     * @param idempotencyKey - Optional idempotency key for the request.
+     * @returns The response from the extract operation.
+     */
+    asyncExtract(urls: string[], params?: ExtractParams, idempotencyKey?: string): Promise<ExtractResponse | ErrorResponse>;
+    /**
+     * Retrieves the status of an extract job.
+     * @param jobId - The ID of the extract job.
+     * @returns The status of the extract job.
+     */
+    getExtractStatus(jobId: string): Promise<any>;
     /**
      * Prepares the headers for an API request.
      * @param idempotencyKey - Optional key to ensure idempotency.

package/dist/index.js

@@ -228,23 +228,35 @@ var FirecrawlApp = class {
           if ("data" in statusData) {
             let data = statusData.data;
             while (typeof statusData === "object" && "next" in statusData) {
+              if (data.length === 0) {
+                break;
+              }
               statusData = (await this.getRequest(statusData.next, headers)).data;
               data = data.concat(statusData.data);
             }
             allData = data;
           }
         }
-        return {
+        let resp = {
           success: response.data.success,
           status: response.data.status,
           total: response.data.total,
           completed: response.data.completed,
           creditsUsed: response.data.creditsUsed,
           expiresAt: new Date(response.data.expiresAt),
-          next: response.data.next,
-          data: allData,
-          error: response.data.error
+          data: allData
         };
+        if (!response.data.success && response.data.error) {
+          resp = {
+            ...resp,
+            success: false,
+            error: response.data.error
+          };
+        }
+        if (response.data.next) {
+          resp.next = response.data.next;
+        }
+        return resp;
       } else {
         this.handleError(response, "check crawl status");
       }
@@ -423,23 +435,35 @@ var FirecrawlApp = class {
           if ("data" in statusData) {
             let data = statusData.data;
             while (typeof statusData === "object" && "next" in statusData) {
+              if (data.length === 0) {
+                break;
+              }
               statusData = (await this.getRequest(statusData.next, headers)).data;
               data = data.concat(statusData.data);
             }
             allData = data;
           }
         }
-        return {
+        let resp = {
           success: response.data.success,
           status: response.data.status,
           total: response.data.total,
           completed: response.data.completed,
           creditsUsed: response.data.creditsUsed,
           expiresAt: new Date(response.data.expiresAt),
-          next: response.data.next,
-          data: allData,
-          error: response.data.error
+          data: allData
         };
+        if (!response.data.success && response.data.error) {
+          resp = {
+            ...resp,
+            success: false,
+            error: response.data.error
+          };
+        }
+        if (response.data.next) {
+          resp.next = response.data.next;
+        }
+        return resp;
       } else {
         this.handleError(response, "check batch scrape status");
       }
@@ -473,21 +497,34 @@ var FirecrawlApp = class {
     try {
       const response = await this.postRequest(
         this.apiUrl + `/v1/extract`,
-        { ...jsonData, schema: jsonSchema },
+        { ...jsonData, schema: jsonSchema, origin: "api-sdk" },
         headers
       );
       if (response.status === 200) {
-        const responseData = response.data;
-        if (responseData.success) {
-          return {
-            success: true,
-            data: responseData.data,
-            warning: responseData.warning,
-            error: responseData.error
-          };
-        } else {
-          throw new FirecrawlError(`Failed to scrape URL. Error: ${responseData.error}`, response.status);
-        }
+        const jobId = response.data.id;
+        let extractStatus;
+        do {
+          const statusResponse = await this.getRequest(
+            `${this.apiUrl}/v1/extract/${jobId}`,
+            headers
+          );
+          extractStatus = statusResponse.data;
+          if (extractStatus.status === "completed") {
+            if (extractStatus.success) {
+              return {
+                success: true,
+                data: extractStatus.data,
+                warning: extractStatus.warning,
+                error: extractStatus.error
+              };
+            } else {
+              throw new FirecrawlError(`Failed to extract data. Error: ${extractStatus.error}`, statusResponse.status);
+            }
+          } else if (extractStatus.status === "failed" || extractStatus.status === "cancelled") {
+            throw new FirecrawlError(`Extract job ${extractStatus.status}. Error: ${extractStatus.error}`, statusResponse.status);
+          }
+          await new Promise((resolve) => setTimeout(resolve, 1e3));
+        } while (extractStatus.status !== "completed");
       } else {
         this.handleError(response, "extract");
       }
@@ -496,6 +533,62 @@ var FirecrawlApp = class {
     }
     return { success: false, error: "Internal server error." };
   }
+  /**
+   * Initiates an asynchronous extract job for a URL using the Firecrawl API.
+   * @param url - The URL to extract data from.
+   * @param params - Additional parameters for the extract request.
+   * @param idempotencyKey - Optional idempotency key for the request.
+   * @returns The response from the extract operation.
+   */
+  async asyncExtract(urls, params, idempotencyKey) {
+    const headers = this.prepareHeaders(idempotencyKey);
+    let jsonData = { urls, ...params };
+    let jsonSchema;
+    try {
+      if (params?.schema instanceof zt.ZodType) {
+        jsonSchema = zodToJsonSchema(params.schema);
+      } else {
+        jsonSchema = params?.schema;
+      }
+    } catch (error) {
+      throw new FirecrawlError("Invalid schema. Schema must be either a valid Zod schema or JSON schema object.", 400);
+    }
+    try {
+      const response = await this.postRequest(
+        this.apiUrl + `/v1/extract`,
+        { ...jsonData, schema: jsonSchema },
+        headers
+      );
+      if (response.status === 200) {
+        return response.data;
+      } else {
+        this.handleError(response, "start extract job");
+      }
+    } catch (error) {
+      throw new FirecrawlError(error.message, 500);
+    }
+    return { success: false, error: "Internal server error." };
+  }
+  /**
+   * Retrieves the status of an extract job.
+   * @param jobId - The ID of the extract job.
+   * @returns The status of the extract job.
+   */
+  async getExtractStatus(jobId) {
+    try {
+      const response = await this.getRequest(
+        `${this.apiUrl}/v1/extract/${jobId}`,
+        this.prepareHeaders()
+      );
+      if (response.status === 200) {
+        return response.data;
+      } else {
+        this.handleError(response, "get extract status");
+      }
+    } catch (error) {
+      throw new FirecrawlError(error.message, 500);
+    }
+  }
   /**
    * Prepares the headers for an API request.
    * @param idempotencyKey - Optional key to ensure idempotency.
@@ -573,6 +666,9 @@ var FirecrawlApp = class {
             if ("data" in statusData) {
               let data = statusData.data;
               while (typeof statusData === "object" && "next" in statusData) {
+                if (data.length === 0) {
+                  break;
+                }
                 statusResponse = await this.getRequest(statusData.next, headers);
                 statusData = statusResponse.data;
                 data = data.concat(statusData.data);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "firecrawl",
-  "version": "1.11.2",
+  "version": "1.13.0",
   "description": "JavaScript SDK for Firecrawl API",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

package/src/index.ts

@@ -565,23 +565,39 @@ export default class FirecrawlApp {
           if ("data" in statusData) {
             let data = statusData.data;
             while (typeof statusData === 'object' && 'next' in statusData) {
+              if (data.length === 0) {
+                break
+              }
               statusData = (await this.getRequest(statusData.next, headers)).data;
               data = data.concat(statusData.data);
             }
             allData = data;
           }
         }
-        return ({
+
+        let resp: CrawlStatusResponse | ErrorResponse = {
           success: response.data.success,
           status: response.data.status,
           total: response.data.total,
           completed: response.data.completed,
           creditsUsed: response.data.creditsUsed,
           expiresAt: new Date(response.data.expiresAt),
-          next: response.data.next,
-          data: allData,
-          error: response.data.error,
-        })
+          data: allData
+        }
+
+        if (!response.data.success && response.data.error) {
+          resp = {
+            ...resp,
+            success: false,
+            error: response.data.error
+          } as ErrorResponse;
+        }
+
+        if (response.data.next) {
+          (resp as CrawlStatusResponse).next = response.data.next;
+        }
+        
+        return resp;
       } else {
         this.handleError(response, "check crawl status");
       }
@@ -799,23 +815,39 @@ export default class FirecrawlApp {
           if ("data" in statusData) {
             let data = statusData.data;
             while (typeof statusData === 'object' && 'next' in statusData) {
+              if (data.length === 0) {
+                break
+              }
               statusData = (await this.getRequest(statusData.next, headers)).data;
               data = data.concat(statusData.data);
             }
             allData = data;
           }
         }
-        return ({
+
+        let resp: BatchScrapeStatusResponse | ErrorResponse = {
           success: response.data.success,
           status: response.data.status,
           total: response.data.total,
           completed: response.data.completed,
           creditsUsed: response.data.creditsUsed,
           expiresAt: new Date(response.data.expiresAt),
-          next: response.data.next,
-          data: allData,
-          error: response.data.error,
-        })
+          data: allData
+        }
+
+        if (!response.data.success && response.data.error) {
+          resp = {
+            ...resp,
+            success: false,
+            error: response.data.error
+          } as ErrorResponse;
+        }
+
+        if (response.data.next) {
+          (resp as BatchScrapeStatusResponse).next = response.data.next;
+        }
+        
+        return resp;
       } else {
         this.handleError(response, "check batch scrape status");
       }
@@ -852,21 +884,35 @@ export default class FirecrawlApp {
     try {
       const response: AxiosResponse = await this.postRequest(
         this.apiUrl + `/v1/extract`,
-        { ...jsonData, schema: jsonSchema },
+        { ...jsonData, schema: jsonSchema, origin: "api-sdk" },
         headers
       );
+
       if (response.status === 200) {
-        const responseData = response.data as ExtractResponse<T>;
-        if (responseData.success) {
-          return {
-            success: true,
-            data: responseData.data,
-            warning: responseData.warning,
-            error: responseData.error
-          };
-        } else {
-          throw new FirecrawlError(`Failed to scrape URL. Error: ${responseData.error}`, response.status);
-        }
+        const jobId = response.data.id;
+        let extractStatus;
+        do {
+          const statusResponse: AxiosResponse = await this.getRequest(
+            `${this.apiUrl}/v1/extract/${jobId}`,
+            headers
+          );
+          extractStatus = statusResponse.data;
+          if (extractStatus.status === "completed") {
+            if (extractStatus.success) {
+              return {
+                success: true,
+                data: extractStatus.data,
+                warning: extractStatus.warning,
+                error: extractStatus.error
+              };
+            } else {
+              throw new FirecrawlError(`Failed to extract data. Error: ${extractStatus.error}`, statusResponse.status);
+            }
+          } else if (extractStatus.status === "failed" || extractStatus.status === "cancelled") {
+            throw new FirecrawlError(`Extract job ${extractStatus.status}. Error: ${extractStatus.error}`, statusResponse.status);
+          }
+          await new Promise(resolve => setTimeout(resolve, 1000)); // Polling interval
+        } while (extractStatus.status !== "completed");
       } else {
         this.handleError(response, "extract");
       }
@@ -876,6 +922,72 @@ export default class FirecrawlApp {
     return { success: false, error: "Internal server error." };
   }
 
+  /**
+   * Initiates an asynchronous extract job for a URL using the Firecrawl API.
+   * @param url - The URL to extract data from.
+   * @param params - Additional parameters for the extract request.
+   * @param idempotencyKey - Optional idempotency key for the request.
+   * @returns The response from the extract operation.
+   */
+  async asyncExtract(
+    urls: string[],
+    params?: ExtractParams,
+    idempotencyKey?: string
+  ): Promise<ExtractResponse | ErrorResponse> {
+    const headers = this.prepareHeaders(idempotencyKey);
+    let jsonData: any = { urls, ...params };
+    let jsonSchema: any;
+
+    try {
+      if (params?.schema instanceof zt.ZodType) {
+        jsonSchema = zodToJsonSchema(params.schema);
+      } else {
+        jsonSchema = params?.schema;
+      }
+    } catch (error: any) {
+      throw new FirecrawlError("Invalid schema. Schema must be either a valid Zod schema or JSON schema object.", 400);
+    }
+
+    try {
+      const response: AxiosResponse = await this.postRequest(
+        this.apiUrl + `/v1/extract`,
+        { ...jsonData, schema: jsonSchema },
+        headers
+      );
+
+      if (response.status === 200) {
+        return response.data;
+      } else {
+        this.handleError(response, "start extract job");
+      }
+    } catch (error: any) {
+      throw new FirecrawlError(error.message, 500);
+    }
+    return { success: false, error: "Internal server error." };
+  }
+
+  /**
+   * Retrieves the status of an extract job.
+   * @param jobId - The ID of the extract job.
+   * @returns The status of the extract job.
+   */
+  async getExtractStatus(jobId: string): Promise<any> {
+    try {
+      const response: AxiosResponse = await this.getRequest(
+        `${this.apiUrl}/v1/extract/${jobId}`,
+        this.prepareHeaders()
+      );
+
+      if (response.status === 200) {
+        return response.data;
+      } else {
+        this.handleError(response, "get extract status");
+      }
+    } catch (error: any) {
+      throw new FirecrawlError(error.message, 500);
+    }
+  }
+
   /**
    * Prepares the headers for an API request.
    * @param idempotencyKey - Optional key to ensure idempotency.
@@ -971,6 +1083,9 @@ export default class FirecrawlApp {
               if ("data" in statusData) {
                 let data = statusData.data;
                 while (typeof statusData === 'object' && 'next' in statusData) {
+                  if (data.length === 0) {
+                    break
+                  }
                   statusResponse = await this.getRequest(statusData.next, headers);
                   statusData = statusResponse.data;
                   data = data.concat(statusData.data);