firecrawl 1.11.1 → 1.12.0

package/dist/index.cjs

@@ -36,6 +36,7 @@ __export(src_exports, {
 });
 module.exports = __toCommonJS(src_exports);
 var import_axios = __toESM(require("axios"), 1);
+var zt = __toESM(require("zod"), 1);
 var import_zod_to_json_schema = require("zod-to-json-schema");
 var import_isows = require("isows");
 var import_typescript_event_target = require("typescript-event-target");
@@ -263,23 +264,35 @@ var FirecrawlApp = class {
           if ("data" in statusData) {
             let data = statusData.data;
             while (typeof statusData === "object" && "next" in statusData) {
+              if (data.length === 0) {
+                break;
+              }
               statusData = (await this.getRequest(statusData.next, headers)).data;
               data = data.concat(statusData.data);
             }
             allData = data;
           }
         }
-        return {
+        let resp = {
           success: response.data.success,
           status: response.data.status,
           total: response.data.total,
           completed: response.data.completed,
           creditsUsed: response.data.creditsUsed,
           expiresAt: new Date(response.data.expiresAt),
-          next: response.data.next,
-          data: allData,
-          error: response.data.error
+          data: allData
         };
+        if (!response.data.success && response.data.error) {
+          resp = {
+            ...resp,
+            success: false,
+            error: response.data.error
+          };
+        }
+        if (response.data.next) {
+          resp.next = response.data.next;
+        }
+        return resp;
       } else {
         this.handleError(response, "check crawl status");
       }
@@ -458,23 +471,35 @@ var FirecrawlApp = class {
           if ("data" in statusData) {
             let data = statusData.data;
             while (typeof statusData === "object" && "next" in statusData) {
+              if (data.length === 0) {
+                break;
+              }
               statusData = (await this.getRequest(statusData.next, headers)).data;
               data = data.concat(statusData.data);
             }
             allData = data;
           }
         }
-        return {
+        let resp = {
           success: response.data.success,
           status: response.data.status,
           total: response.data.total,
           completed: response.data.completed,
           creditsUsed: response.data.creditsUsed,
           expiresAt: new Date(response.data.expiresAt),
-          next: response.data.next,
-          data: allData,
-          error: response.data.error
+          data: allData
         };
+        if (!response.data.success && response.data.error) {
+          resp = {
+            ...resp,
+            success: false,
+            error: response.data.error
+          };
+        }
+        if (response.data.next) {
+          resp.next = response.data.next;
+        }
+        return resp;
       } else {
         this.handleError(response, "check batch scrape status");
       }
@@ -495,28 +520,47 @@ var FirecrawlApp = class {
     let jsonData = { urls, ...params };
     let jsonSchema;
     try {
-      jsonSchema = params?.schema ? (0, import_zod_to_json_schema.zodToJsonSchema)(params.schema) : void 0;
+      if (!params?.schema) {
+        jsonSchema = void 0;
+      } else if (params.schema instanceof zt.ZodType) {
+        jsonSchema = (0, import_zod_to_json_schema.zodToJsonSchema)(params.schema);
+      } else {
+        jsonSchema = params.schema;
+      }
     } catch (error) {
-      throw new FirecrawlError("Invalid schema. Use a valid Zod schema.", 400);
+      throw new FirecrawlError("Invalid schema. Schema must be either a valid Zod schema or JSON schema object.", 400);
     }
     try {
       const response = await this.postRequest(
         this.apiUrl + `/v1/extract`,
-        { ...jsonData, schema: jsonSchema },
+        { ...jsonData, schema: jsonSchema, origin: "api-sdk" },
         headers
       );
       if (response.status === 200) {
-        const responseData = response.data;
-        if (responseData.success) {
-          return {
-            success: true,
-            data: responseData.data,
-            warning: responseData.warning,
-            error: responseData.error
-          };
-        } else {
-          throw new FirecrawlError(`Failed to scrape URL. Error: ${responseData.error}`, response.status);
-        }
+        const jobId = response.data.id;
+        let extractStatus;
+        do {
+          const statusResponse = await this.getRequest(
+            `${this.apiUrl}/v1/extract/${jobId}`,
+            headers
+          );
+          extractStatus = statusResponse.data;
+          if (extractStatus.status === "completed") {
+            if (extractStatus.success) {
+              return {
+                success: true,
+                data: extractStatus.data,
+                warning: extractStatus.warning,
+                error: extractStatus.error
+              };
+            } else {
+              throw new FirecrawlError(`Failed to extract data. Error: ${extractStatus.error}`, statusResponse.status);
+            }
+          } else if (extractStatus.status === "failed" || extractStatus.status === "cancelled") {
+            throw new FirecrawlError(`Extract job ${extractStatus.status}. Error: ${extractStatus.error}`, statusResponse.status);
+          }
+          await new Promise((resolve) => setTimeout(resolve, 1e3));
+        } while (extractStatus.status !== "completed");
       } else {
         this.handleError(response, "extract");
       }
@@ -525,6 +569,62 @@ var FirecrawlApp = class {
     }
     return { success: false, error: "Internal server error." };
   }
+  /**
+   * Initiates an asynchronous extract job for a URL using the Firecrawl API.
+   * @param url - The URL to extract data from.
+   * @param params - Additional parameters for the extract request.
+   * @param idempotencyKey - Optional idempotency key for the request.
+   * @returns The response from the extract operation.
+   */
+  async asyncExtract(url, params, idempotencyKey) {
+    const headers = this.prepareHeaders(idempotencyKey);
+    let jsonData = { url, ...params };
+    let jsonSchema;
+    try {
+      if (params?.schema instanceof zt.ZodType) {
+        jsonSchema = (0, import_zod_to_json_schema.zodToJsonSchema)(params.schema);
+      } else {
+        jsonSchema = params?.schema;
+      }
+    } catch (error) {
+      throw new FirecrawlError("Invalid schema. Schema must be either a valid Zod schema or JSON schema object.", 400);
+    }
+    try {
+      const response = await this.postRequest(
+        this.apiUrl + `/v1/extract`,
+        { ...jsonData, schema: jsonSchema },
+        headers
+      );
+      if (response.status === 200) {
+        return response.data;
+      } else {
+        this.handleError(response, "start extract job");
+      }
+    } catch (error) {
+      throw new FirecrawlError(error.message, 500);
+    }
+    return { success: false, error: "Internal server error." };
+  }
+  /**
+   * Retrieves the status of an extract job.
+   * @param jobId - The ID of the extract job.
+   * @returns The status of the extract job.
+   */
+  async getExtractStatus(jobId) {
+    try {
+      const response = await this.getRequest(
+        `${this.apiUrl}/v1/extract/${jobId}`,
+        this.prepareHeaders()
+      );
+      if (response.status === 200) {
+        return response.data;
+      } else {
+        this.handleError(response, "get extract status");
+      }
+    } catch (error) {
+      throw new FirecrawlError(error.message, 500);
+    }
+  }
   /**
    * Prepares the headers for an API request.
    * @param idempotencyKey - Optional key to ensure idempotency.
@@ -602,6 +702,9 @@ var FirecrawlApp = class {
             if ("data" in statusData) {
               let data = statusData.data;
               while (typeof statusData === "object" && "next" in statusData) {
+                if (data.length === 0) {
+                  break;
+                }
                 statusResponse = await this.getRequest(statusData.next, headers);
                 statusData = statusResponse.data;
                 data = data.concat(statusData.data);

package/dist/index.d.cts

@@ -229,7 +229,7 @@ interface MapResponse {
  */
 interface ExtractParams<LLMSchema extends zt.ZodSchema = any> {
     prompt?: string;
-    schema?: LLMSchema;
+    schema?: LLMSchema | object;
     systemPrompt?: string;
     allowExternalLinks?: boolean;
     includeSubdomains?: boolean;
@@ -384,6 +384,20 @@ declare class FirecrawlApp {
      * @returns The response from the extract operation.
      */
     extract<T extends zt.ZodSchema = any>(urls: string[], params?: ExtractParams<T>): Promise<ExtractResponse<zt.infer<T>> | ErrorResponse>;
+    /**
+     * Initiates an asynchronous extract job for a URL using the Firecrawl API.
+     * @param url - The URL to extract data from.
+     * @param params - Additional parameters for the extract request.
+     * @param idempotencyKey - Optional idempotency key for the request.
+     * @returns The response from the extract operation.
+     */
+    asyncExtract(url: string, params?: ExtractParams, idempotencyKey?: string): Promise<ExtractResponse | ErrorResponse>;
+    /**
+     * Retrieves the status of an extract job.
+     * @param jobId - The ID of the extract job.
+     * @returns The status of the extract job.
+     */
+    getExtractStatus(jobId: string): Promise<any>;
     /**
      * Prepares the headers for an API request.
      * @param idempotencyKey - Optional key to ensure idempotency.

package/dist/index.d.ts

@@ -229,7 +229,7 @@ interface MapResponse {
  */
 interface ExtractParams<LLMSchema extends zt.ZodSchema = any> {
     prompt?: string;
-    schema?: LLMSchema;
+    schema?: LLMSchema | object;
     systemPrompt?: string;
     allowExternalLinks?: boolean;
     includeSubdomains?: boolean;
@@ -384,6 +384,20 @@ declare class FirecrawlApp {
      * @returns The response from the extract operation.
      */
     extract<T extends zt.ZodSchema = any>(urls: string[], params?: ExtractParams<T>): Promise<ExtractResponse<zt.infer<T>> | ErrorResponse>;
+    /**
+     * Initiates an asynchronous extract job for a URL using the Firecrawl API.
+     * @param url - The URL to extract data from.
+     * @param params - Additional parameters for the extract request.
+     * @param idempotencyKey - Optional idempotency key for the request.
+     * @returns The response from the extract operation.
+     */
+    asyncExtract(url: string, params?: ExtractParams, idempotencyKey?: string): Promise<ExtractResponse | ErrorResponse>;
+    /**
+     * Retrieves the status of an extract job.
+     * @param jobId - The ID of the extract job.
+     * @returns The status of the extract job.
+     */
+    getExtractStatus(jobId: string): Promise<any>;
     /**
      * Prepares the headers for an API request.
      * @param idempotencyKey - Optional key to ensure idempotency.

package/dist/index.js

@@ -1,5 +1,6 @@
 // src/index.ts
 import axios, { AxiosError } from "axios";
+import * as zt from "zod";
 import { zodToJsonSchema } from "zod-to-json-schema";
 import { WebSocket } from "isows";
 import { TypedEventTarget } from "typescript-event-target";
@@ -227,23 +228,35 @@ var FirecrawlApp = class {
           if ("data" in statusData) {
             let data = statusData.data;
             while (typeof statusData === "object" && "next" in statusData) {
+              if (data.length === 0) {
+                break;
+              }
               statusData = (await this.getRequest(statusData.next, headers)).data;
               data = data.concat(statusData.data);
             }
             allData = data;
           }
         }
-        return {
+        let resp = {
           success: response.data.success,
           status: response.data.status,
           total: response.data.total,
           completed: response.data.completed,
           creditsUsed: response.data.creditsUsed,
           expiresAt: new Date(response.data.expiresAt),
-          next: response.data.next,
-          data: allData,
-          error: response.data.error
+          data: allData
         };
+        if (!response.data.success && response.data.error) {
+          resp = {
+            ...resp,
+            success: false,
+            error: response.data.error
+          };
+        }
+        if (response.data.next) {
+          resp.next = response.data.next;
+        }
+        return resp;
       } else {
         this.handleError(response, "check crawl status");
       }
@@ -422,23 +435,35 @@ var FirecrawlApp = class {
           if ("data" in statusData) {
             let data = statusData.data;
             while (typeof statusData === "object" && "next" in statusData) {
+              if (data.length === 0) {
+                break;
+              }
               statusData = (await this.getRequest(statusData.next, headers)).data;
               data = data.concat(statusData.data);
             }
             allData = data;
           }
         }
-        return {
+        let resp = {
           success: response.data.success,
           status: response.data.status,
           total: response.data.total,
           completed: response.data.completed,
           creditsUsed: response.data.creditsUsed,
           expiresAt: new Date(response.data.expiresAt),
-          next: response.data.next,
-          data: allData,
-          error: response.data.error
+          data: allData
         };
+        if (!response.data.success && response.data.error) {
+          resp = {
+            ...resp,
+            success: false,
+            error: response.data.error
+          };
+        }
+        if (response.data.next) {
+          resp.next = response.data.next;
+        }
+        return resp;
       } else {
         this.handleError(response, "check batch scrape status");
       }
@@ -459,28 +484,47 @@ var FirecrawlApp = class {
     let jsonData = { urls, ...params };
     let jsonSchema;
     try {
-      jsonSchema = params?.schema ? zodToJsonSchema(params.schema) : void 0;
+      if (!params?.schema) {
+        jsonSchema = void 0;
+      } else if (params.schema instanceof zt.ZodType) {
+        jsonSchema = zodToJsonSchema(params.schema);
+      } else {
+        jsonSchema = params.schema;
+      }
     } catch (error) {
-      throw new FirecrawlError("Invalid schema. Use a valid Zod schema.", 400);
+      throw new FirecrawlError("Invalid schema. Schema must be either a valid Zod schema or JSON schema object.", 400);
     }
     try {
       const response = await this.postRequest(
         this.apiUrl + `/v1/extract`,
-        { ...jsonData, schema: jsonSchema },
+        { ...jsonData, schema: jsonSchema, origin: "api-sdk" },
         headers
       );
       if (response.status === 200) {
-        const responseData = response.data;
-        if (responseData.success) {
-          return {
-            success: true,
-            data: responseData.data,
-            warning: responseData.warning,
-            error: responseData.error
-          };
-        } else {
-          throw new FirecrawlError(`Failed to scrape URL. Error: ${responseData.error}`, response.status);
-        }
+        const jobId = response.data.id;
+        let extractStatus;
+        do {
+          const statusResponse = await this.getRequest(
+            `${this.apiUrl}/v1/extract/${jobId}`,
+            headers
+          );
+          extractStatus = statusResponse.data;
+          if (extractStatus.status === "completed") {
+            if (extractStatus.success) {
+              return {
+                success: true,
+                data: extractStatus.data,
+                warning: extractStatus.warning,
+                error: extractStatus.error
+              };
+            } else {
+              throw new FirecrawlError(`Failed to extract data. Error: ${extractStatus.error}`, statusResponse.status);
+            }
+          } else if (extractStatus.status === "failed" || extractStatus.status === "cancelled") {
+            throw new FirecrawlError(`Extract job ${extractStatus.status}. Error: ${extractStatus.error}`, statusResponse.status);
+          }
+          await new Promise((resolve) => setTimeout(resolve, 1e3));
+        } while (extractStatus.status !== "completed");
       } else {
         this.handleError(response, "extract");
       }
@@ -489,6 +533,62 @@ var FirecrawlApp = class {
     }
     return { success: false, error: "Internal server error." };
   }
+  /**
+   * Initiates an asynchronous extract job for a URL using the Firecrawl API.
+   * @param url - The URL to extract data from.
+   * @param params - Additional parameters for the extract request.
+   * @param idempotencyKey - Optional idempotency key for the request.
+   * @returns The response from the extract operation.
+   */
+  async asyncExtract(url, params, idempotencyKey) {
+    const headers = this.prepareHeaders(idempotencyKey);
+    let jsonData = { url, ...params };
+    let jsonSchema;
+    try {
+      if (params?.schema instanceof zt.ZodType) {
+        jsonSchema = zodToJsonSchema(params.schema);
+      } else {
+        jsonSchema = params?.schema;
+      }
+    } catch (error) {
+      throw new FirecrawlError("Invalid schema. Schema must be either a valid Zod schema or JSON schema object.", 400);
+    }
+    try {
+      const response = await this.postRequest(
+        this.apiUrl + `/v1/extract`,
+        { ...jsonData, schema: jsonSchema },
+        headers
+      );
+      if (response.status === 200) {
+        return response.data;
+      } else {
+        this.handleError(response, "start extract job");
+      }
+    } catch (error) {
+      throw new FirecrawlError(error.message, 500);
+    }
+    return { success: false, error: "Internal server error." };
+  }
+  /**
+   * Retrieves the status of an extract job.
+   * @param jobId - The ID of the extract job.
+   * @returns The status of the extract job.
+   */
+  async getExtractStatus(jobId) {
+    try {
+      const response = await this.getRequest(
+        `${this.apiUrl}/v1/extract/${jobId}`,
+        this.prepareHeaders()
+      );
+      if (response.status === 200) {
+        return response.data;
+      } else {
+        this.handleError(response, "get extract status");
+      }
+    } catch (error) {
+      throw new FirecrawlError(error.message, 500);
+    }
+  }
   /**
    * Prepares the headers for an API request.
    * @param idempotencyKey - Optional key to ensure idempotency.
@@ -566,6 +666,9 @@ var FirecrawlApp = class {
             if ("data" in statusData) {
               let data = statusData.data;
               while (typeof statusData === "object" && "next" in statusData) {
+                if (data.length === 0) {
+                  break;
+                }
                 statusResponse = await this.getRequest(statusData.next, headers);
                 statusData = statusResponse.data;
                 data = data.concat(statusData.data);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "firecrawl",
-  "version": "1.11.1",
+  "version": "1.12.0",
   "description": "JavaScript SDK for Firecrawl API",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",