firecrawl 1.19.1 → 1.20.0

package/dist/index.cjs

@@ -607,7 +607,7 @@ var FirecrawlApp = class {
   /**
    * Extracts information from URLs using the Firecrawl API.
    * Currently in Beta. Expect breaking changes on future minor versions.
-   * @param url - The URL to extract information from.
+   * @param urls - The URLs to extract information from. Optional if using other methods for data extraction.
    * @param params - Additional parameters for the extract request.
    * @returns The response from the extract operation.
    */
@@ -788,12 +788,14 @@ var FirecrawlApp = class {
    */
   async monitorJobStatus(id, headers, checkInterval) {
     try {
+      let failedTries = 0;
       while (true) {
         let statusResponse = await this.getRequest(
           `${this.apiUrl}/v1/crawl/${id}`,
           headers
         );
         if (statusResponse.status === 200) {
+          failedTries = 0;
           let statusData = statusResponse.data;
           if (statusData.status === "completed") {
             if ("data" in statusData) {
@@ -823,7 +825,10 @@ var FirecrawlApp = class {
             );
           }
         } else {
-          this.handleError(statusResponse, "check crawl status");
+          failedTries++;
+          if (failedTries >= 3) {
+            this.handleError(statusResponse, "check crawl status");
+          }
         }
       }
     } catch (error) {

package/dist/index.d.cts

@@ -149,6 +149,7 @@ interface CrawlParams {
     includePaths?: string[];
     excludePaths?: string[];
     maxDepth?: number;
+    maxDiscoveryDepth?: number;
     limit?: number;
     allowBackwardLinks?: boolean;
     allowExternalLinks?: boolean;
@@ -542,11 +543,11 @@ declare class FirecrawlApp {
     /**
      * Extracts information from URLs using the Firecrawl API.
      * Currently in Beta. Expect breaking changes on future minor versions.
-     * @param url - The URL to extract information from.
+     * @param urls - The URLs to extract information from. Optional if using other methods for data extraction.
      * @param params - Additional parameters for the extract request.
      * @returns The response from the extract operation.
      */
-    extract<T extends zt.ZodSchema = any>(urls: string[], params?: ExtractParams<T>): Promise<ExtractResponse<zt.infer<T>> | ErrorResponse>;
+    extract<T extends zt.ZodSchema = any>(urls?: string[], params?: ExtractParams<T>): Promise<ExtractResponse<zt.infer<T>> | ErrorResponse>;
     /**
      * Initiates an asynchronous extract job for a URL using the Firecrawl API.
      * @param url - The URL to extract data from.

package/dist/index.d.ts

@@ -149,6 +149,7 @@ interface CrawlParams {
     includePaths?: string[];
     excludePaths?: string[];
     maxDepth?: number;
+    maxDiscoveryDepth?: number;
     limit?: number;
     allowBackwardLinks?: boolean;
     allowExternalLinks?: boolean;
@@ -542,11 +543,11 @@ declare class FirecrawlApp {
     /**
      * Extracts information from URLs using the Firecrawl API.
      * Currently in Beta. Expect breaking changes on future minor versions.
-     * @param url - The URL to extract information from.
+     * @param urls - The URLs to extract information from. Optional if using other methods for data extraction.
      * @param params - Additional parameters for the extract request.
      * @returns The response from the extract operation.
      */
-    extract<T extends zt.ZodSchema = any>(urls: string[], params?: ExtractParams<T>): Promise<ExtractResponse<zt.infer<T>> | ErrorResponse>;
+    extract<T extends zt.ZodSchema = any>(urls?: string[], params?: ExtractParams<T>): Promise<ExtractResponse<zt.infer<T>> | ErrorResponse>;
     /**
      * Initiates an asynchronous extract job for a URL using the Firecrawl API.
      * @param url - The URL to extract data from.

package/dist/index.js

@@ -571,7 +571,7 @@ var FirecrawlApp = class {
   /**
    * Extracts information from URLs using the Firecrawl API.
    * Currently in Beta. Expect breaking changes on future minor versions.
-   * @param url - The URL to extract information from.
+   * @param urls - The URLs to extract information from. Optional if using other methods for data extraction.
    * @param params - Additional parameters for the extract request.
    * @returns The response from the extract operation.
    */
@@ -752,12 +752,14 @@ var FirecrawlApp = class {
    */
   async monitorJobStatus(id, headers, checkInterval) {
     try {
+      let failedTries = 0;
       while (true) {
         let statusResponse = await this.getRequest(
           `${this.apiUrl}/v1/crawl/${id}`,
           headers
         );
         if (statusResponse.status === 200) {
+          failedTries = 0;
           let statusData = statusResponse.data;
           if (statusData.status === "completed") {
             if ("data" in statusData) {
@@ -787,7 +789,10 @@ var FirecrawlApp = class {
             );
           }
         } else {
-          this.handleError(statusResponse, "check crawl status");
+          failedTries++;
+          if (failedTries >= 3) {
+            this.handleError(statusResponse, "check crawl status");
+          }
         }
       }
     } catch (error) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "firecrawl",
-  "version": "1.19.1",
+  "version": "1.20.0",
   "description": "JavaScript SDK for Firecrawl API",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

package/src/index.ts

@@ -160,6 +160,7 @@ export interface CrawlParams {
   includePaths?: string[];
   excludePaths?: string[];
   maxDepth?: number;
+  maxDiscoveryDepth?: number;
   limit?: number;
   allowBackwardLinks?: boolean;
   allowExternalLinks?: boolean;
@@ -1118,14 +1119,14 @@ export default class FirecrawlApp {
   /**
    * Extracts information from URLs using the Firecrawl API.
    * Currently in Beta. Expect breaking changes on future minor versions.
-   * @param url - The URL to extract information from.
+   * @param urls - The URLs to extract information from. Optional if using other methods for data extraction.
    * @param params - Additional parameters for the extract request.
    * @returns The response from the extract operation.
    */
-  async extract<T extends zt.ZodSchema = any>(urls: string[], params?: ExtractParams<T>): Promise<ExtractResponse<zt.infer<T>> | ErrorResponse> {
+  async extract<T extends zt.ZodSchema = any>(urls?: string[], params?: ExtractParams<T>): Promise<ExtractResponse<zt.infer<T>> | ErrorResponse> {
     const headers = this.prepareHeaders();
 
-    let jsonData: { urls: string[] } & ExtractParams<T> = { urls,  ...params };
+    let jsonData: { urls?: string[] } & ExtractParams<T> = { urls: urls,  ...params };
     let jsonSchema: any;
     try {
       if (!params?.schema) {
@@ -1331,12 +1332,14 @@ export default class FirecrawlApp {
     checkInterval: number
   ): Promise<CrawlStatusResponse | ErrorResponse> {
     try {
+      let failedTries = 0;
       while (true) {
         let statusResponse: AxiosResponse = await this.getRequest(
           `${this.apiUrl}/v1/crawl/${id}`,
           headers
         );
         if (statusResponse.status === 200) {
+          failedTries = 0;
           let statusData = statusResponse.data;
             if (statusData.status === "completed") {
               if ("data" in statusData) {
@@ -1368,7 +1371,10 @@ export default class FirecrawlApp {
             );
           }
         } else {
-          this.handleError(statusResponse, "check crawl status");
+          failedTries++;
+          if (failedTries >= 3) {
+            this.handleError(statusResponse, "check crawl status");
+          }
         }
       }
     } catch (error: any) {