@mendable/firecrawl 1.19.1 → 1.20.1

package/dist/index.cjs

@@ -607,7 +607,7 @@ var FirecrawlApp = class {
   /**
    * Extracts information from URLs using the Firecrawl API.
    * Currently in Beta. Expect breaking changes on future minor versions.
-   * @param url - The URL to extract information from.
+   * @param urls - The URLs to extract information from. Optional if using other methods for data extraction.
    * @param params - Additional parameters for the extract request.
    * @returns The response from the extract operation.
    */
@@ -788,12 +788,14 @@ var FirecrawlApp = class {
    */
   async monitorJobStatus(id, headers, checkInterval) {
     try {
+      let failedTries = 0;
       while (true) {
         let statusResponse = await this.getRequest(
           `${this.apiUrl}/v1/crawl/${id}`,
           headers
         );
         if (statusResponse.status === 200) {
+          failedTries = 0;
           let statusData = statusResponse.data;
           if (statusData.status === "completed") {
             if ("data" in statusData) {
@@ -823,7 +825,10 @@ var FirecrawlApp = class {
             );
           }
         } else {
-          this.handleError(statusResponse, "check crawl status");
+          failedTries++;
+          if (failedTries >= 3) {
+            this.handleError(statusResponse, "check crawl status");
+          }
         }
       }
     } catch (error) {
@@ -836,7 +841,7 @@ var FirecrawlApp = class {
    * @param {string} action - The action being performed when the error occurred.
    */
   handleError(response, action) {
-    if ([400, 402, 408, 409, 500].includes(response.status)) {
+    if ([400, 402, 403, 408, 409, 500].includes(response.status)) {
       const errorMessage = response.data.error || "Unknown error occurred";
       const details = response.data.details ? ` - ${JSON.stringify(response.data.details)}` : "";
       throw new FirecrawlError(

package/dist/index.d.cts

@@ -97,6 +97,7 @@ type Action = {
 } | {
     type: "click";
     selector: string;
+    all?: boolean;
 } | {
     type: "screenshot";
     fullPage?: boolean;
@@ -149,6 +150,7 @@ interface CrawlParams {
     includePaths?: string[];
     excludePaths?: string[];
     maxDepth?: number;
+    maxDiscoveryDepth?: number;
     limit?: number;
     allowBackwardLinks?: boolean;
     allowExternalLinks?: boolean;
@@ -542,11 +544,11 @@ declare class FirecrawlApp {
     /**
      * Extracts information from URLs using the Firecrawl API.
      * Currently in Beta. Expect breaking changes on future minor versions.
-     * @param url - The URL to extract information from.
+     * @param urls - The URLs to extract information from. Optional if using other methods for data extraction.
      * @param params - Additional parameters for the extract request.
      * @returns The response from the extract operation.
      */
-    extract<T extends zt.ZodSchema = any>(urls: string[], params?: ExtractParams<T>): Promise<ExtractResponse<zt.infer<T>> | ErrorResponse>;
+    extract<T extends zt.ZodSchema = any>(urls?: string[], params?: ExtractParams<T>): Promise<ExtractResponse<zt.infer<T>> | ErrorResponse>;
     /**
      * Initiates an asynchronous extract job for a URL using the Firecrawl API.
      * @param url - The URL to extract data from.

package/dist/index.d.ts

@@ -97,6 +97,7 @@ type Action = {
 } | {
     type: "click";
     selector: string;
+    all?: boolean;
 } | {
     type: "screenshot";
     fullPage?: boolean;
@@ -149,6 +150,7 @@ interface CrawlParams {
     includePaths?: string[];
     excludePaths?: string[];
     maxDepth?: number;
+    maxDiscoveryDepth?: number;
     limit?: number;
     allowBackwardLinks?: boolean;
     allowExternalLinks?: boolean;
@@ -542,11 +544,11 @@ declare class FirecrawlApp {
     /**
      * Extracts information from URLs using the Firecrawl API.
      * Currently in Beta. Expect breaking changes on future minor versions.
-     * @param url - The URL to extract information from.
+     * @param urls - The URLs to extract information from. Optional if using other methods for data extraction.
      * @param params - Additional parameters for the extract request.
      * @returns The response from the extract operation.
      */
-    extract<T extends zt.ZodSchema = any>(urls: string[], params?: ExtractParams<T>): Promise<ExtractResponse<zt.infer<T>> | ErrorResponse>;
+    extract<T extends zt.ZodSchema = any>(urls?: string[], params?: ExtractParams<T>): Promise<ExtractResponse<zt.infer<T>> | ErrorResponse>;
     /**
      * Initiates an asynchronous extract job for a URL using the Firecrawl API.
      * @param url - The URL to extract data from.

package/dist/index.js

@@ -571,7 +571,7 @@ var FirecrawlApp = class {
   /**
    * Extracts information from URLs using the Firecrawl API.
    * Currently in Beta. Expect breaking changes on future minor versions.
-   * @param url - The URL to extract information from.
+   * @param urls - The URLs to extract information from. Optional if using other methods for data extraction.
    * @param params - Additional parameters for the extract request.
    * @returns The response from the extract operation.
    */
@@ -752,12 +752,14 @@ var FirecrawlApp = class {
    */
   async monitorJobStatus(id, headers, checkInterval) {
     try {
+      let failedTries = 0;
       while (true) {
         let statusResponse = await this.getRequest(
           `${this.apiUrl}/v1/crawl/${id}`,
           headers
         );
         if (statusResponse.status === 200) {
+          failedTries = 0;
           let statusData = statusResponse.data;
           if (statusData.status === "completed") {
             if ("data" in statusData) {
@@ -787,7 +789,10 @@ var FirecrawlApp = class {
             );
           }
         } else {
-          this.handleError(statusResponse, "check crawl status");
+          failedTries++;
+          if (failedTries >= 3) {
+            this.handleError(statusResponse, "check crawl status");
+          }
         }
       }
     } catch (error) {
@@ -800,7 +805,7 @@ var FirecrawlApp = class {
    * @param {string} action - The action being performed when the error occurred.
    */
   handleError(response, action) {
-    if ([400, 402, 408, 409, 500].includes(response.status)) {
+    if ([400, 402, 403, 408, 409, 500].includes(response.status)) {
       const errorMessage = response.data.error || "Unknown error occurred";
       const details = response.data.details ? ` - ${JSON.stringify(response.data.details)}` : "";
       throw new FirecrawlError(

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mendable/firecrawl",
-  "version": "1.19.1",
+  "version": "1.20.1",
   "description": "JavaScript SDK for Firecrawl API",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

package/src/index.ts

@@ -104,6 +104,7 @@ export type Action = {
 } | {
   type: "click",
   selector: string,
+  all?: boolean,
 } | {
   type: "screenshot",
   fullPage?: boolean,
@@ -160,6 +161,7 @@ export interface CrawlParams {
   includePaths?: string[];
   excludePaths?: string[];
   maxDepth?: number;
+  maxDiscoveryDepth?: number;
   limit?: number;
   allowBackwardLinks?: boolean;
   allowExternalLinks?: boolean;
@@ -1118,14 +1120,14 @@ export default class FirecrawlApp {
   /**
    * Extracts information from URLs using the Firecrawl API.
    * Currently in Beta. Expect breaking changes on future minor versions.
-   * @param url - The URL to extract information from.
+   * @param urls - The URLs to extract information from. Optional if using other methods for data extraction.
    * @param params - Additional parameters for the extract request.
    * @returns The response from the extract operation.
    */
-  async extract<T extends zt.ZodSchema = any>(urls: string[], params?: ExtractParams<T>): Promise<ExtractResponse<zt.infer<T>> | ErrorResponse> {
+  async extract<T extends zt.ZodSchema = any>(urls?: string[], params?: ExtractParams<T>): Promise<ExtractResponse<zt.infer<T>> | ErrorResponse> {
     const headers = this.prepareHeaders();
 
-    let jsonData: { urls: string[] } & ExtractParams<T> = { urls,  ...params };
+    let jsonData: { urls?: string[] } & ExtractParams<T> = { urls: urls,  ...params };
     let jsonSchema: any;
     try {
       if (!params?.schema) {
@@ -1331,12 +1333,14 @@ export default class FirecrawlApp {
     checkInterval: number
   ): Promise<CrawlStatusResponse | ErrorResponse> {
     try {
+      let failedTries = 0;
       while (true) {
         let statusResponse: AxiosResponse = await this.getRequest(
           `${this.apiUrl}/v1/crawl/${id}`,
           headers
         );
         if (statusResponse.status === 200) {
+          failedTries = 0;
           let statusData = statusResponse.data;
             if (statusData.status === "completed") {
               if ("data" in statusData) {
@@ -1368,7 +1372,10 @@ export default class FirecrawlApp {
             );
           }
         } else {
-          this.handleError(statusResponse, "check crawl status");
+          failedTries++;
+          if (failedTries >= 3) {
+            this.handleError(statusResponse, "check crawl status");
+          }
         }
       }
     } catch (error: any) {
@@ -1382,7 +1389,7 @@ export default class FirecrawlApp {
    * @param {string} action - The action being performed when the error occurred.
    */
   handleError(response: AxiosResponse, action: string): void {
-    if ([400, 402, 408, 409, 500].includes(response.status)) {
+    if ([400, 402, 403, 408, 409, 500].includes(response.status)) {
       const errorMessage: string =
         response.data.error || "Unknown error occurred";
       const details = response.data.details ? ` - ${JSON.stringify(response.data.details)}` : '';