@mendable/firecrawl 1.29.2 → 1.29.3

package/dist/index.cjs

@@ -35,7 +35,7 @@ var require_package = __commonJS({
   "package.json"(exports2, module2) {
     module2.exports = {
       name: "@mendable/firecrawl-js",
-      version: "1.29.2",
+      version: "1.29.3",
       description: "JavaScript SDK for Firecrawl API",
       main: "dist/index.js",
       types: "dist/index.d.ts",
@@ -51,7 +51,8 @@ var require_package = __commonJS({
         build: "tsup",
         "build-and-publish": "npm run build && npm publish --access public",
         "publish-beta": "npm run build && npm publish --access public --tag beta",
-        test: "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/v1/**/*.test.ts"
+        test: "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/v1/**/*.test.ts",
+        "test:unit": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/v1/unit/*.test.ts"
       },
       repository: {
         type: "git",
@@ -894,15 +895,18 @@ var FirecrawlApp = class {
    * @returns The final job status or data.
    */
   async monitorJobStatus(id, headers, checkInterval) {
-    try {
-      let failedTries = 0;
-      while (true) {
+    let failedTries = 0;
+    let networkRetries = 0;
+    const maxNetworkRetries = 3;
+    while (true) {
+      try {
         let statusResponse = await this.getRequest(
           `${this.apiUrl}/v1/crawl/${id}`,
           headers
         );
         if (statusResponse.status === 200) {
           failedTries = 0;
+          networkRetries = 0;
           let statusData = statusResponse.data;
           if (statusData.status === "completed") {
             if ("data" in statusData) {
@@ -937,10 +941,44 @@ var FirecrawlApp = class {
             this.handleError(statusResponse, "check crawl status");
           }
         }
+      } catch (error) {
+        if (this.isRetryableError(error) && networkRetries < maxNetworkRetries) {
+          networkRetries++;
+          const backoffDelay = Math.min(1e3 * Math.pow(2, networkRetries - 1), 1e4);
+          await new Promise((resolve) => setTimeout(resolve, backoffDelay));
+          continue;
+        }
+        throw new FirecrawlError(error, 500);
+      }
+    }
+  }
+  /**
+   * Determines if an error is retryable (transient network error)
+   * @param error - The error to check
+   * @returns True if the error should be retried
+   */
+  isRetryableError(error) {
+    if (error instanceof import_axios.AxiosError) {
+      if (!error.response) {
+        const code = error.code;
+        const message = error.message?.toLowerCase() || "";
+        return code === "ECONNRESET" || code === "ETIMEDOUT" || code === "ENOTFOUND" || code === "ECONNREFUSED" || message.includes("socket hang up") || message.includes("network error") || message.includes("timeout");
+      }
+      if (error.response?.status === 408 || error.response?.status === 504) {
+        return true;
+      }
+    }
+    if (error && typeof error === "object") {
+      const code = error.code;
+      const message = error.message?.toLowerCase() || "";
+      if (code === "ECONNRESET" || code === "ETIMEDOUT" || code === "ENOTFOUND" || code === "ECONNREFUSED" || message.includes("socket hang up") || message.includes("network error") || message.includes("timeout")) {
+        return true;
+      }
+      if (error.response?.status === 408 || error.response?.status === 504) {
+        return true;
       }
-    } catch (error) {
-      throw new FirecrawlError(error, 500);
     }
+    return false;
   }
   /**
    * Handles errors from API responses.

package/dist/index.d.cts

@@ -698,6 +698,12 @@ declare class FirecrawlApp {
      * @returns The final job status or data.
      */
     monitorJobStatus(id: string, headers: AxiosRequestHeaders, checkInterval: number): Promise<CrawlStatusResponse | ErrorResponse>;
+    /**
+     * Determines if an error is retryable (transient network error)
+     * @param error - The error to check
+     * @returns True if the error should be retried
+     */
+    private isRetryableError;
     /**
      * Handles errors from API responses.
      * @param {AxiosResponse} response - The response from the API.

package/dist/index.d.ts

@@ -698,6 +698,12 @@ declare class FirecrawlApp {
      * @returns The final job status or data.
      */
     monitorJobStatus(id: string, headers: AxiosRequestHeaders, checkInterval: number): Promise<CrawlStatusResponse | ErrorResponse>;
+    /**
+     * Determines if an error is retryable (transient network error)
+     * @param error - The error to check
+     * @returns True if the error should be retried
+     */
+    private isRetryableError;
     /**
      * Handles errors from API responses.
      * @param {AxiosResponse} response - The response from the API.

package/dist/index.js

@@ -29,7 +29,7 @@ var FirecrawlApp = class {
   }
   async getVersion() {
     try {
-      const packageJson = await import("./package-E7ICGMY6.js");
+      const packageJson = await import("./package-Z6F7JDXI.js");
       return packageJson.default.version;
     } catch (error) {
       console.error("Error getting version:", error);
@@ -779,15 +779,18 @@ var FirecrawlApp = class {
    * @returns The final job status or data.
    */
   async monitorJobStatus(id, headers, checkInterval) {
-    try {
-      let failedTries = 0;
-      while (true) {
+    let failedTries = 0;
+    let networkRetries = 0;
+    const maxNetworkRetries = 3;
+    while (true) {
+      try {
         let statusResponse = await this.getRequest(
           `${this.apiUrl}/v1/crawl/${id}`,
           headers
         );
         if (statusResponse.status === 200) {
           failedTries = 0;
+          networkRetries = 0;
           let statusData = statusResponse.data;
           if (statusData.status === "completed") {
             if ("data" in statusData) {
@@ -822,10 +825,44 @@ var FirecrawlApp = class {
             this.handleError(statusResponse, "check crawl status");
           }
         }
+      } catch (error) {
+        if (this.isRetryableError(error) && networkRetries < maxNetworkRetries) {
+          networkRetries++;
+          const backoffDelay = Math.min(1e3 * Math.pow(2, networkRetries - 1), 1e4);
+          await new Promise((resolve) => setTimeout(resolve, backoffDelay));
+          continue;
+        }
+        throw new FirecrawlError(error, 500);
+      }
+    }
+  }
+  /**
+   * Determines if an error is retryable (transient network error)
+   * @param error - The error to check
+   * @returns True if the error should be retried
+   */
+  isRetryableError(error) {
+    if (error instanceof AxiosError) {
+      if (!error.response) {
+        const code = error.code;
+        const message = error.message?.toLowerCase() || "";
+        return code === "ECONNRESET" || code === "ETIMEDOUT" || code === "ENOTFOUND" || code === "ECONNREFUSED" || message.includes("socket hang up") || message.includes("network error") || message.includes("timeout");
+      }
+      if (error.response?.status === 408 || error.response?.status === 504) {
+        return true;
+      }
+    }
+    if (error && typeof error === "object") {
+      const code = error.code;
+      const message = error.message?.toLowerCase() || "";
+      if (code === "ECONNRESET" || code === "ETIMEDOUT" || code === "ENOTFOUND" || code === "ECONNREFUSED" || message.includes("socket hang up") || message.includes("network error") || message.includes("timeout")) {
+        return true;
+      }
+      if (error.response?.status === 408 || error.response?.status === 504) {
+        return true;
       }
-    } catch (error) {
-      throw new FirecrawlError(error, 500);
     }
+    return false;
   }
   /**
    * Handles errors from API responses.

package/dist/{package-E7ICGMY6.js → package-Z6F7JDXI.js}

@@ -1,6 +1,6 @@
 // package.json
 var name = "@mendable/firecrawl-js";
-var version = "1.29.2";
+var version = "1.29.3";
 var description = "JavaScript SDK for Firecrawl API";
 var main = "dist/index.js";
 var types = "dist/index.d.ts";
@@ -16,7 +16,8 @@ var scripts = {
   build: "tsup",
   "build-and-publish": "npm run build && npm publish --access public",
   "publish-beta": "npm run build && npm publish --access public --tag beta",
-  test: "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/v1/**/*.test.ts"
+  test: "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/v1/**/*.test.ts",
+  "test:unit": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/v1/unit/*.test.ts"
 };
 var repository = {
   type: "git",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mendable/firecrawl",
-  "version": "1.29.2",
+  "version": "1.29.3",
   "description": "JavaScript SDK for Firecrawl API",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -16,7 +16,8 @@
     "build": "tsup",
     "build-and-publish": "npm run build && npm publish --access public",
     "publish-beta": "npm run build && npm publish --access public --tag beta",
-    "test": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/v1/**/*.test.ts"
+    "test": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/v1/**/*.test.ts",
+    "test:unit": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/v1/unit/*.test.ts"
   },
   "repository": {
     "type": "git",

package/src/__tests__/v1/e2e_withAuth/index.test.ts

@@ -297,4 +297,5 @@ describe('FirecrawlApp E2E Tests', () => {
     const app = new FirecrawlApp({ apiUrl: API_URL, apiKey: "invalid_api_key" });
     await expect(app.search("test query")).rejects.toThrow("Request failed with status code 401");
   });
+
 });

package/src/__tests__/v1/unit/monitor-job-status-retry.test.ts

@@ -0,0 +1,154 @@
+import FirecrawlApp from '../../../index';
+import { describe, test, expect, jest, beforeEach, afterEach } from '@jest/globals';
+
+describe('monitorJobStatus retry logic', () => {
+  let app: FirecrawlApp;
+  let originalConsoleWarn: typeof console.warn;
+  
+  beforeEach(() => {
+    app = new FirecrawlApp({ apiKey: 'test-key', apiUrl: 'https://test.com' });
+    originalConsoleWarn = console.warn;
+    console.warn = jest.fn();
+  });
+
+  afterEach(() => {
+    console.warn = originalConsoleWarn;
+    jest.clearAllMocks();
+  });
+
+  test('should retry on socket hang up error', async () => {
+    const socketHangUpError = new Error('socket hang up') as any;
+    socketHangUpError.code = 'ECONNRESET';
+    
+    const successResponse = {
+      status: 200,
+      data: { status: 'completed', data: [{ url: 'test.com', markdown: 'test' }] }
+    };
+
+    const originalGetRequest = app.getRequest;
+    let callCount = 0;
+    
+    app.getRequest = async function(url: string, headers: any) {
+      callCount++;
+      if (callCount === 1) {
+        throw socketHangUpError;
+      }
+      return successResponse;
+    };
+
+    const result = await app.monitorJobStatus('test-id', {}, 1);
+    
+    expect(callCount).toBe(2);
+    expect(result).toEqual(successResponse.data);
+    expect(console.warn).toHaveBeenCalledWith(
+      expect.stringContaining('Network error during job status check (attempt 1/3): socket hang up')
+    );
+  });
+
+  test('should retry on ETIMEDOUT error', async () => {
+    const timeoutError = new Error('timeout') as any;
+    timeoutError.code = 'ETIMEDOUT';
+    
+    const successResponse = {
+      status: 200,
+      data: { status: 'completed', data: [{ url: 'test.com', markdown: 'test' }] }
+    };
+
+    const originalGetRequest = app.getRequest;
+    let callCount = 0;
+    
+    app.getRequest = async function(url: string, headers: any) {
+      callCount++;
+      if (callCount === 1) {
+        throw timeoutError;
+      }
+      return successResponse;
+    };
+
+    const result = await app.monitorJobStatus('test-id', {}, 1);
+    
+    expect(callCount).toBe(2);
+    expect(result).toEqual(successResponse.data);
+  });
+
+  test('should fail after max retries exceeded', async () => {
+    const socketHangUpError = new Error('socket hang up') as any;
+    socketHangUpError.code = 'ECONNRESET';
+    
+    app.getRequest = async function(url: string, headers: any) {
+      throw socketHangUpError;
+    };
+
+    await expect(app.monitorJobStatus('test-id', {}, 1)).rejects.toThrow('socket hang up');
+    
+    expect(console.warn).toHaveBeenCalledTimes(3);
+  }, 15000);
+
+  test('should not retry on non-retryable errors', async () => {
+    const authError = new Error('Unauthorized') as any;
+    authError.response = { status: 401, data: { error: 'Unauthorized' } };
+    
+    app.getRequest = async function(url: string, headers: any) {
+      throw authError;
+    };
+
+    await expect(app.monitorJobStatus('test-id', {}, 1)).rejects.toThrow('Unauthorized');
+    
+    expect(console.warn).not.toHaveBeenCalled();
+  });
+
+  test('should retry on HTTP timeout status codes', async () => {
+    const timeoutError = new Error('Request timeout') as any;
+    timeoutError.response = { status: 408, data: { error: 'Request timeout' } };
+    
+    const successResponse = {
+      status: 200,
+      data: { status: 'completed', data: [{ url: 'test.com', markdown: 'test' }] }
+    };
+
+    const originalGetRequest = app.getRequest;
+    let callCount = 0;
+    
+    app.getRequest = async function(url: string, headers: any) {
+      callCount++;
+      if (callCount === 1) {
+        throw timeoutError;
+      }
+      return successResponse;
+    };
+
+    const result = await app.monitorJobStatus('test-id', {}, 1);
+    
+    expect(callCount).toBe(2);
+    expect(result).toEqual(successResponse.data);
+  });
+
+  test('should use exponential backoff for retries', async () => {
+    const socketHangUpError = new Error('socket hang up') as any;
+    socketHangUpError.code = 'ECONNRESET';
+    
+    const successResponse = {
+      status: 200,
+      data: { status: 'completed', data: [{ url: 'test.com', markdown: 'test' }] }
+    };
+
+    const originalGetRequest = app.getRequest;
+    let callCount = 0;
+    
+    app.getRequest = async function(url: string, headers: any) {
+      callCount++;
+      if (callCount <= 2) {
+        throw socketHangUpError;
+      }
+      return successResponse;
+    };
+
+    const startTime = Date.now();
+    const result = await app.monitorJobStatus('test-id', {}, 1);
+    const endTime = Date.now();
+    
+    expect(callCount).toBe(3);
+    expect(result).toEqual(successResponse.data);
+    expect(endTime - startTime).toBeGreaterThan(3000);
+  });
+});

package/src/index.ts

@@ -1455,33 +1455,39 @@ export default class FirecrawlApp {
     headers: AxiosRequestHeaders,
     checkInterval: number
   ): Promise<CrawlStatusResponse | ErrorResponse> {
-    try {
-      let failedTries = 0;
-      while (true) {
+    let failedTries = 0;
+    let networkRetries = 0;
+    const maxNetworkRetries = 3;
+    
+    while (true) {
+      try {
         let statusResponse: AxiosResponse = await this.getRequest(
           `${this.apiUrl}/v1/crawl/${id}`,
           headers
         );
+        
         if (statusResponse.status === 200) {
           failedTries = 0;
+          networkRetries = 0;
           let statusData = statusResponse.data;
-            if (statusData.status === "completed") {
-              if ("data" in statusData) {
-                let data = statusData.data;
-                while (typeof statusData === 'object' && 'next' in statusData) {
-                  if (data.length === 0) {
-                    break
-                  }
-                  statusResponse = await this.getRequest(statusData.next, headers);
-                  statusData = statusResponse.data;
-                  data = data.concat(statusData.data);
+          
+          if (statusData.status === "completed") {
+            if ("data" in statusData) {
+              let data = statusData.data;
+              while (typeof statusData === 'object' && 'next' in statusData) {
+                if (data.length === 0) {
+                  break
                 }
-                statusData.data = data;
-                return statusData;
-              } else {
-                throw new FirecrawlError("Crawl job completed but no data was returned", 500);
+                statusResponse = await this.getRequest(statusData.next, headers);
+                statusData = statusResponse.data;
+                data = data.concat(statusData.data);
               }
-            } else if (
+              statusData.data = data;
+              return statusData;
+            } else {
+              throw new FirecrawlError("Crawl job completed but no data was returned", 500);
+            }
+          } else if (
             ["active", "paused", "pending", "queued", "waiting", "scraping"].includes(statusData.status)
           ) {
             checkInterval = Math.max(checkInterval, 2);
@@ -1500,12 +1506,69 @@ export default class FirecrawlApp {
             this.handleError(statusResponse, "check crawl status");
           }
         }
+      } catch (error: any) {
+        if (this.isRetryableError(error) && networkRetries < maxNetworkRetries) {
+          networkRetries++;
+          const backoffDelay = Math.min(1000 * Math.pow(2, networkRetries - 1), 10000);
+          
+          await new Promise((resolve) => setTimeout(resolve, backoffDelay));
+          continue;
+        }
+        
+        throw new FirecrawlError(error, 500);
       }
-    } catch (error: any) {
-      throw new FirecrawlError(error, 500);
     }
   }
 
+  /**
+   * Determines if an error is retryable (transient network error)
+   * @param error - The error to check
+   * @returns True if the error should be retried
+   */
+  private isRetryableError(error: any): boolean {
+    if (error instanceof AxiosError) {
+      if (!error.response) {
+        const code = error.code;
+        const message = error.message?.toLowerCase() || '';
+        
+        return (
+          code === 'ECONNRESET' ||
+          code === 'ETIMEDOUT' ||
+          code === 'ENOTFOUND' ||
+          code === 'ECONNREFUSED' ||
+          message.includes('socket hang up') ||
+          message.includes('network error') ||
+          message.includes('timeout')
+        );
+      }
+      
+      if (error.response?.status === 408 || error.response?.status === 504) {
+        return true;
+      }
+    }
+    
+    if (error && typeof error === 'object') {
+      const code = error.code;
+      const message = error.message?.toLowerCase() || '';
+      
+      if (code === 'ECONNRESET' ||
+          code === 'ETIMEDOUT' ||
+          code === 'ENOTFOUND' ||
+          code === 'ECONNREFUSED' ||
+          message.includes('socket hang up') ||
+          message.includes('network error') ||
+          message.includes('timeout')) {
+        return true;
+      }
+      
+      if (error.response?.status === 408 || error.response?.status === 504) {
+        return true;
+      }
+    }
+    
+    return false;
+  }
+
   /**
    * Handles errors from API responses.
    * @param {AxiosResponse} response - The response from the API.