@mendable/firecrawl 1.29.1 → 1.29.3

package/dist/index.cjs

@@ -35,7 +35,7 @@ var require_package = __commonJS({
   "package.json"(exports2, module2) {
     module2.exports = {
       name: "@mendable/firecrawl-js",
-      version: "1.29.1",
+      version: "1.29.3",
       description: "JavaScript SDK for Firecrawl API",
       main: "dist/index.js",
       types: "dist/index.d.ts",
@@ -51,7 +51,8 @@ var require_package = __commonJS({
         build: "tsup",
         "build-and-publish": "npm run build && npm publish --access public",
         "publish-beta": "npm run build && npm publish --access public --tag beta",
-        test: "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/v1/**/*.test.ts"
+        test: "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/v1/**/*.test.ts",
+        "test:unit": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/v1/unit/*.test.ts"
       },
       repository: {
         type: "git",
@@ -60,27 +61,26 @@ var require_package = __commonJS({
       author: "Mendable.ai",
       license: "MIT",
       dependencies: {
+        axios: "^1.11.0",
         "typescript-event-target": "^1.1.1",
         zod: "^3.23.8",
-        "zod-to-json-schema": "^3.23.0",
-        axios: "^1.6.8"
+        "zod-to-json-schema": "^3.23.0"
       },
       bugs: {
         url: "https://github.com/mendableai/firecrawl/issues"
       },
       homepage: "https://github.com/mendableai/firecrawl#readme",
       devDependencies: {
-        "@jest/globals": "^29.7.0",
-        "@types/axios": "^0.14.0",
+        "@jest/globals": "^30.0.5",
         "@types/dotenv": "^8.2.0",
-        "@types/jest": "^29.5.14",
+        "@types/jest": "^30.0.0",
         "@types/mocha": "^10.0.6",
         "@types/node": "^20.12.12",
         "@types/uuid": "^9.0.8",
         dotenv: "^16.4.5",
-        jest: "^29.7.0",
-        "ts-jest": "^29.2.2",
-        tsup: "^8.2.4",
+        jest: "^30.0.5",
+        "ts-jest": "^29.4.0",
+        tsup: "^8.5.0",
         typescript: "^5.4.5",
         uuid: "^9.0.1"
       },
@@ -95,19 +95,26 @@ var require_package = __commonJS({
       ],
       engines: {
         node: ">=22.0.0"
+      },
+      pnpm: {
+        overrides: {
+          "@babel/helpers@<7.26.10": ">=7.26.10",
+          "brace-expansion@>=1.0.0 <=1.1.11": ">=1.1.12",
+          "brace-expansion@>=2.0.0 <=2.0.1": ">=2.0.2"
+        }
       }
     };
   }
 });
 
 // src/index.ts
-var src_exports = {};
-__export(src_exports, {
+var index_exports = {};
+__export(index_exports, {
   CrawlWatcher: () => CrawlWatcher,
   FirecrawlError: () => FirecrawlError,
   default: () => FirecrawlApp
 });
-module.exports = __toCommonJS(src_exports);
+module.exports = __toCommonJS(index_exports);
 var import_axios = __toESM(require("axios"), 1);
 var zt = require("zod");
 var import_zod_to_json_schema = require("zod-to-json-schema");
@@ -888,15 +895,18 @@ var FirecrawlApp = class {
    * @returns The final job status or data.
    */
   async monitorJobStatus(id, headers, checkInterval) {
-    try {
-      let failedTries = 0;
-      while (true) {
+    let failedTries = 0;
+    let networkRetries = 0;
+    const maxNetworkRetries = 3;
+    while (true) {
+      try {
         let statusResponse = await this.getRequest(
           `${this.apiUrl}/v1/crawl/${id}`,
           headers
         );
         if (statusResponse.status === 200) {
           failedTries = 0;
+          networkRetries = 0;
           let statusData = statusResponse.data;
           if (statusData.status === "completed") {
             if ("data" in statusData) {
@@ -931,10 +941,44 @@ var FirecrawlApp = class {
             this.handleError(statusResponse, "check crawl status");
           }
         }
+      } catch (error) {
+        if (this.isRetryableError(error) && networkRetries < maxNetworkRetries) {
+          networkRetries++;
+          const backoffDelay = Math.min(1e3 * Math.pow(2, networkRetries - 1), 1e4);
+          await new Promise((resolve) => setTimeout(resolve, backoffDelay));
+          continue;
+        }
+        throw new FirecrawlError(error, 500);
+      }
+    }
+  }
+  /**
+   * Determines if an error is retryable (transient network error)
+   * @param error - The error to check
+   * @returns True if the error should be retried
+   */
+  isRetryableError(error) {
+    if (error instanceof import_axios.AxiosError) {
+      if (!error.response) {
+        const code = error.code;
+        const message = error.message?.toLowerCase() || "";
+        return code === "ECONNRESET" || code === "ETIMEDOUT" || code === "ENOTFOUND" || code === "ECONNREFUSED" || message.includes("socket hang up") || message.includes("network error") || message.includes("timeout");
+      }
+      if (error.response?.status === 408 || error.response?.status === 504) {
+        return true;
+      }
+    }
+    if (error && typeof error === "object") {
+      const code = error.code;
+      const message = error.message?.toLowerCase() || "";
+      if (code === "ECONNRESET" || code === "ETIMEDOUT" || code === "ENOTFOUND" || code === "ECONNREFUSED" || message.includes("socket hang up") || message.includes("network error") || message.includes("timeout")) {
+        return true;
+      }
+      if (error.response?.status === 408 || error.response?.status === 504) {
+        return true;
       }
-    } catch (error) {
-      throw new FirecrawlError(error, 500);
     }
+    return false;
   }
   /**
    * Handles errors from API responses.

package/dist/index.d.cts

@@ -698,6 +698,12 @@ declare class FirecrawlApp {
      * @returns The final job status or data.
      */
     monitorJobStatus(id: string, headers: AxiosRequestHeaders, checkInterval: number): Promise<CrawlStatusResponse | ErrorResponse>;
+    /**
+     * Determines if an error is retryable (transient network error)
+     * @param error - The error to check
+     * @returns True if the error should be retried
+     */
+    private isRetryableError;
     /**
      * Handles errors from API responses.
      * @param {AxiosResponse} response - The response from the API.

package/dist/index.d.ts

@@ -698,6 +698,12 @@ declare class FirecrawlApp {
      * @returns The final job status or data.
      */
     monitorJobStatus(id: string, headers: AxiosRequestHeaders, checkInterval: number): Promise<CrawlStatusResponse | ErrorResponse>;
+    /**
+     * Determines if an error is retryable (transient network error)
+     * @param error - The error to check
+     * @returns True if the error should be retried
+     */
+    private isRetryableError;
     /**
      * Handles errors from API responses.
      * @param {AxiosResponse} response - The response from the API.

package/dist/index.js

@@ -29,7 +29,7 @@ var FirecrawlApp = class {
   }
   async getVersion() {
     try {
-      const packageJson = await import("./package-TKKTR5R7.js");
+      const packageJson = await import("./package-Z6F7JDXI.js");
       return packageJson.default.version;
     } catch (error) {
       console.error("Error getting version:", error);
@@ -779,15 +779,18 @@ var FirecrawlApp = class {
    * @returns The final job status or data.
    */
   async monitorJobStatus(id, headers, checkInterval) {
-    try {
-      let failedTries = 0;
-      while (true) {
+    let failedTries = 0;
+    let networkRetries = 0;
+    const maxNetworkRetries = 3;
+    while (true) {
+      try {
         let statusResponse = await this.getRequest(
           `${this.apiUrl}/v1/crawl/${id}`,
           headers
         );
         if (statusResponse.status === 200) {
           failedTries = 0;
+          networkRetries = 0;
           let statusData = statusResponse.data;
           if (statusData.status === "completed") {
             if ("data" in statusData) {
@@ -822,10 +825,44 @@ var FirecrawlApp = class {
             this.handleError(statusResponse, "check crawl status");
           }
         }
+      } catch (error) {
+        if (this.isRetryableError(error) && networkRetries < maxNetworkRetries) {
+          networkRetries++;
+          const backoffDelay = Math.min(1e3 * Math.pow(2, networkRetries - 1), 1e4);
+          await new Promise((resolve) => setTimeout(resolve, backoffDelay));
+          continue;
+        }
+        throw new FirecrawlError(error, 500);
+      }
+    }
+  }
+  /**
+   * Determines if an error is retryable (transient network error)
+   * @param error - The error to check
+   * @returns True if the error should be retried
+   */
+  isRetryableError(error) {
+    if (error instanceof AxiosError) {
+      if (!error.response) {
+        const code = error.code;
+        const message = error.message?.toLowerCase() || "";
+        return code === "ECONNRESET" || code === "ETIMEDOUT" || code === "ENOTFOUND" || code === "ECONNREFUSED" || message.includes("socket hang up") || message.includes("network error") || message.includes("timeout");
+      }
+      if (error.response?.status === 408 || error.response?.status === 504) {
+        return true;
+      }
+    }
+    if (error && typeof error === "object") {
+      const code = error.code;
+      const message = error.message?.toLowerCase() || "";
+      if (code === "ECONNRESET" || code === "ETIMEDOUT" || code === "ENOTFOUND" || code === "ECONNREFUSED" || message.includes("socket hang up") || message.includes("network error") || message.includes("timeout")) {
+        return true;
+      }
+      if (error.response?.status === 408 || error.response?.status === 504) {
+        return true;
       }
-    } catch (error) {
-      throw new FirecrawlError(error, 500);
     }
+    return false;
   }
   /**
    * Handles errors from API responses.

package/dist/{package-TKKTR5R7.js → package-Z6F7JDXI.js}

@@ -1,6 +1,6 @@
 // package.json
 var name = "@mendable/firecrawl-js";
-var version = "1.29.1";
+var version = "1.29.3";
 var description = "JavaScript SDK for Firecrawl API";
 var main = "dist/index.js";
 var types = "dist/index.d.ts";
@@ -16,7 +16,8 @@ var scripts = {
   build: "tsup",
   "build-and-publish": "npm run build && npm publish --access public",
   "publish-beta": "npm run build && npm publish --access public --tag beta",
-  test: "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/v1/**/*.test.ts"
+  test: "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/v1/**/*.test.ts",
+  "test:unit": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/v1/unit/*.test.ts"
 };
 var repository = {
   type: "git",
@@ -25,27 +26,26 @@ var repository = {
 var author = "Mendable.ai";
 var license = "MIT";
 var dependencies = {
+  axios: "^1.11.0",
   "typescript-event-target": "^1.1.1",
   zod: "^3.23.8",
-  "zod-to-json-schema": "^3.23.0",
-  axios: "^1.6.8"
+  "zod-to-json-schema": "^3.23.0"
 };
 var bugs = {
   url: "https://github.com/mendableai/firecrawl/issues"
 };
 var homepage = "https://github.com/mendableai/firecrawl#readme";
 var devDependencies = {
-  "@jest/globals": "^29.7.0",
-  "@types/axios": "^0.14.0",
+  "@jest/globals": "^30.0.5",
   "@types/dotenv": "^8.2.0",
-  "@types/jest": "^29.5.14",
+  "@types/jest": "^30.0.0",
   "@types/mocha": "^10.0.6",
   "@types/node": "^20.12.12",
   "@types/uuid": "^9.0.8",
   dotenv: "^16.4.5",
-  jest: "^29.7.0",
-  "ts-jest": "^29.2.2",
-  tsup: "^8.2.4",
+  jest: "^30.0.5",
+  "ts-jest": "^29.4.0",
+  tsup: "^8.5.0",
   typescript: "^5.4.5",
   uuid: "^9.0.1"
 };
@@ -61,6 +61,13 @@ var keywords = [
 var engines = {
   node: ">=22.0.0"
 };
+var pnpm = {
+  overrides: {
+    "@babel/helpers@<7.26.10": ">=7.26.10",
+    "brace-expansion@>=1.0.0 <=1.1.11": ">=1.1.12",
+    "brace-expansion@>=2.0.0 <=2.0.1": ">=2.0.2"
+  }
+};
 var package_default = {
   name,
   version,
@@ -78,7 +85,8 @@ var package_default = {
   homepage,
   devDependencies,
   keywords,
-  engines
+  engines,
+  pnpm
 };
 export {
   author,
@@ -94,6 +102,7 @@ export {
   license,
   main,
   name,
+  pnpm,
   repository,
   scripts,
   type,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mendable/firecrawl",
-  "version": "1.29.1",
+  "version": "1.29.3",
   "description": "JavaScript SDK for Firecrawl API",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -16,7 +16,8 @@
     "build": "tsup",
     "build-and-publish": "npm run build && npm publish --access public",
     "publish-beta": "npm run build && npm publish --access public --tag beta",
-    "test": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/v1/**/*.test.ts"
+    "test": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/v1/**/*.test.ts",
+    "test:unit": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/v1/unit/*.test.ts"
   },
   "repository": {
     "type": "git",
@@ -25,27 +26,26 @@
   "author": "Mendable.ai",
   "license": "MIT",
   "dependencies": {
+    "axios": "^1.11.0",
     "typescript-event-target": "^1.1.1",
     "zod": "^3.23.8",
-    "zod-to-json-schema": "^3.23.0",
-    "axios": "^1.6.8"
+    "zod-to-json-schema": "^3.23.0"
   },
   "bugs": {
     "url": "https://github.com/mendableai/firecrawl/issues"
   },
   "homepage": "https://github.com/mendableai/firecrawl#readme",
   "devDependencies": {
-    "@jest/globals": "^29.7.0",
-    "@types/axios": "^0.14.0",
+    "@jest/globals": "^30.0.5",
     "@types/dotenv": "^8.2.0",
-    "@types/jest": "^29.5.14",
+    "@types/jest": "^30.0.0",
     "@types/mocha": "^10.0.6",
     "@types/node": "^20.12.12",
     "@types/uuid": "^9.0.8",
     "dotenv": "^16.4.5",
-    "jest": "^29.7.0",
-    "ts-jest": "^29.2.2",
-    "tsup": "^8.2.4",
+    "jest": "^30.0.5",
+    "ts-jest": "^29.4.0",
+    "tsup": "^8.5.0",
     "typescript": "^5.4.5",
     "uuid": "^9.0.1"
   },
@@ -60,5 +60,12 @@
   ],
   "engines": {
     "node": ">=22.0.0"
+  },
+  "pnpm": {
+    "overrides": {
+      "@babel/helpers@<7.26.10": ">=7.26.10",
+      "brace-expansion@>=1.0.0 <=1.1.11": ">=1.1.12",
+      "brace-expansion@>=2.0.0 <=2.0.1": ">=2.0.2"
+    }
   }
 }

package/src/__tests__/v1/e2e_withAuth/index.test.ts

@@ -297,4 +297,5 @@ describe('FirecrawlApp E2E Tests', () => {
     const app = new FirecrawlApp({ apiUrl: API_URL, apiKey: "invalid_api_key" });
     await expect(app.search("test query")).rejects.toThrow("Request failed with status code 401");
   });
+
 });

package/src/__tests__/v1/unit/monitor-job-status-retry.test.ts

@@ -0,0 +1,154 @@
+import FirecrawlApp from '../../../index';
+import { describe, test, expect, jest, beforeEach, afterEach } from '@jest/globals';
+
+describe('monitorJobStatus retry logic', () => {
+  let app: FirecrawlApp;
+  let originalConsoleWarn: typeof console.warn;
+  
+  beforeEach(() => {
+    app = new FirecrawlApp({ apiKey: 'test-key', apiUrl: 'https://test.com' });
+    originalConsoleWarn = console.warn;
+    console.warn = jest.fn();
+  });
+
+  afterEach(() => {
+    console.warn = originalConsoleWarn;
+    jest.clearAllMocks();
+  });
+
+  test('should retry on socket hang up error', async () => {
+    const socketHangUpError = new Error('socket hang up') as any;
+    socketHangUpError.code = 'ECONNRESET';
+    
+    const successResponse = {
+      status: 200,
+      data: { status: 'completed', data: [{ url: 'test.com', markdown: 'test' }] }
+    };
+
+    const originalGetRequest = app.getRequest;
+    let callCount = 0;
+    
+    app.getRequest = async function(url: string, headers: any) {
+      callCount++;
+      if (callCount === 1) {
+        throw socketHangUpError;
+      }
+      return successResponse;
+    };
+
+    const result = await app.monitorJobStatus('test-id', {}, 1);
+    
+    expect(callCount).toBe(2);
+    expect(result).toEqual(successResponse.data);
+    expect(console.warn).toHaveBeenCalledWith(
+      expect.stringContaining('Network error during job status check (attempt 1/3): socket hang up')
+    );
+  });
+
+  test('should retry on ETIMEDOUT error', async () => {
+    const timeoutError = new Error('timeout') as any;
+    timeoutError.code = 'ETIMEDOUT';
+    
+    const successResponse = {
+      status: 200,
+      data: { status: 'completed', data: [{ url: 'test.com', markdown: 'test' }] }
+    };
+
+    const originalGetRequest = app.getRequest;
+    let callCount = 0;
+    
+    app.getRequest = async function(url: string, headers: any) {
+      callCount++;
+      if (callCount === 1) {
+        throw timeoutError;
+      }
+      return successResponse;
+    };
+
+    const result = await app.monitorJobStatus('test-id', {}, 1);
+    
+    expect(callCount).toBe(2);
+    expect(result).toEqual(successResponse.data);
+  });
+
+  test('should fail after max retries exceeded', async () => {
+    const socketHangUpError = new Error('socket hang up') as any;
+    socketHangUpError.code = 'ECONNRESET';
+    
+    app.getRequest = async function(url: string, headers: any) {
+      throw socketHangUpError;
+    };
+
+    await expect(app.monitorJobStatus('test-id', {}, 1)).rejects.toThrow('socket hang up');
+    
+    expect(console.warn).toHaveBeenCalledTimes(3);
+  }, 15000);
+
+  test('should not retry on non-retryable errors', async () => {
+    const authError = new Error('Unauthorized') as any;
+    authError.response = { status: 401, data: { error: 'Unauthorized' } };
+    
+    app.getRequest = async function(url: string, headers: any) {
+      throw authError;
+    };
+
+    await expect(app.monitorJobStatus('test-id', {}, 1)).rejects.toThrow('Unauthorized');
+    
+    expect(console.warn).not.toHaveBeenCalled();
+  });
+
+  test('should retry on HTTP timeout status codes', async () => {
+    const timeoutError = new Error('Request timeout') as any;
+    timeoutError.response = { status: 408, data: { error: 'Request timeout' } };
+    
+    const successResponse = {
+      status: 200,
+      data: { status: 'completed', data: [{ url: 'test.com', markdown: 'test' }] }
+    };
+
+    const originalGetRequest = app.getRequest;
+    let callCount = 0;
+    
+    app.getRequest = async function(url: string, headers: any) {
+      callCount++;
+      if (callCount === 1) {
+        throw timeoutError;
+      }
+      return successResponse;
+    };
+
+    const result = await app.monitorJobStatus('test-id', {}, 1);
+    
+    expect(callCount).toBe(2);
+    expect(result).toEqual(successResponse.data);
+  });
+
+  test('should use exponential backoff for retries', async () => {
+    const socketHangUpError = new Error('socket hang up') as any;
+    socketHangUpError.code = 'ECONNRESET';
+    
+    const successResponse = {
+      status: 200,
+      data: { status: 'completed', data: [{ url: 'test.com', markdown: 'test' }] }
+    };
+
+    const originalGetRequest = app.getRequest;
+    let callCount = 0;
+    
+    app.getRequest = async function(url: string, headers: any) {
+      callCount++;
+      if (callCount <= 2) {
+        throw socketHangUpError;
+      }
+      return successResponse;
+    };
+
+    const startTime = Date.now();
+    const result = await app.monitorJobStatus('test-id', {}, 1);
+    const endTime = Date.now();
+    
+    expect(callCount).toBe(3);
+    expect(result).toEqual(successResponse.data);
+    expect(endTime - startTime).toBeGreaterThan(3000);
+  });
+});

package/src/index.ts

@@ -1455,33 +1455,39 @@ export default class FirecrawlApp {
     headers: AxiosRequestHeaders,
     checkInterval: number
   ): Promise<CrawlStatusResponse | ErrorResponse> {
-    try {
-      let failedTries = 0;
-      while (true) {
+    let failedTries = 0;
+    let networkRetries = 0;
+    const maxNetworkRetries = 3;
+    
+    while (true) {
+      try {
         let statusResponse: AxiosResponse = await this.getRequest(
           `${this.apiUrl}/v1/crawl/${id}`,
           headers
         );
+        
         if (statusResponse.status === 200) {
           failedTries = 0;
+          networkRetries = 0;
           let statusData = statusResponse.data;
-            if (statusData.status === "completed") {
-              if ("data" in statusData) {
-                let data = statusData.data;
-                while (typeof statusData === 'object' && 'next' in statusData) {
-                  if (data.length === 0) {
-                    break
-                  }
-                  statusResponse = await this.getRequest(statusData.next, headers);
-                  statusData = statusResponse.data;
-                  data = data.concat(statusData.data);
+          
+          if (statusData.status === "completed") {
+            if ("data" in statusData) {
+              let data = statusData.data;
+              while (typeof statusData === 'object' && 'next' in statusData) {
+                if (data.length === 0) {
+                  break
                 }
-                statusData.data = data;
-                return statusData;
-              } else {
-                throw new FirecrawlError("Crawl job completed but no data was returned", 500);
+                statusResponse = await this.getRequest(statusData.next, headers);
+                statusData = statusResponse.data;
+                data = data.concat(statusData.data);
               }
-            } else if (
+              statusData.data = data;
+              return statusData;
+            } else {
+              throw new FirecrawlError("Crawl job completed but no data was returned", 500);
+            }
+          } else if (
             ["active", "paused", "pending", "queued", "waiting", "scraping"].includes(statusData.status)
           ) {
             checkInterval = Math.max(checkInterval, 2);
@@ -1500,12 +1506,69 @@ export default class FirecrawlApp {
             this.handleError(statusResponse, "check crawl status");
           }
         }
+      } catch (error: any) {
+        if (this.isRetryableError(error) && networkRetries < maxNetworkRetries) {
+          networkRetries++;
+          const backoffDelay = Math.min(1000 * Math.pow(2, networkRetries - 1), 10000);
+          
+          await new Promise((resolve) => setTimeout(resolve, backoffDelay));
+          continue;
+        }
+        
+        throw new FirecrawlError(error, 500);
       }
-    } catch (error: any) {
-      throw new FirecrawlError(error, 500);
     }
   }
 
+  /**
+   * Determines if an error is retryable (transient network error)
+   * @param error - The error to check
+   * @returns True if the error should be retried
+   */
+  private isRetryableError(error: any): boolean {
+    if (error instanceof AxiosError) {
+      if (!error.response) {
+        const code = error.code;
+        const message = error.message?.toLowerCase() || '';
+        
+        return (
+          code === 'ECONNRESET' ||
+          code === 'ETIMEDOUT' ||
+          code === 'ENOTFOUND' ||
+          code === 'ECONNREFUSED' ||
+          message.includes('socket hang up') ||
+          message.includes('network error') ||
+          message.includes('timeout')
+        );
+      }
+      
+      if (error.response?.status === 408 || error.response?.status === 504) {
+        return true;
+      }
+    }
+    
+    if (error && typeof error === 'object') {
+      const code = error.code;
+      const message = error.message?.toLowerCase() || '';
+      
+      if (code === 'ECONNRESET' ||
+          code === 'ETIMEDOUT' ||
+          code === 'ENOTFOUND' ||
+          code === 'ECONNREFUSED' ||
+          message.includes('socket hang up') ||
+          message.includes('network error') ||
+          message.includes('timeout')) {
+        return true;
+      }
+      
+      if (error.response?.status === 408 || error.response?.status === 504) {
+        return true;
+      }
+    }
+    
+    return false;
+  }
+
   /**
    * Handles errors from API responses.
    * @param {AxiosResponse} response - The response from the API.