firecrawl 1.13.0 → 1.14.1

package/dist/index.cjs

@@ -42,9 +42,11 @@ var import_isows = require("isows");
 var import_typescript_event_target = require("typescript-event-target");
 var FirecrawlError = class extends Error {
   statusCode;
-  constructor(message, statusCode) {
+  details;
+  constructor(message, statusCode, details) {
     super(message);
     this.statusCode = statusCode;
+    this.details = details;
   }
 };
 var FirecrawlApp = class {
@@ -245,16 +247,26 @@ var FirecrawlApp = class {
    * Checks the status of a crawl job using the Firecrawl API.
    * @param id - The ID of the crawl operation.
    * @param getAllData - Paginate through all the pages of documents, returning the full list of all documents. (default: `false`)
+   * @param nextURL - The `next` URL from the previous crawl status. Only required if you're not manually increasing `skip`. Only used when `getAllData = false`.
+   * @param skip - How many entries to skip to paginate. Only required if you're not providing `nextURL`. Only used when `getAllData = false`.
+   * @param limit - How many entries to return. Only used when `getAllData = false`.
    * @returns The response containing the job status.
    */
-  async checkCrawlStatus(id, getAllData = false) {
+  async checkCrawlStatus(id, getAllData = false, nextURL, skip, limit) {
     if (!id) {
       throw new FirecrawlError("No crawl ID provided", 400);
     }
     const headers = this.prepareHeaders();
+    const targetURL = new URL(nextURL ?? `${this.apiUrl}/v1/crawl/${id}`);
+    if (skip !== void 0) {
+      targetURL.searchParams.set("skip", skip.toString());
+    }
+    if (limit !== void 0) {
+      targetURL.searchParams.set("skip", limit.toString());
+    }
     try {
       const response = await this.getRequest(
-        `${this.apiUrl}/v1/crawl/${id}`,
+        targetURL.href,
         headers
       );
       if (response.status === 200) {
@@ -279,6 +291,7 @@ var FirecrawlApp = class {
           total: response.data.total,
           completed: response.data.completed,
           creditsUsed: response.data.creditsUsed,
+          next: getAllData ? void 0 : response.data.next,
           expiresAt: new Date(response.data.expiresAt),
           data: allData
         };
@@ -452,16 +465,26 @@ var FirecrawlApp = class {
    * Checks the status of a batch scrape job using the Firecrawl API.
    * @param id - The ID of the batch scrape operation.
    * @param getAllData - Paginate through all the pages of documents, returning the full list of all documents. (default: `false`)
+   * @param nextURL - The `next` URL from the previous batch scrape status. Only required if you're not manually increasing `skip`. Only used when `getAllData = false`.
+   * @param skip - How many entries to skip to paginate. Only used when `getAllData = false`.
+   * @param limit - How many entries to return. Only used when `getAllData = false`.
    * @returns The response containing the job status.
    */
-  async checkBatchScrapeStatus(id, getAllData = false) {
+  async checkBatchScrapeStatus(id, getAllData = false, nextURL, skip, limit) {
     if (!id) {
       throw new FirecrawlError("No batch scrape ID provided", 400);
     }
     const headers = this.prepareHeaders();
+    const targetURL = new URL(nextURL ?? `${this.apiUrl}/v1/batch/scrape/${id}`);
+    if (skip !== void 0) {
+      targetURL.searchParams.set("skip", skip.toString());
+    }
+    if (limit !== void 0) {
+      targetURL.searchParams.set("skip", limit.toString());
+    }
     try {
       const response = await this.getRequest(
-        `${this.apiUrl}/v1/batch/scrape/${id}`,
+        targetURL.href,
         headers
       );
       if (response.status === 200) {
@@ -486,6 +509,7 @@ var FirecrawlApp = class {
           total: response.data.total,
           completed: response.data.completed,
           creditsUsed: response.data.creditsUsed,
+          next: getAllData ? void 0 : response.data.next,
           expiresAt: new Date(response.data.expiresAt),
           data: allData
         };
@@ -565,7 +589,7 @@ var FirecrawlApp = class {
         this.handleError(response, "extract");
       }
     } catch (error) {
-      throw new FirecrawlError(error.message, 500);
+      throw new FirecrawlError(error.message, 500, error.response?.data?.details);
     }
     return { success: false, error: "Internal server error." };
   }
@@ -601,7 +625,7 @@ var FirecrawlApp = class {
         this.handleError(response, "start extract job");
       }
     } catch (error) {
-      throw new FirecrawlError(error.message, 500);
+      throw new FirecrawlError(error.message, 500, error.response?.data?.details);
     }
     return { success: false, error: "Internal server error." };
   }

package/dist/index.d.cts

@@ -258,7 +258,8 @@ interface ErrorResponse {
  */
 declare class FirecrawlError extends Error {
     statusCode: number;
-    constructor(message: string, statusCode: number);
+    details?: any;
+    constructor(message: string, statusCode: number, details?: any);
 }
 /**
  * Parameters for search operations.
@@ -326,9 +327,12 @@ declare class FirecrawlApp {
      * Checks the status of a crawl job using the Firecrawl API.
      * @param id - The ID of the crawl operation.
      * @param getAllData - Paginate through all the pages of documents, returning the full list of all documents. (default: `false`)
+     * @param nextURL - The `next` URL from the previous crawl status. Only required if you're not manually increasing `skip`. Only used when `getAllData = false`.
+     * @param skip - How many entries to skip to paginate. Only required if you're not providing `nextURL`. Only used when `getAllData = false`.
+     * @param limit - How many entries to return. Only used when `getAllData = false`.
      * @returns The response containing the job status.
      */
-    checkCrawlStatus(id?: string, getAllData?: boolean): Promise<CrawlStatusResponse | ErrorResponse>;
+    checkCrawlStatus(id?: string, getAllData?: boolean, nextURL?: string, skip?: number, limit?: number): Promise<CrawlStatusResponse | ErrorResponse>;
     /**
      * Cancels a crawl job using the Firecrawl API.
      * @param id - The ID of the crawl operation.
@@ -373,9 +377,12 @@ declare class FirecrawlApp {
      * Checks the status of a batch scrape job using the Firecrawl API.
      * @param id - The ID of the batch scrape operation.
      * @param getAllData - Paginate through all the pages of documents, returning the full list of all documents. (default: `false`)
+     * @param nextURL - The `next` URL from the previous batch scrape status. Only required if you're not manually increasing `skip`. Only used when `getAllData = false`.
+     * @param skip - How many entries to skip to paginate. Only used when `getAllData = false`.
+     * @param limit - How many entries to return. Only used when `getAllData = false`.
      * @returns The response containing the job status.
      */
-    checkBatchScrapeStatus(id?: string, getAllData?: boolean): Promise<BatchScrapeStatusResponse | ErrorResponse>;
+    checkBatchScrapeStatus(id?: string, getAllData?: boolean, nextURL?: string, skip?: number, limit?: number): Promise<BatchScrapeStatusResponse | ErrorResponse>;
     /**
      * Extracts information from URLs using the Firecrawl API.
      * Currently in Beta. Expect breaking changes on future minor versions.

package/dist/index.d.ts

@@ -258,7 +258,8 @@ interface ErrorResponse {
  */
 declare class FirecrawlError extends Error {
     statusCode: number;
-    constructor(message: string, statusCode: number);
+    details?: any;
+    constructor(message: string, statusCode: number, details?: any);
 }
 /**
  * Parameters for search operations.
@@ -326,9 +327,12 @@ declare class FirecrawlApp {
      * Checks the status of a crawl job using the Firecrawl API.
      * @param id - The ID of the crawl operation.
      * @param getAllData - Paginate through all the pages of documents, returning the full list of all documents. (default: `false`)
+     * @param nextURL - The `next` URL from the previous crawl status. Only required if you're not manually increasing `skip`. Only used when `getAllData = false`.
+     * @param skip - How many entries to skip to paginate. Only required if you're not providing `nextURL`. Only used when `getAllData = false`.
+     * @param limit - How many entries to return. Only used when `getAllData = false`.
      * @returns The response containing the job status.
      */
-    checkCrawlStatus(id?: string, getAllData?: boolean): Promise<CrawlStatusResponse | ErrorResponse>;
+    checkCrawlStatus(id?: string, getAllData?: boolean, nextURL?: string, skip?: number, limit?: number): Promise<CrawlStatusResponse | ErrorResponse>;
     /**
      * Cancels a crawl job using the Firecrawl API.
      * @param id - The ID of the crawl operation.
@@ -373,9 +377,12 @@ declare class FirecrawlApp {
      * Checks the status of a batch scrape job using the Firecrawl API.
      * @param id - The ID of the batch scrape operation.
      * @param getAllData - Paginate through all the pages of documents, returning the full list of all documents. (default: `false`)
+     * @param nextURL - The `next` URL from the previous batch scrape status. Only required if you're not manually increasing `skip`. Only used when `getAllData = false`.
+     * @param skip - How many entries to skip to paginate. Only used when `getAllData = false`.
+     * @param limit - How many entries to return. Only used when `getAllData = false`.
      * @returns The response containing the job status.
      */
-    checkBatchScrapeStatus(id?: string, getAllData?: boolean): Promise<BatchScrapeStatusResponse | ErrorResponse>;
+    checkBatchScrapeStatus(id?: string, getAllData?: boolean, nextURL?: string, skip?: number, limit?: number): Promise<BatchScrapeStatusResponse | ErrorResponse>;
     /**
      * Extracts information from URLs using the Firecrawl API.
      * Currently in Beta. Expect breaking changes on future minor versions.

package/dist/index.js

@@ -6,9 +6,11 @@ import { WebSocket } from "isows";
 import { TypedEventTarget } from "typescript-event-target";
 var FirecrawlError = class extends Error {
   statusCode;
-  constructor(message, statusCode) {
+  details;
+  constructor(message, statusCode, details) {
     super(message);
     this.statusCode = statusCode;
+    this.details = details;
   }
 };
 var FirecrawlApp = class {
@@ -209,16 +211,26 @@ var FirecrawlApp = class {
    * Checks the status of a crawl job using the Firecrawl API.
    * @param id - The ID of the crawl operation.
    * @param getAllData - Paginate through all the pages of documents, returning the full list of all documents. (default: `false`)
+   * @param nextURL - The `next` URL from the previous crawl status. Only required if you're not manually increasing `skip`. Only used when `getAllData = false`.
+   * @param skip - How many entries to skip to paginate. Only required if you're not providing `nextURL`. Only used when `getAllData = false`.
+   * @param limit - How many entries to return. Only used when `getAllData = false`.
    * @returns The response containing the job status.
    */
-  async checkCrawlStatus(id, getAllData = false) {
+  async checkCrawlStatus(id, getAllData = false, nextURL, skip, limit) {
     if (!id) {
       throw new FirecrawlError("No crawl ID provided", 400);
     }
     const headers = this.prepareHeaders();
+    const targetURL = new URL(nextURL ?? `${this.apiUrl}/v1/crawl/${id}`);
+    if (skip !== void 0) {
+      targetURL.searchParams.set("skip", skip.toString());
+    }
+    if (limit !== void 0) {
+      targetURL.searchParams.set("skip", limit.toString());
+    }
     try {
       const response = await this.getRequest(
-        `${this.apiUrl}/v1/crawl/${id}`,
+        targetURL.href,
         headers
       );
       if (response.status === 200) {
@@ -243,6 +255,7 @@ var FirecrawlApp = class {
           total: response.data.total,
           completed: response.data.completed,
           creditsUsed: response.data.creditsUsed,
+          next: getAllData ? void 0 : response.data.next,
           expiresAt: new Date(response.data.expiresAt),
           data: allData
         };
@@ -416,16 +429,26 @@ var FirecrawlApp = class {
    * Checks the status of a batch scrape job using the Firecrawl API.
    * @param id - The ID of the batch scrape operation.
    * @param getAllData - Paginate through all the pages of documents, returning the full list of all documents. (default: `false`)
+   * @param nextURL - The `next` URL from the previous batch scrape status. Only required if you're not manually increasing `skip`. Only used when `getAllData = false`.
+   * @param skip - How many entries to skip to paginate. Only used when `getAllData = false`.
+   * @param limit - How many entries to return. Only used when `getAllData = false`.
    * @returns The response containing the job status.
    */
-  async checkBatchScrapeStatus(id, getAllData = false) {
+  async checkBatchScrapeStatus(id, getAllData = false, nextURL, skip, limit) {
     if (!id) {
       throw new FirecrawlError("No batch scrape ID provided", 400);
     }
     const headers = this.prepareHeaders();
+    const targetURL = new URL(nextURL ?? `${this.apiUrl}/v1/batch/scrape/${id}`);
+    if (skip !== void 0) {
+      targetURL.searchParams.set("skip", skip.toString());
+    }
+    if (limit !== void 0) {
+      targetURL.searchParams.set("skip", limit.toString());
+    }
     try {
       const response = await this.getRequest(
-        `${this.apiUrl}/v1/batch/scrape/${id}`,
+        targetURL.href,
         headers
       );
       if (response.status === 200) {
@@ -450,6 +473,7 @@ var FirecrawlApp = class {
           total: response.data.total,
           completed: response.data.completed,
           creditsUsed: response.data.creditsUsed,
+          next: getAllData ? void 0 : response.data.next,
           expiresAt: new Date(response.data.expiresAt),
           data: allData
         };
@@ -529,7 +553,7 @@ var FirecrawlApp = class {
         this.handleError(response, "extract");
       }
     } catch (error) {
-      throw new FirecrawlError(error.message, 500);
+      throw new FirecrawlError(error.message, 500, error.response?.data?.details);
     }
     return { success: false, error: "Internal server error." };
   }
@@ -565,7 +589,7 @@ var FirecrawlApp = class {
         this.handleError(response, "start extract job");
       }
     } catch (error) {
-      throw new FirecrawlError(error.message, 500);
+      throw new FirecrawlError(error.message, 500, error.response?.data?.details);
     }
     return { success: false, error: "Internal server error." };
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "firecrawl",
-  "version": "1.13.0",
+  "version": "1.14.1",
   "description": "JavaScript SDK for Firecrawl API",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

package/src/index.ts

@@ -279,9 +279,11 @@ export interface ErrorResponse {
  */
 export class FirecrawlError extends Error {
   statusCode: number;
-  constructor(message: string, statusCode: number) {
+  details?: any;
+  constructor(message: string, statusCode: number, details?: any) {
     super(message);
     this.statusCode = statusCode;
+    this.details = details;
   }
 }
 
@@ -545,17 +547,28 @@ export default class FirecrawlApp {
    * Checks the status of a crawl job using the Firecrawl API.
    * @param id - The ID of the crawl operation.
    * @param getAllData - Paginate through all the pages of documents, returning the full list of all documents. (default: `false`)
+   * @param nextURL - The `next` URL from the previous crawl status. Only required if you're not manually increasing `skip`. Only used when `getAllData = false`.
+   * @param skip - How many entries to skip to paginate. Only required if you're not providing `nextURL`. Only used when `getAllData = false`.
+   * @param limit - How many entries to return. Only used when `getAllData = false`.
    * @returns The response containing the job status.
    */
-  async checkCrawlStatus(id?: string, getAllData = false): Promise<CrawlStatusResponse | ErrorResponse> {
+  async checkCrawlStatus(id?: string, getAllData = false, nextURL?: string, skip?: number, limit?: number): Promise<CrawlStatusResponse | ErrorResponse> {
     if (!id) {
       throw new FirecrawlError("No crawl ID provided", 400);
     }
 
     const headers: AxiosRequestHeaders = this.prepareHeaders();
+    const targetURL = new URL(nextURL ?? `${this.apiUrl}/v1/crawl/${id}`);
+    if (skip !== undefined) {
+      targetURL.searchParams.set("skip", skip.toString());
+    }
+    if (limit !== undefined) {
+      targetURL.searchParams.set("skip", limit.toString());
+    }
+
     try {
       const response: AxiosResponse = await this.getRequest(
-        `${this.apiUrl}/v1/crawl/${id}`,
+        targetURL.href,
         headers
       );
       if (response.status === 200) {
@@ -581,6 +594,7 @@ export default class FirecrawlApp {
           total: response.data.total,
           completed: response.data.completed,
           creditsUsed: response.data.creditsUsed,
+          next: getAllData ? undefined : response.data.next,
           expiresAt: new Date(response.data.expiresAt),
           data: allData
         }
@@ -795,17 +809,28 @@ export default class FirecrawlApp {
    * Checks the status of a batch scrape job using the Firecrawl API.
    * @param id - The ID of the batch scrape operation.
    * @param getAllData - Paginate through all the pages of documents, returning the full list of all documents. (default: `false`)
+   * @param nextURL - The `next` URL from the previous batch scrape status. Only required if you're not manually increasing `skip`. Only used when `getAllData = false`.
+   * @param skip - How many entries to skip to paginate. Only used when `getAllData = false`.
+   * @param limit - How many entries to return. Only used when `getAllData = false`.
    * @returns The response containing the job status.
    */
-  async checkBatchScrapeStatus(id?: string, getAllData = false): Promise<BatchScrapeStatusResponse | ErrorResponse> {
+  async checkBatchScrapeStatus(id?: string, getAllData = false, nextURL?: string, skip?: number, limit?: number): Promise<BatchScrapeStatusResponse | ErrorResponse> {
     if (!id) {
       throw new FirecrawlError("No batch scrape ID provided", 400);
     }
 
     const headers: AxiosRequestHeaders = this.prepareHeaders();
+    const targetURL = new URL(nextURL ?? `${this.apiUrl}/v1/batch/scrape/${id}`);
+    if (skip !== undefined) {
+      targetURL.searchParams.set("skip", skip.toString());
+    }
+    if (limit !== undefined) {
+      targetURL.searchParams.set("skip", limit.toString());
+    }
+
     try {
       const response: AxiosResponse = await this.getRequest(
-        `${this.apiUrl}/v1/batch/scrape/${id}`,
+        targetURL.href,
         headers
       );
       if (response.status === 200) {
@@ -831,6 +856,7 @@ export default class FirecrawlApp {
           total: response.data.total,
           completed: response.data.completed,
           creditsUsed: response.data.creditsUsed,
+          next: getAllData ? undefined : response.data.next,
           expiresAt: new Date(response.data.expiresAt),
           data: allData
         }
@@ -917,9 +943,9 @@ export default class FirecrawlApp {
         this.handleError(response, "extract");
       }
     } catch (error: any) {
-      throw new FirecrawlError(error.message, 500);
+      throw new FirecrawlError(error.message, 500, error.response?.data?.details);
     }
-    return { success: false, error: "Internal server error." };
+    return { success: false, error: "Internal server error."};
   }
 
   /**
@@ -961,7 +987,7 @@ export default class FirecrawlApp {
         this.handleError(response, "start extract job");
       }
     } catch (error: any) {
-      throw new FirecrawlError(error.message, 500);
+      throw new FirecrawlError(error.message, 500, error.response?.data?.details);
     }
     return { success: false, error: "Internal server error." };
   }