firecrawl 1.12.0 → 1.14.0

package/dist/index.cjs

@@ -245,16 +245,26 @@ var FirecrawlApp = class {
    * Checks the status of a crawl job using the Firecrawl API.
    * @param id - The ID of the crawl operation.
    * @param getAllData - Paginate through all the pages of documents, returning the full list of all documents. (default: `false`)
+   * @param nextURL - The `next` URL from the previous crawl status. Only required if you're not manually increasing `skip`. Only used when `getAllData = false`.
+   * @param skip - How many entries to skip to paginate. Only required if you're not providing `nextURL`. Only used when `getAllData = false`.
+   * @param limit - How many entries to return. Only used when `getAllData = false`.
    * @returns The response containing the job status.
    */
-  async checkCrawlStatus(id, getAllData = false) {
+  async checkCrawlStatus(id, getAllData = false, nextURL, skip, limit) {
     if (!id) {
       throw new FirecrawlError("No crawl ID provided", 400);
     }
     const headers = this.prepareHeaders();
+    const targetURL = new URL(nextURL ?? `${this.apiUrl}/v1/crawl/${id}`);
+    if (skip !== void 0) {
+      targetURL.searchParams.set("skip", skip.toString());
+    }
+    if (limit !== void 0) {
+      targetURL.searchParams.set("skip", limit.toString());
+    }
     try {
       const response = await this.getRequest(
-        `${this.apiUrl}/v1/crawl/${id}`,
+        targetURL.href,
         headers
       );
       if (response.status === 200) {
@@ -279,6 +289,7 @@ var FirecrawlApp = class {
           total: response.data.total,
           completed: response.data.completed,
           creditsUsed: response.data.creditsUsed,
+          next: getAllData ? void 0 : response.data.next,
           expiresAt: new Date(response.data.expiresAt),
           data: allData
         };
@@ -452,16 +463,26 @@ var FirecrawlApp = class {
    * Checks the status of a batch scrape job using the Firecrawl API.
    * @param id - The ID of the batch scrape operation.
    * @param getAllData - Paginate through all the pages of documents, returning the full list of all documents. (default: `false`)
+   * @param nextURL - The `next` URL from the previous batch scrape status. Only required if you're not manually increasing `skip`. Only used when `getAllData = false`.
+   * @param skip - How many entries to skip to paginate. Only used when `getAllData = false`.
+   * @param limit - How many entries to return. Only used when `getAllData = false`.
    * @returns The response containing the job status.
    */
-  async checkBatchScrapeStatus(id, getAllData = false) {
+  async checkBatchScrapeStatus(id, getAllData = false, nextURL, skip, limit) {
     if (!id) {
       throw new FirecrawlError("No batch scrape ID provided", 400);
     }
     const headers = this.prepareHeaders();
+    const targetURL = new URL(nextURL ?? `${this.apiUrl}/v1/batch/scrape/${id}`);
+    if (skip !== void 0) {
+      targetURL.searchParams.set("skip", skip.toString());
+    }
+    if (limit !== void 0) {
+      targetURL.searchParams.set("skip", limit.toString());
+    }
     try {
       const response = await this.getRequest(
-        `${this.apiUrl}/v1/batch/scrape/${id}`,
+        targetURL.href,
         headers
       );
       if (response.status === 200) {
@@ -486,6 +507,7 @@ var FirecrawlApp = class {
           total: response.data.total,
           completed: response.data.completed,
           creditsUsed: response.data.creditsUsed,
+          next: getAllData ? void 0 : response.data.next,
           expiresAt: new Date(response.data.expiresAt),
           data: allData
         };
@@ -576,9 +598,9 @@ var FirecrawlApp = class {
    * @param idempotencyKey - Optional idempotency key for the request.
    * @returns The response from the extract operation.
    */
-  async asyncExtract(url, params, idempotencyKey) {
+  async asyncExtract(urls, params, idempotencyKey) {
     const headers = this.prepareHeaders(idempotencyKey);
-    let jsonData = { url, ...params };
+    let jsonData = { urls, ...params };
     let jsonSchema;
     try {
       if (params?.schema instanceof zt.ZodType) {

package/dist/index.d.cts

@@ -326,9 +326,12 @@ declare class FirecrawlApp {
      * Checks the status of a crawl job using the Firecrawl API.
      * @param id - The ID of the crawl operation.
      * @param getAllData - Paginate through all the pages of documents, returning the full list of all documents. (default: `false`)
+     * @param nextURL - The `next` URL from the previous crawl status. Only required if you're not manually increasing `skip`. Only used when `getAllData = false`.
+     * @param skip - How many entries to skip to paginate. Only required if you're not providing `nextURL`. Only used when `getAllData = false`.
+     * @param limit - How many entries to return. Only used when `getAllData = false`.
      * @returns The response containing the job status.
      */
-    checkCrawlStatus(id?: string, getAllData?: boolean): Promise<CrawlStatusResponse | ErrorResponse>;
+    checkCrawlStatus(id?: string, getAllData?: boolean, nextURL?: string, skip?: number, limit?: number): Promise<CrawlStatusResponse | ErrorResponse>;
     /**
      * Cancels a crawl job using the Firecrawl API.
      * @param id - The ID of the crawl operation.
@@ -373,9 +376,12 @@ declare class FirecrawlApp {
      * Checks the status of a batch scrape job using the Firecrawl API.
      * @param id - The ID of the batch scrape operation.
      * @param getAllData - Paginate through all the pages of documents, returning the full list of all documents. (default: `false`)
+     * @param nextURL - The `next` URL from the previous batch scrape status. Only required if you're not manually increasing `skip`. Only used when `getAllData = false`.
+     * @param skip - How many entries to skip to paginate. Only used when `getAllData = false`.
+     * @param limit - How many entries to return. Only used when `getAllData = false`.
      * @returns The response containing the job status.
      */
-    checkBatchScrapeStatus(id?: string, getAllData?: boolean): Promise<BatchScrapeStatusResponse | ErrorResponse>;
+    checkBatchScrapeStatus(id?: string, getAllData?: boolean, nextURL?: string, skip?: number, limit?: number): Promise<BatchScrapeStatusResponse | ErrorResponse>;
     /**
      * Extracts information from URLs using the Firecrawl API.
      * Currently in Beta. Expect breaking changes on future minor versions.
@@ -391,7 +397,7 @@ declare class FirecrawlApp {
      * @param idempotencyKey - Optional idempotency key for the request.
      * @returns The response from the extract operation.
      */
-    asyncExtract(url: string, params?: ExtractParams, idempotencyKey?: string): Promise<ExtractResponse | ErrorResponse>;
+    asyncExtract(urls: string[], params?: ExtractParams, idempotencyKey?: string): Promise<ExtractResponse | ErrorResponse>;
     /**
      * Retrieves the status of an extract job.
      * @param jobId - The ID of the extract job.

package/dist/index.d.ts

@@ -326,9 +326,12 @@ declare class FirecrawlApp {
      * Checks the status of a crawl job using the Firecrawl API.
      * @param id - The ID of the crawl operation.
      * @param getAllData - Paginate through all the pages of documents, returning the full list of all documents. (default: `false`)
+     * @param nextURL - The `next` URL from the previous crawl status. Only required if you're not manually increasing `skip`. Only used when `getAllData = false`.
+     * @param skip - How many entries to skip to paginate. Only required if you're not providing `nextURL`. Only used when `getAllData = false`.
+     * @param limit - How many entries to return. Only used when `getAllData = false`.
      * @returns The response containing the job status.
      */
-    checkCrawlStatus(id?: string, getAllData?: boolean): Promise<CrawlStatusResponse | ErrorResponse>;
+    checkCrawlStatus(id?: string, getAllData?: boolean, nextURL?: string, skip?: number, limit?: number): Promise<CrawlStatusResponse | ErrorResponse>;
     /**
      * Cancels a crawl job using the Firecrawl API.
      * @param id - The ID of the crawl operation.
@@ -373,9 +376,12 @@ declare class FirecrawlApp {
      * Checks the status of a batch scrape job using the Firecrawl API.
      * @param id - The ID of the batch scrape operation.
      * @param getAllData - Paginate through all the pages of documents, returning the full list of all documents. (default: `false`)
+     * @param nextURL - The `next` URL from the previous batch scrape status. Only required if you're not manually increasing `skip`. Only used when `getAllData = false`.
+     * @param skip - How many entries to skip to paginate. Only used when `getAllData = false`.
+     * @param limit - How many entries to return. Only used when `getAllData = false`.
      * @returns The response containing the job status.
      */
-    checkBatchScrapeStatus(id?: string, getAllData?: boolean): Promise<BatchScrapeStatusResponse | ErrorResponse>;
+    checkBatchScrapeStatus(id?: string, getAllData?: boolean, nextURL?: string, skip?: number, limit?: number): Promise<BatchScrapeStatusResponse | ErrorResponse>;
     /**
      * Extracts information from URLs using the Firecrawl API.
      * Currently in Beta. Expect breaking changes on future minor versions.
@@ -391,7 +397,7 @@ declare class FirecrawlApp {
      * @param idempotencyKey - Optional idempotency key for the request.
      * @returns The response from the extract operation.
      */
-    asyncExtract(url: string, params?: ExtractParams, idempotencyKey?: string): Promise<ExtractResponse | ErrorResponse>;
+    asyncExtract(urls: string[], params?: ExtractParams, idempotencyKey?: string): Promise<ExtractResponse | ErrorResponse>;
     /**
      * Retrieves the status of an extract job.
      * @param jobId - The ID of the extract job.

package/dist/index.js

@@ -209,16 +209,26 @@ var FirecrawlApp = class {
    * Checks the status of a crawl job using the Firecrawl API.
    * @param id - The ID of the crawl operation.
    * @param getAllData - Paginate through all the pages of documents, returning the full list of all documents. (default: `false`)
+   * @param nextURL - The `next` URL from the previous crawl status. Only required if you're not manually increasing `skip`. Only used when `getAllData = false`.
+   * @param skip - How many entries to skip to paginate. Only required if you're not providing `nextURL`. Only used when `getAllData = false`.
+   * @param limit - How many entries to return. Only used when `getAllData = false`.
    * @returns The response containing the job status.
    */
-  async checkCrawlStatus(id, getAllData = false) {
+  async checkCrawlStatus(id, getAllData = false, nextURL, skip, limit) {
     if (!id) {
       throw new FirecrawlError("No crawl ID provided", 400);
     }
     const headers = this.prepareHeaders();
+    const targetURL = new URL(nextURL ?? `${this.apiUrl}/v1/crawl/${id}`);
+    if (skip !== void 0) {
+      targetURL.searchParams.set("skip", skip.toString());
+    }
+    if (limit !== void 0) {
+      targetURL.searchParams.set("skip", limit.toString());
+    }
     try {
       const response = await this.getRequest(
-        `${this.apiUrl}/v1/crawl/${id}`,
+        targetURL.href,
         headers
       );
       if (response.status === 200) {
@@ -243,6 +253,7 @@ var FirecrawlApp = class {
           total: response.data.total,
           completed: response.data.completed,
           creditsUsed: response.data.creditsUsed,
+          next: getAllData ? void 0 : response.data.next,
           expiresAt: new Date(response.data.expiresAt),
           data: allData
         };
@@ -416,16 +427,26 @@ var FirecrawlApp = class {
    * Checks the status of a batch scrape job using the Firecrawl API.
    * @param id - The ID of the batch scrape operation.
    * @param getAllData - Paginate through all the pages of documents, returning the full list of all documents. (default: `false`)
+   * @param nextURL - The `next` URL from the previous batch scrape status. Only required if you're not manually increasing `skip`. Only used when `getAllData = false`.
+   * @param skip - How many entries to skip to paginate. Only used when `getAllData = false`.
+   * @param limit - How many entries to return. Only used when `getAllData = false`.
    * @returns The response containing the job status.
    */
-  async checkBatchScrapeStatus(id, getAllData = false) {
+  async checkBatchScrapeStatus(id, getAllData = false, nextURL, skip, limit) {
     if (!id) {
       throw new FirecrawlError("No batch scrape ID provided", 400);
     }
     const headers = this.prepareHeaders();
+    const targetURL = new URL(nextURL ?? `${this.apiUrl}/v1/batch/scrape/${id}`);
+    if (skip !== void 0) {
+      targetURL.searchParams.set("skip", skip.toString());
+    }
+    if (limit !== void 0) {
+      targetURL.searchParams.set("skip", limit.toString());
+    }
     try {
       const response = await this.getRequest(
-        `${this.apiUrl}/v1/batch/scrape/${id}`,
+        targetURL.href,
         headers
       );
       if (response.status === 200) {
@@ -450,6 +471,7 @@ var FirecrawlApp = class {
           total: response.data.total,
           completed: response.data.completed,
           creditsUsed: response.data.creditsUsed,
+          next: getAllData ? void 0 : response.data.next,
           expiresAt: new Date(response.data.expiresAt),
           data: allData
         };
@@ -540,9 +562,9 @@ var FirecrawlApp = class {
    * @param idempotencyKey - Optional idempotency key for the request.
    * @returns The response from the extract operation.
    */
-  async asyncExtract(url, params, idempotencyKey) {
+  async asyncExtract(urls, params, idempotencyKey) {
     const headers = this.prepareHeaders(idempotencyKey);
-    let jsonData = { url, ...params };
+    let jsonData = { urls, ...params };
     let jsonSchema;
     try {
       if (params?.schema instanceof zt.ZodType) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "firecrawl",
-  "version": "1.12.0",
+  "version": "1.14.0",
   "description": "JavaScript SDK for Firecrawl API",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

package/src/index.ts

@@ -545,17 +545,28 @@ export default class FirecrawlApp {
    * Checks the status of a crawl job using the Firecrawl API.
    * @param id - The ID of the crawl operation.
    * @param getAllData - Paginate through all the pages of documents, returning the full list of all documents. (default: `false`)
+   * @param nextURL - The `next` URL from the previous crawl status. Only required if you're not manually increasing `skip`. Only used when `getAllData = false`.
+   * @param skip - How many entries to skip to paginate. Only required if you're not providing `nextURL`. Only used when `getAllData = false`.
+   * @param limit - How many entries to return. Only used when `getAllData = false`.
    * @returns The response containing the job status.
    */
-  async checkCrawlStatus(id?: string, getAllData = false): Promise<CrawlStatusResponse | ErrorResponse> {
+  async checkCrawlStatus(id?: string, getAllData = false, nextURL?: string, skip?: number, limit?: number): Promise<CrawlStatusResponse | ErrorResponse> {
     if (!id) {
       throw new FirecrawlError("No crawl ID provided", 400);
     }
 
     const headers: AxiosRequestHeaders = this.prepareHeaders();
+    const targetURL = new URL(nextURL ?? `${this.apiUrl}/v1/crawl/${id}`);
+    if (skip !== undefined) {
+      targetURL.searchParams.set("skip", skip.toString());
+    }
+    if (limit !== undefined) {
+      targetURL.searchParams.set("skip", limit.toString());
+    }
+
     try {
       const response: AxiosResponse = await this.getRequest(
-        `${this.apiUrl}/v1/crawl/${id}`,
+        targetURL.href,
         headers
       );
       if (response.status === 200) {
@@ -581,6 +592,7 @@ export default class FirecrawlApp {
           total: response.data.total,
           completed: response.data.completed,
           creditsUsed: response.data.creditsUsed,
+          next: getAllData ? undefined : response.data.next,
           expiresAt: new Date(response.data.expiresAt),
           data: allData
         }
@@ -795,17 +807,28 @@ export default class FirecrawlApp {
    * Checks the status of a batch scrape job using the Firecrawl API.
    * @param id - The ID of the batch scrape operation.
    * @param getAllData - Paginate through all the pages of documents, returning the full list of all documents. (default: `false`)
+   * @param nextURL - The `next` URL from the previous batch scrape status. Only required if you're not manually increasing `skip`. Only used when `getAllData = false`.
+   * @param skip - How many entries to skip to paginate. Only used when `getAllData = false`.
+   * @param limit - How many entries to return. Only used when `getAllData = false`.
    * @returns The response containing the job status.
    */
-  async checkBatchScrapeStatus(id?: string, getAllData = false): Promise<BatchScrapeStatusResponse | ErrorResponse> {
+  async checkBatchScrapeStatus(id?: string, getAllData = false, nextURL?: string, skip?: number, limit?: number): Promise<BatchScrapeStatusResponse | ErrorResponse> {
     if (!id) {
       throw new FirecrawlError("No batch scrape ID provided", 400);
     }
 
     const headers: AxiosRequestHeaders = this.prepareHeaders();
+    const targetURL = new URL(nextURL ?? `${this.apiUrl}/v1/batch/scrape/${id}`);
+    if (skip !== undefined) {
+      targetURL.searchParams.set("skip", skip.toString());
+    }
+    if (limit !== undefined) {
+      targetURL.searchParams.set("skip", limit.toString());
+    }
+
     try {
       const response: AxiosResponse = await this.getRequest(
-        `${this.apiUrl}/v1/batch/scrape/${id}`,
+        targetURL.href,
         headers
       );
       if (response.status === 200) {
@@ -831,6 +854,7 @@ export default class FirecrawlApp {
           total: response.data.total,
           completed: response.data.completed,
           creditsUsed: response.data.creditsUsed,
+          next: getAllData ? undefined : response.data.next,
           expiresAt: new Date(response.data.expiresAt),
           data: allData
         }
@@ -930,12 +954,12 @@ export default class FirecrawlApp {
    * @returns The response from the extract operation.
    */
   async asyncExtract(
-    url: string,
+    urls: string[],
     params?: ExtractParams,
     idempotencyKey?: string
   ): Promise<ExtractResponse | ErrorResponse> {
     const headers = this.prepareHeaders(idempotencyKey);
-    let jsonData: any = { url, ...params };
+    let jsonData: any = { urls, ...params };
     let jsonSchema: any;
 
     try {