firecrawl 1.9.7 → 1.10.1

package/dist/index.cjs

@@ -198,7 +198,7 @@ var FirecrawlApp = class {
           let statusData = response.data;
           if ("data" in statusData) {
             let data = statusData.data;
-            while ("next" in statusData) {
+            while (typeof statusData === "object" && "next" in statusData) {
               statusData = (await this.getRequest(statusData.next, headers)).data;
               data = data.concat(statusData.data);
             }
@@ -295,9 +295,9 @@ var FirecrawlApp = class {
    * @param webhook - Optional webhook for the batch scrape.
    * @returns The response from the crawl operation.
    */
-  async batchScrapeUrls(urls, params, pollInterval = 2, idempotencyKey, webhook, ignoreInvalidURLs) {
+  async batchScrapeUrls(urls, params, pollInterval = 2, idempotencyKey, webhook) {
     const headers = this.prepareHeaders(idempotencyKey);
-    let jsonData = { urls, webhook, ignoreInvalidURLs, ...params };
+    let jsonData = { urls, ...params };
     if (jsonData?.extract?.schema) {
       let schema = jsonData.extract.schema;
       try {
@@ -333,9 +333,9 @@ var FirecrawlApp = class {
     }
     return { success: false, error: "Internal server error." };
   }
-  async asyncBatchScrapeUrls(urls, params, idempotencyKey, webhook, ignoreInvalidURLs) {
+  async asyncBatchScrapeUrls(urls, params, idempotencyKey) {
     const headers = this.prepareHeaders(idempotencyKey);
-    let jsonData = { urls, webhook, ignoreInvalidURLs, ...params ?? {} };
+    let jsonData = { urls, ...params ?? {} };
     try {
       const response = await this.postRequest(
         this.apiUrl + `/v1/batch/scrape`,
@@ -363,8 +363,8 @@ var FirecrawlApp = class {
    * @param idempotencyKey - Optional idempotency key for the request.
    * @returns A CrawlWatcher instance to monitor the crawl job.
    */
-  async batchScrapeUrlsAndWatch(urls, params, idempotencyKey, webhook, ignoreInvalidURLs) {
-    const crawl = await this.asyncBatchScrapeUrls(urls, params, idempotencyKey, webhook, ignoreInvalidURLs);
+  async batchScrapeUrlsAndWatch(urls, params, idempotencyKey) {
+    const crawl = await this.asyncBatchScrapeUrls(urls, params, idempotencyKey);
     if (crawl.success && crawl.id) {
       const id = crawl.id;
       return new CrawlWatcher(id, this);
@@ -393,7 +393,7 @@ var FirecrawlApp = class {
           let statusData = response.data;
           if ("data" in statusData) {
             let data = statusData.data;
-            while ("next" in statusData) {
+            while (typeof statusData === "object" && "next" in statusData) {
               statusData = (await this.getRequest(statusData.next, headers)).data;
               data = data.concat(statusData.data);
             }
@@ -529,40 +529,44 @@ var FirecrawlApp = class {
    * @returns The final job status or data.
    */
   async monitorJobStatus(id, headers, checkInterval) {
-    while (true) {
-      let statusResponse = await this.getRequest(
-        `${this.apiUrl}/v1/crawl/${id}`,
-        headers
-      );
-      if (statusResponse.status === 200) {
-        let statusData = statusResponse.data;
-        if (statusData.status === "completed") {
-          if ("data" in statusData) {
-            let data = statusData.data;
-            while ("next" in statusData) {
-              statusResponse = await this.getRequest(statusData.next, headers);
-              statusData = statusResponse.data;
-              data = data.concat(statusData.data);
+    try {
+      while (true) {
+        let statusResponse = await this.getRequest(
+          `${this.apiUrl}/v1/crawl/${id}`,
+          headers
+        );
+        if (statusResponse.status === 200) {
+          let statusData = statusResponse.data;
+          if (statusData.status === "completed") {
+            if ("data" in statusData) {
+              let data = statusData.data;
+              while (typeof statusData === "object" && "next" in statusData) {
+                statusResponse = await this.getRequest(statusData.next, headers);
+                statusData = statusResponse.data;
+                data = data.concat(statusData.data);
+              }
+              statusData.data = data;
+              return statusData;
+            } else {
+              throw new FirecrawlError("Crawl job completed but no data was returned", 500);
             }
-            statusData.data = data;
-            return statusData;
+          } else if (["active", "paused", "pending", "queued", "waiting", "scraping"].includes(statusData.status)) {
+            checkInterval = Math.max(checkInterval, 2);
+            await new Promise(
+              (resolve) => setTimeout(resolve, checkInterval * 1e3)
+            );
           } else {
-            throw new FirecrawlError("Crawl job completed but no data was returned", 500);
+            throw new FirecrawlError(
+              `Crawl job failed or was stopped. Status: ${statusData.status}`,
+              500
+            );
           }
-        } else if (["active", "paused", "pending", "queued", "waiting", "scraping"].includes(statusData.status)) {
-          checkInterval = Math.max(checkInterval, 2);
-          await new Promise(
-            (resolve) => setTimeout(resolve, checkInterval * 1e3)
-          );
         } else {
-          throw new FirecrawlError(
-            `Crawl job failed or was stopped. Status: ${statusData.status}`,
-            500
-          );
+          this.handleError(statusResponse, "check crawl status");
         }
-      } else {
-        this.handleError(statusResponse, "check crawl status");
       }
+    } catch (error) {
+      throw new FirecrawlError(error, 500);
     }
   }
   /**
@@ -589,10 +593,8 @@ var CrawlWatcher = class extends import_typescript_event_target.TypedEventTarget
   ws;
   data;
   status;
-  id;
   constructor(id, app) {
     super();
-    this.id = id;
     this.ws = new import_isows.WebSocket(`${app.apiUrl}/v1/crawl/${id}`, app.apiKey);
     this.status = "scraping";
     this.data = [];
@@ -602,8 +604,7 @@ var CrawlWatcher = class extends import_typescript_event_target.TypedEventTarget
         this.dispatchTypedEvent("done", new CustomEvent("done", {
           detail: {
             status: this.status,
-            data: this.data,
-            id: this.id
+            data: this.data
           }
         }));
       } else if (msg.type === "error") {
@@ -612,8 +613,7 @@ var CrawlWatcher = class extends import_typescript_event_target.TypedEventTarget
           detail: {
             status: this.status,
             data: this.data,
-            error: msg.error,
-            id: this.id
+            error: msg.error
           }
         }));
       } else if (msg.type === "catchup") {
@@ -621,18 +621,12 @@ var CrawlWatcher = class extends import_typescript_event_target.TypedEventTarget
         this.data.push(...msg.data.data ?? []);
         for (const doc of this.data) {
           this.dispatchTypedEvent("document", new CustomEvent("document", {
-            detail: {
-              ...doc,
-              id: this.id
-            }
+            detail: doc
           }));
         }
       } else if (msg.type === "document") {
         this.dispatchTypedEvent("document", new CustomEvent("document", {
-          detail: {
-            ...msg.data,
-            id: this.id
-          }
+          detail: msg.data
         }));
       }
     };
@@ -654,8 +648,7 @@ var CrawlWatcher = class extends import_typescript_event_target.TypedEventTarget
         detail: {
           status: this.status,
           data: this.data,
-          error: "WebSocket error",
-          id: this.id
+          error: "WebSocket error"
         }
       }));
     }).bind(this);

package/dist/index.d.cts

@@ -171,7 +171,6 @@ interface BatchScrapeResponse {
     url?: string;
     success: true;
     error?: string;
-    invalidURLs?: string[];
 }
 /**
  * Response interface for job status checks.
@@ -226,11 +225,10 @@ interface MapResponse {
  * Defines options for extracting information from URLs.
  */
 interface ExtractParams<LLMSchema extends zt.ZodSchema = any> {
-    prompt?: string;
+    prompt: string;
     schema?: LLMSchema;
     systemPrompt?: string;
     allowExternalLinks?: boolean;
-    includeSubdomains?: boolean;
 }
 /**
  * Response interface for extracting information from URLs.
@@ -331,8 +329,8 @@ declare class FirecrawlApp {
      * @param webhook - Optional webhook for the batch scrape.
      * @returns The response from the crawl operation.
      */
-    batchScrapeUrls(urls: string[], params?: ScrapeParams, pollInterval?: number, idempotencyKey?: string, webhook?: CrawlParams["webhook"], ignoreInvalidURLs?: boolean): Promise<BatchScrapeStatusResponse | ErrorResponse>;
-    asyncBatchScrapeUrls(urls: string[], params?: ScrapeParams, idempotencyKey?: string, webhook?: CrawlParams["webhook"], ignoreInvalidURLs?: boolean): Promise<BatchScrapeResponse | ErrorResponse>;
+    batchScrapeUrls(urls: string[], params?: ScrapeParams, pollInterval?: number, idempotencyKey?: string, webhook?: CrawlParams["webhook"]): Promise<BatchScrapeStatusResponse | ErrorResponse>;
+    asyncBatchScrapeUrls(urls: string[], params?: ScrapeParams, idempotencyKey?: string): Promise<BatchScrapeResponse | ErrorResponse>;
     /**
      * Initiates a batch scrape job and returns a CrawlWatcher to monitor the job via WebSocket.
      * @param urls - The URL to scrape.
@@ -340,7 +338,7 @@ declare class FirecrawlApp {
      * @param idempotencyKey - Optional idempotency key for the request.
      * @returns A CrawlWatcher instance to monitor the crawl job.
      */
-    batchScrapeUrlsAndWatch(urls: string[], params?: ScrapeParams, idempotencyKey?: string, webhook?: CrawlParams["webhook"], ignoreInvalidURLs?: boolean): Promise<CrawlWatcher>;
+    batchScrapeUrlsAndWatch(urls: string[], params?: ScrapeParams, idempotencyKey?: string): Promise<CrawlWatcher>;
     /**
      * Checks the status of a batch scrape job using the Firecrawl API.
      * @param id - The ID of the batch scrape operation.
@@ -416,7 +414,6 @@ declare class CrawlWatcher extends TypedEventTarget<CrawlWatcherEvents> {
     private ws;
     data: FirecrawlDocument<undefined>[];
     status: CrawlStatusResponse["status"];
-    id: string;
     constructor(id: string, app: FirecrawlApp);
     close(): void;
 }

package/dist/index.d.ts

@@ -171,7 +171,6 @@ interface BatchScrapeResponse {
     url?: string;
     success: true;
     error?: string;
-    invalidURLs?: string[];
 }
 /**
  * Response interface for job status checks.
@@ -226,11 +225,10 @@ interface MapResponse {
  * Defines options for extracting information from URLs.
  */
 interface ExtractParams<LLMSchema extends zt.ZodSchema = any> {
-    prompt?: string;
+    prompt: string;
     schema?: LLMSchema;
     systemPrompt?: string;
     allowExternalLinks?: boolean;
-    includeSubdomains?: boolean;
 }
 /**
  * Response interface for extracting information from URLs.
@@ -331,8 +329,8 @@ declare class FirecrawlApp {
      * @param webhook - Optional webhook for the batch scrape.
      * @returns The response from the crawl operation.
      */
-    batchScrapeUrls(urls: string[], params?: ScrapeParams, pollInterval?: number, idempotencyKey?: string, webhook?: CrawlParams["webhook"], ignoreInvalidURLs?: boolean): Promise<BatchScrapeStatusResponse | ErrorResponse>;
-    asyncBatchScrapeUrls(urls: string[], params?: ScrapeParams, idempotencyKey?: string, webhook?: CrawlParams["webhook"], ignoreInvalidURLs?: boolean): Promise<BatchScrapeResponse | ErrorResponse>;
+    batchScrapeUrls(urls: string[], params?: ScrapeParams, pollInterval?: number, idempotencyKey?: string, webhook?: CrawlParams["webhook"]): Promise<BatchScrapeStatusResponse | ErrorResponse>;
+    asyncBatchScrapeUrls(urls: string[], params?: ScrapeParams, idempotencyKey?: string): Promise<BatchScrapeResponse | ErrorResponse>;
     /**
      * Initiates a batch scrape job and returns a CrawlWatcher to monitor the job via WebSocket.
      * @param urls - The URL to scrape.
@@ -340,7 +338,7 @@ declare class FirecrawlApp {
      * @param idempotencyKey - Optional idempotency key for the request.
      * @returns A CrawlWatcher instance to monitor the crawl job.
      */
-    batchScrapeUrlsAndWatch(urls: string[], params?: ScrapeParams, idempotencyKey?: string, webhook?: CrawlParams["webhook"], ignoreInvalidURLs?: boolean): Promise<CrawlWatcher>;
+    batchScrapeUrlsAndWatch(urls: string[], params?: ScrapeParams, idempotencyKey?: string): Promise<CrawlWatcher>;
     /**
      * Checks the status of a batch scrape job using the Firecrawl API.
      * @param id - The ID of the batch scrape operation.
@@ -416,7 +414,6 @@ declare class CrawlWatcher extends TypedEventTarget<CrawlWatcherEvents> {
     private ws;
     data: FirecrawlDocument<undefined>[];
     status: CrawlStatusResponse["status"];
-    id: string;
     constructor(id: string, app: FirecrawlApp);
     close(): void;
 }

package/dist/index.js

@@ -162,7 +162,7 @@ var FirecrawlApp = class {
           let statusData = response.data;
           if ("data" in statusData) {
             let data = statusData.data;
-            while ("next" in statusData) {
+            while (typeof statusData === "object" && "next" in statusData) {
               statusData = (await this.getRequest(statusData.next, headers)).data;
               data = data.concat(statusData.data);
             }
@@ -259,9 +259,9 @@ var FirecrawlApp = class {
    * @param webhook - Optional webhook for the batch scrape.
    * @returns The response from the crawl operation.
    */
-  async batchScrapeUrls(urls, params, pollInterval = 2, idempotencyKey, webhook, ignoreInvalidURLs) {
+  async batchScrapeUrls(urls, params, pollInterval = 2, idempotencyKey, webhook) {
     const headers = this.prepareHeaders(idempotencyKey);
-    let jsonData = { urls, webhook, ignoreInvalidURLs, ...params };
+    let jsonData = { urls, ...params };
     if (jsonData?.extract?.schema) {
       let schema = jsonData.extract.schema;
       try {
@@ -297,9 +297,9 @@ var FirecrawlApp = class {
     }
     return { success: false, error: "Internal server error." };
   }
-  async asyncBatchScrapeUrls(urls, params, idempotencyKey, webhook, ignoreInvalidURLs) {
+  async asyncBatchScrapeUrls(urls, params, idempotencyKey) {
     const headers = this.prepareHeaders(idempotencyKey);
-    let jsonData = { urls, webhook, ignoreInvalidURLs, ...params ?? {} };
+    let jsonData = { urls, ...params ?? {} };
     try {
       const response = await this.postRequest(
         this.apiUrl + `/v1/batch/scrape`,
@@ -327,8 +327,8 @@ var FirecrawlApp = class {
    * @param idempotencyKey - Optional idempotency key for the request.
    * @returns A CrawlWatcher instance to monitor the crawl job.
    */
-  async batchScrapeUrlsAndWatch(urls, params, idempotencyKey, webhook, ignoreInvalidURLs) {
-    const crawl = await this.asyncBatchScrapeUrls(urls, params, idempotencyKey, webhook, ignoreInvalidURLs);
+  async batchScrapeUrlsAndWatch(urls, params, idempotencyKey) {
+    const crawl = await this.asyncBatchScrapeUrls(urls, params, idempotencyKey);
     if (crawl.success && crawl.id) {
       const id = crawl.id;
       return new CrawlWatcher(id, this);
@@ -357,7 +357,7 @@ var FirecrawlApp = class {
           let statusData = response.data;
           if ("data" in statusData) {
             let data = statusData.data;
-            while ("next" in statusData) {
+            while (typeof statusData === "object" && "next" in statusData) {
               statusData = (await this.getRequest(statusData.next, headers)).data;
               data = data.concat(statusData.data);
             }
@@ -493,40 +493,44 @@ var FirecrawlApp = class {
    * @returns The final job status or data.
    */
   async monitorJobStatus(id, headers, checkInterval) {
-    while (true) {
-      let statusResponse = await this.getRequest(
-        `${this.apiUrl}/v1/crawl/${id}`,
-        headers
-      );
-      if (statusResponse.status === 200) {
-        let statusData = statusResponse.data;
-        if (statusData.status === "completed") {
-          if ("data" in statusData) {
-            let data = statusData.data;
-            while ("next" in statusData) {
-              statusResponse = await this.getRequest(statusData.next, headers);
-              statusData = statusResponse.data;
-              data = data.concat(statusData.data);
+    try {
+      while (true) {
+        let statusResponse = await this.getRequest(
+          `${this.apiUrl}/v1/crawl/${id}`,
+          headers
+        );
+        if (statusResponse.status === 200) {
+          let statusData = statusResponse.data;
+          if (statusData.status === "completed") {
+            if ("data" in statusData) {
+              let data = statusData.data;
+              while (typeof statusData === "object" && "next" in statusData) {
+                statusResponse = await this.getRequest(statusData.next, headers);
+                statusData = statusResponse.data;
+                data = data.concat(statusData.data);
+              }
+              statusData.data = data;
+              return statusData;
+            } else {
+              throw new FirecrawlError("Crawl job completed but no data was returned", 500);
             }
-            statusData.data = data;
-            return statusData;
+          } else if (["active", "paused", "pending", "queued", "waiting", "scraping"].includes(statusData.status)) {
+            checkInterval = Math.max(checkInterval, 2);
+            await new Promise(
+              (resolve) => setTimeout(resolve, checkInterval * 1e3)
+            );
           } else {
-            throw new FirecrawlError("Crawl job completed but no data was returned", 500);
+            throw new FirecrawlError(
+              `Crawl job failed or was stopped. Status: ${statusData.status}`,
+              500
+            );
           }
-        } else if (["active", "paused", "pending", "queued", "waiting", "scraping"].includes(statusData.status)) {
-          checkInterval = Math.max(checkInterval, 2);
-          await new Promise(
-            (resolve) => setTimeout(resolve, checkInterval * 1e3)
-          );
         } else {
-          throw new FirecrawlError(
-            `Crawl job failed or was stopped. Status: ${statusData.status}`,
-            500
-          );
+          this.handleError(statusResponse, "check crawl status");
         }
-      } else {
-        this.handleError(statusResponse, "check crawl status");
       }
+    } catch (error) {
+      throw new FirecrawlError(error, 500);
     }
   }
   /**
@@ -553,10 +557,8 @@ var CrawlWatcher = class extends TypedEventTarget {
   ws;
   data;
   status;
-  id;
   constructor(id, app) {
     super();
-    this.id = id;
     this.ws = new WebSocket(`${app.apiUrl}/v1/crawl/${id}`, app.apiKey);
     this.status = "scraping";
     this.data = [];
@@ -566,8 +568,7 @@ var CrawlWatcher = class extends TypedEventTarget {
         this.dispatchTypedEvent("done", new CustomEvent("done", {
           detail: {
             status: this.status,
-            data: this.data,
-            id: this.id
+            data: this.data
           }
         }));
       } else if (msg.type === "error") {
@@ -576,8 +577,7 @@ var CrawlWatcher = class extends TypedEventTarget {
           detail: {
             status: this.status,
             data: this.data,
-            error: msg.error,
-            id: this.id
+            error: msg.error
           }
         }));
       } else if (msg.type === "catchup") {
@@ -585,18 +585,12 @@ var CrawlWatcher = class extends TypedEventTarget {
         this.data.push(...msg.data.data ?? []);
         for (const doc of this.data) {
           this.dispatchTypedEvent("document", new CustomEvent("document", {
-            detail: {
-              ...doc,
-              id: this.id
-            }
+            detail: doc
           }));
         }
       } else if (msg.type === "document") {
         this.dispatchTypedEvent("document", new CustomEvent("document", {
-          detail: {
-            ...msg.data,
-            id: this.id
-          }
+          detail: msg.data
         }));
       }
     };
@@ -618,8 +612,7 @@ var CrawlWatcher = class extends TypedEventTarget {
         detail: {
           status: this.status,
           data: this.data,
-          error: "WebSocket error",
-          id: this.id
+          error: "WebSocket error"
         }
       }));
     }).bind(this);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "firecrawl",
-  "version": "1.9.7",
+  "version": "1.10.1",
   "description": "JavaScript SDK for Firecrawl API",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

package/src/index.ts

@@ -183,7 +183,6 @@ export interface BatchScrapeResponse {
   url?: string;
   success: true;
   error?: string;
-  invalidURLs?: string[];
 }
 
 /**
@@ -243,11 +242,10 @@ export interface MapResponse {
  * Defines options for extracting information from URLs.
  */
 export interface ExtractParams<LLMSchema extends zt.ZodSchema = any> {
-  prompt?: string;
+  prompt: string;
   schema?: LLMSchema;
   systemPrompt?: string;
   allowExternalLinks?: boolean;
-  includeSubdomains?: boolean;
 }
 
 /**
@@ -464,7 +462,7 @@ export default class FirecrawlApp {
           let statusData = response.data
           if ("data" in statusData) {
             let data = statusData.data;
-            while ('next' in statusData) {
+            while (typeof statusData === 'object' && 'next' in statusData) {
               statusData = (await this.getRequest(statusData.next, headers)).data;
               data = data.concat(statusData.data);
             }
@@ -578,10 +576,9 @@ export default class FirecrawlApp {
     pollInterval: number = 2,
     idempotencyKey?: string,
     webhook?: CrawlParams["webhook"],
-    ignoreInvalidURLs?: boolean,
   ): Promise<BatchScrapeStatusResponse | ErrorResponse> {
     const headers = this.prepareHeaders(idempotencyKey);
-    let jsonData: any = { urls, webhook, ignoreInvalidURLs, ...params };
+    let jsonData: any = { urls, ...params };
     if (jsonData?.extract?.schema) {
       let schema = jsonData.extract.schema;
 
@@ -624,12 +621,10 @@ export default class FirecrawlApp {
   async asyncBatchScrapeUrls(
     urls: string[],
     params?: ScrapeParams,
-    idempotencyKey?: string,
-    webhook?: CrawlParams["webhook"],
-    ignoreInvalidURLs?: boolean,
+    idempotencyKey?: string
   ): Promise<BatchScrapeResponse | ErrorResponse> {
     const headers = this.prepareHeaders(idempotencyKey);
-    let jsonData: any = { urls, webhook, ignoreInvalidURLs, ...(params ?? {}) };
+    let jsonData: any = { urls, ...(params ?? {}) };
     try {
       const response: AxiosResponse = await this.postRequest(
         this.apiUrl + `/v1/batch/scrape`,
@@ -662,10 +657,8 @@ export default class FirecrawlApp {
     urls: string[],
     params?: ScrapeParams,
     idempotencyKey?: string,
-    webhook?: CrawlParams["webhook"],
-    ignoreInvalidURLs?: boolean,
   ) {
-    const crawl = await this.asyncBatchScrapeUrls(urls, params, idempotencyKey, webhook, ignoreInvalidURLs);
+    const crawl = await this.asyncBatchScrapeUrls(urls, params, idempotencyKey);
 
     if (crawl.success && crawl.id) {
       const id = crawl.id;
@@ -698,7 +691,7 @@ export default class FirecrawlApp {
           let statusData = response.data
           if ("data" in statusData) {
             let data = statusData.data;
-            while ('next' in statusData) {
+            while (typeof statusData === 'object' && 'next' in statusData) {
               statusData = (await this.getRequest(statusData.next, headers)).data;
               data = data.concat(statusData.data);
             }
@@ -857,42 +850,46 @@ export default class FirecrawlApp {
     headers: AxiosRequestHeaders,
     checkInterval: number
   ): Promise<CrawlStatusResponse | ErrorResponse> {
-    while (true) {
-      let statusResponse: AxiosResponse = await this.getRequest(
-        `${this.apiUrl}/v1/crawl/${id}`,
-        headers
-      );
-      if (statusResponse.status === 200) {
-        let statusData = statusResponse.data;
-          if (statusData.status === "completed") {
-            if ("data" in statusData) {
-              let data = statusData.data;
-              while ('next' in statusData) {
-                statusResponse = await this.getRequest(statusData.next, headers);
-                statusData = statusResponse.data;
-                data = data.concat(statusData.data);
+    try {
+      while (true) {
+        let statusResponse: AxiosResponse = await this.getRequest(
+          `${this.apiUrl}/v1/crawl/${id}`,
+          headers
+        );
+        if (statusResponse.status === 200) {
+          let statusData = statusResponse.data;
+            if (statusData.status === "completed") {
+              if ("data" in statusData) {
+                let data = statusData.data;
+                while (typeof statusData === 'object' && 'next' in statusData) {
+                  statusResponse = await this.getRequest(statusData.next, headers);
+                  statusData = statusResponse.data;
+                  data = data.concat(statusData.data);
+                }
+                statusData.data = data;
+                return statusData;
+              } else {
+                throw new FirecrawlError("Crawl job completed but no data was returned", 500);
               }
-              statusData.data = data;
-              return statusData;
-            } else {
-              throw new FirecrawlError("Crawl job completed but no data was returned", 500);
-            }
-          } else if (
-          ["active", "paused", "pending", "queued", "waiting", "scraping"].includes(statusData.status)
-        ) {
-          checkInterval = Math.max(checkInterval, 2);
-          await new Promise((resolve) =>
-            setTimeout(resolve, checkInterval * 1000)
-          );
+            } else if (
+            ["active", "paused", "pending", "queued", "waiting", "scraping"].includes(statusData.status)
+          ) {
+            checkInterval = Math.max(checkInterval, 2);
+            await new Promise((resolve) =>
+              setTimeout(resolve, checkInterval * 1000)
+            );
+          } else {
+            throw new FirecrawlError(
+              `Crawl job failed or was stopped. Status: ${statusData.status}`,
+              500
+            );
+          }
         } else {
-          throw new FirecrawlError(
-            `Crawl job failed or was stopped. Status: ${statusData.status}`,
-            500
-          );
+          this.handleError(statusResponse, "check crawl status");
         }
-      } else {
-        this.handleError(statusResponse, "check crawl status");
       }
+    } catch (error: any) {
+      throw new FirecrawlError(error, 500);
     }
   }
 
@@ -935,11 +932,9 @@ export class CrawlWatcher extends TypedEventTarget<CrawlWatcherEvents> {
   private ws: WebSocket;
   public data: FirecrawlDocument<undefined>[];
   public status: CrawlStatusResponse["status"];
-  public id: string;
 
   constructor(id: string, app: FirecrawlApp) {
     super();
-    this.id = id;
     this.ws = new WebSocket(`${app.apiUrl}/v1/crawl/${id}`, app.apiKey);
     this.status = "scraping";
     this.data = [];
@@ -970,7 +965,6 @@ export class CrawlWatcher extends TypedEventTarget<CrawlWatcherEvents> {
           detail: {
             status: this.status,
             data: this.data,
-            id: this.id,
           },
         }));
       } else if (msg.type === "error") {
@@ -980,7 +974,6 @@ export class CrawlWatcher extends TypedEventTarget<CrawlWatcherEvents> {
             status: this.status,
             data: this.data,
             error: msg.error,
-            id: this.id,
           },
         }));
       } else if (msg.type === "catchup") {
@@ -988,18 +981,12 @@ export class CrawlWatcher extends TypedEventTarget<CrawlWatcherEvents> {
         this.data.push(...(msg.data.data ?? []));
         for (const doc of this.data) {
           this.dispatchTypedEvent("document", new CustomEvent("document", {
-            detail: {
-              ...doc,
-              id: this.id,
-            },
+            detail: doc,
           }));
         }
       } else if (msg.type === "document") {
         this.dispatchTypedEvent("document", new CustomEvent("document", {
-          detail: {
-            ...msg.data,
-            id: this.id,
-          },
+          detail: msg.data,
         }));
       }
     }
@@ -1026,7 +1013,6 @@ export class CrawlWatcher extends TypedEventTarget<CrawlWatcherEvents> {
           status: this.status,
           data: this.data,
           error: "WebSocket error",
-          id: this.id,
         },
       }));
     }).bind(this);