firecrawl 1.10.0 → 1.10.1

package/dist/index.cjs

@@ -49,20 +49,16 @@ var FirecrawlError = class extends Error {
 var FirecrawlApp = class {
   apiKey;
   apiUrl;
-  isCloudService(url) {
-    return url.includes("api.firecrawl.dev");
-  }
   /**
    * Initializes a new instance of the FirecrawlApp class.
    * @param config - Configuration options for the FirecrawlApp instance.
    */
   constructor({ apiKey = null, apiUrl = null }) {
-    const baseUrl = apiUrl || "https://api.firecrawl.dev";
-    if (this.isCloudService(baseUrl) && typeof apiKey !== "string") {
+    if (typeof apiKey !== "string") {
       throw new FirecrawlError("No API key provided", 401);
     }
-    this.apiKey = apiKey || "";
-    this.apiUrl = baseUrl;
+    this.apiKey = apiKey;
+    this.apiUrl = apiUrl || "https://api.firecrawl.dev";
   }
   /**
    * Scrapes a URL using the Firecrawl API.
@@ -202,7 +198,7 @@ var FirecrawlApp = class {
           let statusData = response.data;
           if ("data" in statusData) {
             let data = statusData.data;
-            while ("next" in statusData) {
+            while (typeof statusData === "object" && "next" in statusData) {
               statusData = (await this.getRequest(statusData.next, headers)).data;
               data = data.concat(statusData.data);
             }
@@ -299,9 +295,9 @@ var FirecrawlApp = class {
    * @param webhook - Optional webhook for the batch scrape.
    * @returns The response from the crawl operation.
    */
-  async batchScrapeUrls(urls, params, pollInterval = 2, idempotencyKey, webhook, ignoreInvalidURLs) {
+  async batchScrapeUrls(urls, params, pollInterval = 2, idempotencyKey, webhook) {
     const headers = this.prepareHeaders(idempotencyKey);
-    let jsonData = { urls, webhook, ignoreInvalidURLs, ...params };
+    let jsonData = { urls, ...params };
     if (jsonData?.extract?.schema) {
       let schema = jsonData.extract.schema;
       try {
@@ -337,9 +333,9 @@ var FirecrawlApp = class {
     }
     return { success: false, error: "Internal server error." };
   }
-  async asyncBatchScrapeUrls(urls, params, idempotencyKey, webhook, ignoreInvalidURLs) {
+  async asyncBatchScrapeUrls(urls, params, idempotencyKey) {
     const headers = this.prepareHeaders(idempotencyKey);
-    let jsonData = { urls, webhook, ignoreInvalidURLs, ...params ?? {} };
+    let jsonData = { urls, ...params ?? {} };
     try {
       const response = await this.postRequest(
         this.apiUrl + `/v1/batch/scrape`,
@@ -367,8 +363,8 @@ var FirecrawlApp = class {
    * @param idempotencyKey - Optional idempotency key for the request.
    * @returns A CrawlWatcher instance to monitor the crawl job.
    */
-  async batchScrapeUrlsAndWatch(urls, params, idempotencyKey, webhook, ignoreInvalidURLs) {
-    const crawl = await this.asyncBatchScrapeUrls(urls, params, idempotencyKey, webhook, ignoreInvalidURLs);
+  async batchScrapeUrlsAndWatch(urls, params, idempotencyKey) {
+    const crawl = await this.asyncBatchScrapeUrls(urls, params, idempotencyKey);
     if (crawl.success && crawl.id) {
       const id = crawl.id;
       return new CrawlWatcher(id, this);
@@ -397,7 +393,7 @@ var FirecrawlApp = class {
           let statusData = response.data;
           if ("data" in statusData) {
             let data = statusData.data;
-            while ("next" in statusData) {
+            while (typeof statusData === "object" && "next" in statusData) {
               statusData = (await this.getRequest(statusData.next, headers)).data;
               data = data.concat(statusData.data);
             }
@@ -533,40 +529,44 @@ var FirecrawlApp = class {
    * @returns The final job status or data.
    */
   async monitorJobStatus(id, headers, checkInterval) {
-    while (true) {
-      let statusResponse = await this.getRequest(
-        `${this.apiUrl}/v1/crawl/${id}`,
-        headers
-      );
-      if (statusResponse.status === 200) {
-        let statusData = statusResponse.data;
-        if (statusData.status === "completed") {
-          if ("data" in statusData) {
-            let data = statusData.data;
-            while ("next" in statusData) {
-              statusResponse = await this.getRequest(statusData.next, headers);
-              statusData = statusResponse.data;
-              data = data.concat(statusData.data);
+    try {
+      while (true) {
+        let statusResponse = await this.getRequest(
+          `${this.apiUrl}/v1/crawl/${id}`,
+          headers
+        );
+        if (statusResponse.status === 200) {
+          let statusData = statusResponse.data;
+          if (statusData.status === "completed") {
+            if ("data" in statusData) {
+              let data = statusData.data;
+              while (typeof statusData === "object" && "next" in statusData) {
+                statusResponse = await this.getRequest(statusData.next, headers);
+                statusData = statusResponse.data;
+                data = data.concat(statusData.data);
+              }
+              statusData.data = data;
+              return statusData;
+            } else {
+              throw new FirecrawlError("Crawl job completed but no data was returned", 500);
             }
-            statusData.data = data;
-            return statusData;
+          } else if (["active", "paused", "pending", "queued", "waiting", "scraping"].includes(statusData.status)) {
+            checkInterval = Math.max(checkInterval, 2);
+            await new Promise(
+              (resolve) => setTimeout(resolve, checkInterval * 1e3)
+            );
           } else {
-            throw new FirecrawlError("Crawl job completed but no data was returned", 500);
+            throw new FirecrawlError(
+              `Crawl job failed or was stopped. Status: ${statusData.status}`,
+              500
+            );
           }
-        } else if (["active", "paused", "pending", "queued", "waiting", "scraping"].includes(statusData.status)) {
-          checkInterval = Math.max(checkInterval, 2);
-          await new Promise(
-            (resolve) => setTimeout(resolve, checkInterval * 1e3)
-          );
         } else {
-          throw new FirecrawlError(
-            `Crawl job failed or was stopped. Status: ${statusData.status}`,
-            500
-          );
+          this.handleError(statusResponse, "check crawl status");
         }
-      } else {
-        this.handleError(statusResponse, "check crawl status");
       }
+    } catch (error) {
+      throw new FirecrawlError(error, 500);
     }
   }
   /**
@@ -593,10 +593,8 @@ var CrawlWatcher = class extends import_typescript_event_target.TypedEventTarget
   ws;
   data;
   status;
-  id;
   constructor(id, app) {
     super();
-    this.id = id;
     this.ws = new import_isows.WebSocket(`${app.apiUrl}/v1/crawl/${id}`, app.apiKey);
     this.status = "scraping";
     this.data = [];
@@ -606,8 +604,7 @@ var CrawlWatcher = class extends import_typescript_event_target.TypedEventTarget
         this.dispatchTypedEvent("done", new CustomEvent("done", {
           detail: {
             status: this.status,
-            data: this.data,
-            id: this.id
+            data: this.data
           }
         }));
       } else if (msg.type === "error") {
@@ -616,8 +613,7 @@ var CrawlWatcher = class extends import_typescript_event_target.TypedEventTarget
           detail: {
             status: this.status,
             data: this.data,
-            error: msg.error,
-            id: this.id
+            error: msg.error
           }
         }));
       } else if (msg.type === "catchup") {
@@ -625,18 +621,12 @@ var CrawlWatcher = class extends import_typescript_event_target.TypedEventTarget
         this.data.push(...msg.data.data ?? []);
         for (const doc of this.data) {
           this.dispatchTypedEvent("document", new CustomEvent("document", {
-            detail: {
-              ...doc,
-              id: this.id
-            }
+            detail: doc
           }));
         }
       } else if (msg.type === "document") {
         this.dispatchTypedEvent("document", new CustomEvent("document", {
-          detail: {
-            ...msg.data,
-            id: this.id
-          }
+          detail: msg.data
         }));
       }
     };
@@ -645,20 +635,12 @@ var CrawlWatcher = class extends import_typescript_event_target.TypedEventTarget
         this.ws.close();
         return;
       }
-      try {
-        const msg = JSON.parse(ev.data);
-        messageHandler(msg);
-      } catch (error) {
-        console.error("Error on message", error);
-      }
+      const msg = JSON.parse(ev.data);
+      messageHandler(msg);
     }).bind(this);
     this.ws.onclose = ((ev) => {
-      try {
-        const msg = JSON.parse(ev.reason);
-        messageHandler(msg);
-      } catch (error) {
-        console.error("Error on close", error);
-      }
+      const msg = JSON.parse(ev.reason);
+      messageHandler(msg);
     }).bind(this);
     this.ws.onerror = ((_) => {
       this.status = "failed";
@@ -666,8 +648,7 @@ var CrawlWatcher = class extends import_typescript_event_target.TypedEventTarget
         detail: {
           status: this.status,
           data: this.data,
-          error: "WebSocket error",
-          id: this.id
+          error: "WebSocket error"
         }
       }));
     }).bind(this);

package/dist/index.d.cts

@@ -171,7 +171,6 @@ interface BatchScrapeResponse {
     url?: string;
     success: true;
     error?: string;
-    invalidURLs?: string[];
 }
 /**
  * Response interface for job status checks.
@@ -226,11 +225,10 @@ interface MapResponse {
  * Defines options for extracting information from URLs.
  */
 interface ExtractParams<LLMSchema extends zt.ZodSchema = any> {
-    prompt?: string;
+    prompt: string;
     schema?: LLMSchema;
     systemPrompt?: string;
     allowExternalLinks?: boolean;
-    includeSubdomains?: boolean;
 }
 /**
  * Response interface for extracting information from URLs.
@@ -265,7 +263,6 @@ declare class FirecrawlError extends Error {
 declare class FirecrawlApp {
     apiKey: string;
     apiUrl: string;
-    private isCloudService;
     /**
      * Initializes a new instance of the FirecrawlApp class.
      * @param config - Configuration options for the FirecrawlApp instance.
@@ -332,8 +329,8 @@ declare class FirecrawlApp {
      * @param webhook - Optional webhook for the batch scrape.
      * @returns The response from the crawl operation.
      */
-    batchScrapeUrls(urls: string[], params?: ScrapeParams, pollInterval?: number, idempotencyKey?: string, webhook?: CrawlParams["webhook"], ignoreInvalidURLs?: boolean): Promise<BatchScrapeStatusResponse | ErrorResponse>;
-    asyncBatchScrapeUrls(urls: string[], params?: ScrapeParams, idempotencyKey?: string, webhook?: CrawlParams["webhook"], ignoreInvalidURLs?: boolean): Promise<BatchScrapeResponse | ErrorResponse>;
+    batchScrapeUrls(urls: string[], params?: ScrapeParams, pollInterval?: number, idempotencyKey?: string, webhook?: CrawlParams["webhook"]): Promise<BatchScrapeStatusResponse | ErrorResponse>;
+    asyncBatchScrapeUrls(urls: string[], params?: ScrapeParams, idempotencyKey?: string): Promise<BatchScrapeResponse | ErrorResponse>;
     /**
      * Initiates a batch scrape job and returns a CrawlWatcher to monitor the job via WebSocket.
      * @param urls - The URL to scrape.
@@ -341,7 +338,7 @@ declare class FirecrawlApp {
      * @param idempotencyKey - Optional idempotency key for the request.
      * @returns A CrawlWatcher instance to monitor the crawl job.
      */
-    batchScrapeUrlsAndWatch(urls: string[], params?: ScrapeParams, idempotencyKey?: string, webhook?: CrawlParams["webhook"], ignoreInvalidURLs?: boolean): Promise<CrawlWatcher>;
+    batchScrapeUrlsAndWatch(urls: string[], params?: ScrapeParams, idempotencyKey?: string): Promise<CrawlWatcher>;
     /**
      * Checks the status of a batch scrape job using the Firecrawl API.
      * @param id - The ID of the batch scrape operation.
@@ -417,7 +414,6 @@ declare class CrawlWatcher extends TypedEventTarget<CrawlWatcherEvents> {
     private ws;
     data: FirecrawlDocument<undefined>[];
     status: CrawlStatusResponse["status"];
-    id: string;
     constructor(id: string, app: FirecrawlApp);
     close(): void;
 }

package/dist/index.d.ts

@@ -171,7 +171,6 @@ interface BatchScrapeResponse {
     url?: string;
     success: true;
     error?: string;
-    invalidURLs?: string[];
 }
 /**
  * Response interface for job status checks.
@@ -226,11 +225,10 @@ interface MapResponse {
  * Defines options for extracting information from URLs.
  */
 interface ExtractParams<LLMSchema extends zt.ZodSchema = any> {
-    prompt?: string;
+    prompt: string;
     schema?: LLMSchema;
     systemPrompt?: string;
     allowExternalLinks?: boolean;
-    includeSubdomains?: boolean;
 }
 /**
  * Response interface for extracting information from URLs.
@@ -265,7 +263,6 @@ declare class FirecrawlError extends Error {
 declare class FirecrawlApp {
     apiKey: string;
     apiUrl: string;
-    private isCloudService;
     /**
      * Initializes a new instance of the FirecrawlApp class.
      * @param config - Configuration options for the FirecrawlApp instance.
@@ -332,8 +329,8 @@ declare class FirecrawlApp {
      * @param webhook - Optional webhook for the batch scrape.
      * @returns The response from the crawl operation.
      */
-    batchScrapeUrls(urls: string[], params?: ScrapeParams, pollInterval?: number, idempotencyKey?: string, webhook?: CrawlParams["webhook"], ignoreInvalidURLs?: boolean): Promise<BatchScrapeStatusResponse | ErrorResponse>;
-    asyncBatchScrapeUrls(urls: string[], params?: ScrapeParams, idempotencyKey?: string, webhook?: CrawlParams["webhook"], ignoreInvalidURLs?: boolean): Promise<BatchScrapeResponse | ErrorResponse>;
+    batchScrapeUrls(urls: string[], params?: ScrapeParams, pollInterval?: number, idempotencyKey?: string, webhook?: CrawlParams["webhook"]): Promise<BatchScrapeStatusResponse | ErrorResponse>;
+    asyncBatchScrapeUrls(urls: string[], params?: ScrapeParams, idempotencyKey?: string): Promise<BatchScrapeResponse | ErrorResponse>;
     /**
      * Initiates a batch scrape job and returns a CrawlWatcher to monitor the job via WebSocket.
      * @param urls - The URL to scrape.
@@ -341,7 +338,7 @@ declare class FirecrawlApp {
      * @param idempotencyKey - Optional idempotency key for the request.
      * @returns A CrawlWatcher instance to monitor the crawl job.
      */
-    batchScrapeUrlsAndWatch(urls: string[], params?: ScrapeParams, idempotencyKey?: string, webhook?: CrawlParams["webhook"], ignoreInvalidURLs?: boolean): Promise<CrawlWatcher>;
+    batchScrapeUrlsAndWatch(urls: string[], params?: ScrapeParams, idempotencyKey?: string): Promise<CrawlWatcher>;
     /**
      * Checks the status of a batch scrape job using the Firecrawl API.
      * @param id - The ID of the batch scrape operation.
@@ -417,7 +414,6 @@ declare class CrawlWatcher extends TypedEventTarget<CrawlWatcherEvents> {
     private ws;
     data: FirecrawlDocument<undefined>[];
     status: CrawlStatusResponse["status"];
-    id: string;
     constructor(id: string, app: FirecrawlApp);
     close(): void;
 }

package/dist/index.js

@@ -13,20 +13,16 @@ var FirecrawlError = class extends Error {
 var FirecrawlApp = class {
   apiKey;
   apiUrl;
-  isCloudService(url) {
-    return url.includes("api.firecrawl.dev");
-  }
   /**
    * Initializes a new instance of the FirecrawlApp class.
    * @param config - Configuration options for the FirecrawlApp instance.
    */
   constructor({ apiKey = null, apiUrl = null }) {
-    const baseUrl = apiUrl || "https://api.firecrawl.dev";
-    if (this.isCloudService(baseUrl) && typeof apiKey !== "string") {
+    if (typeof apiKey !== "string") {
       throw new FirecrawlError("No API key provided", 401);
     }
-    this.apiKey = apiKey || "";
-    this.apiUrl = baseUrl;
+    this.apiKey = apiKey;
+    this.apiUrl = apiUrl || "https://api.firecrawl.dev";
   }
   /**
    * Scrapes a URL using the Firecrawl API.
@@ -166,7 +162,7 @@ var FirecrawlApp = class {
           let statusData = response.data;
           if ("data" in statusData) {
             let data = statusData.data;
-            while ("next" in statusData) {
+            while (typeof statusData === "object" && "next" in statusData) {
               statusData = (await this.getRequest(statusData.next, headers)).data;
               data = data.concat(statusData.data);
             }
@@ -263,9 +259,9 @@ var FirecrawlApp = class {
    * @param webhook - Optional webhook for the batch scrape.
    * @returns The response from the crawl operation.
    */
-  async batchScrapeUrls(urls, params, pollInterval = 2, idempotencyKey, webhook, ignoreInvalidURLs) {
+  async batchScrapeUrls(urls, params, pollInterval = 2, idempotencyKey, webhook) {
     const headers = this.prepareHeaders(idempotencyKey);
-    let jsonData = { urls, webhook, ignoreInvalidURLs, ...params };
+    let jsonData = { urls, ...params };
     if (jsonData?.extract?.schema) {
       let schema = jsonData.extract.schema;
       try {
@@ -301,9 +297,9 @@ var FirecrawlApp = class {
     }
     return { success: false, error: "Internal server error." };
   }
-  async asyncBatchScrapeUrls(urls, params, idempotencyKey, webhook, ignoreInvalidURLs) {
+  async asyncBatchScrapeUrls(urls, params, idempotencyKey) {
     const headers = this.prepareHeaders(idempotencyKey);
-    let jsonData = { urls, webhook, ignoreInvalidURLs, ...params ?? {} };
+    let jsonData = { urls, ...params ?? {} };
     try {
       const response = await this.postRequest(
         this.apiUrl + `/v1/batch/scrape`,
@@ -331,8 +327,8 @@ var FirecrawlApp = class {
    * @param idempotencyKey - Optional idempotency key for the request.
    * @returns A CrawlWatcher instance to monitor the crawl job.
    */
-  async batchScrapeUrlsAndWatch(urls, params, idempotencyKey, webhook, ignoreInvalidURLs) {
-    const crawl = await this.asyncBatchScrapeUrls(urls, params, idempotencyKey, webhook, ignoreInvalidURLs);
+  async batchScrapeUrlsAndWatch(urls, params, idempotencyKey) {
+    const crawl = await this.asyncBatchScrapeUrls(urls, params, idempotencyKey);
     if (crawl.success && crawl.id) {
       const id = crawl.id;
       return new CrawlWatcher(id, this);
@@ -361,7 +357,7 @@ var FirecrawlApp = class {
           let statusData = response.data;
           if ("data" in statusData) {
             let data = statusData.data;
-            while ("next" in statusData) {
+            while (typeof statusData === "object" && "next" in statusData) {
               statusData = (await this.getRequest(statusData.next, headers)).data;
               data = data.concat(statusData.data);
             }
@@ -497,40 +493,44 @@ var FirecrawlApp = class {
    * @returns The final job status or data.
    */
   async monitorJobStatus(id, headers, checkInterval) {
-    while (true) {
-      let statusResponse = await this.getRequest(
-        `${this.apiUrl}/v1/crawl/${id}`,
-        headers
-      );
-      if (statusResponse.status === 200) {
-        let statusData = statusResponse.data;
-        if (statusData.status === "completed") {
-          if ("data" in statusData) {
-            let data = statusData.data;
-            while ("next" in statusData) {
-              statusResponse = await this.getRequest(statusData.next, headers);
-              statusData = statusResponse.data;
-              data = data.concat(statusData.data);
+    try {
+      while (true) {
+        let statusResponse = await this.getRequest(
+          `${this.apiUrl}/v1/crawl/${id}`,
+          headers
+        );
+        if (statusResponse.status === 200) {
+          let statusData = statusResponse.data;
+          if (statusData.status === "completed") {
+            if ("data" in statusData) {
+              let data = statusData.data;
+              while (typeof statusData === "object" && "next" in statusData) {
+                statusResponse = await this.getRequest(statusData.next, headers);
+                statusData = statusResponse.data;
+                data = data.concat(statusData.data);
+              }
+              statusData.data = data;
+              return statusData;
+            } else {
+              throw new FirecrawlError("Crawl job completed but no data was returned", 500);
             }
-            statusData.data = data;
-            return statusData;
+          } else if (["active", "paused", "pending", "queued", "waiting", "scraping"].includes(statusData.status)) {
+            checkInterval = Math.max(checkInterval, 2);
+            await new Promise(
+              (resolve) => setTimeout(resolve, checkInterval * 1e3)
+            );
           } else {
-            throw new FirecrawlError("Crawl job completed but no data was returned", 500);
+            throw new FirecrawlError(
+              `Crawl job failed or was stopped. Status: ${statusData.status}`,
+              500
+            );
           }
-        } else if (["active", "paused", "pending", "queued", "waiting", "scraping"].includes(statusData.status)) {
-          checkInterval = Math.max(checkInterval, 2);
-          await new Promise(
-            (resolve) => setTimeout(resolve, checkInterval * 1e3)
-          );
         } else {
-          throw new FirecrawlError(
-            `Crawl job failed or was stopped. Status: ${statusData.status}`,
-            500
-          );
+          this.handleError(statusResponse, "check crawl status");
         }
-      } else {
-        this.handleError(statusResponse, "check crawl status");
       }
+    } catch (error) {
+      throw new FirecrawlError(error, 500);
     }
   }
   /**
@@ -557,10 +557,8 @@ var CrawlWatcher = class extends TypedEventTarget {
   ws;
   data;
   status;
-  id;
   constructor(id, app) {
     super();
-    this.id = id;
     this.ws = new WebSocket(`${app.apiUrl}/v1/crawl/${id}`, app.apiKey);
     this.status = "scraping";
     this.data = [];
@@ -570,8 +568,7 @@ var CrawlWatcher = class extends TypedEventTarget {
         this.dispatchTypedEvent("done", new CustomEvent("done", {
           detail: {
             status: this.status,
-            data: this.data,
-            id: this.id
+            data: this.data
           }
         }));
       } else if (msg.type === "error") {
@@ -580,8 +577,7 @@ var CrawlWatcher = class extends TypedEventTarget {
           detail: {
             status: this.status,
             data: this.data,
-            error: msg.error,
-            id: this.id
+            error: msg.error
           }
         }));
       } else if (msg.type === "catchup") {
@@ -589,18 +585,12 @@ var CrawlWatcher = class extends TypedEventTarget {
         this.data.push(...msg.data.data ?? []);
         for (const doc of this.data) {
           this.dispatchTypedEvent("document", new CustomEvent("document", {
-            detail: {
-              ...doc,
-              id: this.id
-            }
+            detail: doc
           }));
         }
       } else if (msg.type === "document") {
         this.dispatchTypedEvent("document", new CustomEvent("document", {
-          detail: {
-            ...msg.data,
-            id: this.id
-          }
+          detail: msg.data
         }));
       }
     };
@@ -609,20 +599,12 @@ var CrawlWatcher = class extends TypedEventTarget {
         this.ws.close();
         return;
       }
-      try {
-        const msg = JSON.parse(ev.data);
-        messageHandler(msg);
-      } catch (error) {
-        console.error("Error on message", error);
-      }
+      const msg = JSON.parse(ev.data);
+      messageHandler(msg);
     }).bind(this);
     this.ws.onclose = ((ev) => {
-      try {
-        const msg = JSON.parse(ev.reason);
-        messageHandler(msg);
-      } catch (error) {
-        console.error("Error on close", error);
-      }
+      const msg = JSON.parse(ev.reason);
+      messageHandler(msg);
     }).bind(this);
     this.ws.onerror = ((_) => {
       this.status = "failed";
@@ -630,8 +612,7 @@ var CrawlWatcher = class extends TypedEventTarget {
         detail: {
           status: this.status,
           data: this.data,
-          error: "WebSocket error",
-          id: this.id
+          error: "WebSocket error"
         }
       }));
     }).bind(this);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "firecrawl",
-  "version": "1.10.0",
+  "version": "1.10.1",
   "description": "JavaScript SDK for Firecrawl API",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

package/src/__tests__/index.test.ts

@@ -1,9 +1,9 @@
-import { describe, expect, jest, test } from '@jest/globals';
-
-import FirecrawlApp from '../index';
+import { describe, test, expect, jest } from '@jest/globals';
 import axios from 'axios';
-import { join } from 'path';
+import FirecrawlApp from '../index';
+
 import { readFile } from 'fs/promises';
+import { join } from 'path';
 
 // Mock jest and set the type
 jest.mock('axios');
@@ -14,22 +14,13 @@ async function loadFixture(name: string): Promise<string> {
   return await readFile(join(__dirname, 'fixtures', `${name}.json`), 'utf-8')
 }
 
-const API_URL = process.env.API_URL ?? "https://api.firecrawl.dev";
-
 describe('the firecrawl JS SDK', () => {
 
-  test('Should require an API key only for cloud service', async () => {
-    if (API_URL.includes('api.firecrawl.dev')) {
-      // Should throw for cloud service
-      expect(() => {
-        new FirecrawlApp({ apiKey: undefined, apiUrl: API_URL });
-      }).toThrow('No API key provided');
-    } else {
-      // Should not throw for self-hosted
-      expect(() => {
-        new FirecrawlApp({ apiKey: undefined, apiUrl: API_URL });
-      }).not.toThrow();
-    }
+  test('Should require an API key to instantiate FirecrawlApp', async () => {
+    const fn = () => {
+      new FirecrawlApp({ apiKey: undefined });
+    };
+    expect(fn).toThrow('No API key provided');
   });
 
   test('Should return scraped data from a /scrape API call', async () => {

package/src/__tests__/v1/e2e_withAuth/index.test.ts

@@ -9,28 +9,15 @@ const TEST_API_KEY = process.env.TEST_API_KEY;
 const API_URL = process.env.API_URL ?? "https://api.firecrawl.dev";
 
 describe('FirecrawlApp E2E Tests', () => {
-  test.concurrent('should throw error for no API key only for cloud service', async () => {
-    if (API_URL.includes('api.firecrawl.dev')) {
-      // Should throw for cloud service
-      expect(() => {
-        new FirecrawlApp({ apiKey: null, apiUrl: API_URL });
-      }).toThrow("No API key provided");
-    } else {
-      // Should not throw for self-hosted
-      expect(() => {
-        new FirecrawlApp({ apiKey: null, apiUrl: API_URL });
-      }).not.toThrow();
-    }
+  test.concurrent('should throw error for no API key', async () => {
+    expect(() => {
+      new FirecrawlApp({ apiKey: null, apiUrl: API_URL });
+    }).toThrow("No API key provided");
   });
 
   test.concurrent('should throw error for invalid API key on scrape', async () => {
-    if (API_URL.includes('api.firecrawl.dev')) {
-      const invalidApp = new FirecrawlApp({ apiKey: "invalid_api_key", apiUrl: API_URL });
-      await expect(invalidApp.scrapeUrl('https://roastmywebsite.ai')).rejects.toThrow("Unexpected error occurred while trying to scrape URL. Status code: 404");
-    } else {
-      const invalidApp = new FirecrawlApp({ apiKey: "invalid_api_key", apiUrl: API_URL });
-      await expect(invalidApp.scrapeUrl('https://roastmywebsite.ai')).resolves.not.toThrow();
-    }
+    const invalidApp = new FirecrawlApp({ apiKey: "invalid_api_key", apiUrl: API_URL });
+    await expect(invalidApp.scrapeUrl('https://roastmywebsite.ai')).rejects.toThrow("Request failed with status code 401");
   });
 
   test.concurrent('should throw error for blocklisted URL on scrape', async () => {
@@ -168,13 +155,14 @@ describe('FirecrawlApp E2E Tests', () => {
   }, 30000); // 30 seconds timeout
 
   test.concurrent('should throw error for invalid API key on crawl', async () => {
-    if (API_URL.includes('api.firecrawl.dev')) {
-      const invalidApp = new FirecrawlApp({ apiKey: "invalid_api_key", apiUrl: API_URL });
-      await expect(invalidApp.crawlUrl('https://roastmywebsite.ai')).rejects.toThrow("Request failed with status code 404");
-    } else {
-      const invalidApp = new FirecrawlApp({ apiKey: "invalid_api_key", apiUrl: API_URL });
-      await expect(invalidApp.crawlUrl('https://roastmywebsite.ai')).resolves.not.toThrow();
-    }
+    const invalidApp = new FirecrawlApp({ apiKey: "invalid_api_key", apiUrl: API_URL });
+    await expect(invalidApp.crawlUrl('https://roastmywebsite.ai')).rejects.toThrow("Request failed with status code 401");
+  });
+
+  test.concurrent('should throw error for blocklisted URL on crawl', async () => {
+    const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
+    const blocklistedUrl = "https://twitter.com/fake-test";
+    await expect(app.crawlUrl(blocklistedUrl)).rejects.toThrow("URL is blocked. Firecrawl currently does not support social media scraping due to policy restrictions.");
   });
 
   test.concurrent('should return successful response for crawl and wait for completion', async () => {
@@ -349,13 +337,8 @@ describe('FirecrawlApp E2E Tests', () => {
   }, 60000); // 60 seconds timeout
 
   test.concurrent('should throw error for invalid API key on map', async () => {
-    if (API_URL.includes('api.firecrawl.dev')) {
-      const invalidApp = new FirecrawlApp({ apiKey: "invalid_api_key", apiUrl: API_URL });
-      await expect(invalidApp.mapUrl('https://roastmywebsite.ai')).rejects.toThrow("Request failed with status code 404");
-    } else {
-      const invalidApp = new FirecrawlApp({ apiKey: "invalid_api_key", apiUrl: API_URL });
-      await expect(invalidApp.mapUrl('https://roastmywebsite.ai')).resolves.not.toThrow();
-    }
+    const invalidApp = new FirecrawlApp({ apiKey: "invalid_api_key", apiUrl: API_URL });
+    await expect(invalidApp.mapUrl('https://roastmywebsite.ai')).rejects.toThrow("Request failed with status code 401");
   });
 
   test.concurrent('should throw error for blocklisted URL on map', async () => {
@@ -372,7 +355,8 @@ describe('FirecrawlApp E2E Tests', () => {
   }, 30000); // 30 seconds timeout
 
   test.concurrent('should return successful response for valid map', async () => {
-    const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });    const response = await app.mapUrl('https://roastmywebsite.ai') as MapResponse;
+    const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
+    const response = await app.mapUrl('https://roastmywebsite.ai') as MapResponse;
     expect(response).not.toBeNull();
     
     expect(response.links?.length).toBeGreaterThan(0);

package/src/index.ts

@@ -183,7 +183,6 @@ export interface BatchScrapeResponse {
   url?: string;
   success: true;
   error?: string;
-  invalidURLs?: string[];
 }
 
 /**
@@ -243,11 +242,10 @@ export interface MapResponse {
  * Defines options for extracting information from URLs.
  */
 export interface ExtractParams<LLMSchema extends zt.ZodSchema = any> {
-  prompt?: string;
+  prompt: string;
   schema?: LLMSchema;
   systemPrompt?: string;
   allowExternalLinks?: boolean;
-  includeSubdomains?: boolean;
 }
 
 /**
@@ -290,23 +288,17 @@ export default class FirecrawlApp {
   public apiKey: string;
   public apiUrl: string;
 
-  private isCloudService(url: string): boolean {
-    return url.includes('api.firecrawl.dev');
-  }
-
   /**
    * Initializes a new instance of the FirecrawlApp class.
    * @param config - Configuration options for the FirecrawlApp instance.
    */
   constructor({ apiKey = null, apiUrl = null }: FirecrawlAppConfig) {
-    const baseUrl = apiUrl || "https://api.firecrawl.dev";
-    
-    if (this.isCloudService(baseUrl) && typeof apiKey !== "string") {
+    if (typeof apiKey !== "string") {
       throw new FirecrawlError("No API key provided", 401);
     }
 
-    this.apiKey = apiKey || '';
-    this.apiUrl = baseUrl;
+    this.apiKey = apiKey;
+    this.apiUrl = apiUrl || "https://api.firecrawl.dev";
   }
 
   /**
@@ -470,7 +462,7 @@ export default class FirecrawlApp {
           let statusData = response.data
           if ("data" in statusData) {
             let data = statusData.data;
-            while ('next' in statusData) {
+            while (typeof statusData === 'object' && 'next' in statusData) {
               statusData = (await this.getRequest(statusData.next, headers)).data;
               data = data.concat(statusData.data);
             }
@@ -584,10 +576,9 @@ export default class FirecrawlApp {
     pollInterval: number = 2,
     idempotencyKey?: string,
     webhook?: CrawlParams["webhook"],
-    ignoreInvalidURLs?: boolean,
   ): Promise<BatchScrapeStatusResponse | ErrorResponse> {
     const headers = this.prepareHeaders(idempotencyKey);
-    let jsonData: any = { urls, webhook, ignoreInvalidURLs, ...params };
+    let jsonData: any = { urls, ...params };
     if (jsonData?.extract?.schema) {
       let schema = jsonData.extract.schema;
 
@@ -630,12 +621,10 @@ export default class FirecrawlApp {
   async asyncBatchScrapeUrls(
     urls: string[],
     params?: ScrapeParams,
-    idempotencyKey?: string,
-    webhook?: CrawlParams["webhook"],
-    ignoreInvalidURLs?: boolean,
+    idempotencyKey?: string
   ): Promise<BatchScrapeResponse | ErrorResponse> {
     const headers = this.prepareHeaders(idempotencyKey);
-    let jsonData: any = { urls, webhook, ignoreInvalidURLs, ...(params ?? {}) };
+    let jsonData: any = { urls, ...(params ?? {}) };
     try {
       const response: AxiosResponse = await this.postRequest(
         this.apiUrl + `/v1/batch/scrape`,
@@ -668,10 +657,8 @@ export default class FirecrawlApp {
     urls: string[],
     params?: ScrapeParams,
     idempotencyKey?: string,
-    webhook?: CrawlParams["webhook"],
-    ignoreInvalidURLs?: boolean,
   ) {
-    const crawl = await this.asyncBatchScrapeUrls(urls, params, idempotencyKey, webhook, ignoreInvalidURLs);
+    const crawl = await this.asyncBatchScrapeUrls(urls, params, idempotencyKey);
 
     if (crawl.success && crawl.id) {
       const id = crawl.id;
@@ -704,7 +691,7 @@ export default class FirecrawlApp {
           let statusData = response.data
           if ("data" in statusData) {
             let data = statusData.data;
-            while ('next' in statusData) {
+            while (typeof statusData === 'object' && 'next' in statusData) {
               statusData = (await this.getRequest(statusData.next, headers)).data;
               data = data.concat(statusData.data);
             }
@@ -863,42 +850,46 @@ export default class FirecrawlApp {
     headers: AxiosRequestHeaders,
     checkInterval: number
   ): Promise<CrawlStatusResponse | ErrorResponse> {
-    while (true) {
-      let statusResponse: AxiosResponse = await this.getRequest(
-        `${this.apiUrl}/v1/crawl/${id}`,
-        headers
-      );
-      if (statusResponse.status === 200) {
-        let statusData = statusResponse.data;
-          if (statusData.status === "completed") {
-            if ("data" in statusData) {
-              let data = statusData.data;
-              while ('next' in statusData) {
-                statusResponse = await this.getRequest(statusData.next, headers);
-                statusData = statusResponse.data;
-                data = data.concat(statusData.data);
+    try {
+      while (true) {
+        let statusResponse: AxiosResponse = await this.getRequest(
+          `${this.apiUrl}/v1/crawl/${id}`,
+          headers
+        );
+        if (statusResponse.status === 200) {
+          let statusData = statusResponse.data;
+            if (statusData.status === "completed") {
+              if ("data" in statusData) {
+                let data = statusData.data;
+                while (typeof statusData === 'object' && 'next' in statusData) {
+                  statusResponse = await this.getRequest(statusData.next, headers);
+                  statusData = statusResponse.data;
+                  data = data.concat(statusData.data);
+                }
+                statusData.data = data;
+                return statusData;
+              } else {
+                throw new FirecrawlError("Crawl job completed but no data was returned", 500);
               }
-              statusData.data = data;
-              return statusData;
-            } else {
-              throw new FirecrawlError("Crawl job completed but no data was returned", 500);
-            }
-          } else if (
-          ["active", "paused", "pending", "queued", "waiting", "scraping"].includes(statusData.status)
-        ) {
-          checkInterval = Math.max(checkInterval, 2);
-          await new Promise((resolve) =>
-            setTimeout(resolve, checkInterval * 1000)
-          );
+            } else if (
+            ["active", "paused", "pending", "queued", "waiting", "scraping"].includes(statusData.status)
+          ) {
+            checkInterval = Math.max(checkInterval, 2);
+            await new Promise((resolve) =>
+              setTimeout(resolve, checkInterval * 1000)
+            );
+          } else {
+            throw new FirecrawlError(
+              `Crawl job failed or was stopped. Status: ${statusData.status}`,
+              500
+            );
+          }
         } else {
-          throw new FirecrawlError(
-            `Crawl job failed or was stopped. Status: ${statusData.status}`,
-            500
-          );
+          this.handleError(statusResponse, "check crawl status");
         }
-      } else {
-        this.handleError(statusResponse, "check crawl status");
       }
+    } catch (error: any) {
+      throw new FirecrawlError(error, 500);
     }
   }
 
@@ -941,11 +932,9 @@ export class CrawlWatcher extends TypedEventTarget<CrawlWatcherEvents> {
   private ws: WebSocket;
   public data: FirecrawlDocument<undefined>[];
   public status: CrawlStatusResponse["status"];
-  public id: string;
 
   constructor(id: string, app: FirecrawlApp) {
     super();
-    this.id = id;
     this.ws = new WebSocket(`${app.apiUrl}/v1/crawl/${id}`, app.apiKey);
     this.status = "scraping";
     this.data = [];
@@ -976,7 +965,6 @@ export class CrawlWatcher extends TypedEventTarget<CrawlWatcherEvents> {
           detail: {
             status: this.status,
             data: this.data,
-            id: this.id,
           },
         }));
       } else if (msg.type === "error") {
@@ -986,7 +974,6 @@ export class CrawlWatcher extends TypedEventTarget<CrawlWatcherEvents> {
             status: this.status,
             data: this.data,
             error: msg.error,
-            id: this.id,
           },
         }));
       } else if (msg.type === "catchup") {
@@ -994,18 +981,12 @@ export class CrawlWatcher extends TypedEventTarget<CrawlWatcherEvents> {
         this.data.push(...(msg.data.data ?? []));
         for (const doc of this.data) {
           this.dispatchTypedEvent("document", new CustomEvent("document", {
-            detail: {
-              ...doc,
-              id: this.id,
-            },
+            detail: doc,
           }));
         }
       } else if (msg.type === "document") {
         this.dispatchTypedEvent("document", new CustomEvent("document", {
-          detail: {
-            ...msg.data,
-            id: this.id,
-          },
+          detail: msg.data,
         }));
       }
     }
@@ -1015,21 +996,14 @@ export class CrawlWatcher extends TypedEventTarget<CrawlWatcherEvents> {
         this.ws.close();
         return;
       }
-      try {
-        const msg = JSON.parse(ev.data) as Message;
-        messageHandler(msg);
-      } catch (error) {
-        console.error("Error on message", error);
-      }
+
+      const msg = JSON.parse(ev.data) as Message;
+      messageHandler(msg);
     }).bind(this);
 
     this.ws.onclose = ((ev: CloseEvent) => {
-      try {
-        const msg = JSON.parse(ev.reason) as Message;
-        messageHandler(msg);
-      } catch (error) {
-        console.error("Error on close", error);
-      }
+      const msg = JSON.parse(ev.reason) as Message;
+      messageHandler(msg);
     }).bind(this);
 
     this.ws.onerror = ((_: Event) => {
@@ -1039,7 +1013,6 @@ export class CrawlWatcher extends TypedEventTarget<CrawlWatcherEvents> {
           status: this.status,
           data: this.data,
           error: "WebSocket error",
-          id: this.id,
         },
       }));
     }).bind(this);