firecrawl 4.20.0 → 4.21.0

package/dist/{chunk-UXVHRV2G.js → chunk-R625SBJY.js}

@@ -8,7 +8,7 @@ var require_package = __commonJS({
   "package.json"(exports, module) {
     module.exports = {
       name: "@mendable/firecrawl-js",
-      version: "4.20.0",
+      version: "4.21.0",
       description: "JavaScript SDK for Firecrawl API",
       main: "dist/index.js",
       types: "dist/index.d.ts",

package/dist/index.cjs

@@ -35,7 +35,7 @@ var require_package = __commonJS({
   "package.json"(exports2, module2) {
     module2.exports = {
       name: "@mendable/firecrawl-js",
-      version: "4.20.0",
+      version: "4.21.0",
       description: "JavaScript SDK for Firecrawl API",
       main: "dist/index.js",
       types: "dist/index.d.ts",
@@ -625,11 +625,17 @@ function prepareSearchPayload(req) {
     throw new Error("limit must be positive");
   if (req.timeout != null && req.timeout <= 0)
     throw new Error("timeout must be positive");
+  if (req.includeDomains?.length && req.excludeDomains?.length)
+    throw new Error(
+      "includeDomains and excludeDomains cannot both be specified"
+    );
   const payload = {
     query: req.query
   };
   if (req.sources) payload.sources = req.sources;
   if (req.categories) payload.categories = req.categories;
+  if (req.includeDomains) payload.includeDomains = req.includeDomains;
+  if (req.excludeDomains) payload.excludeDomains = req.excludeDomains;
   if (req.limit != null) payload.limit = req.limit;
   if (req.tbs != null) payload.tbs = req.tbs;
   if (req.location != null) payload.location = req.location;
@@ -1418,23 +1424,41 @@ var Watcher = class extends import_events.EventEmitter {
     return `${wsBase}${path}`;
   }
   async start() {
-    try {
-      const url = this.buildWsUrl();
-      const wsCtor = await getWebSocketCtor();
-      if (!wsCtor) {
-        this.pollLoop();
-        return;
-      }
-      this.ws = new wsCtor(url, this.http.getApiKey());
-      if (this.ws && "binaryType" in this.ws) {
-        this.ws.binaryType = "arraybuffer";
-      }
-      if (this.ws) {
-        this.attachWsHandlers(this.ws);
-      }
-    } catch (err) {
-      this.pollLoop();
-    }
+    return new Promise((resolve, reject) => {
+      const onDone = () => {
+        cleanup();
+        resolve();
+      };
+      const onError = (err) => {
+        cleanup();
+        resolve();
+      };
+      const cleanup = () => {
+        this.removeListener("done", onDone);
+        this.removeListener("error", onError);
+      };
+      this.on("done", onDone);
+      this.on("error", onError);
+      (async () => {
+        try {
+          const url = this.buildWsUrl();
+          const wsCtor = await getWebSocketCtor();
+          if (!wsCtor) {
+            this.pollLoop();
+            return;
+          }
+          this.ws = new wsCtor(url, this.http.getApiKey());
+          if (this.ws && "binaryType" in this.ws) {
+            this.ws.binaryType = "arraybuffer";
+          }
+          if (this.ws) {
+            this.attachWsHandlers(this.ws);
+          }
+        } catch (err) {
+          this.pollLoop();
+        }
+      })();
+    });
   }
   attachWsHandlers(ws) {
     let startTs = Date.now();
@@ -1457,14 +1481,14 @@ var Watcher = class extends import_events.EventEmitter {
         }
         if (type === "document") {
           const doc = body.data;
-          if (doc) this.emit("document", doc);
+          if (doc) this.emitDocuments([doc]);
           return;
         }
         if (type === "done") {
           const payload2 = body.data || body;
           const data = payload2.data || [];
           if (data.length) this.emitDocuments(data);
-          this.emit("done", { status: "completed", data, id: this.jobId });
+          this.emit("done", { status: "completed", data, id: this.jobId, total: payload2.total, completed: payload2.completed, creditsUsed: payload2.creditsUsed });
           this.close();
           return;
         }
@@ -1472,7 +1496,10 @@ var Watcher = class extends import_events.EventEmitter {
         if (payload && payload.status) this.emitSnapshot(payload);
       } catch {
       }
-      if (timeoutMs && Date.now() - startTs > timeoutMs) this.close();
+      if (timeoutMs && Date.now() - startTs > timeoutMs) {
+        this.emit("error", { status: "failed", data: [], error: "Watcher timeout", id: this.jobId });
+        this.close();
+      }
     };
     ws.onerror = () => {
       this.emit("error", { status: "failed", data: [], error: "WebSocket error", id: this.jobId });
@@ -1528,7 +1555,7 @@ var Watcher = class extends import_events.EventEmitter {
     };
     this.emit("snapshot", snap);
     if (["completed", "failed", "cancelled"].includes(status)) {
-      this.emit("done", { status, data, id: this.jobId });
+      this.emit("done", { status, data, id: this.jobId, total: payload.total ?? 0, completed: payload.completed ?? 0, creditsUsed: payload.creditsUsed });
       this.close();
     }
   }
@@ -1541,13 +1568,17 @@ var Watcher = class extends import_events.EventEmitter {
         this.emitDocuments(snap.data || []);
         this.emit("snapshot", snap);
         if (["completed", "failed", "cancelled"].includes(snap.status)) {
-          this.emit("done", { status: snap.status, data: snap.data, id: this.jobId });
+          this.emit("done", { status: snap.status, data: snap.data, id: this.jobId, total: snap.total ?? 0, completed: snap.completed ?? 0, creditsUsed: snap.creditsUsed });
           this.close();
           break;
         }
       } catch {
       }
-      if (timeoutMs && Date.now() - startTs > timeoutMs) break;
+      if (timeoutMs && Date.now() - startTs > timeoutMs) {
+        this.emit("error", { status: "failed", data: [], error: "Watcher timeout", id: this.jobId });
+        this.close();
+        break;
+      }
       await new Promise((r) => setTimeout(r, Math.max(1e3, this.pollInterval * 1e3)));
     }
   }

package/dist/index.d.cts

@@ -395,6 +395,8 @@ interface SearchRequest {
         type: 'web' | 'news' | 'images';
     }>;
     categories?: Array<'github' | 'research' | 'pdf' | CategoryOption>;
+    includeDomains?: string[];
+    excludeDomains?: string[];
     limit?: number;
     tbs?: string;
     location?: string;

package/dist/index.d.ts

@@ -395,6 +395,8 @@ interface SearchRequest {
         type: 'web' | 'news' | 'images';
     }>;
     categories?: Array<'github' | 'research' | 'pdf' | CategoryOption>;
+    includeDomains?: string[];
+    excludeDomains?: string[];
     limit?: number;
     tbs?: string;
     location?: string;

package/dist/index.js

@@ -1,6 +1,6 @@
 import {
   require_package
-} from "./chunk-UXVHRV2G.js";
+} from "./chunk-R625SBJY.js";
 
 // src/v2/utils/httpClient.ts
 import axios from "axios";
@@ -501,11 +501,17 @@ function prepareSearchPayload(req) {
     throw new Error("limit must be positive");
   if (req.timeout != null && req.timeout <= 0)
     throw new Error("timeout must be positive");
+  if (req.includeDomains?.length && req.excludeDomains?.length)
+    throw new Error(
+      "includeDomains and excludeDomains cannot both be specified"
+    );
   const payload = {
     query: req.query
   };
   if (req.sources) payload.sources = req.sources;
   if (req.categories) payload.categories = req.categories;
+  if (req.includeDomains) payload.includeDomains = req.includeDomains;
+  if (req.excludeDomains) payload.excludeDomains = req.excludeDomains;
   if (req.limit != null) payload.limit = req.limit;
   if (req.tbs != null) payload.tbs = req.tbs;
   if (req.location != null) payload.location = req.location;
@@ -1294,23 +1300,41 @@ var Watcher = class extends EventEmitter {
     return `${wsBase}${path}`;
   }
   async start() {
-    try {
-      const url = this.buildWsUrl();
-      const wsCtor = await getWebSocketCtor();
-      if (!wsCtor) {
-        this.pollLoop();
-        return;
-      }
-      this.ws = new wsCtor(url, this.http.getApiKey());
-      if (this.ws && "binaryType" in this.ws) {
-        this.ws.binaryType = "arraybuffer";
-      }
-      if (this.ws) {
-        this.attachWsHandlers(this.ws);
-      }
-    } catch (err) {
-      this.pollLoop();
-    }
+    return new Promise((resolve, reject) => {
+      const onDone = () => {
+        cleanup();
+        resolve();
+      };
+      const onError = (err) => {
+        cleanup();
+        resolve();
+      };
+      const cleanup = () => {
+        this.removeListener("done", onDone);
+        this.removeListener("error", onError);
+      };
+      this.on("done", onDone);
+      this.on("error", onError);
+      (async () => {
+        try {
+          const url = this.buildWsUrl();
+          const wsCtor = await getWebSocketCtor();
+          if (!wsCtor) {
+            this.pollLoop();
+            return;
+          }
+          this.ws = new wsCtor(url, this.http.getApiKey());
+          if (this.ws && "binaryType" in this.ws) {
+            this.ws.binaryType = "arraybuffer";
+          }
+          if (this.ws) {
+            this.attachWsHandlers(this.ws);
+          }
+        } catch (err) {
+          this.pollLoop();
+        }
+      })();
+    });
   }
   attachWsHandlers(ws) {
     let startTs = Date.now();
@@ -1333,14 +1357,14 @@ var Watcher = class extends EventEmitter {
         }
         if (type === "document") {
           const doc = body.data;
-          if (doc) this.emit("document", doc);
+          if (doc) this.emitDocuments([doc]);
           return;
         }
         if (type === "done") {
           const payload2 = body.data || body;
           const data = payload2.data || [];
           if (data.length) this.emitDocuments(data);
-          this.emit("done", { status: "completed", data, id: this.jobId });
+          this.emit("done", { status: "completed", data, id: this.jobId, total: payload2.total, completed: payload2.completed, creditsUsed: payload2.creditsUsed });
           this.close();
           return;
         }
@@ -1348,7 +1372,10 @@ var Watcher = class extends EventEmitter {
         if (payload && payload.status) this.emitSnapshot(payload);
       } catch {
       }
-      if (timeoutMs && Date.now() - startTs > timeoutMs) this.close();
+      if (timeoutMs && Date.now() - startTs > timeoutMs) {
+        this.emit("error", { status: "failed", data: [], error: "Watcher timeout", id: this.jobId });
+        this.close();
+      }
     };
     ws.onerror = () => {
       this.emit("error", { status: "failed", data: [], error: "WebSocket error", id: this.jobId });
@@ -1404,7 +1431,7 @@ var Watcher = class extends EventEmitter {
     };
     this.emit("snapshot", snap);
     if (["completed", "failed", "cancelled"].includes(status)) {
-      this.emit("done", { status, data, id: this.jobId });
+      this.emit("done", { status, data, id: this.jobId, total: payload.total ?? 0, completed: payload.completed ?? 0, creditsUsed: payload.creditsUsed });
       this.close();
     }
   }
@@ -1417,13 +1444,17 @@ var Watcher = class extends EventEmitter {
         this.emitDocuments(snap.data || []);
         this.emit("snapshot", snap);
         if (["completed", "failed", "cancelled"].includes(snap.status)) {
-          this.emit("done", { status: snap.status, data: snap.data, id: this.jobId });
+          this.emit("done", { status: snap.status, data: snap.data, id: this.jobId, total: snap.total ?? 0, completed: snap.completed ?? 0, creditsUsed: snap.creditsUsed });
           this.close();
           break;
         }
       } catch {
       }
-      if (timeoutMs && Date.now() - startTs > timeoutMs) break;
+      if (timeoutMs && Date.now() - startTs > timeoutMs) {
+        this.emit("error", { status: "failed", data: [], error: "Watcher timeout", id: this.jobId });
+        this.close();
+        break;
+      }
       await new Promise((r) => setTimeout(r, Math.max(1e3, this.pollInterval * 1e3)));
     }
   }
@@ -1778,7 +1809,7 @@ var FirecrawlApp = class {
       if (typeof process !== "undefined" && process.env && process.env.npm_package_version) {
         return process.env.npm_package_version;
       }
-      const packageJson = await import("./package-ELM7Z5VP.js");
+      const packageJson = await import("./package-MRLSD24T.js");
       return packageJson.default.version;
     } catch (error) {
       const isTest = typeof process !== "undefined" && (process.env.JEST_WORKER_ID != null || false);

package/dist/{package-ELM7Z5VP.js → package-MRLSD24T.js}

@@ -1,4 +1,4 @@
 import {
   require_package
-} from "./chunk-UXVHRV2G.js";
+} from "./chunk-R625SBJY.js";
 export default require_package();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "firecrawl",
-  "version": "4.20.0",
+  "version": "4.21.0",
   "description": "JavaScript SDK for Firecrawl API",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

package/src/v2/methods/search.ts

@@ -20,11 +20,17 @@ function prepareSearchPayload(req: SearchRequest): Record<string, unknown> {
     throw new Error("limit must be positive");
   if (req.timeout != null && req.timeout <= 0)
     throw new Error("timeout must be positive");
+  if (req.includeDomains?.length && req.excludeDomains?.length)
+    throw new Error(
+      "includeDomains and excludeDomains cannot both be specified",
+    );
   const payload: Record<string, unknown> = {
     query: req.query,
   };
   if (req.sources) payload.sources = req.sources;
   if (req.categories) payload.categories = req.categories;
+  if (req.includeDomains) payload.includeDomains = req.includeDomains;
+  if (req.excludeDomains) payload.excludeDomains = req.excludeDomains;
   if (req.limit != null) payload.limit = req.limit;
   if (req.tbs != null) payload.tbs = req.tbs;
   if (req.location != null) payload.location = req.location;

package/src/v2/types.ts

@@ -508,6 +508,8 @@ export interface SearchRequest {
     'web' | 'news' | 'images' | { type: 'web' | 'news' | 'images' }
   >;
   categories?: Array<'github' | 'research' | 'pdf' | CategoryOption>;
+  includeDomains?: string[];
+  excludeDomains?: string[];
   limit?: number;
   tbs?: string;
   location?: string;

package/src/v2/watcher.ts

@@ -109,24 +109,37 @@ export class Watcher extends EventEmitter {
   }
 
   async start(): Promise<void> {
-    try {
-      const url = this.buildWsUrl();
-      const wsCtor = await getWebSocketCtor();
-      if (!wsCtor) {
-        this.pollLoop();
-        return;
-      }
-      this.ws = new wsCtor(url, this.http.getApiKey()) as any;
-      if (this.ws && "binaryType" in this.ws) {
-        (this.ws as any).binaryType = "arraybuffer";
-      }
-      
-      if (this.ws) {
-        this.attachWsHandlers(this.ws);
-      }
-    } catch (err) {
-      this.pollLoop();
-    }
+    return new Promise<void>((resolve, reject) => {
+      const onDone = () => { cleanup(); resolve(); };
+      const onError = (err: any) => { cleanup(); resolve(); };
+      const cleanup = () => {
+        this.removeListener("done", onDone);
+        this.removeListener("error", onError);
+      };
+      this.on("done", onDone);
+      this.on("error", onError);
+
+      (async () => {
+        try {
+          const url = this.buildWsUrl();
+          const wsCtor = await getWebSocketCtor();
+          if (!wsCtor) {
+            this.pollLoop();
+            return;
+          }
+          this.ws = new wsCtor(url, this.http.getApiKey()) as any;
+          if (this.ws && "binaryType" in this.ws) {
+            (this.ws as any).binaryType = "arraybuffer";
+          }
+
+          if (this.ws) {
+            this.attachWsHandlers(this.ws);
+          }
+        } catch (err) {
+          this.pollLoop();
+        }
+      })();
+    });
   }
 
   private attachWsHandlers(ws: WebSocket) {
@@ -150,14 +163,14 @@ export class Watcher extends EventEmitter {
         }
         if (type === "document") {
           const doc = body.data;
-          if (doc) this.emit("document", doc as Document & { id: string });
+          if (doc) this.emitDocuments([doc]);
           return;
         }
         if (type === "done") {
           const payload = body.data || body;
           const data = (payload.data || []) as Document[];
           if (data.length) this.emitDocuments(data);
-          this.emit("done", { status: "completed", data, id: this.jobId });
+          this.emit("done", { status: "completed", data, id: this.jobId, total: payload.total, completed: payload.completed, creditsUsed: payload.creditsUsed });
           this.close();
           return;
         }
@@ -166,7 +179,10 @@ export class Watcher extends EventEmitter {
       } catch {
         // ignore
       }
-      if (timeoutMs && Date.now() - startTs > timeoutMs) this.close();
+      if (timeoutMs && Date.now() - startTs > timeoutMs) {
+        this.emit("error", { status: "failed", data: [], error: "Watcher timeout", id: this.jobId });
+        this.close();
+      }
     };
     ws.onerror = () => {
       this.emit("error", { status: "failed", data: [], error: "WebSocket error", id: this.jobId });
@@ -227,7 +243,7 @@ export class Watcher extends EventEmitter {
         };
     this.emit("snapshot", snap);
     if (["completed", "failed", "cancelled"].includes(status)) {
-      this.emit("done", { status, data, id: this.jobId });
+      this.emit("done", { status, data, id: this.jobId, total: payload.total ?? 0, completed: payload.completed ?? 0, creditsUsed: payload.creditsUsed });
       this.close();
     }
   }
@@ -243,14 +259,18 @@ export class Watcher extends EventEmitter {
         this.emitDocuments((snap.data || []) as Document[]);
         this.emit("snapshot", snap);
         if (["completed", "failed", "cancelled"].includes(snap.status)) {
-          this.emit("done", { status: snap.status, data: snap.data, id: this.jobId });
+          this.emit("done", { status: snap.status, data: snap.data, id: this.jobId, total: (snap as any).total ?? 0, completed: (snap as any).completed ?? 0, creditsUsed: (snap as any).creditsUsed });
           this.close();
           break;
         }
       } catch {
         // ignore polling errors
       }
-      if (timeoutMs && Date.now() - startTs > timeoutMs) break;
+      if (timeoutMs && Date.now() - startTs > timeoutMs) {
+        this.emit("error", { status: "failed", data: [], error: "Watcher timeout", id: this.jobId });
+        this.close();
+        break;
+      }
       await new Promise((r) => setTimeout(r, Math.max(1000, this.pollInterval * 1000)));
     }
   }