@mendable/firecrawl 4.8.0 → 4.8.1

package/dist/{chunk-SP54QQ5D.js → chunk-SY42GUIL.js}

@@ -8,7 +8,7 @@ var require_package = __commonJS({
   "package.json"(exports, module) {
     module.exports = {
       name: "@mendable/firecrawl-js",
-      version: "4.8.0",
+      version: "4.8.1",
       description: "JavaScript SDK for Firecrawl API",
       main: "dist/index.js",
       types: "dist/index.d.ts",
@@ -36,7 +36,6 @@ var require_package = __commonJS({
       dependencies: {
         axios: "^1.12.2",
         "typescript-event-target": "^1.1.1",
-        ws: "^8.18.3",
         zod: "^3.23.8",
         "zod-to-json-schema": "^3.23.0"
       },
@@ -51,7 +50,6 @@ var require_package = __commonJS({
         "@types/mocha": "^10.0.6",
         "@types/node": "^20.12.12",
         "@types/uuid": "^9.0.8",
-        "@types/ws": "^8.18.1",
         dotenv: "^16.4.5",
         jest: "^30.2.0",
         "ts-jest": "^29.4.5",

package/dist/index.cjs

@@ -35,7 +35,7 @@ var require_package = __commonJS({
   "package.json"(exports2, module2) {
     module2.exports = {
       name: "@mendable/firecrawl-js",
-      version: "4.8.0",
+      version: "4.8.1",
       description: "JavaScript SDK for Firecrawl API",
       main: "dist/index.js",
       types: "dist/index.d.ts",
@@ -63,7 +63,6 @@ var require_package = __commonJS({
       dependencies: {
         axios: "^1.12.2",
         "typescript-event-target": "^1.1.1",
-        ws: "^8.18.3",
         zod: "^3.23.8",
         "zod-to-json-schema": "^3.23.0"
       },
@@ -78,7 +77,6 @@ var require_package = __commonJS({
         "@types/mocha": "^10.0.6",
         "@types/node": "^20.12.12",
         "@types/uuid": "^9.0.8",
-        "@types/ws": "^8.18.1",
         dotenv: "^16.4.5",
         jest: "^30.2.0",
         "ts-jest": "^29.4.5",
@@ -832,7 +830,57 @@ async function getTokenUsageHistorical(http, byApiKey) {
 
 // src/v2/watcher.ts
 var import_events = require("events");
-var import_ws = require("ws");
+var hasGlobalWebSocket = () => {
+  if (typeof globalThis === "undefined") return void 0;
+  const candidate = globalThis.WebSocket;
+  return typeof candidate === "function" ? candidate : void 0;
+};
+var isNodeRuntime = () => typeof process !== "undefined" && !!process.versions?.node;
+var cachedWebSocket;
+var loadPromise;
+var loadNodeWebSocket = async () => {
+  if (!isNodeRuntime()) return void 0;
+  try {
+    const undici = await import("undici");
+    const ctor = undici.WebSocket ?? undici.default?.WebSocket;
+    return typeof ctor === "function" ? ctor : void 0;
+  } catch {
+    return void 0;
+  }
+};
+var getWebSocketCtor = async () => {
+  if (cachedWebSocket) return cachedWebSocket;
+  const globalWs = hasGlobalWebSocket();
+  if (globalWs) {
+    cachedWebSocket = globalWs;
+    return cachedWebSocket;
+  }
+  if (!loadPromise) {
+    loadPromise = loadNodeWebSocket();
+  }
+  cachedWebSocket = await loadPromise;
+  return cachedWebSocket;
+};
+var decoder = typeof TextDecoder !== "undefined" ? new TextDecoder() : void 0;
+var ensureUtf8String = (data) => {
+  if (typeof data === "string") return data;
+  if (typeof Buffer !== "undefined" && Buffer.isBuffer(data)) {
+    return data.toString("utf8");
+  }
+  const convertView = (view) => {
+    if (typeof Buffer !== "undefined") {
+      return Buffer.from(view.buffer, view.byteOffset, view.byteLength).toString("utf8");
+    }
+    return decoder?.decode(view);
+  };
+  if (ArrayBuffer.isView(data)) {
+    return convertView(data);
+  }
+  if (data instanceof ArrayBuffer) {
+    return convertView(new Uint8Array(data));
+  }
+  return void 0;
+};
 var Watcher = class extends import_events.EventEmitter {
   http;
   jobId;
@@ -841,6 +889,7 @@ var Watcher = class extends import_events.EventEmitter {
   timeout;
   ws;
   closed = false;
+  emittedDocumentKeys = /* @__PURE__ */ new Set();
   constructor(http, jobId, opts = {}) {
     super();
     this.http = http;
@@ -858,10 +907,14 @@ var Watcher = class extends import_events.EventEmitter {
   async start() {
     try {
       const url = this.buildWsUrl();
-      if (typeof WebSocket !== "undefined") {
-        this.ws = new WebSocket(url, this.http.getApiKey());
-      } else {
-        this.ws = new import_ws.WebSocket(url, this.http.getApiKey());
+      const wsCtor = await getWebSocketCtor();
+      if (!wsCtor) {
+        this.pollLoop();
+        return;
+      }
+      this.ws = new wsCtor(url, this.http.getApiKey());
+      if (this.ws && "binaryType" in this.ws) {
+        this.ws.binaryType = "arraybuffer";
       }
       if (this.ws) {
         this.attachWsHandlers(this.ws);
@@ -875,8 +928,9 @@ var Watcher = class extends import_events.EventEmitter {
     const timeoutMs = this.timeout ? this.timeout * 1e3 : void 0;
     ws.onmessage = (ev) => {
       try {
-        const body = typeof ev.data === "string" ? JSON.parse(ev.data) : null;
-        if (!body) return;
+        const raw = ensureUtf8String(ev.data);
+        if (!raw) return;
+        const body = JSON.parse(raw);
         const type = body.type;
         if (type === "error") {
           this.emit("error", { status: "failed", data: [], error: body.error, id: this.jobId });
@@ -896,6 +950,7 @@ var Watcher = class extends import_events.EventEmitter {
         if (type === "done") {
           const payload2 = body.data || body;
           const data = payload2.data || [];
+          if (data.length) this.emitDocuments(data);
           this.emit("done", { status: "completed", data, id: this.jobId });
           this.close();
           return;
@@ -914,8 +969,27 @@ var Watcher = class extends import_events.EventEmitter {
       if (!this.closed) this.pollLoop();
     };
   }
+  documentKey(doc) {
+    if (doc && typeof doc === "object") {
+      const explicitId = doc.id ?? doc.docId ?? doc.url;
+      if (typeof explicitId === "string" && explicitId.length) {
+        return explicitId;
+      }
+    }
+    try {
+      return JSON.stringify(doc);
+    } catch {
+      return `${Date.now()}-${Math.random()}`;
+    }
+  }
   emitDocuments(docs) {
-    for (const doc of docs) this.emit("document", { ...doc, id: this.jobId });
+    for (const doc of docs) {
+      if (!doc) continue;
+      const key = this.documentKey(doc);
+      if (this.emittedDocumentKeys.has(key)) continue;
+      this.emittedDocumentKeys.add(key);
+      this.emit("document", { ...doc, id: this.jobId });
+    }
   }
   emitSnapshot(payload) {
     const status = payload.status;
@@ -951,6 +1025,7 @@ var Watcher = class extends import_events.EventEmitter {
     while (!this.closed) {
       try {
         const snap = this.kind === "crawl" ? await getCrawlStatus(this.http, this.jobId) : await getBatchScrapeStatus(this.http, this.jobId);
+        this.emitDocuments(snap.data || []);
         this.emit("snapshot", snap);
         if (["completed", "failed", "cancelled"].includes(snap.status)) {
           this.emit("done", { status: snap.status, data: snap.data, id: this.jobId });

package/dist/index.d.cts

@@ -578,10 +578,12 @@ declare class Watcher extends EventEmitter {
     private readonly timeout?;
     private ws?;
     private closed;
+    private readonly emittedDocumentKeys;
     constructor(http: HttpClient, jobId: string, opts?: WatcherOptions);
     private buildWsUrl;
     start(): Promise<void>;
     private attachWsHandlers;
+    private documentKey;
     private emitDocuments;
     private emitSnapshot;
     private pollLoop;

package/dist/index.d.ts

@@ -578,10 +578,12 @@ declare class Watcher extends EventEmitter {
     private readonly timeout?;
     private ws?;
     private closed;
+    private readonly emittedDocumentKeys;
     constructor(http: HttpClient, jobId: string, opts?: WatcherOptions);
     private buildWsUrl;
     start(): Promise<void>;
     private attachWsHandlers;
+    private documentKey;
     private emitDocuments;
     private emitSnapshot;
     private pollLoop;

package/dist/index.js

@@ -1,6 +1,6 @@
 import {
   require_package
-} from "./chunk-SP54QQ5D.js";
+} from "./chunk-SY42GUIL.js";
 
 // src/v2/utils/httpClient.ts
 import axios from "axios";
@@ -712,7 +712,57 @@ async function getTokenUsageHistorical(http, byApiKey) {
 
 // src/v2/watcher.ts
 import { EventEmitter } from "events";
-import { WebSocket as WS } from "ws";
+var hasGlobalWebSocket = () => {
+  if (typeof globalThis === "undefined") return void 0;
+  const candidate = globalThis.WebSocket;
+  return typeof candidate === "function" ? candidate : void 0;
+};
+var isNodeRuntime = () => typeof process !== "undefined" && !!process.versions?.node;
+var cachedWebSocket;
+var loadPromise;
+var loadNodeWebSocket = async () => {
+  if (!isNodeRuntime()) return void 0;
+  try {
+    const undici = await import("undici");
+    const ctor = undici.WebSocket ?? undici.default?.WebSocket;
+    return typeof ctor === "function" ? ctor : void 0;
+  } catch {
+    return void 0;
+  }
+};
+var getWebSocketCtor = async () => {
+  if (cachedWebSocket) return cachedWebSocket;
+  const globalWs = hasGlobalWebSocket();
+  if (globalWs) {
+    cachedWebSocket = globalWs;
+    return cachedWebSocket;
+  }
+  if (!loadPromise) {
+    loadPromise = loadNodeWebSocket();
+  }
+  cachedWebSocket = await loadPromise;
+  return cachedWebSocket;
+};
+var decoder = typeof TextDecoder !== "undefined" ? new TextDecoder() : void 0;
+var ensureUtf8String = (data) => {
+  if (typeof data === "string") return data;
+  if (typeof Buffer !== "undefined" && Buffer.isBuffer(data)) {
+    return data.toString("utf8");
+  }
+  const convertView = (view) => {
+    if (typeof Buffer !== "undefined") {
+      return Buffer.from(view.buffer, view.byteOffset, view.byteLength).toString("utf8");
+    }
+    return decoder?.decode(view);
+  };
+  if (ArrayBuffer.isView(data)) {
+    return convertView(data);
+  }
+  if (data instanceof ArrayBuffer) {
+    return convertView(new Uint8Array(data));
+  }
+  return void 0;
+};
 var Watcher = class extends EventEmitter {
   http;
   jobId;
@@ -721,6 +771,7 @@ var Watcher = class extends EventEmitter {
   timeout;
   ws;
   closed = false;
+  emittedDocumentKeys = /* @__PURE__ */ new Set();
   constructor(http, jobId, opts = {}) {
     super();
     this.http = http;
@@ -738,10 +789,14 @@ var Watcher = class extends EventEmitter {
   async start() {
     try {
       const url = this.buildWsUrl();
-      if (typeof WebSocket !== "undefined") {
-        this.ws = new WebSocket(url, this.http.getApiKey());
-      } else {
-        this.ws = new WS(url, this.http.getApiKey());
+      const wsCtor = await getWebSocketCtor();
+      if (!wsCtor) {
+        this.pollLoop();
+        return;
+      }
+      this.ws = new wsCtor(url, this.http.getApiKey());
+      if (this.ws && "binaryType" in this.ws) {
+        this.ws.binaryType = "arraybuffer";
       }
       if (this.ws) {
         this.attachWsHandlers(this.ws);
@@ -755,8 +810,9 @@ var Watcher = class extends EventEmitter {
     const timeoutMs = this.timeout ? this.timeout * 1e3 : void 0;
     ws.onmessage = (ev) => {
       try {
-        const body = typeof ev.data === "string" ? JSON.parse(ev.data) : null;
-        if (!body) return;
+        const raw = ensureUtf8String(ev.data);
+        if (!raw) return;
+        const body = JSON.parse(raw);
         const type = body.type;
         if (type === "error") {
           this.emit("error", { status: "failed", data: [], error: body.error, id: this.jobId });
@@ -776,6 +832,7 @@ var Watcher = class extends EventEmitter {
         if (type === "done") {
           const payload2 = body.data || body;
           const data = payload2.data || [];
+          if (data.length) this.emitDocuments(data);
           this.emit("done", { status: "completed", data, id: this.jobId });
           this.close();
           return;
@@ -794,8 +851,27 @@ var Watcher = class extends EventEmitter {
       if (!this.closed) this.pollLoop();
     };
   }
+  documentKey(doc) {
+    if (doc && typeof doc === "object") {
+      const explicitId = doc.id ?? doc.docId ?? doc.url;
+      if (typeof explicitId === "string" && explicitId.length) {
+        return explicitId;
+      }
+    }
+    try {
+      return JSON.stringify(doc);
+    } catch {
+      return `${Date.now()}-${Math.random()}`;
+    }
+  }
   emitDocuments(docs) {
-    for (const doc of docs) this.emit("document", { ...doc, id: this.jobId });
+    for (const doc of docs) {
+      if (!doc) continue;
+      const key = this.documentKey(doc);
+      if (this.emittedDocumentKeys.has(key)) continue;
+      this.emittedDocumentKeys.add(key);
+      this.emit("document", { ...doc, id: this.jobId });
+    }
   }
   emitSnapshot(payload) {
     const status = payload.status;
@@ -831,6 +907,7 @@ var Watcher = class extends EventEmitter {
     while (!this.closed) {
       try {
         const snap = this.kind === "crawl" ? await getCrawlStatus(this.http, this.jobId) : await getBatchScrapeStatus(this.http, this.jobId);
+        this.emitDocuments(snap.data || []);
         this.emit("snapshot", snap);
         if (["completed", "failed", "cancelled"].includes(snap.status)) {
           this.emit("done", { status: snap.status, data: snap.data, id: this.jobId });
@@ -1087,7 +1164,7 @@ var FirecrawlApp = class {
       if (typeof process !== "undefined" && process.env && process.env.npm_package_version) {
         return process.env.npm_package_version;
       }
-      const packageJson = await import("./package-MR6WWGRM.js");
+      const packageJson = await import("./package-G5T62GGJ.js");
       return packageJson.default.version;
     } catch (error) {
       const isTest = typeof process !== "undefined" && (process.env.JEST_WORKER_ID != null || false);

package/dist/{package-MR6WWGRM.js → package-G5T62GGJ.js}

@@ -1,4 +1,4 @@
 import {
   require_package
-} from "./chunk-SP54QQ5D.js";
+} from "./chunk-SY42GUIL.js";
 export default require_package();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mendable/firecrawl",
-  "version": "4.8.0",
+  "version": "4.8.1",
   "description": "JavaScript SDK for Firecrawl API",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -28,7 +28,6 @@
   "dependencies": {
     "axios": "^1.12.2",
     "typescript-event-target": "^1.1.1",
-    "ws": "^8.18.3",
     "zod": "^3.23.8",
     "zod-to-json-schema": "^3.23.0"
   },
@@ -43,7 +42,6 @@
     "@types/mocha": "^10.0.6",
     "@types/node": "^20.12.12",
     "@types/uuid": "^9.0.8",
-    "@types/ws": "^8.18.1",
     "dotenv": "^16.4.5",
     "jest": "^30.2.0",
     "ts-jest": "^29.4.5",

package/src/types/node-undici.d.ts

@@ -0,0 +1,10 @@
+type WebSocketCtor = typeof globalThis.WebSocket;
+
+declare module "node:undici" {
+  export const WebSocket: WebSocketCtor;
+  const _default: {
+    WebSocket: WebSocketCtor;
+  };
+  export default _default;
+}
+

package/src/v2/watcher.ts

@@ -3,7 +3,73 @@ import type { BatchScrapeJob, CrawlJob, Document } from "./types";
 import type { HttpClient } from "./utils/httpClient";
 import { getBatchScrapeStatus } from "./methods/batch";
 import { getCrawlStatus } from "./methods/crawl";
-import { WebSocket as WS } from "ws";
+// Note: browsers/Deno expose globalThis.WebSocket, but many Node runtimes (<22.4 or without
+// experimental flags) do not. We lazily fall back to node:undici.
+
+type WebSocketConstructor = new (url: string, protocols?: string | string[]) => WebSocket;
+
+const hasGlobalWebSocket = (): WebSocketConstructor | undefined => {
+  if (typeof globalThis === "undefined") return undefined;
+  const candidate = (globalThis as any).WebSocket;
+  return typeof candidate === "function" ? (candidate as WebSocketConstructor) : undefined;
+};
+
+const isNodeRuntime = () => typeof process !== "undefined" && !!process.versions?.node;
+
+let cachedWebSocket: WebSocketConstructor | undefined;
+let loadPromise: Promise<WebSocketConstructor | undefined> | undefined;
+
+const loadNodeWebSocket = async (): Promise<WebSocketConstructor | undefined> => {
+  if (!isNodeRuntime()) return undefined;
+  try {
+    const undici = await import("node:undici");
+    const ctor = (undici as any).WebSocket ?? (undici as any).default?.WebSocket;
+    return typeof ctor === "function" ? (ctor as WebSocketConstructor) : undefined;
+  } catch {
+    return undefined;
+  }
+};
+
+const getWebSocketCtor = async (): Promise<WebSocketConstructor | undefined> => {
+  if (cachedWebSocket) return cachedWebSocket;
+  const globalWs = hasGlobalWebSocket();
+  if (globalWs) {
+    cachedWebSocket = globalWs;
+    return cachedWebSocket;
+  }
+  if (!loadPromise) {
+    loadPromise = loadNodeWebSocket();
+  }
+  cachedWebSocket = await loadPromise;
+  return cachedWebSocket;
+};
+
+const decoder = typeof TextDecoder !== "undefined" ? new TextDecoder() : undefined;
+
+const ensureUtf8String = (data: unknown): string | undefined => {
+  if (typeof data === "string") return data;
+
+  if (typeof Buffer !== "undefined" && Buffer.isBuffer(data)) {
+    return data.toString("utf8");
+  }
+
+  const convertView = (view: ArrayBufferView): string | undefined => {
+    if (typeof Buffer !== "undefined") {
+      return Buffer.from(view.buffer, view.byteOffset, view.byteLength).toString("utf8");
+    }
+    return decoder?.decode(view);
+  };
+
+  if (ArrayBuffer.isView(data)) {
+    return convertView(data);
+  }
+
+  if (data instanceof ArrayBuffer) {
+    return convertView(new Uint8Array(data));
+  }
+
+  return undefined;
+};
 
 type JobKind = "crawl" | "batch";
 
@@ -23,6 +89,7 @@ export class Watcher extends EventEmitter {
   private readonly timeout?: number;
   private ws?: WebSocket;
   private closed = false;
+  private readonly emittedDocumentKeys = new Set<string>();
 
   constructor(http: HttpClient, jobId: string, opts: WatcherOptions = {}) {
     super();
@@ -44,11 +111,14 @@ export class Watcher extends EventEmitter {
   async start(): Promise<void> {
     try {
       const url = this.buildWsUrl();
-      
-      if (typeof WebSocket !== 'undefined') {
-        this.ws = new WebSocket(url, this.http.getApiKey()) as any;
-      } else {
-        this.ws = new WS(url, this.http.getApiKey()) as any;
+      const wsCtor = await getWebSocketCtor();
+      if (!wsCtor) {
+        this.pollLoop();
+        return;
+      }
+      this.ws = new wsCtor(url, this.http.getApiKey()) as any;
+      if (this.ws && "binaryType" in this.ws) {
+        (this.ws as any).binaryType = "arraybuffer";
       }
       
       if (this.ws) {
@@ -64,8 +134,9 @@ export class Watcher extends EventEmitter {
     const timeoutMs = this.timeout ? this.timeout * 1000 : undefined;
     ws.onmessage = (ev: MessageEvent) => {
       try {
-        const body = typeof ev.data === "string" ? JSON.parse(ev.data) : null;
-        if (!body) return;
+        const raw = ensureUtf8String(ev.data);
+        if (!raw) return;
+        const body = JSON.parse(raw);
         const type = body.type as string | undefined;
         if (type === "error") {
           this.emit("error", { status: "failed", data: [], error: body.error, id: this.jobId });
@@ -85,6 +156,7 @@ export class Watcher extends EventEmitter {
         if (type === "done") {
           const payload = body.data || body;
           const data = (payload.data || []) as Document[];
+          if (data.length) this.emitDocuments(data);
           this.emit("done", { status: "completed", data, id: this.jobId });
           this.close();
           return;
@@ -105,8 +177,28 @@ export class Watcher extends EventEmitter {
     };
   }
 
+  private documentKey(doc: Document): string {
+    if (doc && typeof doc === "object") {
+      const explicitId = (doc as any).id ?? (doc as any).docId ?? (doc as any).url;
+      if (typeof explicitId === "string" && explicitId.length) {
+        return explicitId;
+      }
+    }
+    try {
+      return JSON.stringify(doc);
+    } catch {
+      return `${Date.now()}-${Math.random()}`;
+    }
+  }
+
   private emitDocuments(docs: Document[]) {
-    for (const doc of docs) this.emit("document", { ...(doc as any), id: this.jobId });
+    for (const doc of docs) {
+      if (!doc) continue;
+      const key = this.documentKey(doc);
+      if (this.emittedDocumentKeys.has(key)) continue;
+      this.emittedDocumentKeys.add(key);
+      this.emit("document", { ...(doc as any), id: this.jobId });
+    }
   }
 
   private emitSnapshot(payload: any) {
@@ -148,6 +240,7 @@ export class Watcher extends EventEmitter {
         const snap = this.kind === "crawl"
           ? await getCrawlStatus(this.http as any, this.jobId)
           : await getBatchScrapeStatus(this.http as any, this.jobId);
+        this.emitDocuments((snap.data || []) as Document[]);
         this.emit("snapshot", snap);
         if (["completed", "failed", "cancelled"].includes(snap.status)) {
           this.emit("done", { status: snap.status, data: snap.data, id: this.jobId });

package/tsup.config.ts

@@ -8,7 +8,6 @@ export default defineConfig({
   clean: true,
   platform: "node",
   target: "node22",
-  external: ["ws"],
   noExternal: ["typescript-event-target"],
   esbuildOptions(options) {
     options.define = {