@mendable/firecrawl 4.5.0 → 4.6.1

package/dist/{chunk-MVAMVUST.js → chunk-47H6QFPY.js}

@@ -8,7 +8,7 @@ var require_package = __commonJS({
   "package.json"(exports, module) {
     module.exports = {
       name: "@mendable/firecrawl-js",
-      version: "4.5.0",
+      version: "4.6.1",
       description: "JavaScript SDK for Firecrawl API",
       main: "dist/index.js",
       types: "dist/index.d.ts",
@@ -36,6 +36,7 @@ var require_package = __commonJS({
       dependencies: {
         axios: "^1.12.2",
         "typescript-event-target": "^1.1.1",
+        ws: "^8.18.3",
         zod: "^3.23.8",
         "zod-to-json-schema": "^3.23.0"
       },
@@ -50,6 +51,7 @@ var require_package = __commonJS({
         "@types/mocha": "^10.0.6",
         "@types/node": "^20.12.12",
         "@types/uuid": "^9.0.8",
+        "@types/ws": "^8.18.1",
         dotenv: "^16.4.5",
         jest: "^30.0.5",
         "ts-jest": "^29.4.0",

package/dist/index.cjs

@@ -35,7 +35,7 @@ var require_package = __commonJS({
   "package.json"(exports2, module2) {
     module2.exports = {
       name: "@mendable/firecrawl-js",
-      version: "4.5.0",
+      version: "4.6.1",
       description: "JavaScript SDK for Firecrawl API",
       main: "dist/index.js",
       types: "dist/index.d.ts",
@@ -63,6 +63,7 @@ var require_package = __commonJS({
       dependencies: {
         axios: "^1.12.2",
         "typescript-event-target": "^1.1.1",
+        ws: "^8.18.3",
         zod: "^3.23.8",
         "zod-to-json-schema": "^3.23.0"
       },
@@ -77,6 +78,7 @@ var require_package = __commonJS({
         "@types/mocha": "^10.0.6",
         "@types/node": "^20.12.12",
         "@types/uuid": "^9.0.8",
+        "@types/ws": "^8.18.1",
         dotenv: "^16.4.5",
         jest: "^30.0.5",
         "ts-jest": "^29.4.0",
@@ -493,6 +495,7 @@ async function getCrawlStatus(http, jobId, pagination) {
     const auto = pagination?.autoPaginate ?? true;
     if (!auto || !body.next) {
       return {
+        id: jobId,
         status: body.status,
         completed: body.completed ?? 0,
         total: body.total ?? 0,
@@ -504,6 +507,7 @@ async function getCrawlStatus(http, jobId, pagination) {
     }
     const aggregated = await fetchAllPages(http, body.next, initialDocs, pagination);
     return {
+      id: jobId,
       status: body.status,
       completed: body.completed ?? 0,
       total: body.total ?? 0,
@@ -622,6 +626,7 @@ async function getBatchScrapeStatus(http, jobId, pagination) {
     const auto = pagination?.autoPaginate ?? true;
     if (!auto || !body.next) {
       return {
+        id: jobId,
         status: body.status,
         completed: body.completed ?? 0,
         total: body.total ?? 0,
@@ -633,6 +638,7 @@ async function getBatchScrapeStatus(http, jobId, pagination) {
     }
     const aggregated = await fetchAllPages(http, body.next, initialDocs, pagination);
     return {
+      id: jobId,
       status: body.status,
       completed: body.completed ?? 0,
       total: body.total ?? 0,
@@ -823,6 +829,7 @@ async function getTokenUsageHistorical(http, byApiKey) {
 
 // src/v2/watcher.ts
 var import_events = require("events");
+var import_ws = require("ws");
 var Watcher = class extends import_events.EventEmitter {
   http;
   jobId;
@@ -848,9 +855,15 @@ var Watcher = class extends import_events.EventEmitter {
   async start() {
     try {
       const url = this.buildWsUrl();
-      this.ws = new WebSocket(url, this.http.getApiKey());
-      this.attachWsHandlers(this.ws);
-    } catch {
+      if (typeof WebSocket !== "undefined") {
+        this.ws = new WebSocket(url, this.http.getApiKey());
+      } else {
+        this.ws = new import_ws.WebSocket(url, this.http.getApiKey());
+      }
+      if (this.ws) {
+        this.attachWsHandlers(this.ws);
+      }
+    } catch (err) {
       this.pollLoop();
     }
   }
@@ -878,7 +891,9 @@ var Watcher = class extends import_events.EventEmitter {
           return;
         }
         if (type === "done") {
-          this.emit("done", { status: "completed", data: [], id: this.jobId });
+          const payload2 = body.data || body;
+          const data = payload2.data || [];
+          this.emit("done", { status: "completed", data, id: this.jobId });
           this.close();
           return;
         }
@@ -903,6 +918,7 @@ var Watcher = class extends import_events.EventEmitter {
     const status = payload.status;
     const data = payload.data || [];
     const snap = this.kind === "crawl" ? {
+      id: this.jobId,
       status,
       completed: payload.completed ?? 0,
       total: payload.total ?? 0,
@@ -911,6 +927,7 @@ var Watcher = class extends import_events.EventEmitter {
       next: payload.next ?? null,
       data
     } : {
+      id: this.jobId,
       status,
       completed: payload.completed ?? 0,
       total: payload.total ?? 0,

package/dist/index.d.cts

@@ -387,6 +387,7 @@ interface CrawlResponse$1 {
     url: string;
 }
 interface CrawlJob {
+    id: string;
     status: 'scraping' | 'completed' | 'failed' | 'cancelled';
     total: number;
     completed: number;
@@ -411,6 +412,7 @@ interface BatchScrapeResponse$1 {
     invalidURLs?: string[];
 }
 interface BatchScrapeJob {
+    id: string;
     status: 'scraping' | 'completed' | 'failed' | 'cancelled';
     completed: number;
     total: number;

package/dist/index.d.ts

@@ -387,6 +387,7 @@ interface CrawlResponse$1 {
     url: string;
 }
 interface CrawlJob {
+    id: string;
     status: 'scraping' | 'completed' | 'failed' | 'cancelled';
     total: number;
     completed: number;
@@ -411,6 +412,7 @@ interface BatchScrapeResponse$1 {
     invalidURLs?: string[];
 }
 interface BatchScrapeJob {
+    id: string;
     status: 'scraping' | 'completed' | 'failed' | 'cancelled';
     completed: number;
     total: number;

package/dist/index.js

@@ -1,6 +1,6 @@
 import {
   require_package
-} from "./chunk-MVAMVUST.js";
+} from "./chunk-47H6QFPY.js";
 
 // src/v2/utils/httpClient.ts
 import axios from "axios";
@@ -377,6 +377,7 @@ async function getCrawlStatus(http, jobId, pagination) {
     const auto = pagination?.autoPaginate ?? true;
     if (!auto || !body.next) {
       return {
+        id: jobId,
         status: body.status,
         completed: body.completed ?? 0,
         total: body.total ?? 0,
@@ -388,6 +389,7 @@ async function getCrawlStatus(http, jobId, pagination) {
     }
     const aggregated = await fetchAllPages(http, body.next, initialDocs, pagination);
     return {
+      id: jobId,
       status: body.status,
       completed: body.completed ?? 0,
       total: body.total ?? 0,
@@ -506,6 +508,7 @@ async function getBatchScrapeStatus(http, jobId, pagination) {
     const auto = pagination?.autoPaginate ?? true;
     if (!auto || !body.next) {
       return {
+        id: jobId,
         status: body.status,
         completed: body.completed ?? 0,
         total: body.total ?? 0,
@@ -517,6 +520,7 @@ async function getBatchScrapeStatus(http, jobId, pagination) {
     }
     const aggregated = await fetchAllPages(http, body.next, initialDocs, pagination);
     return {
+      id: jobId,
       status: body.status,
       completed: body.completed ?? 0,
       total: body.total ?? 0,
@@ -707,6 +711,7 @@ async function getTokenUsageHistorical(http, byApiKey) {
 
 // src/v2/watcher.ts
 import { EventEmitter } from "events";
+import { WebSocket as WS } from "ws";
 var Watcher = class extends EventEmitter {
   http;
   jobId;
@@ -732,9 +737,15 @@ var Watcher = class extends EventEmitter {
   async start() {
     try {
       const url = this.buildWsUrl();
-      this.ws = new WebSocket(url, this.http.getApiKey());
-      this.attachWsHandlers(this.ws);
-    } catch {
+      if (typeof WebSocket !== "undefined") {
+        this.ws = new WebSocket(url, this.http.getApiKey());
+      } else {
+        this.ws = new WS(url, this.http.getApiKey());
+      }
+      if (this.ws) {
+        this.attachWsHandlers(this.ws);
+      }
+    } catch (err) {
       this.pollLoop();
     }
   }
@@ -762,7 +773,9 @@ var Watcher = class extends EventEmitter {
           return;
         }
         if (type === "done") {
-          this.emit("done", { status: "completed", data: [], id: this.jobId });
+          const payload2 = body.data || body;
+          const data = payload2.data || [];
+          this.emit("done", { status: "completed", data, id: this.jobId });
           this.close();
           return;
         }
@@ -787,6 +800,7 @@ var Watcher = class extends EventEmitter {
     const status = payload.status;
     const data = payload.data || [];
     const snap = this.kind === "crawl" ? {
+      id: this.jobId,
       status,
       completed: payload.completed ?? 0,
       total: payload.total ?? 0,
@@ -795,6 +809,7 @@ var Watcher = class extends EventEmitter {
       next: payload.next ?? null,
       data
     } : {
+      id: this.jobId,
       status,
       completed: payload.completed ?? 0,
       total: payload.total ?? 0,
@@ -1071,7 +1086,7 @@ var FirecrawlApp = class {
       if (typeof process !== "undefined" && process.env && process.env.npm_package_version) {
         return process.env.npm_package_version;
       }
-      const packageJson = await import("./package-O4UQBIGU.js");
+      const packageJson = await import("./package-OVF37QHH.js");
       return packageJson.default.version;
     } catch (error) {
       const isTest = typeof process !== "undefined" && (process.env.JEST_WORKER_ID != null || false);

package/dist/{package-O4UQBIGU.js → package-OVF37QHH.js}

@@ -1,4 +1,4 @@
 import {
   require_package
-} from "./chunk-MVAMVUST.js";
+} from "./chunk-47H6QFPY.js";
 export default require_package();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mendable/firecrawl",
-  "version": "4.5.0",
+  "version": "4.6.1",
   "description": "JavaScript SDK for Firecrawl API",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -28,6 +28,7 @@
   "dependencies": {
     "axios": "^1.12.2",
     "typescript-event-target": "^1.1.1",
+    "ws": "^8.18.3",
     "zod": "^3.23.8",
     "zod-to-json-schema": "^3.23.0"
   },
@@ -42,6 +43,7 @@
     "@types/mocha": "^10.0.6",
     "@types/node": "^20.12.12",
     "@types/uuid": "^9.0.8",
+    "@types/ws": "^8.18.1",
     "dotenv": "^16.4.5",
     "jest": "^30.0.5",
     "ts-jest": "^29.4.0",

package/src/__tests__/e2e/v2/batch.test.ts

@@ -29,6 +29,23 @@ describe("v2.batch e2e", () => {
     expect(Array.isArray(job.data)).toBe(true);
   }, 240_000);
 
+  test("batch scrape with wait returns job id for error retrieval", async () => {
+    const urls = [
+      "https://docs.firecrawl.dev",
+      "https://firecrawl.dev",
+    ];
+    const job = await client.batchScrape(urls, { options: { formats: ["markdown"] }, pollInterval: 1, timeout: 180 });
+    // Verify job has id field
+    expect(job.id).toBeDefined();
+    expect(typeof job.id).toBe("string");
+    // Verify we can use the id to retrieve errors
+    const errors = await client.getBatchScrapeErrors(job.id!);
+    expect(errors).toHaveProperty("errors");
+    expect(errors).toHaveProperty("robotsBlocked");
+    expect(Array.isArray(errors.errors)).toBe(true);
+    expect(Array.isArray(errors.robotsBlocked)).toBe(true);
+  }, 240_000);
+
   test("start batch minimal and status", async () => {
     const urls = ["https://docs.firecrawl.dev", "https://firecrawl.dev"]; 
     const start = await client.startBatchScrape(urls, { options: { formats: ["markdown"] }, ignoreInvalidURLs: true });
@@ -37,6 +54,9 @@ describe("v2.batch e2e", () => {
     const status = await client.getBatchScrapeStatus(start.id);
     expect(["scraping", "completed", "failed", "cancelled"]).toContain(status.status);
     expect(status.total).toBeGreaterThanOrEqual(0);
+    // Verify status includes id field
+    expect(status.id).toBeDefined();
+    expect(status.id).toBe(start.id);
   }, 120_000);
 
   test("wait batch with all params", async () => {

package/src/__tests__/e2e/v2/crawl.test.ts

@@ -45,6 +45,9 @@ describe("v2.crawl e2e", () => {
     const status = await client.getCrawlStatus(start.id);
     expect(["scraping", "completed", "failed", "cancelled"]).toContain(status.status);
     expect(status.completed).toBeGreaterThanOrEqual(0);
+    // Verify status includes id field
+    expect(status.id).toBeDefined();
+    expect(status.id).toBe(start.id);
     // next/expiresAt may be null/undefined depending on state; check shape
     expect(Array.isArray(status.data)).toBe(true);
   }, 120_000);
@@ -112,6 +115,20 @@ describe("v2.crawl e2e", () => {
     expect(Array.isArray(job.data)).toBe(true);
   }, 180_000);
 
+  test("crawl with wait returns job id for error retrieval", async () => {
+    if (!client) throw new Error();
+    const job = await client.crawl("https://docs.firecrawl.dev", { limit: 3, maxDiscoveryDepth: 2, pollInterval: 1, timeout: 120 });
+    // Verify job has id field
+    expect(job.id).toBeDefined();
+    expect(typeof job.id).toBe("string");
+    // Verify we can use the id to retrieve errors
+    const errors = await client.getCrawlErrors(job.id!);
+    expect(errors).toHaveProperty("errors");
+    expect(errors).toHaveProperty("robotsBlocked");
+    expect(Array.isArray(errors.errors)).toBe(true);
+    expect(Array.isArray(errors.robotsBlocked)).toBe(true);
+  }, 180_000);
+
   test("crawl with prompt and wait", async () => {
     if (!client) throw new Error();
     const job = await client.crawl("https://docs.firecrawl.dev", { prompt: "Extract all blog posts", limit: 3, pollInterval: 1, timeout: 120 });

package/src/v2/methods/batch.ts

@@ -62,6 +62,7 @@ export async function getBatchScrapeStatus(
     const auto = pagination?.autoPaginate ?? true;
     if (!auto || !body.next) {
       return {
+        id: jobId,
         status: body.status,
         completed: body.completed ?? 0,
         total: body.total ?? 0,
@@ -74,6 +75,7 @@ export async function getBatchScrapeStatus(
 
     const aggregated = await fetchAllPages(http, body.next, initialDocs, pagination);
     return {
+      id: jobId,
       status: body.status,
       completed: body.completed ?? 0,
       total: body.total ?? 0,
@@ -136,4 +138,4 @@ export function chunkUrls(urls: string[], chunkSize = 100): string[][] {
   const chunks: string[][] = [];
   for (let i = 0; i < urls.length; i += chunkSize) chunks.push(urls.slice(i, i + chunkSize));
   return chunks;
-}
+}

package/src/v2/methods/crawl.ts

@@ -72,6 +72,7 @@ export async function getCrawlStatus(
     const auto = pagination?.autoPaginate ?? true;
     if (!auto || !body.next) {
       return {
+        id: jobId,
         status: body.status,
         completed: body.completed ?? 0,
         total: body.total ?? 0,
@@ -85,6 +86,7 @@ export async function getCrawlStatus(
     const aggregated = await fetchAllPages(http, body.next, initialDocs, pagination);
 
     return {
+      id: jobId,
       status: body.status,
       completed: body.completed ?? 0,
       total: body.total ?? 0,
@@ -165,4 +167,4 @@ export async function crawlParamsPreview(http: HttpClient, url: string, prompt:
     if (err?.isAxiosError) return normalizeAxiosError(err, "crawl params preview");
     throw err;
   }
-}
+}

package/src/v2/types.ts

@@ -466,6 +466,7 @@ export interface CrawlResponse {
 }
 
 export interface CrawlJob {
+  id: string;
   status: 'scraping' | 'completed' | 'failed' | 'cancelled';
   total: number;
   completed: number;
@@ -493,6 +494,7 @@ export interface BatchScrapeResponse {
 }
 
 export interface BatchScrapeJob {
+  id: string;
   status: 'scraping' | 'completed' | 'failed' | 'cancelled';
   completed: number;
   total: number;

package/src/v2/watcher.ts

@@ -3,6 +3,7 @@ import type { BatchScrapeJob, CrawlJob, Document } from "./types";
 import type { HttpClient } from "./utils/httpClient";
 import { getBatchScrapeStatus } from "./methods/batch";
 import { getCrawlStatus } from "./methods/crawl";
+import { WebSocket as WS } from "ws";
 
 type JobKind = "crawl" | "batch";
 
@@ -43,11 +44,17 @@ export class Watcher extends EventEmitter {
   async start(): Promise<void> {
     try {
       const url = this.buildWsUrl();
-      // Pass API key as subprotocol for browser compatibility
-      this.ws = new WebSocket(url, this.http.getApiKey());
-      this.attachWsHandlers(this.ws);
-    } catch {
-      // Fallback to polling immediately
+      
+      if (typeof WebSocket !== 'undefined') {
+        this.ws = new WebSocket(url, this.http.getApiKey()) as any;
+      } else {
+        this.ws = new WS(url, this.http.getApiKey()) as any;
+      }
+      
+      if (this.ws) {
+        this.attachWsHandlers(this.ws);
+      }
+    } catch (err) {
       this.pollLoop();
     }
   }
@@ -76,7 +83,9 @@ export class Watcher extends EventEmitter {
           return;
         }
         if (type === "done") {
-          this.emit("done", { status: "completed", data: [], id: this.jobId });
+          const payload = body.data || body;
+          const data = (payload.data || []) as Document[];
+          this.emit("done", { status: "completed", data, id: this.jobId });
           this.close();
           return;
         }
@@ -105,6 +114,7 @@ export class Watcher extends EventEmitter {
     const data = (payload.data || []) as Document[];
     const snap: Snapshot = this.kind === "crawl"
       ? {
+          id: this.jobId,
           status,
           completed: payload.completed ?? 0,
           total: payload.total ?? 0,
@@ -114,6 +124,7 @@ export class Watcher extends EventEmitter {
           data,
         }
       : {
+          id: this.jobId,
           status,
           completed: payload.completed ?? 0,
           total: payload.total ?? 0,

package/tsup.config.ts

@@ -8,6 +8,7 @@ export default defineConfig({
   clean: true,
   platform: "node",
   target: "node22",
+  external: ["ws"],
   noExternal: ["typescript-event-target"],
   esbuildOptions(options) {
     options.define = {