firecrawl 1.29.3 → 3.0.3

package/src/v2/watcher.ts

@@ -0,0 +1,159 @@
+import { EventEmitter } from "events";
+import type { BatchScrapeJob, CrawlJob, Document } from "./types";
+import type { HttpClient } from "./utils/httpClient";
+import { getBatchScrapeStatus } from "./methods/batch";
+import { getCrawlStatus } from "./methods/crawl";
+
+type JobKind = "crawl" | "batch";
+
+export interface WatcherOptions {
+  kind?: JobKind;
+  pollInterval?: number; // seconds
+  timeout?: number; // seconds
+}
+
+type Snapshot = CrawlJob | BatchScrapeJob;
+
+export class Watcher extends EventEmitter {
+  private readonly http: HttpClient;
+  private readonly jobId: string;
+  private readonly kind: JobKind;
+  private readonly pollInterval: number;
+  private readonly timeout?: number;
+  private ws?: WebSocket;
+  private closed = false;
+
+  constructor(http: HttpClient, jobId: string, opts: WatcherOptions = {}) {
+    super();
+    this.http = http;
+    this.jobId = jobId;
+    this.kind = opts.kind ?? "crawl";
+    this.pollInterval = opts.pollInterval ?? 2;
+    this.timeout = opts.timeout;
+  }
+
+  private buildWsUrl(): string {
+    // replace http/https with ws/wss
+    const apiUrl = this.http.getApiUrl();
+    const wsBase = apiUrl.replace(/^http/, "ws");
+    const path = this.kind === "crawl" ? `/v2/crawl/${this.jobId}` : `/v2/batch/scrape/${this.jobId}`;
+    return `${wsBase}${path}`;
+  }
+
+  async start(): Promise<void> {
+    try {
+      const url = this.buildWsUrl();
+      // Pass API key as subprotocol for browser compatibility
+      this.ws = new WebSocket(url, this.http.getApiKey());
+      this.attachWsHandlers(this.ws);
+    } catch {
+      // Fallback to polling immediately
+      this.pollLoop();
+    }
+  }
+
+  private attachWsHandlers(ws: WebSocket) {
+    let startTs = Date.now();
+    const timeoutMs = this.timeout ? this.timeout * 1000 : undefined;
+    ws.onmessage = (ev: MessageEvent) => {
+      try {
+        const body = typeof ev.data === "string" ? JSON.parse(ev.data) : null;
+        if (!body) return;
+        const type = body.type as string | undefined;
+        if (type === "error") {
+          this.emit("error", { status: "failed", data: [], error: body.error, id: this.jobId });
+          return;
+        }
+        if (type === "catchup") {
+          const payload = body.data || {};
+          this.emitDocuments(payload.data || []);
+          this.emitSnapshot(payload);
+          return;
+        }
+        if (type === "document") {
+          const doc = body.data;
+          if (doc) this.emit("document", doc as Document & { id: string });
+          return;
+        }
+        if (type === "done") {
+          this.emit("done", { status: "completed", data: [], id: this.jobId });
+          this.close();
+          return;
+        }
+        const payload = body.data || body;
+        if (payload && payload.status) this.emitSnapshot(payload);
+      } catch {
+        // ignore
+      }
+      if (timeoutMs && Date.now() - startTs > timeoutMs) this.close();
+    };
+    ws.onerror = () => {
+      this.emit("error", { status: "failed", data: [], error: "WebSocket error", id: this.jobId });
+      this.close();
+    };
+    ws.onclose = () => {
+      if (!this.closed) this.pollLoop();
+    };
+  }
+
+  private emitDocuments(docs: Document[]) {
+    for (const doc of docs) this.emit("document", { ...(doc as any), id: this.jobId });
+  }
+
+  private emitSnapshot(payload: any) {
+    const status = payload.status as Snapshot["status"];
+    const data = (payload.data || []) as Document[];
+    const snap: Snapshot = this.kind === "crawl"
+      ? {
+          status,
+          completed: payload.completed ?? 0,
+          total: payload.total ?? 0,
+          creditsUsed: payload.creditsUsed,
+          expiresAt: payload.expiresAt,
+          next: payload.next ?? null,
+          data,
+        }
+      : {
+          status,
+          completed: payload.completed ?? 0,
+          total: payload.total ?? 0,
+          creditsUsed: payload.creditsUsed,
+          expiresAt: payload.expiresAt,
+          next: payload.next ?? null,
+          data,
+        };
+    this.emit("snapshot", snap);
+    if (["completed", "failed", "cancelled"].includes(status)) {
+      this.emit("done", { status, data, id: this.jobId });
+      this.close();
+    }
+  }
+
+  private async pollLoop() {
+    const startTs = Date.now();
+    const timeoutMs = this.timeout ? this.timeout * 1000 : undefined;
+    while (!this.closed) {
+      try {
+        const snap = this.kind === "crawl"
+          ? await getCrawlStatus(this.http as any, this.jobId)
+          : await getBatchScrapeStatus(this.http as any, this.jobId);
+        this.emit("snapshot", snap);
+        if (["completed", "failed", "cancelled"].includes(snap.status)) {
+          this.emit("done", { status: snap.status, data: snap.data, id: this.jobId });
+          this.close();
+          break;
+        }
+      } catch {
+        // ignore polling errors
+      }
+      if (timeoutMs && Date.now() - startTs > timeoutMs) break;
+      await new Promise((r) => setTimeout(r, Math.max(1000, this.pollInterval * 1000)));
+    }
+  }
+
+  close() {
+    this.closed = true;
+    if (this.ws && (this.ws as any).close) (this.ws as any).close();
+  }
+}
+

package/tsconfig.json

@@ -17,7 +17,8 @@
     "noImplicitOverride": true,
 
     /* If NOT transpiling with TypeScript: */
-    "module": "NodeNext",
+    "module": "ESNext",
+    "moduleResolution": "Bundler",
     "noEmit": true,
   },
   "include": ["src/**/*"],

package/dist/package-Z6F7JDXI.js

@@ -1,111 +0,0 @@
-// package.json
-var name = "@mendable/firecrawl-js";
-var version = "1.29.3";
-var description = "JavaScript SDK for Firecrawl API";
-var main = "dist/index.js";
-var types = "dist/index.d.ts";
-var exports = {
-  "./package.json": "./package.json",
-  ".": {
-    import: "./dist/index.js",
-    default: "./dist/index.cjs"
-  }
-};
-var type = "module";
-var scripts = {
-  build: "tsup",
-  "build-and-publish": "npm run build && npm publish --access public",
-  "publish-beta": "npm run build && npm publish --access public --tag beta",
-  test: "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/v1/**/*.test.ts",
-  "test:unit": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/v1/unit/*.test.ts"
-};
-var repository = {
-  type: "git",
-  url: "git+https://github.com/mendableai/firecrawl.git"
-};
-var author = "Mendable.ai";
-var license = "MIT";
-var dependencies = {
-  axios: "^1.11.0",
-  "typescript-event-target": "^1.1.1",
-  zod: "^3.23.8",
-  "zod-to-json-schema": "^3.23.0"
-};
-var bugs = {
-  url: "https://github.com/mendableai/firecrawl/issues"
-};
-var homepage = "https://github.com/mendableai/firecrawl#readme";
-var devDependencies = {
-  "@jest/globals": "^30.0.5",
-  "@types/dotenv": "^8.2.0",
-  "@types/jest": "^30.0.0",
-  "@types/mocha": "^10.0.6",
-  "@types/node": "^20.12.12",
-  "@types/uuid": "^9.0.8",
-  dotenv: "^16.4.5",
-  jest: "^30.0.5",
-  "ts-jest": "^29.4.0",
-  tsup: "^8.5.0",
-  typescript: "^5.4.5",
-  uuid: "^9.0.1"
-};
-var keywords = [
-  "firecrawl",
-  "mendable",
-  "crawler",
-  "web",
-  "scraper",
-  "api",
-  "sdk"
-];
-var engines = {
-  node: ">=22.0.0"
-};
-var pnpm = {
-  overrides: {
-    "@babel/helpers@<7.26.10": ">=7.26.10",
-    "brace-expansion@>=1.0.0 <=1.1.11": ">=1.1.12",
-    "brace-expansion@>=2.0.0 <=2.0.1": ">=2.0.2"
-  }
-};
-var package_default = {
-  name,
-  version,
-  description,
-  main,
-  types,
-  exports,
-  type,
-  scripts,
-  repository,
-  author,
-  license,
-  dependencies,
-  bugs,
-  homepage,
-  devDependencies,
-  keywords,
-  engines,
-  pnpm
-};
-export {
-  author,
-  bugs,
-  package_default as default,
-  dependencies,
-  description,
-  devDependencies,
-  engines,
-  exports,
-  homepage,
-  keywords,
-  license,
-  main,
-  name,
-  pnpm,
-  repository,
-  scripts,
-  type,
-  types,
-  version
-};