@mendable/firecrawl-js 4.19.0 → 4.20.1

package/src/v2/utils/httpClient.ts

@@ -1,4 +1,8 @@
-import axios, { type AxiosInstance, type AxiosRequestConfig, type AxiosResponse } from "axios";
+import axios, {
+  type AxiosInstance,
+  type AxiosRequestConfig,
+  type AxiosResponse,
+} from "axios";
 import { getVersion } from "./getVersion";
 
 export interface HttpClientOptions {
@@ -9,6 +13,11 @@ export interface HttpClientOptions {
   backoffFactor?: number; // seconds factor for 0.5, 1, 2...
 }
 
+export interface RequestOptions {
+  headers?: Record<string, string>;
+  timeoutMs?: number;
+}
+
 export class HttpClient {
   private instance: AxiosInstance;
   private readonly apiKey: string;
@@ -39,7 +48,9 @@ export class HttpClient {
     return this.apiKey;
   }
 
-  private async request<T = any>(config: AxiosRequestConfig): Promise<AxiosResponse<T>> {
+  private async request<T = any>(
+    config: AxiosRequestConfig,
+  ): Promise<AxiosResponse<T>> {
     const version = getVersion();
     config.headers = {
       ...(config.headers || {}),
@@ -64,12 +75,13 @@ export class HttpClient {
           ["post", "put", "patch"].includes(cfg.method.toLowerCase())
         ) {
           const data = (cfg.data ?? {}) as Record<string, unknown>;
-          cfg.data = { ...data, origin: typeof data.origin === "string" && data.origin.includes("mcp") ? data.origin : `js-sdk@${version}` };
-
-          // If timeout is specified in the body, use it to override the request timeout
-          if (typeof data.timeout === "number") {
-            cfg.timeout = data.timeout + 5000;
-          }
+          cfg.data = {
+            ...data,
+            origin:
+              typeof data.origin === "string" && data.origin.includes("mcp")
+                ? data.origin
+                : `js-sdk@${version}`,
+          };
         }
 
         if (isFormDataBody) {
@@ -98,25 +110,34 @@ export class HttpClient {
   }
 
   private sleep(seconds: number): Promise<void> {
-    return new Promise((r) => setTimeout(r, seconds * 1000));
+    return new Promise(r => setTimeout(r, seconds * 1000));
   }
 
-  post<T = any>(endpoint: string, body: Record<string, unknown>, headers?: Record<string, string>) {
-    return this.request<T>({ method: "post", url: endpoint, data: body, headers });
+  post<T = any>(
+    endpoint: string,
+    body: Record<string, unknown>,
+    options?: RequestOptions,
+  ) {
+    return this.request<T>({
+      method: "post",
+      url: endpoint,
+      data: body,
+      headers: options?.headers,
+      timeout: options?.timeoutMs,
+    });
   }
 
   postMultipart<T = any>(
     endpoint: string,
     formData: FormData,
-    headers?: Record<string, string>,
-    timeoutMs?: number,
+    options?: RequestOptions,
   ) {
     return this.request<T>({
       method: "post",
       url: endpoint,
       data: formData,
-      headers,
-      timeout: timeoutMs,
+      headers: options?.headers,
+      timeout: options?.timeoutMs,
     });
   }
 
@@ -134,4 +155,3 @@ export class HttpClient {
     return headers;
   }
 }
-

package/src/v2/watcher.ts

@@ -109,24 +109,37 @@ export class Watcher extends EventEmitter {
   }
 
   async start(): Promise<void> {
-    try {
-      const url = this.buildWsUrl();
-      const wsCtor = await getWebSocketCtor();
-      if (!wsCtor) {
-        this.pollLoop();
-        return;
-      }
-      this.ws = new wsCtor(url, this.http.getApiKey()) as any;
-      if (this.ws && "binaryType" in this.ws) {
-        (this.ws as any).binaryType = "arraybuffer";
-      }
-      
-      if (this.ws) {
-        this.attachWsHandlers(this.ws);
-      }
-    } catch (err) {
-      this.pollLoop();
-    }
+    return new Promise<void>((resolve, reject) => {
+      const onDone = () => { cleanup(); resolve(); };
+      const onError = (err: any) => { cleanup(); resolve(); };
+      const cleanup = () => {
+        this.removeListener("done", onDone);
+        this.removeListener("error", onError);
+      };
+      this.on("done", onDone);
+      this.on("error", onError);
+
+      (async () => {
+        try {
+          const url = this.buildWsUrl();
+          const wsCtor = await getWebSocketCtor();
+          if (!wsCtor) {
+            this.pollLoop();
+            return;
+          }
+          this.ws = new wsCtor(url, this.http.getApiKey()) as any;
+          if (this.ws && "binaryType" in this.ws) {
+            (this.ws as any).binaryType = "arraybuffer";
+          }
+
+          if (this.ws) {
+            this.attachWsHandlers(this.ws);
+          }
+        } catch (err) {
+          this.pollLoop();
+        }
+      })();
+    });
   }
 
   private attachWsHandlers(ws: WebSocket) {
@@ -150,14 +163,14 @@ export class Watcher extends EventEmitter {
         }
         if (type === "document") {
           const doc = body.data;
-          if (doc) this.emit("document", doc as Document & { id: string });
+          if (doc) this.emitDocuments([doc]);
           return;
         }
         if (type === "done") {
           const payload = body.data || body;
           const data = (payload.data || []) as Document[];
           if (data.length) this.emitDocuments(data);
-          this.emit("done", { status: "completed", data, id: this.jobId });
+          this.emit("done", { status: "completed", data, id: this.jobId, total: payload.total, completed: payload.completed, creditsUsed: payload.creditsUsed });
           this.close();
           return;
         }
@@ -166,7 +179,10 @@ export class Watcher extends EventEmitter {
       } catch {
         // ignore
       }
-      if (timeoutMs && Date.now() - startTs > timeoutMs) this.close();
+      if (timeoutMs && Date.now() - startTs > timeoutMs) {
+        this.emit("error", { status: "failed", data: [], error: "Watcher timeout", id: this.jobId });
+        this.close();
+      }
     };
     ws.onerror = () => {
       this.emit("error", { status: "failed", data: [], error: "WebSocket error", id: this.jobId });
@@ -227,7 +243,7 @@ export class Watcher extends EventEmitter {
         };
     this.emit("snapshot", snap);
     if (["completed", "failed", "cancelled"].includes(status)) {
-      this.emit("done", { status, data, id: this.jobId });
+      this.emit("done", { status, data, id: this.jobId, total: payload.total ?? 0, completed: payload.completed ?? 0, creditsUsed: payload.creditsUsed });
       this.close();
     }
   }
@@ -243,14 +259,18 @@ export class Watcher extends EventEmitter {
         this.emitDocuments((snap.data || []) as Document[]);
         this.emit("snapshot", snap);
         if (["completed", "failed", "cancelled"].includes(snap.status)) {
-          this.emit("done", { status: snap.status, data: snap.data, id: this.jobId });
+          this.emit("done", { status: snap.status, data: snap.data, id: this.jobId, total: (snap as any).total ?? 0, completed: (snap as any).completed ?? 0, creditsUsed: (snap as any).creditsUsed });
           this.close();
           break;
         }
       } catch {
         // ignore polling errors
       }
-      if (timeoutMs && Date.now() - startTs > timeoutMs) break;
+      if (timeoutMs && Date.now() - startTs > timeoutMs) {
+        this.emit("error", { status: "failed", data: [], error: "Watcher timeout", id: this.jobId });
+        this.close();
+        break;
+      }
       await new Promise((r) => setTimeout(r, Math.max(1000, this.pollInterval * 1000)));
     }
   }