firecrawl 1.21.0 → 1.22.0

package/dist/index.cjs

@@ -38,8 +38,15 @@ module.exports = __toCommonJS(src_exports);
 var import_axios = __toESM(require("axios"), 1);
 var zt = __toESM(require("zod"), 1);
 var import_zod_to_json_schema = require("zod-to-json-schema");
-var import_isows = require("isows");
-var import_typescript_event_target = require("typescript-event-target");
+
+// node_modules/typescript-event-target/dist/index.mjs
+var e = class extends EventTarget {
+  dispatchTypedEvent(s, t) {
+    return super.dispatchEvent(t);
+  }
+};
+
+// src/index.ts
 var FirecrawlError = class extends Error {
   statusCode;
   details;
@@ -419,6 +426,7 @@ var FirecrawlApp = class {
    * @param pollInterval - Time in seconds for job status checks.
    * @param idempotencyKey - Optional idempotency key for the request.
    * @param webhook - Optional webhook for the batch scrape.
+   * @param ignoreInvalidURLs - Optional flag to ignore invalid URLs.
    * @returns The response from the crawl operation.
    */
   async batchScrapeUrls(urls, params, pollInterval = 2, idempotencyKey, webhook, ignoreInvalidURLs) {
@@ -1191,7 +1199,7 @@ var FirecrawlApp = class {
     return { success: false, error: "Internal server error." };
   }
 };
-var CrawlWatcher = class extends import_typescript_event_target.TypedEventTarget {
+var CrawlWatcher = class extends e {
   ws;
   data;
   status;
@@ -1200,7 +1208,7 @@ var CrawlWatcher = class extends import_typescript_event_target.TypedEventTarget
     super();
     this.id = id;
     const wsUrl = app.apiUrl.replace(/^http/, "ws");
-    this.ws = new import_isows.WebSocket(`${wsUrl}/v1/crawl/${id}`, app.apiKey);
+    this.ws = new WebSocket(`${wsUrl}/v1/crawl/${id}`, app.apiKey);
     this.status = "scraping";
     this.data = [];
     const messageHandler = (msg) => {

package/dist/index.d.cts

@@ -65,6 +65,11 @@ interface FirecrawlDocument<T = any, ActionsSchema extends (ActionsResult | neve
     screenshot?: string;
     metadata?: FirecrawlDocumentMetadata;
     actions: ActionsSchema;
+    compare?: {
+        previousScrapeAt: string | null;
+        changeStatus: "new" | "same" | "changed" | "removed";
+        visibility: "visible" | "hidden";
+    };
     title?: string;
     description?: string;
 }
@@ -73,7 +78,7 @@ interface FirecrawlDocument<T = any, ActionsSchema extends (ActionsResult | neve
  * Defines the options and configurations available for scraping web content.
  */
 interface CrawlScrapeOptions {
-    formats?: ("markdown" | "html" | "rawHtml" | "content" | "links" | "screenshot" | "screenshot@fullPage" | "extract" | "json")[];
+    formats?: ("markdown" | "html" | "rawHtml" | "content" | "links" | "screenshot" | "screenshot@fullPage" | "extract" | "json" | "compare")[];
     headers?: Record<string, string>;
     includeTags?: string[];
     excludeTags?: string[];
@@ -132,6 +137,14 @@ interface ScrapeParams<LLMSchema extends zt.ZodSchema = any, ActionsSchema exten
 }
 interface ActionsResult {
     screenshots: string[];
+    scrapes: ({
+        url: string;
+        html: string;
+    })[];
+    javascriptReturns: {
+        type: string;
+        value: unknown;
+    }[];
 }
 /**
  * Response interface for scraping operations.
@@ -529,6 +542,7 @@ declare class FirecrawlApp {
      * @param pollInterval - Time in seconds for job status checks.
      * @param idempotencyKey - Optional idempotency key for the request.
      * @param webhook - Optional webhook for the batch scrape.
+     * @param ignoreInvalidURLs - Optional flag to ignore invalid URLs.
      * @returns The response from the crawl operation.
      */
     batchScrapeUrls(urls: string[], params?: ScrapeParams, pollInterval?: number, idempotencyKey?: string, webhook?: CrawlParams["webhook"], ignoreInvalidURLs?: boolean): Promise<BatchScrapeStatusResponse | ErrorResponse>;

package/dist/index.d.ts

@@ -65,6 +65,11 @@ interface FirecrawlDocument<T = any, ActionsSchema extends (ActionsResult | neve
     screenshot?: string;
     metadata?: FirecrawlDocumentMetadata;
     actions: ActionsSchema;
+    compare?: {
+        previousScrapeAt: string | null;
+        changeStatus: "new" | "same" | "changed" | "removed";
+        visibility: "visible" | "hidden";
+    };
     title?: string;
     description?: string;
 }
@@ -73,7 +78,7 @@ interface FirecrawlDocument<T = any, ActionsSchema extends (ActionsResult | neve
  * Defines the options and configurations available for scraping web content.
  */
 interface CrawlScrapeOptions {
-    formats?: ("markdown" | "html" | "rawHtml" | "content" | "links" | "screenshot" | "screenshot@fullPage" | "extract" | "json")[];
+    formats?: ("markdown" | "html" | "rawHtml" | "content" | "links" | "screenshot" | "screenshot@fullPage" | "extract" | "json" | "compare")[];
     headers?: Record<string, string>;
     includeTags?: string[];
     excludeTags?: string[];
@@ -132,6 +137,14 @@ interface ScrapeParams<LLMSchema extends zt.ZodSchema = any, ActionsSchema exten
 }
 interface ActionsResult {
     screenshots: string[];
+    scrapes: ({
+        url: string;
+        html: string;
+    })[];
+    javascriptReturns: {
+        type: string;
+        value: unknown;
+    }[];
 }
 /**
  * Response interface for scraping operations.
@@ -529,6 +542,7 @@ declare class FirecrawlApp {
      * @param pollInterval - Time in seconds for job status checks.
      * @param idempotencyKey - Optional idempotency key for the request.
      * @param webhook - Optional webhook for the batch scrape.
+     * @param ignoreInvalidURLs - Optional flag to ignore invalid URLs.
      * @returns The response from the crawl operation.
      */
     batchScrapeUrls(urls: string[], params?: ScrapeParams, pollInterval?: number, idempotencyKey?: string, webhook?: CrawlParams["webhook"], ignoreInvalidURLs?: boolean): Promise<BatchScrapeStatusResponse | ErrorResponse>;

package/dist/index.js

@@ -2,8 +2,15 @@
 import axios, { AxiosError } from "axios";
 import * as zt from "zod";
 import { zodToJsonSchema } from "zod-to-json-schema";
-import { WebSocket } from "isows";
-import { TypedEventTarget } from "typescript-event-target";
+
+// node_modules/typescript-event-target/dist/index.mjs
+var e = class extends EventTarget {
+  dispatchTypedEvent(s, t) {
+    return super.dispatchEvent(t);
+  }
+};
+
+// src/index.ts
 var FirecrawlError = class extends Error {
   statusCode;
   details;
@@ -383,6 +390,7 @@ var FirecrawlApp = class {
    * @param pollInterval - Time in seconds for job status checks.
    * @param idempotencyKey - Optional idempotency key for the request.
    * @param webhook - Optional webhook for the batch scrape.
+   * @param ignoreInvalidURLs - Optional flag to ignore invalid URLs.
    * @returns The response from the crawl operation.
    */
   async batchScrapeUrls(urls, params, pollInterval = 2, idempotencyKey, webhook, ignoreInvalidURLs) {
@@ -1155,7 +1163,7 @@ var FirecrawlApp = class {
     return { success: false, error: "Internal server error." };
   }
 };
-var CrawlWatcher = class extends TypedEventTarget {
+var CrawlWatcher = class extends e {
   ws;
   data;
   status;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "firecrawl",
-  "version": "1.21.0",
+  "version": "1.22.0",
   "description": "JavaScript SDK for Firecrawl API",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -26,7 +26,6 @@
   "license": "MIT",
   "dependencies": {
     "axios": "^1.6.8",
-    "isows": "^1.0.4",
     "typescript-event-target": "^1.1.1",
     "zod": "^3.23.8",
     "zod-to-json-schema": "^3.23.0"
@@ -36,8 +35,6 @@
   },
   "homepage": "https://github.com/mendableai/firecrawl#readme",
   "devDependencies": {
-    "uuid": "^9.0.1",
-    "dotenv": "^16.4.5",
     "@jest/globals": "^29.7.0",
     "@types/axios": "^0.14.0",
     "@types/dotenv": "^8.2.0",
@@ -45,10 +42,12 @@
     "@types/mocha": "^10.0.6",
     "@types/node": "^20.12.12",
     "@types/uuid": "^9.0.8",
+    "dotenv": "^16.4.5",
     "jest": "^29.7.0",
     "ts-jest": "^29.2.2",
     "tsup": "^8.2.4",
-    "typescript": "^5.4.5"
+    "typescript": "^5.4.5",
+    "uuid": "^9.0.1"
   },
   "keywords": [
     "firecrawl",
@@ -58,5 +57,8 @@
     "scraper",
     "api",
     "sdk"
-  ]
+  ],
+  "engines": {
+    "node": ">=22.0.0"
+  }
 }

package/src/index.ts

@@ -1,7 +1,6 @@
 import axios, { type AxiosResponse, type AxiosRequestHeaders, AxiosError } from "axios";
 import * as zt from "zod";
 import { zodToJsonSchema } from "zod-to-json-schema";
-import { WebSocket } from "isows";
 import { TypedEventTarget } from "typescript-event-target";
 
 /**
@@ -69,6 +68,11 @@ export interface FirecrawlDocument<T = any, ActionsSchema extends (ActionsResult
   screenshot?: string;
   metadata?: FirecrawlDocumentMetadata;
   actions: ActionsSchema;
+  compare?: {
+    previousScrapeAt: string | null;
+    changeStatus: "new" | "same" | "changed" | "removed";
+    visibility: "visible" | "hidden";
+  };
   // v1 search only
   title?: string;
   description?: string;
@@ -79,7 +83,7 @@ export interface FirecrawlDocument<T = any, ActionsSchema extends (ActionsResult
  * Defines the options and configurations available for scraping web content.
  */
 export interface CrawlScrapeOptions {
-  formats?: ("markdown" | "html" | "rawHtml" | "content" | "links" | "screenshot" | "screenshot@fullPage" | "extract" | "json")[];
+  formats?: ("markdown" | "html" | "rawHtml" | "content" | "links" | "screenshot" | "screenshot@fullPage" | "extract" | "json" | "compare")[];
   headers?: Record<string, string>;
   includeTags?: string[];
   excludeTags?: string[];
@@ -141,6 +145,14 @@ export interface ScrapeParams<LLMSchema extends zt.ZodSchema = any, ActionsSchem
 
 export interface ActionsResult {
   screenshots: string[];
+  scrapes: ({
+    url: string;
+    html: string;
+  })[];
+  javascriptReturns: {
+    type: string;
+    value: unknown
+  }[];
 }
 
 /**
@@ -914,6 +926,7 @@ export default class FirecrawlApp {
    * @param pollInterval - Time in seconds for job status checks.
    * @param idempotencyKey - Optional idempotency key for the request.
    * @param webhook - Optional webhook for the batch scrape.
+   * @param ignoreInvalidURLs - Optional flag to ignore invalid URLs.
    * @returns The response from the crawl operation.
    */
   async batchScrapeUrls(

package/tsup.config.ts

@@ -6,4 +6,13 @@ export default defineConfig({
   dts: true,
   outDir: "dist",
   clean: true,
+  platform: "node",
+  target: "node22",
+  noExternal: ["typescript-event-target"],
+  esbuildOptions(options) {
+    options.define = {
+      ...options.define,
+      "process.env.NODE_ENV": JSON.stringify(process.env.NODE_ENV || "production"),
+    };
+  },
 });