firecrawl 1.29.3 → 3.0.3

package/dist/package-V5IPFKBE.js

@@ -0,0 +1,4 @@
+import {
+  require_package
+} from "./chunk-OIZ6OKY4.js";
+export default require_package();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "firecrawl",
-  "version": "1.29.3",
+  "version": "3.0.3",
   "description": "JavaScript SDK for Firecrawl API",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -16,12 +16,12 @@
     "build": "tsup",
     "build-and-publish": "npm run build && npm publish --access public",
     "publish-beta": "npm run build && npm publish --access public --tag beta",
-    "test": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/v1/**/*.test.ts",
-    "test:unit": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/v1/unit/*.test.ts"
+    "test": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/e2e/v2/*.test.ts --detectOpenHandles",
+    "test:unit": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/unit/v2/*.test.ts"
   },
   "repository": {
     "type": "git",
-    "url": "git+https://github.com/mendableai/firecrawl.git"
+    "url": "git+https://github.com/firecrawl/firecrawl.git"
   },
   "author": "Mendable.ai",
   "license": "MIT",
@@ -32,9 +32,9 @@
     "zod-to-json-schema": "^3.23.0"
   },
   "bugs": {
-    "url": "https://github.com/mendableai/firecrawl/issues"
+    "url": "https://github.com/firecrawl/firecrawl/issues"
   },
-  "homepage": "https://github.com/mendableai/firecrawl#readme",
+  "homepage": "https://github.com/firecrawl/firecrawl#readme",
   "devDependencies": {
     "@jest/globals": "^30.0.5",
     "@types/dotenv": "^8.2.0",

package/src/__tests__/e2e/v2/batch.test.ts

@@ -0,0 +1,74 @@
+/**
+ * E2E tests for v2 batch scrape (translated from Python tests)
+ */
+import Firecrawl from "../../../index";
+import { config } from "dotenv";
+import { getIdentity, getApiUrl } from "./utils/idmux";
+import { describe, test, expect, beforeAll } from "@jest/globals";
+
+config();
+
+const API_URL = getApiUrl();
+let client: Firecrawl;
+
+beforeAll(async () => {
+  const { apiKey } = await getIdentity({ name: "js-e2e-batch" });
+  client = new Firecrawl({ apiKey, apiUrl: API_URL });
+});
+
+describe("v2.batch e2e", () => {
+  test("batch scrape minimal (wait)", async () => {
+    const urls = [
+      "https://docs.firecrawl.dev",
+      "https://firecrawl.dev",
+    ];
+    const job = await client.batchScrape(urls, { options: { formats: ["markdown"] }, pollInterval: 1, timeout: 180 });
+    expect(["completed", "failed"]).toContain(job.status);
+    expect(job.completed).toBeGreaterThanOrEqual(0);
+    expect(job.total).toBeGreaterThanOrEqual(0);
+    expect(Array.isArray(job.data)).toBe(true);
+  }, 240_000);
+
+  test("start batch minimal and status", async () => {
+    const urls = ["https://docs.firecrawl.dev", "https://firecrawl.dev"]; 
+    const start = await client.startBatchScrape(urls, { options: { formats: ["markdown"] }, ignoreInvalidURLs: true });
+    expect(typeof start.id).toBe("string");
+    expect(typeof start.url).toBe("string");
+    const status = await client.getBatchScrapeStatus(start.id);
+    expect(["scraping", "completed", "failed", "cancelled"]).toContain(status.status);
+    expect(status.total).toBeGreaterThanOrEqual(0);
+  }, 120_000);
+
+  test("wait batch with all params", async () => {
+    const urls = ["https://docs.firecrawl.dev", "https://firecrawl.dev"]; 
+    const job = await client.batchScrape(urls, {
+      options: {
+        formats: [
+          "markdown",
+          { type: "json", prompt: "Extract page title", schema: { type: "object", properties: { title: { type: "string" } }, required: ["title"] } },
+          { type: "changeTracking", prompt: "Track changes", modes: ["json"] },
+        ],
+        onlyMainContent: true,
+        mobile: false,
+      },
+      ignoreInvalidURLs: true,
+      maxConcurrency: 2,
+      zeroDataRetention: false,
+      pollInterval: 1,
+      timeout: 180,
+    });
+    expect(["completed", "failed", "cancelled"]).toContain(job.status);
+    expect(job.completed).toBeGreaterThanOrEqual(0);
+    expect(job.total).toBeGreaterThanOrEqual(0);
+    expect(Array.isArray(job.data)).toBe(true);
+  }, 300_000);
+
+  test("cancel batch", async () => {
+    const urls = ["https://docs.firecrawl.dev", "https://firecrawl.dev"]; 
+    const start = await client.startBatchScrape(urls, { options: { formats: ["markdown"] }, maxConcurrency: 1 });
+    expect(typeof start.id).toBe("string");
+    const cancelled = await client.cancelBatchScrape(start.id);
+    expect(cancelled).toBe(true);
+  }, 120_000);
+});
+

package/src/__tests__/e2e/v2/crawl.test.ts

@@ -0,0 +1,182 @@
+/**
+ * E2E tests for v2 crawl (translated from Python tests)
+ */
+import Firecrawl from "../../../index";
+import { config } from "dotenv";
+import { getIdentity, getApiUrl } from "./utils/idmux";
+import { describe, test, expect, beforeAll } from "@jest/globals";
+
+config();
+
+const API_URL = getApiUrl();
+let client: Firecrawl;
+
+beforeAll(async () => {
+  const { apiKey } = await getIdentity({ name: "js-e2e-crawl" });
+  client = new Firecrawl({ apiKey, apiUrl: API_URL });
+});
+
+describe("v2.crawl e2e", () => {
+
+  test("start crawl minimal request", async () => {
+    if (!client) throw new Error();
+    const job = await client.startCrawl("https://docs.firecrawl.dev", { limit: 3 });
+    expect(typeof job.id).toBe("string");
+    expect(typeof job.url).toBe("string");
+  }, 90_000);
+
+  test("start crawl with options", async () => {
+    if (!client) throw new Error();
+    const job = await client.startCrawl("https://docs.firecrawl.dev", { limit: 5, maxDiscoveryDepth: 2 });
+    expect(typeof job.id).toBe("string");
+    expect(typeof job.url).toBe("string");
+  }, 90_000);
+
+  test("start crawl with prompt", async () => {
+    if (!client) throw new Error();
+    const job = await client.startCrawl("https://firecrawl.dev", { prompt: "Extract all blog posts", limit: 3 });
+    expect(typeof job.id).toBe("string");
+    expect(typeof job.url).toBe("string");
+  }, 90_000);
+
+  test("get crawl status", async () => {
+    if (!client) throw new Error();
+    const start = await client.startCrawl("https://docs.firecrawl.dev", { limit: 3 });
+    const status = await client.getCrawlStatus(start.id);
+    expect(["scraping", "completed", "failed", "cancelled"]).toContain(status.status);
+    expect(status.completed).toBeGreaterThanOrEqual(0);
+    // next/expiresAt may be null/undefined depending on state; check shape
+    expect(Array.isArray(status.data)).toBe(true);
+  }, 120_000);
+
+  test("cancel crawl", async () => {
+    if (!client) throw new Error();
+    const start = await client.startCrawl("https://docs.firecrawl.dev", { limit: 3 });
+    const ok = await client.cancelCrawl(start.id);
+    expect(ok).toBe(true);
+  }, 120_000);
+
+  test("get crawl errors", async () => {
+    if (!client) throw new Error();
+    const start = await client.startCrawl("https://docs.firecrawl.dev", { limit: 3 });
+    const resp = await client.getCrawlErrors(start.id);
+    expect(resp).toHaveProperty("errors");
+    expect(resp).toHaveProperty("robotsBlocked");
+    expect(Array.isArray(resp.errors)).toBe(true);
+    expect(Array.isArray(resp.robotsBlocked)).toBe(true);
+    for (const e of resp.errors) {
+      expect(typeof e.id === "string" || e.id == null).toBe(true);
+      expect(typeof e.timestamp === "string" || e.timestamp == null).toBe(true);
+      expect(typeof e.url === "string" || e.url == null).toBe(true);
+      expect(typeof e.error === "string" || e.error == null).toBe(true);
+    }
+  }, 120_000);
+
+  test("get crawl errors with invalid id should throw", async () => {
+    if (!client) throw new Error();
+    await expect(client.getCrawlErrors("invalid-job-id-12345")).rejects.toThrow();
+  }, 60_000);
+
+  test("get active crawls", async () => {
+    if (!client) throw new Error();
+    const active = await client.getActiveCrawls();
+    expect(typeof active.success).toBe("boolean");
+    expect(Array.isArray(active.crawls)).toBe(true);
+    for (const c of active.crawls) {
+      expect(typeof c.id).toBe("string");
+      expect(typeof c.teamId).toBe("string");
+      expect(typeof c.url).toBe("string");
+      if (c.options != null) {
+        expect(typeof c.options === "object").toBe(true);
+      }
+    }
+  }, 90_000);
+
+  test("get active crawls with running crawl", async () => {
+    if (!client) throw new Error();
+    const start = await client.startCrawl("https://docs.firecrawl.dev", { limit: 5 });
+    await new Promise(resolve => setTimeout(resolve, 300));
+    const active = await client.getActiveCrawls();
+    expect(Array.isArray(active.crawls)).toBe(true);
+    const ids = active.crawls.map(c => c.id);
+    expect(ids.includes(start.id)).toBe(true);
+    await client.cancelCrawl(start.id);
+  }, 120_000);
+
+  test("crawl with wait", async () => {
+    if (!client) throw new Error();
+    const job = await client.crawl("https://docs.firecrawl.dev", { limit: 3, maxDiscoveryDepth: 2, pollInterval: 1, timeout: 120 });
+    expect(["completed", "failed"]).toContain(job.status);
+    expect(job.completed).toBeGreaterThanOrEqual(0);
+    expect(job.total).toBeGreaterThanOrEqual(0);
+    expect(Array.isArray(job.data)).toBe(true);
+  }, 180_000);
+
+  test("crawl with prompt and wait", async () => {
+    if (!client) throw new Error();
+    const job = await client.crawl("https://docs.firecrawl.dev", { prompt: "Extract all blog posts", limit: 3, pollInterval: 1, timeout: 120 });
+    expect(["completed", "failed"]).toContain(job.status);
+    expect(job.completed).toBeGreaterThanOrEqual(0);
+    expect(job.total).toBeGreaterThanOrEqual(0);
+    expect(Array.isArray(job.data)).toBe(true);
+  }, 180_000);
+
+  test("crawl with scrape options", async () => {
+    if (!client) throw new Error();
+    const job = await client.startCrawl("https://docs.firecrawl.dev", {
+      limit: 2,
+      scrapeOptions: { formats: ["markdown", "links"], onlyMainContent: false, mobile: true },
+    });
+    expect(typeof job.id).toBe("string");
+  }, 120_000);
+
+  test("crawl with json format object", async () => {
+    if (!client) throw new Error();
+    const job = await client.startCrawl("https://docs.firecrawl.dev", {
+      limit: 2,
+      scrapeOptions: { formats: [{ type: "json", prompt: "Extract page title", schema: { type: "object", properties: { title: { type: "string" } }, required: ["title"] } }] },
+    });
+    expect(typeof job.id).toBe("string");
+  }, 120_000);
+
+  test("crawl all parameters", async () => {
+    if (!client) throw new Error();
+    const job = await client.startCrawl("https://docs.firecrawl.dev", {
+      prompt: "Extract all blog posts and documentation",
+      includePaths: ["/blog/*", "/docs/*"],
+      excludePaths: ["/admin/*"],
+      maxDiscoveryDepth: 3,
+      sitemap: "skip",
+      ignoreQueryParameters: true,
+      limit: 5,
+      crawlEntireDomain: true,
+      allowExternalLinks: false,
+      allowSubdomains: true,
+      delay: 1,
+      maxConcurrency: 2,
+      webhook: "https://example.com/hook",
+      scrapeOptions: {
+        formats: ["markdown", "html"],
+        headers: { "User-Agent": "Test Bot" },
+        includeTags: ["h1", "h2"],
+        excludeTags: ["nav"],
+        onlyMainContent: false,
+        timeout: 15_000,
+        waitFor: 2000,
+        mobile: true,
+        skipTlsVerification: true,
+        removeBase64Images: false,
+      },
+      zeroDataRetention: false,
+    });
+    expect(typeof job.id).toBe("string");
+  }, 180_000);
+
+  test("crawl params preview", async () => {
+    if (!client) throw new Error();
+    const params = await client.crawlParamsPreview("https://docs.firecrawl.dev", "Extract all blog posts and documentation");
+    expect(params && typeof params === "object").toBe(true);
+    // Optional fields may or may not be present; just assert object shape
+  }, 60_000);
+});
+

package/src/__tests__/e2e/v2/extract.test.ts

@@ -0,0 +1,70 @@
+/**
+ * E2E tests for v2 extract (proxied to v1), translated from Python tests
+ */
+import Firecrawl from "../../../index";
+import { config } from "dotenv";
+import { getIdentity, getApiUrl } from "./utils/idmux";
+import { describe, test, expect, beforeAll } from "@jest/globals";
+import { z } from "zod";
+
+config();
+
+const API_URL = getApiUrl();
+let client: Firecrawl;
+
+beforeAll(async () => {
+  const { apiKey } = await getIdentity({ name: "js-e2e-extract" });
+  client = new Firecrawl({ apiKey, apiUrl: API_URL });
+});
+
+describe("v2.extract e2e", () => {
+  test("extract minimal with prompt", async () => {
+    const resp = await client.extract({ urls: ["https://docs.firecrawl.dev"], prompt: "Extract the main page title" });
+    expect(typeof resp.success === "boolean" || resp.success == null).toBe(true);
+  }, 120_000);
+
+  test("extract with schema", async () => {
+    const schema = {
+      type: "object",
+      properties: { title: { type: "string" } },
+      required: ["title"],
+    } as const;
+    const resp = await client.extract({
+      urls: ["https://docs.firecrawl.dev"],
+      schema,
+      prompt: "Extract the main page title",
+      showSources: true,
+      enableWebSearch: false,
+    });
+    expect(typeof resp.success === "boolean" || resp.success == null).toBe(true);
+    if ((resp as any).sources != null) {
+      expect(typeof (resp as any).sources).toBe("object");
+    }
+    if (resp.data != null) {
+      expect(typeof resp.data).toBe("object");
+      expect((resp.data as any).title).toBeTruthy();
+    }
+  }, 180_000);
+
+  test("extract with zod schema", async () => {
+    const schema = z.object({
+      title: z.string(),
+    });
+    const resp = await client.extract({
+      urls: ["https://docs.firecrawl.dev"],
+      schema: schema,
+      prompt: "Extract the main page title",
+      showSources: true,
+      enableWebSearch: false,
+    });
+    expect(typeof resp.success === "boolean" || resp.success == null).toBe(true);
+    if ((resp as any).sources != null) {
+      expect(typeof (resp as any).sources).toBe("object");
+    }
+    if (resp.data != null) {
+      expect(typeof resp.data).toBe("object");
+      expect(schema.safeParse(resp.data).success).toBe(true);
+    }
+  }, 180_000);
+});
+

package/src/__tests__/e2e/v2/map.test.ts

@@ -0,0 +1,55 @@
+/**
+ * E2E tests for v2 map (translated from Python tests)
+ */
+import Firecrawl from "../../../index";
+import { config } from "dotenv";
+import { getIdentity, getApiUrl } from "./utils/idmux";
+import { describe, test, expect, beforeAll } from "@jest/globals";
+
+config();
+
+const API_URL = getApiUrl();
+let client: Firecrawl;
+
+beforeAll(async () => {
+  const { apiKey } = await getIdentity({ name: "js-e2e-map" });
+  client = new Firecrawl({ apiKey, apiUrl: API_URL });
+});
+
+describe("v2.map e2e", () => {
+
+  test("minimal request", async () => {
+    if (!client) throw new Error();
+    const resp = await client.map("https://docs.firecrawl.dev");
+
+    expect(resp).toBeTruthy();
+    expect(Array.isArray(resp.links)).toBe(true);
+
+    if (resp.links.length > 0) {
+      const first: any = resp.links[0];
+      expect(typeof first.url).toBe("string");
+      expect(first.url.startsWith("http")).toBe(true);
+    }
+  }, 90_000);
+
+  test.each(["only", "skip", "include"]) ("with options sitemap=%s", async (sitemap) => {
+    if (!client) throw new Error();
+    const resp = await client.map("https://docs.firecrawl.dev", {
+      search: "docs",
+      includeSubdomains: true,
+      limit: 10,
+      sitemap: sitemap as any,
+      timeout: 15_000,
+    });
+
+    expect(resp).toBeTruthy();
+    expect(Array.isArray(resp.links)).toBe(true);
+    expect(resp.links.length).toBeLessThanOrEqual(10);
+
+    for (const link of resp.links as any[]) {
+      expect(typeof link.url).toBe("string");
+      expect(link.url.startsWith("http")).toBe(true);
+    }
+  }, 120_000);
+});
+

package/src/__tests__/e2e/v2/scrape.test.ts

@@ -0,0 +1,130 @@
+/**
+ * E2E tests for v2 scrape
+ */
+import Firecrawl from "../../../index";
+import { z } from "zod";
+import { config } from "dotenv";
+import { getIdentity, getApiUrl } from "./utils/idmux";
+import { describe, test, expect, beforeAll } from "@jest/globals";
+
+config();
+
+const API_URL = getApiUrl();
+let client: Firecrawl;
+
+beforeAll(async () => {
+  const { apiKey } = await getIdentity({ name: "js-e2e-scrape" });
+  client = new Firecrawl({ apiKey, apiUrl: API_URL });
+});
+
+describe("v2.scrape e2e", () => {
+
+  const assertValidDocument = (doc: any) => {
+    expect(doc).toBeTruthy();
+    const hasContent = Boolean(doc.markdown?.length) || Boolean(doc.html?.length) || Boolean(doc.rawHtml?.length);
+    expect(hasContent).toBe(true);
+    expect(doc.metadata).toBeTruthy();
+  };
+
+  test("minimal: scrape only required params", async () => {
+    if (!client) throw new Error();
+    const doc = await client.scrape("https://docs.firecrawl.dev");
+    assertValidDocument(doc);
+  }, 60_000);
+
+  test("maximal: scrape with all options", async () => {
+    if (!client) throw new Error();
+    const doc = await client.scrape("https://docs.firecrawl.dev", {
+      formats: [
+        "markdown",
+        "html",
+        "rawHtml",
+        "links",
+        { type: "screenshot", fullPage: true, quality: 80, viewport: { width: 1280, height: 800 } },
+        {
+          type: "json",
+          prompt: "Summarize the page and list links",
+          schema: {
+            type: "object",
+            properties: {
+              summary: { type: "string" },
+              links: { type: "array", items: { type: "string", format: "uri" } },
+            },
+            required: ["summary"],
+          },
+        },
+      ],
+      parsers: ["pdf"],
+      headers: { "User-Agent": "firecrawl-tests" },
+      includeTags: ["article"],
+      excludeTags: ["nav"],
+      onlyMainContent: true,
+      waitFor: 1000,
+      timeout: 30_000,
+      location: { country: "us", languages: ["en"] },
+      mobile: false,
+      skipTlsVerification: false,
+      removeBase64Images: true,
+      blockAds: true,
+      proxy: "auto",
+      storeInCache: true,
+      maxAge: 60_000,
+    });
+    assertValidDocument(doc);
+  }, 90_000);
+
+  test("json format with zod schema (auto-converted internally)", async () => {
+    if (!client) throw new Error();
+    const zodSchema = z.object({
+      title: z.string().min(1),
+      items: z.array(z.string().url()).optional(),
+    });
+
+    const doc = await client.scrape("https://docs.firecrawl.dev", {
+      formats: [
+        {
+          type: "json",
+          prompt: "Extract title and items",
+          schema: zodSchema,
+        },
+      ],
+    });
+    expect(doc).toBeTruthy();
+  }, 90_000);
+
+  test("summary format returns summary string", async () => {
+    if (!client) throw new Error();
+    const doc = await client.scrape("https://firecrawl.dev", { formats: ["summary"] });
+    expect(typeof doc.summary).toBe("string");
+    expect((doc.summary || "").length).toBeGreaterThan(10);
+  }, 90_000);
+
+  test.each([
+    ["markdown", "markdown"],
+    ["html", "html"],
+    ["rawHtml", "rawHtml"],
+    ["links", "links"],
+    ["screenshot", "screenshot"],
+  ])("basic format: %s", async (fmt, expectField) => {
+    if (!client) throw new Error();
+    const doc = await client.scrape("https://docs.firecrawl.dev", { formats: [fmt as any] });
+    if (expectField !== "links" && expectField !== "screenshot") {
+      assertValidDocument(doc);
+    }
+    if (expectField === "markdown") expect(doc.markdown).toBeTruthy();
+    if (expectField === "html") expect(doc.html).toBeTruthy();
+    if (expectField === "rawHtml") expect(doc.rawHtml).toBeTruthy();
+    if (expectField === "screenshot") expect(doc.screenshot).toBeTruthy();
+    if (expectField === "links") {
+      expect(Array.isArray(doc.links)).toBe(true);
+      expect((doc.links || []).length).toBeGreaterThan(0);
+    }
+  }, 90_000);
+
+  test("invalid url should throw", async () => {
+    if (!client) throw new Error();
+    await expect(client.scrape("")).rejects.toThrow("URL cannot be empty");
+    await expect(client.scrape("   ")).rejects.toThrow("URL cannot be empty");
+  });
+});
+