@mendable/firecrawl-js 4.37.0 → 4.39.0

package/src/__tests__/e2e/v2/crawl.test.ts

@@ -4,6 +4,7 @@
 import Firecrawl from "../../../index";
 import { config } from "dotenv";
 import { getIdentity, getApiUrl } from "./utils/idmux";
+import { testTimeoutMs, waitForJob, withRateLimitRetry } from "./utils/rateLimit";
 import { describe, test, expect, beforeAll } from "@jest/globals";
 
 config();
@@ -13,7 +14,7 @@ let client: Firecrawl;
 
 beforeAll(async () => {
   const { apiKey } = await getIdentity({ name: "js-e2e-crawl" });
-  client = new Firecrawl({ apiKey, apiUrl: API_URL });
+  client = withRateLimitRetry(new Firecrawl({ apiKey, apiUrl: API_URL }));
 });
 
 describe("v2.crawl e2e", () => {
@@ -23,21 +24,21 @@ describe("v2.crawl e2e", () => {
     const job = await client.startCrawl("https://docs.firecrawl.dev", { limit: 3 });
     expect(typeof job.id).toBe("string");
     expect(typeof job.url).toBe("string");
-  }, 90_000);
+  }, testTimeoutMs(90_000));
 
   test("start crawl with options", async () => {
     if (!client) throw new Error();
     const job = await client.startCrawl("https://docs.firecrawl.dev", { limit: 5, maxDiscoveryDepth: 2 });
     expect(typeof job.id).toBe("string");
     expect(typeof job.url).toBe("string");
-  }, 90_000);
+  }, testTimeoutMs(90_000));
 
   test("start crawl with prompt", async () => {
     if (!client) throw new Error();
     const job = await client.startCrawl("https://firecrawl.dev", { prompt: "Extract all blog posts", limit: 3 });
     expect(typeof job.id).toBe("string");
     expect(typeof job.url).toBe("string");
-  }, 90_000);
+  }, testTimeoutMs(90_000));
 
   test("get crawl status", async () => {
     if (!client) throw new Error();
@@ -50,14 +51,14 @@ describe("v2.crawl e2e", () => {
     expect(status.id).toBe(start.id);
     // next/expiresAt may be null/undefined depending on state; check shape
     expect(Array.isArray(status.data)).toBe(true);
-  }, 120_000);
+  }, testTimeoutMs(151_000));
 
   test("cancel crawl", async () => {
     if (!client) throw new Error();
     const start = await client.startCrawl("https://docs.firecrawl.dev", { limit: 3 });
     const ok = await client.cancelCrawl(start.id);
     expect(ok).toBe(true);
-  }, 120_000);
+  }, testTimeoutMs(151_000));
 
   test("get crawl errors", async () => {
     if (!client) throw new Error();
@@ -73,12 +74,12 @@ describe("v2.crawl e2e", () => {
       expect(typeof e.url === "string" || e.url == null).toBe(true);
       expect(typeof e.error === "string" || e.error == null).toBe(true);
     }
-  }, 120_000);
+  }, testTimeoutMs(151_000));
 
   test("get crawl errors with invalid id should throw", async () => {
     if (!client) throw new Error();
     await expect(client.getCrawlErrors("invalid-job-id-12345")).rejects.toThrow();
-  }, 60_000);
+  }, testTimeoutMs(60_000));
 
   test("get active crawls", async () => {
     if (!client) throw new Error();
@@ -93,7 +94,7 @@ describe("v2.crawl e2e", () => {
         expect(typeof c.options === "object").toBe(true);
       }
     }
-  }, 90_000);
+  }, testTimeoutMs(90_000));
 
   test("get active crawls with running crawl", async () => {
     if (!client) throw new Error();
@@ -104,20 +105,22 @@ describe("v2.crawl e2e", () => {
     const ids = active.crawls.map(c => c.id);
     expect(ids.includes(start.id)).toBe(true);
     await client.cancelCrawl(start.id);
-  }, 120_000);
+  }, testTimeoutMs(301_000));
 
   test("crawl with wait", async () => {
     if (!client) throw new Error();
-    const job = await client.crawl("https://docs.firecrawl.dev", { limit: 3, maxDiscoveryDepth: 2, pollInterval: 1, timeout: 120 });
+    const start = await client.startCrawl("https://docs.firecrawl.dev", { limit: 3, maxDiscoveryDepth: 2 });
+    const job = await waitForJob(() => client.getCrawlStatus(start.id), { pollInterval: 1, timeout: 120 });
     expect(["completed", "failed"]).toContain(job.status);
     expect(job.completed).toBeGreaterThanOrEqual(0);
     expect(job.total).toBeGreaterThanOrEqual(0);
     expect(Array.isArray(job.data)).toBe(true);
-  }, 180_000);
+  }, testTimeoutMs(272_000));
 
   test("crawl with wait returns job id for error retrieval", async () => {
     if (!client) throw new Error();
-    const job = await client.crawl("https://docs.firecrawl.dev", { limit: 3, maxDiscoveryDepth: 2, pollInterval: 1, timeout: 120 });
+    const start = await client.startCrawl("https://docs.firecrawl.dev", { limit: 3, maxDiscoveryDepth: 2 });
+    const job = await waitForJob(() => client.getCrawlStatus(start.id), { pollInterval: 1, timeout: 120 });
     // Verify job has id field
     expect(job.id).toBeDefined();
     expect(typeof job.id).toBe("string");
@@ -127,16 +130,17 @@ describe("v2.crawl e2e", () => {
     expect(errors).toHaveProperty("robotsBlocked");
     expect(Array.isArray(errors.errors)).toBe(true);
     expect(Array.isArray(errors.robotsBlocked)).toBe(true);
-  }, 180_000);
+  }, testTimeoutMs(422_000));
 
   test("crawl with prompt and wait", async () => {
     if (!client) throw new Error();
-    const job = await client.crawl("https://docs.firecrawl.dev", { prompt: "Extract all blog posts", limit: 3, pollInterval: 1, timeout: 120 });
+    const start = await client.startCrawl("https://docs.firecrawl.dev", { prompt: "Extract all blog posts", limit: 3 });
+    const job = await waitForJob(() => client.getCrawlStatus(start.id), { pollInterval: 1, timeout: 120 });
     expect(["completed", "failed"]).toContain(job.status);
     expect(job.completed).toBeGreaterThanOrEqual(0);
     expect(job.total).toBeGreaterThanOrEqual(0);
     expect(Array.isArray(job.data)).toBe(true);
-  }, 180_000);
+  }, testTimeoutMs(272_000));
 
   test("crawl with scrape options", async () => {
     if (!client) throw new Error();
@@ -145,7 +149,7 @@ describe("v2.crawl e2e", () => {
       scrapeOptions: { formats: ["markdown", "links"], onlyMainContent: false, mobile: true },
     });
     expect(typeof job.id).toBe("string");
-  }, 120_000);
+  }, testTimeoutMs(120_000));
 
   test("crawl with json format object", async () => {
     if (!client) throw new Error();
@@ -154,7 +158,7 @@ describe("v2.crawl e2e", () => {
       scrapeOptions: { formats: [{ type: "json", prompt: "Extract page title", schema: { type: "object", properties: { title: { type: "string" } }, required: ["title"] } }] },
     });
     expect(typeof job.id).toBe("string");
-  }, 120_000);
+  }, testTimeoutMs(120_000));
 
   test("crawl all parameters", async () => {
     if (!client) throw new Error();
@@ -187,13 +191,13 @@ describe("v2.crawl e2e", () => {
       zeroDataRetention: false,
     });
     expect(typeof job.id).toBe("string");
-  }, 180_000);
+  }, testTimeoutMs(180_000));
 
   test("crawl params preview", async () => {
     if (!client) throw new Error();
     const params = await client.crawlParamsPreview("https://docs.firecrawl.dev", "Extract all blog posts and documentation");
     expect(params && typeof params === "object").toBe(true);
     // Optional fields may or may not be present; just assert object shape
-  }, 60_000);
+  }, testTimeoutMs(60_000));
 });
 

package/src/__tests__/e2e/v2/extract.test.ts

@@ -4,6 +4,7 @@
 import Firecrawl from "../../../index";
 import { config } from "dotenv";
 import { getIdentity, getApiUrl } from "./utils/idmux";
+import { testTimeoutMs, waitForJob, withRateLimitRetry } from "./utils/rateLimit";
 import { describe, test, expect, beforeAll } from "@jest/globals";
 import { z } from "zod";
 
@@ -14,14 +15,38 @@ let client: Firecrawl;
 
 beforeAll(async () => {
   const { apiKey } = await getIdentity({ name: "js-e2e-extract" });
-  client = new Firecrawl({ apiKey, apiUrl: API_URL });
+  client = withRateLimitRetry(new Firecrawl({ apiKey, apiUrl: API_URL }));
 });
 
 describe("v2.extract e2e", () => {
+  /**
+   * Starts an extract, then polls it. Mirrors client.extract, but keeps every
+   * request retryable: the wrapper cannot run client.extract again, because a
+   * second run would start a second job.
+   *
+   * The bound is 180s, the time an extract needs, and the same bound batch
+   * uses.
+   *
+   * The bound is not the whole cost of the test. startExtract goes through
+   * the wrapper too, so it can spend the retry budget before the poll starts:
+   * 150s for the start, plus 180 + 2 + 150 = 332s for the poll, so 482s. Each
+   * test below therefore carries a 333_000 ms base, which gives 483s. The
+   * unit test "every wrapped call in a test fits its budget" holds that sum.
+   */
+  async function extractAndWait(
+    args: Parameters<typeof client.startExtract>[0],
+  ) {
+    const started = await client.startExtract(args);
+    if (!started.id) return started;
+    return waitForJob(() => client.getExtractStatus(started.id!), {
+      timeout: 180,
+    });
+  }
+
   test("extract minimal with prompt", async () => {
-    const resp = await client.extract({ urls: ["https://docs.firecrawl.dev"], prompt: "Extract the main page title" });
+    const resp = await extractAndWait({ urls: ["https://docs.firecrawl.dev"], prompt: "Extract the main page title" });
     expect(typeof resp.success === "boolean" || resp.success == null).toBe(true);
-  }, 120_000);
+  }, testTimeoutMs(333_000));
 
   test("extract with schema", async () => {
     const schema = {
@@ -29,7 +54,7 @@ describe("v2.extract e2e", () => {
       properties: { title: { type: "string" } },
       required: ["title"],
     } as const;
-    const resp = await client.extract({
+    const resp = await extractAndWait({
       urls: ["https://docs.firecrawl.dev"],
       schema,
       prompt: "Extract the main page title",
@@ -44,13 +69,13 @@ describe("v2.extract e2e", () => {
       expect(typeof resp.data).toBe("object");
       expect((resp.data as any).title).toBeTruthy();
     }
-  }, 180_000);
+  }, testTimeoutMs(333_000));
 
   test("extract with zod schema", async () => {
     const schema = z.object({
       title: z.string(),
     });
-    const resp = await client.extract({
+    const resp = await extractAndWait({
       urls: ["https://docs.firecrawl.dev"],
       schema: schema,
       prompt: "Extract the main page title",
@@ -65,6 +90,6 @@ describe("v2.extract e2e", () => {
       expect(typeof resp.data).toBe("object");
       expect(schema.safeParse(resp.data).success).toBe(true);
     }
-  }, 180_000);
+  }, testTimeoutMs(333_000));
 });
 

package/src/__tests__/e2e/v2/map.test.ts

@@ -4,6 +4,7 @@
 import Firecrawl from "../../../index";
 import { config } from "dotenv";
 import { getIdentity, getApiUrl } from "./utils/idmux";
+import { testTimeoutMs, withRateLimitRetry } from "./utils/rateLimit";
 import { describe, test, expect, beforeAll } from "@jest/globals";
 
 config();
@@ -13,7 +14,7 @@ let client: Firecrawl;
 
 beforeAll(async () => {
   const { apiKey } = await getIdentity({ name: "js-e2e-map" });
-  client = new Firecrawl({ apiKey, apiUrl: API_URL });
+  client = withRateLimitRetry(new Firecrawl({ apiKey, apiUrl: API_URL }));
 });
 
 describe("v2.map e2e", () => {
@@ -30,7 +31,7 @@ describe("v2.map e2e", () => {
       expect(typeof first.url).toBe("string");
       expect(first.url.startsWith("http")).toBe(true);
     }
-  }, 90_000);
+  }, testTimeoutMs(90_000));
 
   test.each(["only", "skip", "include"]) ("with options sitemap=%s", async (sitemap) => {
     if (!client) throw new Error();
@@ -50,6 +51,6 @@ describe("v2.map e2e", () => {
       expect(typeof link.url).toBe("string");
       expect(link.url.startsWith("http")).toBe(true);
     }
-  }, 120_000);
+  }, testTimeoutMs(120_000));
 });
 

package/src/__tests__/e2e/v2/parse.test.ts

@@ -1,6 +1,7 @@
 import Firecrawl from "../../../index";
 import { config } from "dotenv";
 import { getIdentity, getApiUrl } from "./utils/idmux";
+import { testTimeoutMs, withRateLimitRetry } from "./utils/rateLimit";
 import { describe, test, expect, beforeAll } from "@jest/globals";
 
 config();
@@ -10,7 +11,7 @@ let client: Firecrawl;
 
 beforeAll(async () => {
   const { apiKey } = await getIdentity({ name: "js-e2e-parse" });
-  client = new Firecrawl({ apiKey, apiUrl: API_URL });
+  client = withRateLimitRetry(new Firecrawl({ apiKey, apiUrl: API_URL }));
 });
 
 describe("v2.parse e2e", () => {
@@ -41,7 +42,7 @@ describe("v2.parse e2e", () => {
       expect(doc.markdown).toContain("JS SDK Parse E2E");
       expect(doc.metadata?.creditsUsed).toBe(1);
     },
-    60_000,
+    testTimeoutMs(60_000),
   );
 
   test(
@@ -62,6 +63,6 @@ describe("v2.parse e2e", () => {
         ),
       ).rejects.toThrow();
     },
-    60_000,
+    testTimeoutMs(60_000),
   );
 });

package/src/__tests__/e2e/v2/scrape.test.ts

@@ -5,6 +5,7 @@ import Firecrawl from "../../../index";
 import { z } from "zod";
 import { config } from "dotenv";
 import { getIdentity, getApiUrl } from "./utils/idmux";
+import { testTimeoutMs, withRateLimitRetry } from "./utils/rateLimit";
 import { describe, test, expect, beforeAll } from "@jest/globals";
 
 config();
@@ -14,7 +15,7 @@ let client: Firecrawl;
 
 beforeAll(async () => {
   const { apiKey } = await getIdentity({ name: "js-e2e-scrape" });
-  client = new Firecrawl({ apiKey, apiUrl: API_URL });
+  client = withRateLimitRetry(new Firecrawl({ apiKey, apiUrl: API_URL }));
 });
 
 describe("v2.scrape e2e", () => {
@@ -30,7 +31,7 @@ describe("v2.scrape e2e", () => {
     if (!client) throw new Error();
     const doc = await client.scrape("https://docs.firecrawl.dev");
     assertValidDocument(doc);
-  }, 60_000);
+  }, testTimeoutMs(60_000));
 
   test("maximal: scrape with all options", async () => {
     if (!client) throw new Error();
@@ -71,7 +72,7 @@ describe("v2.scrape e2e", () => {
       maxAge: 60_000,
     });
     assertValidDocument(doc);
-  }, 90_000);
+  }, testTimeoutMs(90_000));
 
   test("json format with zod schema (auto-converted internally)", async () => {
     if (!client) throw new Error();
@@ -90,14 +91,14 @@ describe("v2.scrape e2e", () => {
       ],
     });
     expect(doc).toBeTruthy();
-  }, 90_000);
+  }, testTimeoutMs(90_000));
 
   test("summary format returns summary string", async () => {
     if (!client) throw new Error();
     const doc = await client.scrape("https://firecrawl.dev", { formats: ["summary"] });
     expect(typeof doc.summary).toBe("string");
     expect((doc.summary || "").length).toBeGreaterThan(10);
-  }, 90_000);
+  }, testTimeoutMs(90_000));
 
   test.each([
     ["markdown", "markdown"],
@@ -119,7 +120,7 @@ describe("v2.scrape e2e", () => {
       expect(Array.isArray(doc.links)).toBe(true);
       expect((doc.links || []).length).toBeGreaterThan(0);
     }
-  }, 90_000);
+  }, testTimeoutMs(90_000));
 
   test("images format: extract all images from webpage", async () => {
     if (!client) throw new Error();
@@ -131,7 +132,7 @@ describe("v2.scrape e2e", () => {
     expect(doc.images?.length).toBeGreaterThan(0);
     // Should find firecrawl logo/branding images
     expect(doc.images?.some(img => img.includes("firecrawl") || img.includes("logo"))).toBe(true);
-  }, 60_000);
+  }, testTimeoutMs(60_000));
 
   test("images format: works with multiple formats", async () => {
     if (!client) throw new Error();
@@ -152,12 +153,12 @@ describe("v2.scrape e2e", () => {
     
     // Should discover additional images beyond those with obvious extensions
     expect(doc.images?.length).toBeGreaterThanOrEqual(linkImages.length);
-  }, 60_000);
+  }, testTimeoutMs(60_000));
 
   test("invalid url should throw", async () => {
     if (!client) throw new Error();
     await expect(client.scrape("")).rejects.toThrow("URL cannot be empty");
     await expect(client.scrape("   ")).rejects.toThrow("URL cannot be empty");
-  });
+  }, testTimeoutMs(151_000));
 });
 

package/src/__tests__/e2e/v2/search.test.ts

@@ -5,6 +5,7 @@ import Firecrawl from "../../../index";
 import type { Document, SearchResultWeb, SearchResultNews, SearchResultImages } from "../../../index";
 import { config } from "dotenv";
 import { getIdentity, getApiUrl } from "./utils/idmux";
+import { testTimeoutMs, withRateLimitRetry } from "./utils/rateLimit";
 import { describe, test, expect, beforeAll } from "@jest/globals";
 
 config();
@@ -14,7 +15,7 @@ let client: Firecrawl;
 
 beforeAll(async () => {
   const { apiKey } = await getIdentity({ name: "js-e2e-search" });
-  client = new Firecrawl({ apiKey, apiUrl: API_URL });
+  client = withRateLimitRetry(new Firecrawl({ apiKey, apiUrl: API_URL }));
 });
 
 function collectTexts(entries: any[] | undefined): string[] {
@@ -72,7 +73,7 @@ describe("v2.search e2e", () => {
 
     expect(results.news == null).toBe(true);
     expect(results.images == null).toBe(true);
-  }, 90_000);
+  }, testTimeoutMs(90_000));
 
   test("with sources web+news and limit", async () => {
     if (!client) throw new Error();
@@ -93,7 +94,7 @@ describe("v2.search e2e", () => {
       .map(r => (r.description || "").toString().toLowerCase());
     const allWebText = (webTitles.concat(webDescriptions)).join(" ");
     expect(allWebText.includes("firecrawl")).toBe(true);
-  }, 90_000);
+  }, testTimeoutMs(90_000));
 
   test("result structure", async () => {
     if (!client) throw new Error();
@@ -108,7 +109,7 @@ describe("v2.search e2e", () => {
       expect(typeof result.description === "string" || result.description == null).toBe(true);
       expect(result.url.startsWith("http")).toBe(true);
     }
-  }, 90_000);
+  }, testTimeoutMs(90_000));
 
   test("all parameters (comprehensive)", async () => {
     if (!client) throw new Error();
@@ -196,7 +197,7 @@ describe("v2.search e2e", () => {
         expect(result.url?.startsWith("http")).toBe(true);
       }
     }
-  }, 120_000);
+  }, testTimeoutMs(120_000));
 
   test("formats flexibility: list vs object", async () => {
     if (!client) throw new Error();
@@ -212,7 +213,7 @@ describe("v2.search e2e", () => {
     expect(results2).toBeTruthy();
     expect(results1.web).toBeTruthy();
     expect(results2.web).toBeTruthy();
-  }, 90_000);
+  }, testTimeoutMs(151_000));
 
   test("with json format object", async () => {
     if (!client) throw new Error();
@@ -229,7 +230,7 @@ describe("v2.search e2e", () => {
     });
     expect(results).toBeTruthy();
     expect(Array.isArray(results.web) || results.web == null).toBe(true);
-  }, 90_000);
+  }, testTimeoutMs(90_000));
 
   test("with summary format, documents include summary when present", async () => {
     if (!client) throw new Error();
@@ -242,6 +243,6 @@ describe("v2.search e2e", () => {
       expect(typeof docs[0].summary).toBe("string");
       expect((docs[0].summary || "").length).toBeGreaterThan(5);
     }
-  }, 90_000);
+  }, testTimeoutMs(90_000));
 });
 

package/src/__tests__/e2e/v2/usage.test.ts

@@ -4,6 +4,7 @@
 import Firecrawl from "../../../index";
 import { config } from "dotenv";
 import { getIdentity, getApiUrl } from "./utils/idmux";
+import { testTimeoutMs, withRateLimitRetry } from "./utils/rateLimit";
 import { describe, test, expect, beforeAll } from "@jest/globals";
 
 config();
@@ -13,7 +14,7 @@ let client: Firecrawl;
 
 beforeAll(async () => {
   const { apiKey } = await getIdentity({ name: "js-e2e-usage" });
-  client = new Firecrawl({ apiKey, apiUrl: API_URL });
+  client = withRateLimitRetry(new Firecrawl({ apiKey, apiUrl: API_URL }));
 });
 
 describe("v2.usage e2e", () => {
@@ -21,17 +22,17 @@ describe("v2.usage e2e", () => {
     const resp = await client.getConcurrency();
     expect(typeof resp.concurrency).toBe("number");
     expect(typeof resp.maxConcurrency).toBe("number");
-  }, 60_000);
+  }, testTimeoutMs(60_000));
 
   test("get_credit_usage", async () => {
     const resp = await client.getCreditUsage();
     expect(typeof resp.remainingCredits).toBe("number");
-  }, 60_000);
+  }, testTimeoutMs(60_000));
 
   test("get_token_usage", async () => {
     const resp = await client.getTokenUsage();
     expect(typeof resp.remainingTokens).toBe("number");
-  }, 60_000);
+  }, testTimeoutMs(60_000));
 
   test("get_queue_status", async () => {
     const resp = await client.getQueueStatus();
@@ -39,6 +40,6 @@ describe("v2.usage e2e", () => {
     expect(typeof resp.activeJobsInQueue).toBe("number");
     expect(typeof resp.waitingJobsInQueue).toBe("number");
     expect(typeof resp.maxConcurrency).toBe("number");
-  }, 60_000);
+  }, testTimeoutMs(60_000));
 });