@mendable/firecrawl-js 4.38.0 → 4.40.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. package/README.md +53 -2
  2. package/dist/{chunk-KSTNOGOD.js → chunk-OJ4YJQ4J.js} +1 -1
  3. package/dist/index.cjs +204 -17
  4. package/dist/index.d.cts +289 -7
  5. package/dist/index.d.ts +289 -7
  6. package/dist/index.js +203 -17
  7. package/dist/{package-OS7RONXY.js → package-F6O7J2XI.js} +1 -1
  8. package/package.json +1 -1
  9. package/pnpm-workspace.yaml +3 -2
  10. package/src/__tests__/e2e/v2/batch.test.ts +13 -11
  11. package/src/__tests__/e2e/v2/crawl.test.ts +24 -20
  12. package/src/__tests__/e2e/v2/extract.test.ts +32 -7
  13. package/src/__tests__/e2e/v2/map.test.ts +4 -3
  14. package/src/__tests__/e2e/v2/parse.test.ts +4 -3
  15. package/src/__tests__/e2e/v2/scrape.test.ts +10 -9
  16. package/src/__tests__/e2e/v2/search.test.ts +9 -8
  17. package/src/__tests__/e2e/v2/usage.test.ts +6 -5
  18. package/src/__tests__/e2e/v2/utils/rateLimit.ts +257 -0
  19. package/src/__tests__/e2e/v2/watcher.test.ts +4 -3
  20. package/src/__tests__/unit/v2/agent-effort-trace.unit.test.ts +1 -1
  21. package/src/__tests__/unit/v2/agent-thread.unit.test.ts +164 -0
  22. package/src/__tests__/unit/v2/e2eRateLimitRetry.test.ts +501 -0
  23. package/src/__tests__/unit/v2/extract.unit.test.ts +1 -1
  24. package/src/__tests__/unit/v2/scrapeAutoResume.test.ts +1 -1
  25. package/src/__tests__/unit/v2/search-country.unit.test.ts +32 -0
  26. package/src/__tests__/unit/v2/search-data-property.test.ts +1 -0
  27. package/src/__tests__/unit/v2/threat-protection.unit.test.ts +18 -1
  28. package/src/__tests__/unit/v2/tools.unit.test.ts +240 -0
  29. package/src/__tests__/unit/v2/v1-aliases.test.ts +1 -0
  30. package/src/v2/client.ts +52 -4
  31. package/src/v2/methods/agent.ts +23 -1
  32. package/src/v2/methods/research.ts +14 -8
  33. package/src/v2/methods/scrape.ts +3 -1
  34. package/src/v2/methods/search.ts +8 -1
  35. package/src/v2/methods/tools.ts +137 -0
  36. package/src/v2/types.ts +307 -35
  37. package/src/v2/utils/errorHandler.ts +19 -3
package/dist/index.js CHANGED
@@ -1,6 +1,6 @@
1
1
  import {
2
2
  require_package
3
- } from "./chunk-KSTNOGOD.js";
3
+ } from "./chunk-OJ4YJQ4J.js";
4
4
 
5
5
  // src/v2/utils/httpClient.ts
6
6
  import axios from "axios";
@@ -138,18 +138,35 @@ var HttpClient = class {
138
138
  };
139
139
 
140
140
  // src/v2/types.ts
141
+ function parseRequiresAction(value) {
142
+ if (!value || typeof value !== "object") return void 0;
143
+ const record = value;
144
+ if (typeof record.type !== "string") return void 0;
145
+ const action = { type: record.type };
146
+ if (typeof record.terms === "string") action.terms = record.terms;
147
+ if (typeof record.version === "string") action.version = record.version;
148
+ if (typeof record.url === "string") action.url = record.url;
149
+ return action;
150
+ }
141
151
  var SdkError = class extends Error {
152
+ requestId;
142
153
  status;
143
154
  code;
144
155
  details;
145
156
  jobId;
146
- constructor(message, status, code, details, jobId) {
157
+ /** Present on exchange-mediated scrape failures that already captured credits. */
158
+ chargeId;
159
+ /** Present when the API needs an out-of-band step first, such as accepting provider terms. */
160
+ requiresAction;
161
+ constructor(message, status, code, details, jobId, chargeId, requiresAction) {
147
162
  super(message);
148
163
  this.name = "FirecrawlSdkError";
149
164
  this.status = status;
150
165
  this.code = code;
151
166
  this.details = details;
152
167
  this.jobId = jobId;
168
+ this.chargeId = chargeId;
169
+ this.requiresAction = requiresAction;
153
170
  }
154
171
  };
155
172
  var JobTimeoutError = class extends SdkError {
@@ -401,14 +418,30 @@ function throwForBadResponse(resp, action) {
401
418
  const status = resp.status;
402
419
  const body = resp.data || {};
403
420
  const msg = body?.error || body?.message || `Request failed (${status}) while trying to ${action}`;
404
- throw new SdkError(msg, status, void 0, body?.details);
421
+ throw new SdkError(
422
+ msg,
423
+ status,
424
+ body?.code,
425
+ body?.details,
426
+ void 0,
427
+ body?.chargeId,
428
+ parseRequiresAction(body?.requiresAction)
429
+ );
405
430
  }
406
431
  function normalizeAxiosError(err, action) {
407
432
  const status = err.response?.status;
408
433
  const body = err.response?.data;
409
434
  const message = body?.error || err.message || `Request failed${status ? ` (${status})` : ""} while trying to ${action}`;
410
435
  const code = body?.code || err.code;
411
- throw new SdkError(message, status, code, body?.details ?? body);
436
+ throw new SdkError(
437
+ message,
438
+ status,
439
+ code,
440
+ body?.details ?? body,
441
+ void 0,
442
+ body?.chargeId,
443
+ parseRequiresAction(body?.requiresAction)
444
+ );
412
445
  }
413
446
  function isRetryableError(err) {
414
447
  if (err instanceof JobTimeoutError) {
@@ -454,9 +487,10 @@ async function scrape(http, url, options, deps = {}) {
454
487
  throw new Error("URL cannot be empty");
455
488
  }
456
489
  if (options) ensureValidScrapeOptions(options);
457
- const { autoResume, ...requestOptions } = options ?? {};
490
+ const { autoResume, domainTools, ...requestOptions } = options ?? {};
458
491
  const payload = { url: url.trim() };
459
492
  Object.assign(payload, requestOptions);
493
+ if (domainTools) payload.domainTools = domainTools;
460
494
  const requestTimeout = autoResume === false ? typeof options?.timeout === "number" ? { timeoutMs: options.timeout + 5e3 } : {} : {
461
495
  timeoutMs: Math.max(
462
496
  typeof options?.timeout === "number" ? options.timeout : 3e5,
@@ -613,6 +647,7 @@ function prepareSearchPayload(req) {
613
647
  const payload = {
614
648
  query: req.query
615
649
  };
650
+ if (req.domainTools != null) payload.domainTools = req.domainTools;
616
651
  if (req.sources) payload.sources = req.sources;
617
652
  if (req.categories) payload.categories = req.categories;
618
653
  if (req.includeDomains) payload.includeDomains = req.includeDomains;
@@ -620,6 +655,7 @@ function prepareSearchPayload(req) {
620
655
  if (req.limit != null) payload.limit = req.limit;
621
656
  if (req.tbs != null) payload.tbs = req.tbs;
622
657
  if (req.location != null) payload.location = req.location;
658
+ if (req.country != null) payload.country = req.country;
623
659
  if (req.ignoreInvalidURLs != null)
624
660
  payload.ignoreInvalidURLs = req.ignoreInvalidURLs;
625
661
  if (req.timeout != null) payload.timeout = req.timeout;
@@ -664,10 +700,12 @@ async function search(http, request) {
664
700
  }
665
701
  const data = res.data.data || {};
666
702
  const out = {};
703
+ if (res.data.warning) out.warning = res.data.warning;
667
704
  if (data.web) out.web = transformArray(data.web);
668
705
  if (data.news) out.news = transformArray(data.news);
669
706
  if (data.images)
670
707
  out.images = transformArray(data.images);
708
+ if (data.tools) out.tools = data.tools;
671
709
  Object.defineProperty(out, "data", {
672
710
  get() {
673
711
  const parts = [];
@@ -675,7 +713,9 @@ async function search(http, request) {
675
713
  if (out.news?.length) parts.push(`.news (${out.news.length} results)`);
676
714
  if (out.images?.length)
677
715
  parts.push(`.images (${out.images.length} results)`);
678
- const available = parts.length ? parts.join(", ") : ".web, .news, or .images";
716
+ if (out.tools?.length)
717
+ parts.push(`.tools (${out.tools.length} results)`);
718
+ const available = parts.length ? parts.join(", ") : ".web, .news, .images, or .tools";
679
719
  throw new Error(
680
720
  `SearchData has no '.data'. Results are grouped by source: ${available}`
681
721
  );
@@ -690,6 +730,101 @@ async function search(http, request) {
690
730
  }
691
731
  }
692
732
 
733
+ // src/v2/methods/tools.ts
734
+ var ALEXANDRIA_MAX_CALLS = 10;
735
+ function prepareAlexandriaPayload(calls, opts) {
736
+ if (!Array.isArray(calls) || calls.length === 0) {
737
+ throw new Error("alexandria requires at least one call");
738
+ }
739
+ if (calls.length > ALEXANDRIA_MAX_CALLS) {
740
+ throw new Error(`alexandria accepts at most ${ALEXANDRIA_MAX_CALLS} calls`);
741
+ }
742
+ const alexandria = calls.map((call, index) => {
743
+ if (!call || typeof call.provider !== "string" || !call.provider.trim()) {
744
+ throw new Error(`alexandria[${index}].provider cannot be empty`);
745
+ }
746
+ if (typeof call.capability !== "string" || !call.capability.trim()) {
747
+ throw new Error(`alexandria[${index}].capability cannot be empty`);
748
+ }
749
+ if (Object.keys(call).some(
750
+ (key) => !["provider", "capability", "options"].includes(key)
751
+ ))
752
+ throw new Error("Unknown alexandria call option");
753
+ const item = {
754
+ provider: call.provider.trim(),
755
+ capability: call.capability.trim()
756
+ };
757
+ if (call.options != null) {
758
+ if (typeof call.options !== "object" || Array.isArray(call.options)) {
759
+ throw new Error(`alexandria[${index}].options must be an object`);
760
+ }
761
+ item.options = call.options;
762
+ }
763
+ return item;
764
+ });
765
+ if (opts.timeout != null && (!Number.isInteger(opts.timeout) || opts.timeout <= 0)) {
766
+ throw new Error("timeout must be a positive integer");
767
+ }
768
+ const payload = { alexandria };
769
+ if (opts.timeout != null) payload.timeout = opts.timeout;
770
+ if (opts.integration && opts.integration.trim())
771
+ payload.integration = opts.integration.trim();
772
+ if (opts.origin) payload.origin = opts.origin;
773
+ return payload;
774
+ }
775
+ async function scrapeAlexandria(http, calls, opts = {}) {
776
+ const payload = prepareAlexandriaPayload(calls, opts);
777
+ const requestId = opts.requestId ?? crypto.randomUUID();
778
+ if (!/^[A-Za-z0-9._:-]{1,128}$/.test(requestId))
779
+ throw new Error("Invalid requestId");
780
+ try {
781
+ const res = await http.post("/v2/scrape", payload, {
782
+ headers: { "x-request-id": requestId },
783
+ // Allow response delivery after the API's capped execution deadline.
784
+ timeoutMs: Math.min(opts.timeout ?? 5e4, 5e4) + 3e4
785
+ });
786
+ if (res.status !== 200 || !res.data?.success) {
787
+ throwForBadResponse(res, "alexandria");
788
+ }
789
+ const data = res.data.data;
790
+ if (!data || !Array.isArray(data.alexandria) || typeof data.creditsCost !== "number" || !Number.isInteger(data.creditsCost) || data.creditsCost < 0) {
791
+ throw new SdkError("Invalid alexandria response");
792
+ }
793
+ return {
794
+ scrapeId: res.data.scrape_id ?? "",
795
+ requestId,
796
+ alexandria: data.alexandria,
797
+ creditsCost: data.creditsCost
798
+ };
799
+ } catch (err) {
800
+ try {
801
+ if (err?.isAxiosError) normalizeAxiosError(err, "alexandria");
802
+ throw err;
803
+ } catch (error) {
804
+ if (error && typeof error === "object")
805
+ Object.assign(error, { requestId });
806
+ throw error;
807
+ }
808
+ }
809
+ }
810
+ async function findTools(http, options = {}) {
811
+ const result = await scrapeAlexandria(http, [
812
+ {
813
+ provider: "firecrawl",
814
+ capability: "find-tools",
815
+ options: { ...options }
816
+ }
817
+ ]);
818
+ const item = result.alexandria[0];
819
+ if (!item) throw new SdkError("Missing Find Tools result");
820
+ if (item.error)
821
+ throw Object.assign(
822
+ new SdkError(item.error.message, item.error.status, item.error.code),
823
+ { requestId: result.requestId }
824
+ );
825
+ return item.data;
826
+ }
827
+
693
828
  // src/v2/methods/developer.ts
694
829
  var ENDPOINT = "/v2/search/developer";
695
830
  async function developerSearch(http, query, options = {}) {
@@ -1240,6 +1375,9 @@ function prepareAgentPayload(args) {
1240
1375
  body.threatProtection = args.threatProtection;
1241
1376
  if (args.auditMetadata != null)
1242
1377
  body.auditMetadata = args.auditMetadata;
1378
+ if (args.threadId != null) body.threadId = args.threadId;
1379
+ if (args.mode != null) body.mode = args.mode;
1380
+ if (args.exchange != null) body.exchange = args.exchange;
1243
1381
  return body;
1244
1382
  }
1245
1383
  async function startAgent(http, args) {
@@ -1310,6 +1448,17 @@ async function getAgentSnapshot(http, jobId, snapshotId) {
1310
1448
  throw err;
1311
1449
  }
1312
1450
  }
1451
+ async function getAgentThread(http, threadId, options) {
1452
+ try {
1453
+ const query = options?.includeData ? "?includeData=true" : "";
1454
+ const res = await http.get(`/v2/agent/threads/${threadId}${query}`);
1455
+ if (res.status !== 200) throwForBadResponse(res, "agent thread");
1456
+ return res.data;
1457
+ } catch (err) {
1458
+ if (err?.isAxiosError) return normalizeAxiosError(err, "agent thread");
1459
+ throw err;
1460
+ }
1461
+ }
1313
1462
  async function cancelAgent(http, jobId) {
1314
1463
  try {
1315
1464
  const res = await http.delete(`/v2/agent/${jobId}`);
@@ -1550,9 +1699,7 @@ var ResearchClient = class {
1550
1699
  appendParam(params, "query", options.query);
1551
1700
  appendParam(params, "k", options.k);
1552
1701
  try {
1553
- const res = await this.http.get(
1554
- withQuery(`${BASE}/papers/${encodeURIComponent(id)}`, params)
1555
- );
1702
+ const res = await this.http.get(withQuery(`${BASE}/papers/${encodeURIComponent(id)}`, params));
1556
1703
  if (res.status !== 200) throwForBadResponse(res, "get paper");
1557
1704
  return res.data;
1558
1705
  } catch (err) {
@@ -1578,10 +1725,7 @@ var ResearchClient = class {
1578
1725
  appendParam(params, "anchor", options.anchor);
1579
1726
  try {
1580
1727
  const res = await this.http.get(
1581
- withQuery(
1582
- `${BASE}/papers/${encodeURIComponent(id)}/similar`,
1583
- params
1584
- )
1728
+ withQuery(`${BASE}/papers/${encodeURIComponent(id)}/similar`, params)
1585
1729
  );
1586
1730
  if (res.status !== 200) throwForBadResponse(res, "find similar papers");
1587
1731
  return res.data;
@@ -1590,7 +1734,16 @@ var ResearchClient = class {
1590
1734
  }
1591
1735
  }
1592
1736
  /**
1593
- * Search GitHub issue/PR history and repository readmes.
1737
+ * Search the research index GitHub slice: issue/PR history and repository
1738
+ * readmes.
1739
+ *
1740
+ * @deprecated Use `developerSearch()` on the client instead. This
1741
+ * endpoint stops responding after 2026-11-03. The developer index searches
1742
+ * GitHub issues, pull requests and readmes plus curated documentation
1743
+ * sources, returns matched passages, and adds filters for repo, language,
1744
+ * license and stars. It does not carry over the `scores` breakdown or the
1745
+ * `resultType: "web"` fallback results. See
1746
+ * https://docs.firecrawl.dev/features/developer.
1594
1747
  * @param query Search query.
1595
1748
  * @param options Optional `k`.
1596
1749
  */
@@ -1999,7 +2152,28 @@ var FirecrawlClient = class {
1999
2152
  });
2000
2153
  }
2001
2154
  async scrape(url, options) {
2002
- return scrape(this.http, url, options);
2155
+ if (typeof url === "string") return scrape(this.http, url, options);
2156
+ if (!url || options !== void 0 || Object.keys(url).some(
2157
+ (key) => ![
2158
+ "alexandria",
2159
+ "requestId",
2160
+ "timeout",
2161
+ "integration",
2162
+ "origin"
2163
+ ].includes(key)
2164
+ )) {
2165
+ throw new Error("Provide an alexandria request without URL scrape options");
2166
+ }
2167
+ const { alexandria, ...opts } = url;
2168
+ return scrapeAlexandria(
2169
+ this.http,
2170
+ Array.isArray(alexandria) ? alexandria : [alexandria],
2171
+ opts
2172
+ );
2173
+ }
2174
+ /** Explore the catalogue without executing the tools it returns. */
2175
+ async findTools(options) {
2176
+ return findTools(this.http, options);
2003
2177
  }
2004
2178
  /**
2005
2179
  * Interact with the browser session associated with a scrape job.
@@ -2079,6 +2253,9 @@ var FirecrawlClient = class {
2079
2253
  * Access the v2 research endpoints — Firecrawl's **research paper index**
2080
2254
  * (~43M paper abstracts) plus GitHub history/readmes.
2081
2255
  *
2256
+ * `research.searchGithub()` is deprecated and stops responding after
2257
+ * 2026-11-03. Use `developerSearch()` instead.
2258
+ *
2082
2259
  * The paper corpus is roughly 90% biomedical and life sciences — PubMed,
2083
2260
  * bioRxiv and medRxiv — with arXiv covering physics, mathematics and
2084
2261
  * computer science.
@@ -2338,6 +2515,14 @@ var FirecrawlClient = class {
2338
2515
  async getAgentSnapshot(jobId, snapshotId) {
2339
2516
  return getAgentSnapshot(this.http, jobId, snapshotId);
2340
2517
  }
2518
+ /**
2519
+ * Get a thread and its runs, oldest turn first.
2520
+ * @param threadId Thread id, as returned by startAgent or getAgentStatus.
2521
+ * @param options.includeData Inline each succeeded run's data.
2522
+ */
2523
+ async getAgentThread(threadId, options) {
2524
+ return getAgentThread(this.http, threadId, options);
2525
+ }
2341
2526
  // Browser
2342
2527
  /**
2343
2528
  * Create a new browser session.
@@ -2480,7 +2665,7 @@ var FirecrawlApp = class {
2480
2665
  if (typeof process !== "undefined" && process.env && process.env.npm_package_version) {
2481
2666
  return process.env.npm_package_version;
2482
2667
  }
2483
- const packageJson = await import("./package-OS7RONXY.js");
2668
+ const packageJson = await import("./package-F6O7J2XI.js");
2484
2669
  return packageJson.default.version;
2485
2670
  } catch (error) {
2486
2671
  const isTest = typeof process !== "undefined" && (process.env.JEST_WORKER_ID != null || false);
@@ -3880,5 +4065,6 @@ export {
3880
4065
  ResearchClient,
3881
4066
  SdkError,
3882
4067
  Watcher,
3883
- index_default as default
4068
+ index_default as default,
4069
+ parseRequiresAction
3884
4070
  };
@@ -1,4 +1,4 @@
1
1
  import {
2
2
  require_package
3
- } from "./chunk-KSTNOGOD.js";
3
+ } from "./chunk-OJ4YJQ4J.js";
4
4
  export default require_package();
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@mendable/firecrawl-js",
3
- "version": "4.38.0",
3
+ "version": "4.40.0",
4
4
  "description": "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
5
5
  "main": "dist/index.js",
6
6
  "types": "dist/index.d.ts",
@@ -8,10 +8,11 @@ overrides:
8
8
  "picomatch@<4.0.4": ">=4.0.4"
9
9
  handlebars: ">=4.7.9"
10
10
  "brace-expansion@>=5.0.0 <5.0.9": "5.0.9"
11
- "js-yaml@>=3.0.0 <3.15.1": "3.15.1"
12
- "js-yaml@>=4.0.0 <4.3.1": "4.3.1"
11
+ "js-yaml@>=3.0.0 <3.15.2": "3.15.2"
12
+ "js-yaml@>=4.0.0 <4.3.2": "4.3.2"
13
13
  "axios@>=1.0.0 <1.16.0": "1.16.1"
14
14
  "follow-redirects@<1.16.0": ">=1.16.0 <2.0.0"
15
15
  "form-data@>=4.0.0 <4.0.6": "4.0.6"
16
16
  "@babel/core@>=7.0.0 <7.29.6": "7.29.7"
17
17
  "esbuild@>=0.17.0 <0.28.1": "0.28.1"
18
+ "browserslist@<=4.28.6": ">=4.28.7 <5.0.0"
@@ -4,6 +4,7 @@
4
4
  import Firecrawl from "../../../index";
5
5
  import { config } from "dotenv";
6
6
  import { getIdentity, getApiUrl } from "./utils/idmux";
7
+ import { testTimeoutMs, waitForJob, withRateLimitRetry } from "./utils/rateLimit";
7
8
  import { describe, test, expect, beforeAll } from "@jest/globals";
8
9
 
9
10
  config();
@@ -13,7 +14,7 @@ let client: Firecrawl;
13
14
 
14
15
  beforeAll(async () => {
15
16
  const { apiKey } = await getIdentity({ name: "js-e2e-batch" });
16
- client = new Firecrawl({ apiKey, apiUrl: API_URL });
17
+ client = withRateLimitRetry(new Firecrawl({ apiKey, apiUrl: API_URL }));
17
18
  });
18
19
 
19
20
  describe("v2.batch e2e", () => {
@@ -22,19 +23,21 @@ describe("v2.batch e2e", () => {
22
23
  "https://docs.firecrawl.dev",
23
24
  "https://firecrawl.dev",
24
25
  ];
25
- const job = await client.batchScrape(urls, { options: { formats: ["markdown"] }, pollInterval: 1, timeout: 180 });
26
+ const start = await client.startBatchScrape(urls, { options: { formats: ["markdown"] } });
27
+ const job = await waitForJob(() => client.getBatchScrapeStatus(start.id), { pollInterval: 1, timeout: 210 });
26
28
  expect(["completed", "failed"]).toContain(job.status);
27
29
  expect(job.completed).toBeGreaterThanOrEqual(0);
28
30
  expect(job.total).toBeGreaterThanOrEqual(0);
29
31
  expect(Array.isArray(job.data)).toBe(true);
30
- }, 240_000);
32
+ }, testTimeoutMs(362_000));
31
33
 
32
34
  test("batch scrape with wait returns job id for error retrieval", async () => {
33
35
  const urls = [
34
36
  "https://docs.firecrawl.dev",
35
37
  "https://firecrawl.dev",
36
38
  ];
37
- const job = await client.batchScrape(urls, { options: { formats: ["markdown"] }, pollInterval: 1, timeout: 180 });
39
+ const start = await client.startBatchScrape(urls, { options: { formats: ["markdown"] } });
40
+ const job = await waitForJob(() => client.getBatchScrapeStatus(start.id), { pollInterval: 1, timeout: 210 });
38
41
  // Verify job has id field
39
42
  expect(job.id).toBeDefined();
40
43
  expect(typeof job.id).toBe("string");
@@ -44,7 +47,7 @@ describe("v2.batch e2e", () => {
44
47
  expect(errors).toHaveProperty("robotsBlocked");
45
48
  expect(Array.isArray(errors.errors)).toBe(true);
46
49
  expect(Array.isArray(errors.robotsBlocked)).toBe(true);
47
- }, 240_000);
50
+ }, testTimeoutMs(512_000));
48
51
 
49
52
  test("start batch minimal and status", async () => {
50
53
  const urls = ["https://docs.firecrawl.dev", "https://firecrawl.dev"];
@@ -57,11 +60,11 @@ describe("v2.batch e2e", () => {
57
60
  // Verify status includes id field
58
61
  expect(status.id).toBeDefined();
59
62
  expect(status.id).toBe(start.id);
60
- }, 120_000);
63
+ }, testTimeoutMs(151_000));
61
64
 
62
65
  test("wait batch with all params", async () => {
63
66
  const urls = ["https://docs.firecrawl.dev", "https://firecrawl.dev"];
64
- const job = await client.batchScrape(urls, {
67
+ const start = await client.startBatchScrape(urls, {
65
68
  options: {
66
69
  formats: [
67
70
  "markdown",
@@ -74,14 +77,13 @@ describe("v2.batch e2e", () => {
74
77
  ignoreInvalidURLs: true,
75
78
  maxConcurrency: 2,
76
79
  zeroDataRetention: false,
77
- pollInterval: 1,
78
- timeout: 180,
79
80
  });
81
+ const job = await waitForJob(() => client.getBatchScrapeStatus(start.id), { pollInterval: 1, timeout: 210 });
80
82
  expect(["completed", "failed", "cancelled"]).toContain(job.status);
81
83
  expect(job.completed).toBeGreaterThanOrEqual(0);
82
84
  expect(job.total).toBeGreaterThanOrEqual(0);
83
85
  expect(Array.isArray(job.data)).toBe(true);
84
- }, 300_000);
86
+ }, testTimeoutMs(362_000));
85
87
 
86
88
  test("cancel batch", async () => {
87
89
  const urls = ["https://docs.firecrawl.dev", "https://firecrawl.dev"];
@@ -89,6 +91,6 @@ describe("v2.batch e2e", () => {
89
91
  expect(typeof start.id).toBe("string");
90
92
  const cancelled = await client.cancelBatchScrape(start.id);
91
93
  expect(cancelled).toBe(true);
92
- }, 120_000);
94
+ }, testTimeoutMs(151_000));
93
95
  });
94
96
 
@@ -4,6 +4,7 @@
4
4
  import Firecrawl from "../../../index";
5
5
  import { config } from "dotenv";
6
6
  import { getIdentity, getApiUrl } from "./utils/idmux";
7
+ import { testTimeoutMs, waitForJob, withRateLimitRetry } from "./utils/rateLimit";
7
8
  import { describe, test, expect, beforeAll } from "@jest/globals";
8
9
 
9
10
  config();
@@ -13,7 +14,7 @@ let client: Firecrawl;
13
14
 
14
15
  beforeAll(async () => {
15
16
  const { apiKey } = await getIdentity({ name: "js-e2e-crawl" });
16
- client = new Firecrawl({ apiKey, apiUrl: API_URL });
17
+ client = withRateLimitRetry(new Firecrawl({ apiKey, apiUrl: API_URL }));
17
18
  });
18
19
 
19
20
  describe("v2.crawl e2e", () => {
@@ -23,21 +24,21 @@ describe("v2.crawl e2e", () => {
23
24
  const job = await client.startCrawl("https://docs.firecrawl.dev", { limit: 3 });
24
25
  expect(typeof job.id).toBe("string");
25
26
  expect(typeof job.url).toBe("string");
26
- }, 90_000);
27
+ }, testTimeoutMs(90_000));
27
28
 
28
29
  test("start crawl with options", async () => {
29
30
  if (!client) throw new Error();
30
31
  const job = await client.startCrawl("https://docs.firecrawl.dev", { limit: 5, maxDiscoveryDepth: 2 });
31
32
  expect(typeof job.id).toBe("string");
32
33
  expect(typeof job.url).toBe("string");
33
- }, 90_000);
34
+ }, testTimeoutMs(90_000));
34
35
 
35
36
  test("start crawl with prompt", async () => {
36
37
  if (!client) throw new Error();
37
38
  const job = await client.startCrawl("https://firecrawl.dev", { prompt: "Extract all blog posts", limit: 3 });
38
39
  expect(typeof job.id).toBe("string");
39
40
  expect(typeof job.url).toBe("string");
40
- }, 90_000);
41
+ }, testTimeoutMs(90_000));
41
42
 
42
43
  test("get crawl status", async () => {
43
44
  if (!client) throw new Error();
@@ -50,14 +51,14 @@ describe("v2.crawl e2e", () => {
50
51
  expect(status.id).toBe(start.id);
51
52
  // next/expiresAt may be null/undefined depending on state; check shape
52
53
  expect(Array.isArray(status.data)).toBe(true);
53
- }, 120_000);
54
+ }, testTimeoutMs(151_000));
54
55
 
55
56
  test("cancel crawl", async () => {
56
57
  if (!client) throw new Error();
57
58
  const start = await client.startCrawl("https://docs.firecrawl.dev", { limit: 3 });
58
59
  const ok = await client.cancelCrawl(start.id);
59
60
  expect(ok).toBe(true);
60
- }, 120_000);
61
+ }, testTimeoutMs(151_000));
61
62
 
62
63
  test("get crawl errors", async () => {
63
64
  if (!client) throw new Error();
@@ -73,12 +74,12 @@ describe("v2.crawl e2e", () => {
73
74
  expect(typeof e.url === "string" || e.url == null).toBe(true);
74
75
  expect(typeof e.error === "string" || e.error == null).toBe(true);
75
76
  }
76
- }, 120_000);
77
+ }, testTimeoutMs(151_000));
77
78
 
78
79
  test("get crawl errors with invalid id should throw", async () => {
79
80
  if (!client) throw new Error();
80
81
  await expect(client.getCrawlErrors("invalid-job-id-12345")).rejects.toThrow();
81
- }, 60_000);
82
+ }, testTimeoutMs(60_000));
82
83
 
83
84
  test("get active crawls", async () => {
84
85
  if (!client) throw new Error();
@@ -93,7 +94,7 @@ describe("v2.crawl e2e", () => {
93
94
  expect(typeof c.options === "object").toBe(true);
94
95
  }
95
96
  }
96
- }, 90_000);
97
+ }, testTimeoutMs(90_000));
97
98
 
98
99
  test("get active crawls with running crawl", async () => {
99
100
  if (!client) throw new Error();
@@ -104,20 +105,22 @@ describe("v2.crawl e2e", () => {
104
105
  const ids = active.crawls.map(c => c.id);
105
106
  expect(ids.includes(start.id)).toBe(true);
106
107
  await client.cancelCrawl(start.id);
107
- }, 120_000);
108
+ }, testTimeoutMs(301_000));
108
109
 
109
110
  test("crawl with wait", async () => {
110
111
  if (!client) throw new Error();
111
- const job = await client.crawl("https://docs.firecrawl.dev", { limit: 3, maxDiscoveryDepth: 2, pollInterval: 1, timeout: 120 });
112
+ const start = await client.startCrawl("https://docs.firecrawl.dev", { limit: 3, maxDiscoveryDepth: 2 });
113
+ const job = await waitForJob(() => client.getCrawlStatus(start.id), { pollInterval: 1, timeout: 120 });
112
114
  expect(["completed", "failed"]).toContain(job.status);
113
115
  expect(job.completed).toBeGreaterThanOrEqual(0);
114
116
  expect(job.total).toBeGreaterThanOrEqual(0);
115
117
  expect(Array.isArray(job.data)).toBe(true);
116
- }, 180_000);
118
+ }, testTimeoutMs(272_000));
117
119
 
118
120
  test("crawl with wait returns job id for error retrieval", async () => {
119
121
  if (!client) throw new Error();
120
- const job = await client.crawl("https://docs.firecrawl.dev", { limit: 3, maxDiscoveryDepth: 2, pollInterval: 1, timeout: 120 });
122
+ const start = await client.startCrawl("https://docs.firecrawl.dev", { limit: 3, maxDiscoveryDepth: 2 });
123
+ const job = await waitForJob(() => client.getCrawlStatus(start.id), { pollInterval: 1, timeout: 120 });
121
124
  // Verify job has id field
122
125
  expect(job.id).toBeDefined();
123
126
  expect(typeof job.id).toBe("string");
@@ -127,16 +130,17 @@ describe("v2.crawl e2e", () => {
127
130
  expect(errors).toHaveProperty("robotsBlocked");
128
131
  expect(Array.isArray(errors.errors)).toBe(true);
129
132
  expect(Array.isArray(errors.robotsBlocked)).toBe(true);
130
- }, 180_000);
133
+ }, testTimeoutMs(422_000));
131
134
 
132
135
  test("crawl with prompt and wait", async () => {
133
136
  if (!client) throw new Error();
134
- const job = await client.crawl("https://docs.firecrawl.dev", { prompt: "Extract all blog posts", limit: 3, pollInterval: 1, timeout: 120 });
137
+ const start = await client.startCrawl("https://docs.firecrawl.dev", { prompt: "Extract all blog posts", limit: 3 });
138
+ const job = await waitForJob(() => client.getCrawlStatus(start.id), { pollInterval: 1, timeout: 120 });
135
139
  expect(["completed", "failed"]).toContain(job.status);
136
140
  expect(job.completed).toBeGreaterThanOrEqual(0);
137
141
  expect(job.total).toBeGreaterThanOrEqual(0);
138
142
  expect(Array.isArray(job.data)).toBe(true);
139
- }, 180_000);
143
+ }, testTimeoutMs(272_000));
140
144
 
141
145
  test("crawl with scrape options", async () => {
142
146
  if (!client) throw new Error();
@@ -145,7 +149,7 @@ describe("v2.crawl e2e", () => {
145
149
  scrapeOptions: { formats: ["markdown", "links"], onlyMainContent: false, mobile: true },
146
150
  });
147
151
  expect(typeof job.id).toBe("string");
148
- }, 120_000);
152
+ }, testTimeoutMs(120_000));
149
153
 
150
154
  test("crawl with json format object", async () => {
151
155
  if (!client) throw new Error();
@@ -154,7 +158,7 @@ describe("v2.crawl e2e", () => {
154
158
  scrapeOptions: { formats: [{ type: "json", prompt: "Extract page title", schema: { type: "object", properties: { title: { type: "string" } }, required: ["title"] } }] },
155
159
  });
156
160
  expect(typeof job.id).toBe("string");
157
- }, 120_000);
161
+ }, testTimeoutMs(120_000));
158
162
 
159
163
  test("crawl all parameters", async () => {
160
164
  if (!client) throw new Error();
@@ -187,13 +191,13 @@ describe("v2.crawl e2e", () => {
187
191
  zeroDataRetention: false,
188
192
  });
189
193
  expect(typeof job.id).toBe("string");
190
- }, 180_000);
194
+ }, testTimeoutMs(180_000));
191
195
 
192
196
  test("crawl params preview", async () => {
193
197
  if (!client) throw new Error();
194
198
  const params = await client.crawlParamsPreview("https://docs.firecrawl.dev", "Extract all blog posts and documentation");
195
199
  expect(params && typeof params === "object").toBe(true);
196
200
  // Optional fields may or may not be present; just assert object shape
197
- }, 60_000);
201
+ }, testTimeoutMs(60_000));
198
202
  });
199
203