firecrawl 4.42.2 → 4.42.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -12,7 +12,7 @@ var require_package = __commonJS({
12
12
  "package.json"(exports, module) {
13
13
  module.exports = {
14
14
  name: "@mendable/firecrawl-js",
15
- version: "4.42.2",
15
+ version: "4.42.3",
16
16
  description: "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
17
17
  main: "dist/index.js",
18
18
  types: "dist/index.d.ts",
@@ -29,7 +29,7 @@ var require_package = __commonJS({
29
29
  "build-and-publish": "pnpm run build && pnpm publish --access public",
30
30
  "publish-beta": "pnpm run build && pnpm publish --access public --tag beta",
31
31
  test: "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/e2e/v2/*.test.ts --detectOpenHandles",
32
- "test:unit": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/unit/v2/*.test.ts"
32
+ "test:unit": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/unit"
33
33
  },
34
34
  repository: {
35
35
  type: "git",
package/dist/index.cjs CHANGED
@@ -39,7 +39,7 @@ var require_package = __commonJS({
39
39
  "package.json"(exports2, module2) {
40
40
  module2.exports = {
41
41
  name: "@mendable/firecrawl-js",
42
- version: "4.42.2",
42
+ version: "4.42.3",
43
43
  description: "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
44
44
  main: "dist/index.js",
45
45
  types: "dist/index.d.ts",
@@ -56,7 +56,7 @@ var require_package = __commonJS({
56
56
  "build-and-publish": "pnpm run build && pnpm publish --access public",
57
57
  "publish-beta": "pnpm run build && pnpm publish --access public --tag beta",
58
58
  test: "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/e2e/v2/*.test.ts --detectOpenHandles",
59
- "test:unit": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/unit/v2/*.test.ts"
59
+ "test:unit": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/unit"
60
60
  },
61
61
  repository: {
62
62
  type: "git",
@@ -3233,7 +3233,7 @@ var FirecrawlApp = class {
3233
3233
  );
3234
3234
  if (response.status === 200) {
3235
3235
  const id = response.data.id;
3236
- return this.monitorJobStatus(id, headers, pollInterval);
3236
+ return this.monitorJobStatus(id, headers, pollInterval, "batch");
3237
3237
  } else {
3238
3238
  this.handleError(response, "start batch scrape job");
3239
3239
  }
@@ -3537,21 +3537,23 @@ var FirecrawlApp = class {
3537
3537
  }
3538
3538
  }
3539
3539
  /**
3540
- * Monitors the status of a crawl job until completion or failure.
3541
- * @param id - The ID of the crawl operation.
3540
+ * Monitors the status of a crawl or batch scrape job until completion or failure.
3541
+ * @param id - The ID of the crawl or batch scrape operation.
3542
3542
  * @param headers - The headers for the request.
3543
3543
  * @param checkInterval - Interval in seconds for job status checks.
3544
- * @param checkUrl - Optional URL to check the status (used for v1 API)
3544
+ * @param jobType - Which status endpoint to poll. Defaults to `"crawl"`.
3545
3545
  * @returns The final job status or data.
3546
3546
  */
3547
- async monitorJobStatus(id, headers, checkInterval) {
3547
+ async monitorJobStatus(id, headers, checkInterval, jobType = "crawl") {
3548
+ const statusPath = jobType === "batch" ? "batch/scrape" : "crawl";
3549
+ const jobLabel = jobType === "batch" ? "Batch scrape" : "Crawl";
3548
3550
  let failedTries = 0;
3549
3551
  let networkRetries = 0;
3550
3552
  const maxNetworkRetries = 3;
3551
3553
  while (true) {
3552
3554
  try {
3553
3555
  let statusResponse = await this.getRequest(
3554
- `${this.apiUrl}/v1/crawl/${id}`,
3556
+ `${this.apiUrl}/v1/${statusPath}/${id}`,
3555
3557
  headers
3556
3558
  );
3557
3559
  if (statusResponse.status === 200) {
@@ -3572,7 +3574,7 @@ var FirecrawlApp = class {
3572
3574
  statusData.data = data;
3573
3575
  return statusData;
3574
3576
  } else {
3575
- throw new FirecrawlError("Crawl job completed but no data was returned", 500);
3577
+ throw new FirecrawlError(`${jobLabel} job completed but no data was returned`, 500);
3576
3578
  }
3577
3579
  } else if (["active", "paused", "pending", "queued", "waiting", "scraping"].includes(statusData.status)) {
3578
3580
  checkInterval = Math.max(checkInterval, 2);
@@ -3581,14 +3583,14 @@ var FirecrawlApp = class {
3581
3583
  );
3582
3584
  } else {
3583
3585
  throw new FirecrawlError(
3584
- `Crawl job failed or was stopped. Status: ${statusData.status}`,
3586
+ `${jobLabel} job failed or was stopped. Status: ${statusData.status}`,
3585
3587
  500
3586
3588
  );
3587
3589
  }
3588
3590
  } else {
3589
3591
  failedTries++;
3590
3592
  if (failedTries >= 3) {
3591
- this.handleError(statusResponse, "check crawl status");
3593
+ this.handleError(statusResponse, `check ${jobLabel.toLowerCase()} status`);
3592
3594
  }
3593
3595
  }
3594
3596
  } catch (error) {
package/dist/index.d.cts CHANGED
@@ -3388,14 +3388,14 @@ declare class FirecrawlApp {
3388
3388
  */
3389
3389
  deleteRequest(url: string, headers: AxiosRequestHeaders): Promise<AxiosResponse>;
3390
3390
  /**
3391
- * Monitors the status of a crawl job until completion or failure.
3392
- * @param id - The ID of the crawl operation.
3391
+ * Monitors the status of a crawl or batch scrape job until completion or failure.
3392
+ * @param id - The ID of the crawl or batch scrape operation.
3393
3393
  * @param headers - The headers for the request.
3394
3394
  * @param checkInterval - Interval in seconds for job status checks.
3395
- * @param checkUrl - Optional URL to check the status (used for v1 API)
3395
+ * @param jobType - Which status endpoint to poll. Defaults to `"crawl"`.
3396
3396
  * @returns The final job status or data.
3397
3397
  */
3398
- monitorJobStatus(id: string, headers: AxiosRequestHeaders, checkInterval: number): Promise<CrawlStatusResponse | ErrorResponse>;
3398
+ monitorJobStatus(id: string, headers: AxiosRequestHeaders, checkInterval: number, jobType?: "crawl" | "batch"): Promise<CrawlStatusResponse | ErrorResponse>;
3399
3399
  /**
3400
3400
  * Determines if an error is retryable (transient network error)
3401
3401
  * @param error - The error to check
package/dist/index.d.ts CHANGED
@@ -3388,14 +3388,14 @@ declare class FirecrawlApp {
3388
3388
  */
3389
3389
  deleteRequest(url: string, headers: AxiosRequestHeaders): Promise<AxiosResponse>;
3390
3390
  /**
3391
- * Monitors the status of a crawl job until completion or failure.
3392
- * @param id - The ID of the crawl operation.
3391
+ * Monitors the status of a crawl or batch scrape job until completion or failure.
3392
+ * @param id - The ID of the crawl or batch scrape operation.
3393
3393
  * @param headers - The headers for the request.
3394
3394
  * @param checkInterval - Interval in seconds for job status checks.
3395
- * @param checkUrl - Optional URL to check the status (used for v1 API)
3395
+ * @param jobType - Which status endpoint to poll. Defaults to `"crawl"`.
3396
3396
  * @returns The final job status or data.
3397
3397
  */
3398
- monitorJobStatus(id: string, headers: AxiosRequestHeaders, checkInterval: number): Promise<CrawlStatusResponse | ErrorResponse>;
3398
+ monitorJobStatus(id: string, headers: AxiosRequestHeaders, checkInterval: number, jobType?: "crawl" | "batch"): Promise<CrawlStatusResponse | ErrorResponse>;
3399
3399
  /**
3400
3400
  * Determines if an error is retryable (transient network error)
3401
3401
  * @param error - The error to check
package/dist/index.js CHANGED
@@ -1,6 +1,6 @@
1
1
  import {
2
2
  require_package
3
- } from "./chunk-XZ5XI5DK.js";
3
+ } from "./chunk-AMV6GR2H.js";
4
4
 
5
5
  // src/v2/utils/httpClient.ts
6
6
  import axios from "axios";
@@ -2706,7 +2706,7 @@ var FirecrawlApp = class {
2706
2706
  if (typeof process !== "undefined" && process.env && process.env.npm_package_version) {
2707
2707
  return process.env.npm_package_version;
2708
2708
  }
2709
- const packageJson = await import("./package-JVVNUZPT.js");
2709
+ const packageJson = await import("./package-RQDQRVLO.js");
2710
2710
  return packageJson.default.version;
2711
2711
  } catch (error) {
2712
2712
  const isTest = typeof process !== "undefined" && (process.env.JEST_WORKER_ID != null || false);
@@ -3101,7 +3101,7 @@ var FirecrawlApp = class {
3101
3101
  );
3102
3102
  if (response.status === 200) {
3103
3103
  const id = response.data.id;
3104
- return this.monitorJobStatus(id, headers, pollInterval);
3104
+ return this.monitorJobStatus(id, headers, pollInterval, "batch");
3105
3105
  } else {
3106
3106
  this.handleError(response, "start batch scrape job");
3107
3107
  }
@@ -3405,21 +3405,23 @@ var FirecrawlApp = class {
3405
3405
  }
3406
3406
  }
3407
3407
  /**
3408
- * Monitors the status of a crawl job until completion or failure.
3409
- * @param id - The ID of the crawl operation.
3408
+ * Monitors the status of a crawl or batch scrape job until completion or failure.
3409
+ * @param id - The ID of the crawl or batch scrape operation.
3410
3410
  * @param headers - The headers for the request.
3411
3411
  * @param checkInterval - Interval in seconds for job status checks.
3412
- * @param checkUrl - Optional URL to check the status (used for v1 API)
3412
+ * @param jobType - Which status endpoint to poll. Defaults to `"crawl"`.
3413
3413
  * @returns The final job status or data.
3414
3414
  */
3415
- async monitorJobStatus(id, headers, checkInterval) {
3415
+ async monitorJobStatus(id, headers, checkInterval, jobType = "crawl") {
3416
+ const statusPath = jobType === "batch" ? "batch/scrape" : "crawl";
3417
+ const jobLabel = jobType === "batch" ? "Batch scrape" : "Crawl";
3416
3418
  let failedTries = 0;
3417
3419
  let networkRetries = 0;
3418
3420
  const maxNetworkRetries = 3;
3419
3421
  while (true) {
3420
3422
  try {
3421
3423
  let statusResponse = await this.getRequest(
3422
- `${this.apiUrl}/v1/crawl/${id}`,
3424
+ `${this.apiUrl}/v1/${statusPath}/${id}`,
3423
3425
  headers
3424
3426
  );
3425
3427
  if (statusResponse.status === 200) {
@@ -3440,7 +3442,7 @@ var FirecrawlApp = class {
3440
3442
  statusData.data = data;
3441
3443
  return statusData;
3442
3444
  } else {
3443
- throw new FirecrawlError("Crawl job completed but no data was returned", 500);
3445
+ throw new FirecrawlError(`${jobLabel} job completed but no data was returned`, 500);
3444
3446
  }
3445
3447
  } else if (["active", "paused", "pending", "queued", "waiting", "scraping"].includes(statusData.status)) {
3446
3448
  checkInterval = Math.max(checkInterval, 2);
@@ -3449,14 +3451,14 @@ var FirecrawlApp = class {
3449
3451
  );
3450
3452
  } else {
3451
3453
  throw new FirecrawlError(
3452
- `Crawl job failed or was stopped. Status: ${statusData.status}`,
3454
+ `${jobLabel} job failed or was stopped. Status: ${statusData.status}`,
3453
3455
  500
3454
3456
  );
3455
3457
  }
3456
3458
  } else {
3457
3459
  failedTries++;
3458
3460
  if (failedTries >= 3) {
3459
- this.handleError(statusResponse, "check crawl status");
3461
+ this.handleError(statusResponse, `check ${jobLabel.toLowerCase()} status`);
3460
3462
  }
3461
3463
  }
3462
3464
  } catch (error) {
@@ -1,4 +1,4 @@
1
1
  import {
2
2
  require_package
3
- } from "./chunk-XZ5XI5DK.js";
3
+ } from "./chunk-AMV6GR2H.js";
4
4
  export default require_package();
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "firecrawl",
3
- "version": "4.42.2",
3
+ "version": "4.42.3",
4
4
  "description": "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
5
5
  "main": "dist/index.js",
6
6
  "types": "dist/index.d.ts",
@@ -73,6 +73,6 @@
73
73
  "build-and-publish": "pnpm run build && pnpm publish --access public",
74
74
  "publish-beta": "pnpm run build && pnpm publish --access public --tag beta",
75
75
  "test": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/e2e/v2/*.test.ts --detectOpenHandles",
76
- "test:unit": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/unit/v2/*.test.ts"
76
+ "test:unit": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/unit"
77
77
  }
78
78
  }
@@ -0,0 +1,91 @@
1
+ import { describe, test, expect, jest, afterEach, beforeEach } from "@jest/globals";
2
+ import axios from "axios";
3
+ import FirecrawlApp from "../../../v1";
4
+
5
+ const API_URL = "https://api.firecrawl.dev";
6
+ const API_KEY = "fc-test";
7
+
8
+ const scrapingPage = () => ({ success: true, status: "scraping", total: 2, completed: 0, data: [] });
9
+ const completedPage = (next: string) => ({
10
+ success: true,
11
+ status: "completed",
12
+ total: 2,
13
+ completed: 2,
14
+ expiresAt: "2030-01-01T00:00:00Z",
15
+ data: [{ markdown: "a" }],
16
+ next,
17
+ });
18
+ const lastPage = () => ({ success: true, status: "completed", data: [{ markdown: "b" }] });
19
+
20
+ function mockPoll(path: string, nextOrigin = API_URL) {
21
+ const sent: string[] = [];
22
+ let polls = 0;
23
+ jest.spyOn(axios, "get").mockImplementation(async (url: string) => {
24
+ sent.push(url);
25
+ if (new URL(url).searchParams.has("skip")) return { status: 200, data: lastPage() };
26
+ polls++;
27
+ return { status: 200, data: polls === 1 ? scrapingPage() : completedPage(`${nextOrigin}${path}?skip=1`) };
28
+ });
29
+ return sent;
30
+ }
31
+
32
+ describe("v1 batch scrape waits on the batch scrape status endpoint", () => {
33
+ const app = new FirecrawlApp({ apiKey: API_KEY, apiUrl: API_URL });
34
+
35
+ beforeEach(() => {
36
+ jest.spyOn(global, "setTimeout").mockImplementation(((fn: () => void) => {
37
+ fn();
38
+ return 0;
39
+ }) as any);
40
+ });
41
+
42
+ afterEach(() => {
43
+ jest.restoreAllMocks();
44
+ });
45
+
46
+ test("batchScrapeUrls polls /v1/batch/scrape/{id} and follows pinned next pages", async () => {
47
+ const post = jest.spyOn(axios, "post").mockResolvedValue({ status: 200, data: { success: true, id: "abc" } });
48
+ const path = "/v1/batch/scrape/abc";
49
+ const sent = mockPoll(path, "https://evil.example");
50
+
51
+ const res: any = await app.batchScrapeUrls(["https://example.com", "https://example.org"], {});
52
+
53
+ expect(post.mock.calls[0][0]).toBe(`${API_URL}/v1/batch/scrape`);
54
+ expect(sent).toEqual([`${API_URL}${path}`, `${API_URL}${path}`, `${API_URL}${path}?skip=1`]);
55
+ expect(sent.some(url => url.includes("/v1/crawl/"))).toBe(false);
56
+ expect(res.status).toBe("completed");
57
+ expect(res.data).toHaveLength(2);
58
+ });
59
+
60
+ test("batchScrapeUrls reports batch scrape failures as batch scrape errors", async () => {
61
+ jest.spyOn(axios, "post").mockResolvedValue({ status: 200, data: { success: true, id: "abc" } });
62
+ const sent: string[] = [];
63
+ jest.spyOn(axios, "get").mockImplementation(async (url: string) => {
64
+ sent.push(url);
65
+ return { status: 200, data: { success: true, status: "failed", data: [] } };
66
+ });
67
+
68
+ await expect(app.batchScrapeUrls(["https://example.com"], {})).rejects.toThrow("Batch scrape job failed or was stopped");
69
+ expect(sent.every(url => url === `${API_URL}/v1/batch/scrape/abc`)).toBe(true);
70
+ });
71
+
72
+ test("monitorJobStatus still polls /v1/crawl/{id} by default", async () => {
73
+ const path = "/v1/crawl/abc";
74
+ const sent = mockPoll(path);
75
+
76
+ const res: any = await app.monitorJobStatus("abc", app.prepareHeaders(), 0);
77
+
78
+ expect(sent).toEqual([`${API_URL}${path}`, `${API_URL}${path}`, `${API_URL}${path}?skip=1`]);
79
+ expect(res.data).toHaveLength(2);
80
+ });
81
+
82
+ test("monitorJobStatus polls /v1/batch/scrape/{id} for batch jobs", async () => {
83
+ const path = "/v1/batch/scrape/abc";
84
+ const sent = mockPoll(path);
85
+
86
+ const res: any = await app.monitorJobStatus("abc", app.prepareHeaders(), 0, "batch");
87
+
88
+ expect(sent).toEqual([`${API_URL}${path}`, `${API_URL}${path}`, `${API_URL}${path}?skip=1`]);
89
+ expect(res.data).toHaveLength(2);
90
+ });
91
+ });
@@ -1,154 +1,107 @@
1
- import FirecrawlApp from '../../../index';
1
+ import FirecrawlApp from '../../../v1';
2
2
  import { describe, test, expect, jest, beforeEach, afterEach } from '@jest/globals';
3
3
 
4
+ const successResponse = {
5
+ status: 200,
6
+ data: { status: 'completed', data: [{ url: 'test.com', markdown: 'test' }] },
7
+ };
8
+
9
+ function networkError(message: string, code: string) {
10
+ return Object.assign(new Error(message), { code });
11
+ }
12
+
13
+ function httpError(message: string, status: number) {
14
+ return Object.assign(new Error(message), { response: { status, data: { error: message } } });
15
+ }
16
+
4
17
  describe('monitorJobStatus retry logic', () => {
5
18
  let app: FirecrawlApp;
6
- let originalConsoleWarn: typeof console.warn;
7
-
19
+ let delays: number[];
20
+
8
21
  beforeEach(() => {
9
22
  app = new FirecrawlApp({ apiKey: 'test-key', apiUrl: 'https://test.com' });
10
- originalConsoleWarn = console.warn;
11
- console.warn = jest.fn();
23
+ delays = [];
24
+ jest.useFakeTimers();
25
+ const fakeSetTimeout = globalThis.setTimeout;
26
+ jest.spyOn(globalThis, 'setTimeout').mockImplementation(((fn: () => void, ms?: number) => {
27
+ delays.push(ms ?? 0);
28
+ return fakeSetTimeout(fn, ms);
29
+ }) as any);
12
30
  });
13
31
 
14
32
  afterEach(() => {
15
- console.warn = originalConsoleWarn;
16
- jest.clearAllMocks();
33
+ jest.restoreAllMocks();
34
+ jest.useRealTimers();
17
35
  });
18
36
 
19
- test('should retry on socket hang up error', async () => {
20
- const socketHangUpError = new Error('socket hang up') as any;
21
- socketHangUpError.code = 'ECONNRESET';
22
-
23
- const successResponse = {
24
- status: 200,
25
- data: { status: 'completed', data: [{ url: 'test.com', markdown: 'test' }] }
26
- };
27
-
28
- const originalGetRequest = app.getRequest;
29
- let callCount = 0;
30
-
31
- app.getRequest = async function(url: string, headers: any) {
32
- callCount++;
33
- if (callCount === 1) {
34
- throw socketHangUpError;
35
- }
36
- return successResponse;
37
- };
38
-
39
- const result = await app.monitorJobStatus('test-id', {}, 1);
40
-
41
- expect(callCount).toBe(2);
37
+ function failFirst(n: number, error: Error, statuses: string[] = []) {
38
+ let calls = 0;
39
+ app.getRequest = (async () => {
40
+ calls++;
41
+ if (calls <= n) throw error;
42
+ const status = statuses.shift();
43
+ return status ? { status: 200, data: { status } } : successResponse;
44
+ }) as any;
45
+ return () => calls;
46
+ }
47
+
48
+ async function monitor() {
49
+ const result = app.monitorJobStatus('test-id', {} as any, 1);
50
+ result.catch(() => {});
51
+ await jest.runAllTimersAsync();
52
+ return result;
53
+ }
54
+
55
+ test.each([
56
+ ['socket hang up', networkError('socket hang up', 'ECONNRESET')],
57
+ ['ETIMEDOUT', networkError('timeout', 'ETIMEDOUT')],
58
+ ['HTTP 408', httpError('Request timeout', 408)],
59
+ ['HTTP 504', httpError('Gateway timeout', 504)],
60
+ ])('retries once after a %s error and returns the job data', async (_label, error) => {
61
+ const calls = failFirst(1, error);
62
+
63
+ const result = await monitor();
64
+
65
+ expect(calls()).toBe(2);
42
66
  expect(result).toEqual(successResponse.data);
43
- expect(console.warn).toHaveBeenCalledWith(
44
- expect.stringContaining('Network error during job status check (attempt 1/3): socket hang up')
45
- );
67
+ expect(delays).toEqual([1000]);
46
68
  });
47
69
 
48
- test('should retry on ETIMEDOUT error', async () => {
49
- const timeoutError = new Error('timeout') as any;
50
- timeoutError.code = 'ETIMEDOUT';
51
-
52
- const successResponse = {
53
- status: 200,
54
- data: { status: 'completed', data: [{ url: 'test.com', markdown: 'test' }] }
55
- };
56
-
57
- const originalGetRequest = app.getRequest;
58
- let callCount = 0;
59
-
60
- app.getRequest = async function(url: string, headers: any) {
61
- callCount++;
62
- if (callCount === 1) {
63
- throw timeoutError;
64
- }
65
- return successResponse;
66
- };
67
-
68
- const result = await app.monitorJobStatus('test-id', {}, 1);
69
-
70
- expect(callCount).toBe(2);
70
+ test('uses exponential backoff between retries', async () => {
71
+ const calls = failFirst(2, networkError('socket hang up', 'ECONNRESET'));
72
+
73
+ const result = await monitor();
74
+
75
+ expect(calls()).toBe(3);
71
76
  expect(result).toEqual(successResponse.data);
77
+ expect(delays).toEqual([1000, 2000]);
72
78
  });
73
79
 
74
- test('should fail after max retries exceeded', async () => {
75
- const socketHangUpError = new Error('socket hang up') as any;
76
- socketHangUpError.code = 'ECONNRESET';
77
-
78
- app.getRequest = async function(url: string, headers: any) {
79
- throw socketHangUpError;
80
- };
81
-
82
- await expect(app.monitorJobStatus('test-id', {}, 1)).rejects.toThrow('socket hang up');
83
-
84
- expect(console.warn).toHaveBeenCalledTimes(3);
85
- }, 15000);
86
-
87
- test('should not retry on non-retryable errors', async () => {
88
- const authError = new Error('Unauthorized') as any;
89
- authError.response = { status: 401, data: { error: 'Unauthorized' } };
90
-
91
- app.getRequest = async function(url: string, headers: any) {
92
- throw authError;
93
- };
94
-
95
- await expect(app.monitorJobStatus('test-id', {}, 1)).rejects.toThrow('Unauthorized');
96
-
97
- expect(console.warn).not.toHaveBeenCalled();
98
- });
80
+ test('records both the retry backoff and the 2s poll wait for an active job', async () => {
81
+ const calls = failFirst(1, networkError('socket hang up', 'ECONNRESET'), ['active']);
82
+
83
+ const result = await monitor();
99
84
 
100
- test('should retry on HTTP timeout status codes', async () => {
101
- const timeoutError = new Error('Request timeout') as any;
102
- timeoutError.response = { status: 408, data: { error: 'Request timeout' } };
103
-
104
- const successResponse = {
105
- status: 200,
106
- data: { status: 'completed', data: [{ url: 'test.com', markdown: 'test' }] }
107
- };
108
-
109
- const originalGetRequest = app.getRequest;
110
- let callCount = 0;
111
-
112
- app.getRequest = async function(url: string, headers: any) {
113
- callCount++;
114
- if (callCount === 1) {
115
- throw timeoutError;
116
- }
117
- return successResponse;
118
- };
119
-
120
- const result = await app.monitorJobStatus('test-id', {}, 1);
121
-
122
- expect(callCount).toBe(2);
85
+ expect(calls()).toBe(3);
123
86
  expect(result).toEqual(successResponse.data);
87
+ expect(delays).toEqual([1000, 2000]);
124
88
  });
125
89
 
126
- test('should use exponential backoff for retries', async () => {
127
- const socketHangUpError = new Error('socket hang up') as any;
128
- socketHangUpError.code = 'ECONNRESET';
129
-
130
- const successResponse = {
131
- status: 200,
132
- data: { status: 'completed', data: [{ url: 'test.com', markdown: 'test' }] }
133
- };
134
-
135
- const originalGetRequest = app.getRequest;
136
- let callCount = 0;
137
-
138
- app.getRequest = async function(url: string, headers: any) {
139
- callCount++;
140
- if (callCount <= 2) {
141
- throw socketHangUpError;
142
- }
143
- return successResponse;
144
- };
145
-
146
- const startTime = Date.now();
147
- const result = await app.monitorJobStatus('test-id', {}, 1);
148
- const endTime = Date.now();
149
-
150
- expect(callCount).toBe(3);
151
- expect(result).toEqual(successResponse.data);
152
- expect(endTime - startTime).toBeGreaterThan(3000);
90
+ test('fails after max retries are exceeded', async () => {
91
+ const calls = failFirst(Infinity, networkError('socket hang up', 'ECONNRESET'));
92
+
93
+ await expect(monitor()).rejects.toThrow('socket hang up');
94
+
95
+ expect(calls()).toBe(4);
96
+ expect(delays).toEqual([1000, 2000, 4000]);
97
+ });
98
+
99
+ test('does not retry non-retryable errors', async () => {
100
+ const calls = failFirst(Infinity, httpError('Unauthorized', 401));
101
+
102
+ await expect(monitor()).rejects.toThrow('Unauthorized');
103
+
104
+ expect(calls()).toBe(1);
105
+ expect(delays).toEqual([]);
153
106
  });
154
107
  });
@@ -1,11 +1,60 @@
1
- /**
2
- * Minimal unit test for v2 scrape (no mocking; sanity check payload path)
3
- */
1
+ import { describe, test, expect } from "@jest/globals";
2
+ import axios, { type AxiosAdapter } from "axios";
4
3
  import { FirecrawlClient } from "../../../v2/client";
5
4
 
5
+ const API_URL = "https://api.firecrawl.dev";
6
+
7
+ function makeClient(apiKey: string) {
8
+ const client = new FirecrawlClient({ apiKey, apiUrl: API_URL });
9
+ const sent: Array<{ method?: string; url: string; authorization: unknown; body: unknown }> = [];
10
+ const adapter: AxiosAdapter = async config => {
11
+ sent.push({
12
+ method: config.method,
13
+ url: axios.getUri(config),
14
+ authorization: config.headers.Authorization,
15
+ body: JSON.parse(config.data),
16
+ });
17
+ return {
18
+ data: { success: true, data: { markdown: "# hello" } },
19
+ status: 200,
20
+ statusText: "OK",
21
+ headers: {},
22
+ config,
23
+ };
24
+ };
25
+ (client as any).http.instance.defaults.adapter = adapter;
26
+ return { client, sent };
27
+ }
28
+
6
29
  describe("v2.scrape unit", () => {
7
- test("constructor requires apiKey", () => {
8
- expect(() => new FirecrawlClient({ apiKey: "", apiUrl: "https://api.firecrawl.dev" })).toThrow();
30
+ test("constructs without an API key and scrapes keyless, without an Authorization header", async () => {
31
+ const { client, sent } = makeClient("");
32
+ const doc = await client.scrape(" https://example.com ", { formats: ["markdown"] });
33
+
34
+ expect(doc.markdown).toBe("# hello");
35
+ expect(sent).toHaveLength(1);
36
+ expect(sent[0]!.method).toBe("post");
37
+ expect(sent[0]!.url).toBe(`${API_URL}/v2/scrape`);
38
+ expect(sent[0]!.authorization).toBeUndefined();
39
+ expect(sent[0]!.body).toEqual({
40
+ url: "https://example.com",
41
+ formats: ["markdown"],
42
+ origin: expect.stringMatching(/^js-sdk@/),
43
+ });
9
44
  });
10
- });
11
45
 
46
+ test("sends the API key as a bearer token when one is set", async () => {
47
+ const { client, sent } = makeClient("fc-test");
48
+ await client.scrape("https://example.com");
49
+
50
+ expect(sent).toHaveLength(1);
51
+ expect(sent[0]!.authorization).toBe("Bearer fc-test");
52
+ });
53
+
54
+ test("rejects an empty URL before sending a request", async () => {
55
+ const { client, sent } = makeClient("fc-test");
56
+
57
+ await expect(client.scrape(" ")).rejects.toThrow("URL cannot be empty");
58
+ expect(sent).toHaveLength(0);
59
+ });
60
+ });
package/src/v1/index.ts CHANGED
@@ -1128,7 +1128,7 @@ export default class FirecrawlApp {
1128
1128
  );
1129
1129
  if (response.status === 200) {
1130
1130
  const id: string = response.data.id;
1131
- return this.monitorJobStatus(id, headers, pollInterval);
1131
+ return this.monitorJobStatus(id, headers, pollInterval, "batch");
1132
1132
  } else {
1133
1133
  this.handleError(response, "start batch scrape job");
1134
1134
  }
@@ -1484,18 +1484,21 @@ export default class FirecrawlApp {
1484
1484
  }
1485
1485
 
1486
1486
  /**
1487
- * Monitors the status of a crawl job until completion or failure.
1488
- * @param id - The ID of the crawl operation.
1487
+ * Monitors the status of a crawl or batch scrape job until completion or failure.
1488
+ * @param id - The ID of the crawl or batch scrape operation.
1489
1489
  * @param headers - The headers for the request.
1490
1490
  * @param checkInterval - Interval in seconds for job status checks.
1491
- * @param checkUrl - Optional URL to check the status (used for v1 API)
1491
+ * @param jobType - Which status endpoint to poll. Defaults to `"crawl"`.
1492
1492
  * @returns The final job status or data.
1493
1493
  */
1494
1494
  async monitorJobStatus(
1495
1495
  id: string,
1496
1496
  headers: AxiosRequestHeaders,
1497
- checkInterval: number
1497
+ checkInterval: number,
1498
+ jobType: "crawl" | "batch" = "crawl"
1498
1499
  ): Promise<CrawlStatusResponse | ErrorResponse> {
1500
+ const statusPath = jobType === "batch" ? "batch/scrape" : "crawl";
1501
+ const jobLabel = jobType === "batch" ? "Batch scrape" : "Crawl";
1499
1502
  let failedTries = 0;
1500
1503
  let networkRetries = 0;
1501
1504
  const maxNetworkRetries = 3;
@@ -1503,7 +1506,7 @@ export default class FirecrawlApp {
1503
1506
  while (true) {
1504
1507
  try {
1505
1508
  let statusResponse: AxiosResponse = await this.getRequest(
1506
- `${this.apiUrl}/v1/crawl/${id}`,
1509
+ `${this.apiUrl}/v1/${statusPath}/${id}`,
1507
1510
  headers
1508
1511
  );
1509
1512
 
@@ -1526,7 +1529,7 @@ export default class FirecrawlApp {
1526
1529
  statusData.data = data;
1527
1530
  return statusData;
1528
1531
  } else {
1529
- throw new FirecrawlError("Crawl job completed but no data was returned", 500);
1532
+ throw new FirecrawlError(`${jobLabel} job completed but no data was returned`, 500);
1530
1533
  }
1531
1534
  } else if (
1532
1535
  ["active", "paused", "pending", "queued", "waiting", "scraping"].includes(statusData.status)
@@ -1537,14 +1540,14 @@ export default class FirecrawlApp {
1537
1540
  );
1538
1541
  } else {
1539
1542
  throw new FirecrawlError(
1540
- `Crawl job failed or was stopped. Status: ${statusData.status}`,
1543
+ `${jobLabel} job failed or was stopped. Status: ${statusData.status}`,
1541
1544
  500
1542
1545
  );
1543
1546
  }
1544
1547
  } else {
1545
1548
  failedTries++;
1546
1549
  if (failedTries >= 3) {
1547
- this.handleError(statusResponse, "check crawl status");
1550
+ this.handleError(statusResponse, `check ${jobLabel.toLowerCase()} status`);
1548
1551
  }
1549
1552
  }
1550
1553
  } catch (error: any) {