firecrawl 4.42.2 → 4.42.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-XZ5XI5DK.js → chunk-AMV6GR2H.js} +2 -2
- package/dist/index.cjs +13 -11
- package/dist/index.d.cts +4 -4
- package/dist/index.d.ts +4 -4
- package/dist/index.js +13 -11
- package/dist/{package-JVVNUZPT.js → package-RQDQRVLO.js} +1 -1
- package/package.json +2 -2
- package/src/__tests__/unit/v1/batch-scrape-poll.test.ts +91 -0
- package/src/__tests__/unit/v1/monitor-job-status-retry.test.ts +84 -131
- package/src/__tests__/unit/v2/scrape.unit.test.ts +55 -6
- package/src/v1/index.ts +12 -9
|
@@ -12,7 +12,7 @@ var require_package = __commonJS({
|
|
|
12
12
|
"package.json"(exports, module) {
|
|
13
13
|
module.exports = {
|
|
14
14
|
name: "@mendable/firecrawl-js",
|
|
15
|
-
version: "4.42.
|
|
15
|
+
version: "4.42.3",
|
|
16
16
|
description: "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
|
|
17
17
|
main: "dist/index.js",
|
|
18
18
|
types: "dist/index.d.ts",
|
|
@@ -29,7 +29,7 @@ var require_package = __commonJS({
|
|
|
29
29
|
"build-and-publish": "pnpm run build && pnpm publish --access public",
|
|
30
30
|
"publish-beta": "pnpm run build && pnpm publish --access public --tag beta",
|
|
31
31
|
test: "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/e2e/v2/*.test.ts --detectOpenHandles",
|
|
32
|
-
"test:unit": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/unit
|
|
32
|
+
"test:unit": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/unit"
|
|
33
33
|
},
|
|
34
34
|
repository: {
|
|
35
35
|
type: "git",
|
package/dist/index.cjs
CHANGED
|
@@ -39,7 +39,7 @@ var require_package = __commonJS({
|
|
|
39
39
|
"package.json"(exports2, module2) {
|
|
40
40
|
module2.exports = {
|
|
41
41
|
name: "@mendable/firecrawl-js",
|
|
42
|
-
version: "4.42.
|
|
42
|
+
version: "4.42.3",
|
|
43
43
|
description: "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
|
|
44
44
|
main: "dist/index.js",
|
|
45
45
|
types: "dist/index.d.ts",
|
|
@@ -56,7 +56,7 @@ var require_package = __commonJS({
|
|
|
56
56
|
"build-and-publish": "pnpm run build && pnpm publish --access public",
|
|
57
57
|
"publish-beta": "pnpm run build && pnpm publish --access public --tag beta",
|
|
58
58
|
test: "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/e2e/v2/*.test.ts --detectOpenHandles",
|
|
59
|
-
"test:unit": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/unit
|
|
59
|
+
"test:unit": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/unit"
|
|
60
60
|
},
|
|
61
61
|
repository: {
|
|
62
62
|
type: "git",
|
|
@@ -3233,7 +3233,7 @@ var FirecrawlApp = class {
|
|
|
3233
3233
|
);
|
|
3234
3234
|
if (response.status === 200) {
|
|
3235
3235
|
const id = response.data.id;
|
|
3236
|
-
return this.monitorJobStatus(id, headers, pollInterval);
|
|
3236
|
+
return this.monitorJobStatus(id, headers, pollInterval, "batch");
|
|
3237
3237
|
} else {
|
|
3238
3238
|
this.handleError(response, "start batch scrape job");
|
|
3239
3239
|
}
|
|
@@ -3537,21 +3537,23 @@ var FirecrawlApp = class {
|
|
|
3537
3537
|
}
|
|
3538
3538
|
}
|
|
3539
3539
|
/**
|
|
3540
|
-
* Monitors the status of a crawl job until completion or failure.
|
|
3541
|
-
* @param id - The ID of the crawl operation.
|
|
3540
|
+
* Monitors the status of a crawl or batch scrape job until completion or failure.
|
|
3541
|
+
* @param id - The ID of the crawl or batch scrape operation.
|
|
3542
3542
|
* @param headers - The headers for the request.
|
|
3543
3543
|
* @param checkInterval - Interval in seconds for job status checks.
|
|
3544
|
-
* @param
|
|
3544
|
+
* @param jobType - Which status endpoint to poll. Defaults to `"crawl"`.
|
|
3545
3545
|
* @returns The final job status or data.
|
|
3546
3546
|
*/
|
|
3547
|
-
async monitorJobStatus(id, headers, checkInterval) {
|
|
3547
|
+
async monitorJobStatus(id, headers, checkInterval, jobType = "crawl") {
|
|
3548
|
+
const statusPath = jobType === "batch" ? "batch/scrape" : "crawl";
|
|
3549
|
+
const jobLabel = jobType === "batch" ? "Batch scrape" : "Crawl";
|
|
3548
3550
|
let failedTries = 0;
|
|
3549
3551
|
let networkRetries = 0;
|
|
3550
3552
|
const maxNetworkRetries = 3;
|
|
3551
3553
|
while (true) {
|
|
3552
3554
|
try {
|
|
3553
3555
|
let statusResponse = await this.getRequest(
|
|
3554
|
-
`${this.apiUrl}/v1
|
|
3556
|
+
`${this.apiUrl}/v1/${statusPath}/${id}`,
|
|
3555
3557
|
headers
|
|
3556
3558
|
);
|
|
3557
3559
|
if (statusResponse.status === 200) {
|
|
@@ -3572,7 +3574,7 @@ var FirecrawlApp = class {
|
|
|
3572
3574
|
statusData.data = data;
|
|
3573
3575
|
return statusData;
|
|
3574
3576
|
} else {
|
|
3575
|
-
throw new FirecrawlError(
|
|
3577
|
+
throw new FirecrawlError(`${jobLabel} job completed but no data was returned`, 500);
|
|
3576
3578
|
}
|
|
3577
3579
|
} else if (["active", "paused", "pending", "queued", "waiting", "scraping"].includes(statusData.status)) {
|
|
3578
3580
|
checkInterval = Math.max(checkInterval, 2);
|
|
@@ -3581,14 +3583,14 @@ var FirecrawlApp = class {
|
|
|
3581
3583
|
);
|
|
3582
3584
|
} else {
|
|
3583
3585
|
throw new FirecrawlError(
|
|
3584
|
-
|
|
3586
|
+
`${jobLabel} job failed or was stopped. Status: ${statusData.status}`,
|
|
3585
3587
|
500
|
|
3586
3588
|
);
|
|
3587
3589
|
}
|
|
3588
3590
|
} else {
|
|
3589
3591
|
failedTries++;
|
|
3590
3592
|
if (failedTries >= 3) {
|
|
3591
|
-
this.handleError(statusResponse,
|
|
3593
|
+
this.handleError(statusResponse, `check ${jobLabel.toLowerCase()} status`);
|
|
3592
3594
|
}
|
|
3593
3595
|
}
|
|
3594
3596
|
} catch (error) {
|
package/dist/index.d.cts
CHANGED
|
@@ -3388,14 +3388,14 @@ declare class FirecrawlApp {
|
|
|
3388
3388
|
*/
|
|
3389
3389
|
deleteRequest(url: string, headers: AxiosRequestHeaders): Promise<AxiosResponse>;
|
|
3390
3390
|
/**
|
|
3391
|
-
* Monitors the status of a crawl job until completion or failure.
|
|
3392
|
-
* @param id - The ID of the crawl operation.
|
|
3391
|
+
* Monitors the status of a crawl or batch scrape job until completion or failure.
|
|
3392
|
+
* @param id - The ID of the crawl or batch scrape operation.
|
|
3393
3393
|
* @param headers - The headers for the request.
|
|
3394
3394
|
* @param checkInterval - Interval in seconds for job status checks.
|
|
3395
|
-
* @param
|
|
3395
|
+
* @param jobType - Which status endpoint to poll. Defaults to `"crawl"`.
|
|
3396
3396
|
* @returns The final job status or data.
|
|
3397
3397
|
*/
|
|
3398
|
-
monitorJobStatus(id: string, headers: AxiosRequestHeaders, checkInterval: number): Promise<CrawlStatusResponse | ErrorResponse>;
|
|
3398
|
+
monitorJobStatus(id: string, headers: AxiosRequestHeaders, checkInterval: number, jobType?: "crawl" | "batch"): Promise<CrawlStatusResponse | ErrorResponse>;
|
|
3399
3399
|
/**
|
|
3400
3400
|
* Determines if an error is retryable (transient network error)
|
|
3401
3401
|
* @param error - The error to check
|
package/dist/index.d.ts
CHANGED
|
@@ -3388,14 +3388,14 @@ declare class FirecrawlApp {
|
|
|
3388
3388
|
*/
|
|
3389
3389
|
deleteRequest(url: string, headers: AxiosRequestHeaders): Promise<AxiosResponse>;
|
|
3390
3390
|
/**
|
|
3391
|
-
* Monitors the status of a crawl job until completion or failure.
|
|
3392
|
-
* @param id - The ID of the crawl operation.
|
|
3391
|
+
* Monitors the status of a crawl or batch scrape job until completion or failure.
|
|
3392
|
+
* @param id - The ID of the crawl or batch scrape operation.
|
|
3393
3393
|
* @param headers - The headers for the request.
|
|
3394
3394
|
* @param checkInterval - Interval in seconds for job status checks.
|
|
3395
|
-
* @param
|
|
3395
|
+
* @param jobType - Which status endpoint to poll. Defaults to `"crawl"`.
|
|
3396
3396
|
* @returns The final job status or data.
|
|
3397
3397
|
*/
|
|
3398
|
-
monitorJobStatus(id: string, headers: AxiosRequestHeaders, checkInterval: number): Promise<CrawlStatusResponse | ErrorResponse>;
|
|
3398
|
+
monitorJobStatus(id: string, headers: AxiosRequestHeaders, checkInterval: number, jobType?: "crawl" | "batch"): Promise<CrawlStatusResponse | ErrorResponse>;
|
|
3399
3399
|
/**
|
|
3400
3400
|
* Determines if an error is retryable (transient network error)
|
|
3401
3401
|
* @param error - The error to check
|
package/dist/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
require_package
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-AMV6GR2H.js";
|
|
4
4
|
|
|
5
5
|
// src/v2/utils/httpClient.ts
|
|
6
6
|
import axios from "axios";
|
|
@@ -2706,7 +2706,7 @@ var FirecrawlApp = class {
|
|
|
2706
2706
|
if (typeof process !== "undefined" && process.env && process.env.npm_package_version) {
|
|
2707
2707
|
return process.env.npm_package_version;
|
|
2708
2708
|
}
|
|
2709
|
-
const packageJson = await import("./package-
|
|
2709
|
+
const packageJson = await import("./package-RQDQRVLO.js");
|
|
2710
2710
|
return packageJson.default.version;
|
|
2711
2711
|
} catch (error) {
|
|
2712
2712
|
const isTest = typeof process !== "undefined" && (process.env.JEST_WORKER_ID != null || false);
|
|
@@ -3101,7 +3101,7 @@ var FirecrawlApp = class {
|
|
|
3101
3101
|
);
|
|
3102
3102
|
if (response.status === 200) {
|
|
3103
3103
|
const id = response.data.id;
|
|
3104
|
-
return this.monitorJobStatus(id, headers, pollInterval);
|
|
3104
|
+
return this.monitorJobStatus(id, headers, pollInterval, "batch");
|
|
3105
3105
|
} else {
|
|
3106
3106
|
this.handleError(response, "start batch scrape job");
|
|
3107
3107
|
}
|
|
@@ -3405,21 +3405,23 @@ var FirecrawlApp = class {
|
|
|
3405
3405
|
}
|
|
3406
3406
|
}
|
|
3407
3407
|
/**
|
|
3408
|
-
* Monitors the status of a crawl job until completion or failure.
|
|
3409
|
-
* @param id - The ID of the crawl operation.
|
|
3408
|
+
* Monitors the status of a crawl or batch scrape job until completion or failure.
|
|
3409
|
+
* @param id - The ID of the crawl or batch scrape operation.
|
|
3410
3410
|
* @param headers - The headers for the request.
|
|
3411
3411
|
* @param checkInterval - Interval in seconds for job status checks.
|
|
3412
|
-
* @param
|
|
3412
|
+
* @param jobType - Which status endpoint to poll. Defaults to `"crawl"`.
|
|
3413
3413
|
* @returns The final job status or data.
|
|
3414
3414
|
*/
|
|
3415
|
-
async monitorJobStatus(id, headers, checkInterval) {
|
|
3415
|
+
async monitorJobStatus(id, headers, checkInterval, jobType = "crawl") {
|
|
3416
|
+
const statusPath = jobType === "batch" ? "batch/scrape" : "crawl";
|
|
3417
|
+
const jobLabel = jobType === "batch" ? "Batch scrape" : "Crawl";
|
|
3416
3418
|
let failedTries = 0;
|
|
3417
3419
|
let networkRetries = 0;
|
|
3418
3420
|
const maxNetworkRetries = 3;
|
|
3419
3421
|
while (true) {
|
|
3420
3422
|
try {
|
|
3421
3423
|
let statusResponse = await this.getRequest(
|
|
3422
|
-
`${this.apiUrl}/v1
|
|
3424
|
+
`${this.apiUrl}/v1/${statusPath}/${id}`,
|
|
3423
3425
|
headers
|
|
3424
3426
|
);
|
|
3425
3427
|
if (statusResponse.status === 200) {
|
|
@@ -3440,7 +3442,7 @@ var FirecrawlApp = class {
|
|
|
3440
3442
|
statusData.data = data;
|
|
3441
3443
|
return statusData;
|
|
3442
3444
|
} else {
|
|
3443
|
-
throw new FirecrawlError(
|
|
3445
|
+
throw new FirecrawlError(`${jobLabel} job completed but no data was returned`, 500);
|
|
3444
3446
|
}
|
|
3445
3447
|
} else if (["active", "paused", "pending", "queued", "waiting", "scraping"].includes(statusData.status)) {
|
|
3446
3448
|
checkInterval = Math.max(checkInterval, 2);
|
|
@@ -3449,14 +3451,14 @@ var FirecrawlApp = class {
|
|
|
3449
3451
|
);
|
|
3450
3452
|
} else {
|
|
3451
3453
|
throw new FirecrawlError(
|
|
3452
|
-
|
|
3454
|
+
`${jobLabel} job failed or was stopped. Status: ${statusData.status}`,
|
|
3453
3455
|
500
|
|
3454
3456
|
);
|
|
3455
3457
|
}
|
|
3456
3458
|
} else {
|
|
3457
3459
|
failedTries++;
|
|
3458
3460
|
if (failedTries >= 3) {
|
|
3459
|
-
this.handleError(statusResponse,
|
|
3461
|
+
this.handleError(statusResponse, `check ${jobLabel.toLowerCase()} status`);
|
|
3460
3462
|
}
|
|
3461
3463
|
}
|
|
3462
3464
|
} catch (error) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "firecrawl",
|
|
3
|
-
"version": "4.42.
|
|
3
|
+
"version": "4.42.3",
|
|
4
4
|
"description": "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -73,6 +73,6 @@
|
|
|
73
73
|
"build-and-publish": "pnpm run build && pnpm publish --access public",
|
|
74
74
|
"publish-beta": "pnpm run build && pnpm publish --access public --tag beta",
|
|
75
75
|
"test": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/e2e/v2/*.test.ts --detectOpenHandles",
|
|
76
|
-
"test:unit": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/unit
|
|
76
|
+
"test:unit": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/unit"
|
|
77
77
|
}
|
|
78
78
|
}
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
import { describe, test, expect, jest, afterEach, beforeEach } from "@jest/globals";
|
|
2
|
+
import axios from "axios";
|
|
3
|
+
import FirecrawlApp from "../../../v1";
|
|
4
|
+
|
|
5
|
+
const API_URL = "https://api.firecrawl.dev";
|
|
6
|
+
const API_KEY = "fc-test";
|
|
7
|
+
|
|
8
|
+
const scrapingPage = () => ({ success: true, status: "scraping", total: 2, completed: 0, data: [] });
|
|
9
|
+
const completedPage = (next: string) => ({
|
|
10
|
+
success: true,
|
|
11
|
+
status: "completed",
|
|
12
|
+
total: 2,
|
|
13
|
+
completed: 2,
|
|
14
|
+
expiresAt: "2030-01-01T00:00:00Z",
|
|
15
|
+
data: [{ markdown: "a" }],
|
|
16
|
+
next,
|
|
17
|
+
});
|
|
18
|
+
const lastPage = () => ({ success: true, status: "completed", data: [{ markdown: "b" }] });
|
|
19
|
+
|
|
20
|
+
function mockPoll(path: string, nextOrigin = API_URL) {
|
|
21
|
+
const sent: string[] = [];
|
|
22
|
+
let polls = 0;
|
|
23
|
+
jest.spyOn(axios, "get").mockImplementation(async (url: string) => {
|
|
24
|
+
sent.push(url);
|
|
25
|
+
if (new URL(url).searchParams.has("skip")) return { status: 200, data: lastPage() };
|
|
26
|
+
polls++;
|
|
27
|
+
return { status: 200, data: polls === 1 ? scrapingPage() : completedPage(`${nextOrigin}${path}?skip=1`) };
|
|
28
|
+
});
|
|
29
|
+
return sent;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
describe("v1 batch scrape waits on the batch scrape status endpoint", () => {
|
|
33
|
+
const app = new FirecrawlApp({ apiKey: API_KEY, apiUrl: API_URL });
|
|
34
|
+
|
|
35
|
+
beforeEach(() => {
|
|
36
|
+
jest.spyOn(global, "setTimeout").mockImplementation(((fn: () => void) => {
|
|
37
|
+
fn();
|
|
38
|
+
return 0;
|
|
39
|
+
}) as any);
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
afterEach(() => {
|
|
43
|
+
jest.restoreAllMocks();
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
test("batchScrapeUrls polls /v1/batch/scrape/{id} and follows pinned next pages", async () => {
|
|
47
|
+
const post = jest.spyOn(axios, "post").mockResolvedValue({ status: 200, data: { success: true, id: "abc" } });
|
|
48
|
+
const path = "/v1/batch/scrape/abc";
|
|
49
|
+
const sent = mockPoll(path, "https://evil.example");
|
|
50
|
+
|
|
51
|
+
const res: any = await app.batchScrapeUrls(["https://example.com", "https://example.org"], {});
|
|
52
|
+
|
|
53
|
+
expect(post.mock.calls[0][0]).toBe(`${API_URL}/v1/batch/scrape`);
|
|
54
|
+
expect(sent).toEqual([`${API_URL}${path}`, `${API_URL}${path}`, `${API_URL}${path}?skip=1`]);
|
|
55
|
+
expect(sent.some(url => url.includes("/v1/crawl/"))).toBe(false);
|
|
56
|
+
expect(res.status).toBe("completed");
|
|
57
|
+
expect(res.data).toHaveLength(2);
|
|
58
|
+
});
|
|
59
|
+
|
|
60
|
+
test("batchScrapeUrls reports batch scrape failures as batch scrape errors", async () => {
|
|
61
|
+
jest.spyOn(axios, "post").mockResolvedValue({ status: 200, data: { success: true, id: "abc" } });
|
|
62
|
+
const sent: string[] = [];
|
|
63
|
+
jest.spyOn(axios, "get").mockImplementation(async (url: string) => {
|
|
64
|
+
sent.push(url);
|
|
65
|
+
return { status: 200, data: { success: true, status: "failed", data: [] } };
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
await expect(app.batchScrapeUrls(["https://example.com"], {})).rejects.toThrow("Batch scrape job failed or was stopped");
|
|
69
|
+
expect(sent.every(url => url === `${API_URL}/v1/batch/scrape/abc`)).toBe(true);
|
|
70
|
+
});
|
|
71
|
+
|
|
72
|
+
test("monitorJobStatus still polls /v1/crawl/{id} by default", async () => {
|
|
73
|
+
const path = "/v1/crawl/abc";
|
|
74
|
+
const sent = mockPoll(path);
|
|
75
|
+
|
|
76
|
+
const res: any = await app.monitorJobStatus("abc", app.prepareHeaders(), 0);
|
|
77
|
+
|
|
78
|
+
expect(sent).toEqual([`${API_URL}${path}`, `${API_URL}${path}`, `${API_URL}${path}?skip=1`]);
|
|
79
|
+
expect(res.data).toHaveLength(2);
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
test("monitorJobStatus polls /v1/batch/scrape/{id} for batch jobs", async () => {
|
|
83
|
+
const path = "/v1/batch/scrape/abc";
|
|
84
|
+
const sent = mockPoll(path);
|
|
85
|
+
|
|
86
|
+
const res: any = await app.monitorJobStatus("abc", app.prepareHeaders(), 0, "batch");
|
|
87
|
+
|
|
88
|
+
expect(sent).toEqual([`${API_URL}${path}`, `${API_URL}${path}`, `${API_URL}${path}?skip=1`]);
|
|
89
|
+
expect(res.data).toHaveLength(2);
|
|
90
|
+
});
|
|
91
|
+
});
|
|
@@ -1,154 +1,107 @@
|
|
|
1
|
-
import FirecrawlApp from '../../../
|
|
1
|
+
import FirecrawlApp from '../../../v1';
|
|
2
2
|
import { describe, test, expect, jest, beforeEach, afterEach } from '@jest/globals';
|
|
3
3
|
|
|
4
|
+
const successResponse = {
|
|
5
|
+
status: 200,
|
|
6
|
+
data: { status: 'completed', data: [{ url: 'test.com', markdown: 'test' }] },
|
|
7
|
+
};
|
|
8
|
+
|
|
9
|
+
function networkError(message: string, code: string) {
|
|
10
|
+
return Object.assign(new Error(message), { code });
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
function httpError(message: string, status: number) {
|
|
14
|
+
return Object.assign(new Error(message), { response: { status, data: { error: message } } });
|
|
15
|
+
}
|
|
16
|
+
|
|
4
17
|
describe('monitorJobStatus retry logic', () => {
|
|
5
18
|
let app: FirecrawlApp;
|
|
6
|
-
let
|
|
7
|
-
|
|
19
|
+
let delays: number[];
|
|
20
|
+
|
|
8
21
|
beforeEach(() => {
|
|
9
22
|
app = new FirecrawlApp({ apiKey: 'test-key', apiUrl: 'https://test.com' });
|
|
10
|
-
|
|
11
|
-
|
|
23
|
+
delays = [];
|
|
24
|
+
jest.useFakeTimers();
|
|
25
|
+
const fakeSetTimeout = globalThis.setTimeout;
|
|
26
|
+
jest.spyOn(globalThis, 'setTimeout').mockImplementation(((fn: () => void, ms?: number) => {
|
|
27
|
+
delays.push(ms ?? 0);
|
|
28
|
+
return fakeSetTimeout(fn, ms);
|
|
29
|
+
}) as any);
|
|
12
30
|
});
|
|
13
31
|
|
|
14
32
|
afterEach(() => {
|
|
15
|
-
|
|
16
|
-
jest.
|
|
33
|
+
jest.restoreAllMocks();
|
|
34
|
+
jest.useRealTimers();
|
|
17
35
|
});
|
|
18
36
|
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
status
|
|
25
|
-
|
|
26
|
-
};
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
37
|
+
function failFirst(n: number, error: Error, statuses: string[] = []) {
|
|
38
|
+
let calls = 0;
|
|
39
|
+
app.getRequest = (async () => {
|
|
40
|
+
calls++;
|
|
41
|
+
if (calls <= n) throw error;
|
|
42
|
+
const status = statuses.shift();
|
|
43
|
+
return status ? { status: 200, data: { status } } : successResponse;
|
|
44
|
+
}) as any;
|
|
45
|
+
return () => calls;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
async function monitor() {
|
|
49
|
+
const result = app.monitorJobStatus('test-id', {} as any, 1);
|
|
50
|
+
result.catch(() => {});
|
|
51
|
+
await jest.runAllTimersAsync();
|
|
52
|
+
return result;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
test.each([
|
|
56
|
+
['socket hang up', networkError('socket hang up', 'ECONNRESET')],
|
|
57
|
+
['ETIMEDOUT', networkError('timeout', 'ETIMEDOUT')],
|
|
58
|
+
['HTTP 408', httpError('Request timeout', 408)],
|
|
59
|
+
['HTTP 504', httpError('Gateway timeout', 504)],
|
|
60
|
+
])('retries once after a %s error and returns the job data', async (_label, error) => {
|
|
61
|
+
const calls = failFirst(1, error);
|
|
62
|
+
|
|
63
|
+
const result = await monitor();
|
|
64
|
+
|
|
65
|
+
expect(calls()).toBe(2);
|
|
42
66
|
expect(result).toEqual(successResponse.data);
|
|
43
|
-
expect(
|
|
44
|
-
expect.stringContaining('Network error during job status check (attempt 1/3): socket hang up')
|
|
45
|
-
);
|
|
67
|
+
expect(delays).toEqual([1000]);
|
|
46
68
|
});
|
|
47
69
|
|
|
48
|
-
test('
|
|
49
|
-
const
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
data: { status: 'completed', data: [{ url: 'test.com', markdown: 'test' }] }
|
|
55
|
-
};
|
|
56
|
-
|
|
57
|
-
const originalGetRequest = app.getRequest;
|
|
58
|
-
let callCount = 0;
|
|
59
|
-
|
|
60
|
-
app.getRequest = async function(url: string, headers: any) {
|
|
61
|
-
callCount++;
|
|
62
|
-
if (callCount === 1) {
|
|
63
|
-
throw timeoutError;
|
|
64
|
-
}
|
|
65
|
-
return successResponse;
|
|
66
|
-
};
|
|
67
|
-
|
|
68
|
-
const result = await app.monitorJobStatus('test-id', {}, 1);
|
|
69
|
-
|
|
70
|
-
expect(callCount).toBe(2);
|
|
70
|
+
test('uses exponential backoff between retries', async () => {
|
|
71
|
+
const calls = failFirst(2, networkError('socket hang up', 'ECONNRESET'));
|
|
72
|
+
|
|
73
|
+
const result = await monitor();
|
|
74
|
+
|
|
75
|
+
expect(calls()).toBe(3);
|
|
71
76
|
expect(result).toEqual(successResponse.data);
|
|
77
|
+
expect(delays).toEqual([1000, 2000]);
|
|
72
78
|
});
|
|
73
79
|
|
|
74
|
-
test('
|
|
75
|
-
const
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
app.getRequest = async function(url: string, headers: any) {
|
|
79
|
-
throw socketHangUpError;
|
|
80
|
-
};
|
|
81
|
-
|
|
82
|
-
await expect(app.monitorJobStatus('test-id', {}, 1)).rejects.toThrow('socket hang up');
|
|
83
|
-
|
|
84
|
-
expect(console.warn).toHaveBeenCalledTimes(3);
|
|
85
|
-
}, 15000);
|
|
86
|
-
|
|
87
|
-
test('should not retry on non-retryable errors', async () => {
|
|
88
|
-
const authError = new Error('Unauthorized') as any;
|
|
89
|
-
authError.response = { status: 401, data: { error: 'Unauthorized' } };
|
|
90
|
-
|
|
91
|
-
app.getRequest = async function(url: string, headers: any) {
|
|
92
|
-
throw authError;
|
|
93
|
-
};
|
|
94
|
-
|
|
95
|
-
await expect(app.monitorJobStatus('test-id', {}, 1)).rejects.toThrow('Unauthorized');
|
|
96
|
-
|
|
97
|
-
expect(console.warn).not.toHaveBeenCalled();
|
|
98
|
-
});
|
|
80
|
+
test('records both the retry backoff and the 2s poll wait for an active job', async () => {
|
|
81
|
+
const calls = failFirst(1, networkError('socket hang up', 'ECONNRESET'), ['active']);
|
|
82
|
+
|
|
83
|
+
const result = await monitor();
|
|
99
84
|
|
|
100
|
-
|
|
101
|
-
const timeoutError = new Error('Request timeout') as any;
|
|
102
|
-
timeoutError.response = { status: 408, data: { error: 'Request timeout' } };
|
|
103
|
-
|
|
104
|
-
const successResponse = {
|
|
105
|
-
status: 200,
|
|
106
|
-
data: { status: 'completed', data: [{ url: 'test.com', markdown: 'test' }] }
|
|
107
|
-
};
|
|
108
|
-
|
|
109
|
-
const originalGetRequest = app.getRequest;
|
|
110
|
-
let callCount = 0;
|
|
111
|
-
|
|
112
|
-
app.getRequest = async function(url: string, headers: any) {
|
|
113
|
-
callCount++;
|
|
114
|
-
if (callCount === 1) {
|
|
115
|
-
throw timeoutError;
|
|
116
|
-
}
|
|
117
|
-
return successResponse;
|
|
118
|
-
};
|
|
119
|
-
|
|
120
|
-
const result = await app.monitorJobStatus('test-id', {}, 1);
|
|
121
|
-
|
|
122
|
-
expect(callCount).toBe(2);
|
|
85
|
+
expect(calls()).toBe(3);
|
|
123
86
|
expect(result).toEqual(successResponse.data);
|
|
87
|
+
expect(delays).toEqual([1000, 2000]);
|
|
124
88
|
});
|
|
125
89
|
|
|
126
|
-
test('
|
|
127
|
-
const
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
}
|
|
143
|
-
return successResponse;
|
|
144
|
-
};
|
|
145
|
-
|
|
146
|
-
const startTime = Date.now();
|
|
147
|
-
const result = await app.monitorJobStatus('test-id', {}, 1);
|
|
148
|
-
const endTime = Date.now();
|
|
149
|
-
|
|
150
|
-
expect(callCount).toBe(3);
|
|
151
|
-
expect(result).toEqual(successResponse.data);
|
|
152
|
-
expect(endTime - startTime).toBeGreaterThan(3000);
|
|
90
|
+
test('fails after max retries are exceeded', async () => {
|
|
91
|
+
const calls = failFirst(Infinity, networkError('socket hang up', 'ECONNRESET'));
|
|
92
|
+
|
|
93
|
+
await expect(monitor()).rejects.toThrow('socket hang up');
|
|
94
|
+
|
|
95
|
+
expect(calls()).toBe(4);
|
|
96
|
+
expect(delays).toEqual([1000, 2000, 4000]);
|
|
97
|
+
});
|
|
98
|
+
|
|
99
|
+
test('does not retry non-retryable errors', async () => {
|
|
100
|
+
const calls = failFirst(Infinity, httpError('Unauthorized', 401));
|
|
101
|
+
|
|
102
|
+
await expect(monitor()).rejects.toThrow('Unauthorized');
|
|
103
|
+
|
|
104
|
+
expect(calls()).toBe(1);
|
|
105
|
+
expect(delays).toEqual([]);
|
|
153
106
|
});
|
|
154
107
|
});
|
|
@@ -1,11 +1,60 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
*/
|
|
1
|
+
import { describe, test, expect } from "@jest/globals";
|
|
2
|
+
import axios, { type AxiosAdapter } from "axios";
|
|
4
3
|
import { FirecrawlClient } from "../../../v2/client";
|
|
5
4
|
|
|
5
|
+
const API_URL = "https://api.firecrawl.dev";
|
|
6
|
+
|
|
7
|
+
function makeClient(apiKey: string) {
|
|
8
|
+
const client = new FirecrawlClient({ apiKey, apiUrl: API_URL });
|
|
9
|
+
const sent: Array<{ method?: string; url: string; authorization: unknown; body: unknown }> = [];
|
|
10
|
+
const adapter: AxiosAdapter = async config => {
|
|
11
|
+
sent.push({
|
|
12
|
+
method: config.method,
|
|
13
|
+
url: axios.getUri(config),
|
|
14
|
+
authorization: config.headers.Authorization,
|
|
15
|
+
body: JSON.parse(config.data),
|
|
16
|
+
});
|
|
17
|
+
return {
|
|
18
|
+
data: { success: true, data: { markdown: "# hello" } },
|
|
19
|
+
status: 200,
|
|
20
|
+
statusText: "OK",
|
|
21
|
+
headers: {},
|
|
22
|
+
config,
|
|
23
|
+
};
|
|
24
|
+
};
|
|
25
|
+
(client as any).http.instance.defaults.adapter = adapter;
|
|
26
|
+
return { client, sent };
|
|
27
|
+
}
|
|
28
|
+
|
|
6
29
|
describe("v2.scrape unit", () => {
|
|
7
|
-
test("
|
|
8
|
-
|
|
30
|
+
test("constructs without an API key and scrapes keyless, without an Authorization header", async () => {
|
|
31
|
+
const { client, sent } = makeClient("");
|
|
32
|
+
const doc = await client.scrape(" https://example.com ", { formats: ["markdown"] });
|
|
33
|
+
|
|
34
|
+
expect(doc.markdown).toBe("# hello");
|
|
35
|
+
expect(sent).toHaveLength(1);
|
|
36
|
+
expect(sent[0]!.method).toBe("post");
|
|
37
|
+
expect(sent[0]!.url).toBe(`${API_URL}/v2/scrape`);
|
|
38
|
+
expect(sent[0]!.authorization).toBeUndefined();
|
|
39
|
+
expect(sent[0]!.body).toEqual({
|
|
40
|
+
url: "https://example.com",
|
|
41
|
+
formats: ["markdown"],
|
|
42
|
+
origin: expect.stringMatching(/^js-sdk@/),
|
|
43
|
+
});
|
|
9
44
|
});
|
|
10
|
-
});
|
|
11
45
|
|
|
46
|
+
test("sends the API key as a bearer token when one is set", async () => {
|
|
47
|
+
const { client, sent } = makeClient("fc-test");
|
|
48
|
+
await client.scrape("https://example.com");
|
|
49
|
+
|
|
50
|
+
expect(sent).toHaveLength(1);
|
|
51
|
+
expect(sent[0]!.authorization).toBe("Bearer fc-test");
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
test("rejects an empty URL before sending a request", async () => {
|
|
55
|
+
const { client, sent } = makeClient("fc-test");
|
|
56
|
+
|
|
57
|
+
await expect(client.scrape(" ")).rejects.toThrow("URL cannot be empty");
|
|
58
|
+
expect(sent).toHaveLength(0);
|
|
59
|
+
});
|
|
60
|
+
});
|
package/src/v1/index.ts
CHANGED
|
@@ -1128,7 +1128,7 @@ export default class FirecrawlApp {
|
|
|
1128
1128
|
);
|
|
1129
1129
|
if (response.status === 200) {
|
|
1130
1130
|
const id: string = response.data.id;
|
|
1131
|
-
return this.monitorJobStatus(id, headers, pollInterval);
|
|
1131
|
+
return this.monitorJobStatus(id, headers, pollInterval, "batch");
|
|
1132
1132
|
} else {
|
|
1133
1133
|
this.handleError(response, "start batch scrape job");
|
|
1134
1134
|
}
|
|
@@ -1484,18 +1484,21 @@ export default class FirecrawlApp {
|
|
|
1484
1484
|
}
|
|
1485
1485
|
|
|
1486
1486
|
/**
|
|
1487
|
-
* Monitors the status of a crawl job until completion or failure.
|
|
1488
|
-
* @param id - The ID of the crawl operation.
|
|
1487
|
+
* Monitors the status of a crawl or batch scrape job until completion or failure.
|
|
1488
|
+
* @param id - The ID of the crawl or batch scrape operation.
|
|
1489
1489
|
* @param headers - The headers for the request.
|
|
1490
1490
|
* @param checkInterval - Interval in seconds for job status checks.
|
|
1491
|
-
* @param
|
|
1491
|
+
* @param jobType - Which status endpoint to poll. Defaults to `"crawl"`.
|
|
1492
1492
|
* @returns The final job status or data.
|
|
1493
1493
|
*/
|
|
1494
1494
|
async monitorJobStatus(
|
|
1495
1495
|
id: string,
|
|
1496
1496
|
headers: AxiosRequestHeaders,
|
|
1497
|
-
checkInterval: number
|
|
1497
|
+
checkInterval: number,
|
|
1498
|
+
jobType: "crawl" | "batch" = "crawl"
|
|
1498
1499
|
): Promise<CrawlStatusResponse | ErrorResponse> {
|
|
1500
|
+
const statusPath = jobType === "batch" ? "batch/scrape" : "crawl";
|
|
1501
|
+
const jobLabel = jobType === "batch" ? "Batch scrape" : "Crawl";
|
|
1499
1502
|
let failedTries = 0;
|
|
1500
1503
|
let networkRetries = 0;
|
|
1501
1504
|
const maxNetworkRetries = 3;
|
|
@@ -1503,7 +1506,7 @@ export default class FirecrawlApp {
|
|
|
1503
1506
|
while (true) {
|
|
1504
1507
|
try {
|
|
1505
1508
|
let statusResponse: AxiosResponse = await this.getRequest(
|
|
1506
|
-
`${this.apiUrl}/v1
|
|
1509
|
+
`${this.apiUrl}/v1/${statusPath}/${id}`,
|
|
1507
1510
|
headers
|
|
1508
1511
|
);
|
|
1509
1512
|
|
|
@@ -1526,7 +1529,7 @@ export default class FirecrawlApp {
|
|
|
1526
1529
|
statusData.data = data;
|
|
1527
1530
|
return statusData;
|
|
1528
1531
|
} else {
|
|
1529
|
-
throw new FirecrawlError(
|
|
1532
|
+
throw new FirecrawlError(`${jobLabel} job completed but no data was returned`, 500);
|
|
1530
1533
|
}
|
|
1531
1534
|
} else if (
|
|
1532
1535
|
["active", "paused", "pending", "queued", "waiting", "scraping"].includes(statusData.status)
|
|
@@ -1537,14 +1540,14 @@ export default class FirecrawlApp {
|
|
|
1537
1540
|
);
|
|
1538
1541
|
} else {
|
|
1539
1542
|
throw new FirecrawlError(
|
|
1540
|
-
|
|
1543
|
+
`${jobLabel} job failed or was stopped. Status: ${statusData.status}`,
|
|
1541
1544
|
500
|
|
1542
1545
|
);
|
|
1543
1546
|
}
|
|
1544
1547
|
} else {
|
|
1545
1548
|
failedTries++;
|
|
1546
1549
|
if (failedTries >= 3) {
|
|
1547
|
-
this.handleError(statusResponse,
|
|
1550
|
+
this.handleError(statusResponse, `check ${jobLabel.toLowerCase()} status`);
|
|
1548
1551
|
}
|
|
1549
1552
|
}
|
|
1550
1553
|
} catch (error: any) {
|