firecrawl 4.45.0 → 4.46.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/audit-ci.jsonc +1 -15
- package/dist/{chunk-LKCOCWYW.js → chunk-R4XT7PME.js} +5 -9
- package/dist/index.cjs +6 -10
- package/dist/index.d.cts +8 -0
- package/dist/index.d.ts +8 -0
- package/dist/index.js +3 -3
- package/dist/{package-C6PK3CK7.js → package-EI4Q33HD.js} +1 -1
- package/package.json +5 -9
- package/pnpm-workspace.yaml +0 -6
- package/src/__tests__/e2e/v1/index.test.ts +1 -1
- package/src/__tests__/e2e/v2/batch.test.ts +1 -1
- package/src/__tests__/e2e/v2/crawl.test.ts +1 -1
- package/src/__tests__/e2e/v2/extract.test.ts +1 -1
- package/src/__tests__/e2e/v2/map.test.ts +1 -1
- package/src/__tests__/e2e/v2/parse.test.ts +1 -1
- package/src/__tests__/e2e/v2/scrape.test.ts +1 -1
- package/src/__tests__/e2e/v2/search.test.ts +1 -1
- package/src/__tests__/e2e/v2/usage.test.ts +1 -1
- package/src/__tests__/e2e/v2/utils/rateLimit.ts +2 -2
- package/src/__tests__/e2e/v2/watcher.test.ts +1 -1
- package/src/__tests__/unit/v1/batch-scrape-poll.test.ts +7 -7
- package/src/__tests__/unit/v1/monitor-job-status-retry.test.ts +6 -6
- package/src/__tests__/unit/v1/next-url-pinning.test.ts +3 -3
- package/src/__tests__/unit/v1/optional-params.test.ts +4 -4
- package/src/__tests__/unit/v2/agent-effort-trace.unit.test.ts +6 -6
- package/src/__tests__/unit/v2/agent-hints.test.ts +3 -3
- package/src/__tests__/unit/v2/agent-list.unit.test.ts +4 -4
- package/src/__tests__/unit/v2/agent-thread.unit.test.ts +7 -7
- package/src/__tests__/unit/v2/agent.test.ts +1 -1
- package/src/__tests__/unit/v2/audit-metadata.unit.test.ts +3 -3
- package/src/__tests__/unit/v2/branding.test.ts +2 -2
- package/src/__tests__/unit/v2/clientOptions.test.ts +1 -0
- package/src/__tests__/unit/v2/crawl-cancel-abort.unit.test.ts +1 -1
- package/src/__tests__/unit/v2/developer.test.ts +4 -4
- package/src/__tests__/unit/v2/e2eRateLimitRetry.test.ts +16 -16
- package/src/__tests__/unit/v2/errorHandler.test.ts +1 -1
- package/src/__tests__/unit/v2/extract.unit.test.ts +4 -4
- package/src/__tests__/unit/v2/gov.test.ts +4 -4
- package/src/__tests__/unit/v2/menu.test.ts +2 -2
- package/src/__tests__/unit/v2/monitor-search-target.test.ts +1 -0
- package/src/__tests__/unit/v2/next-url-pinning.test.ts +1 -1
- package/src/__tests__/unit/v2/pagination.test.ts +6 -6
- package/src/__tests__/unit/v2/parse-formats.unit.test.ts +1 -1
- package/src/__tests__/unit/v2/parse.unit.test.ts +1 -1
- package/src/__tests__/unit/v2/product.test.ts +2 -2
- package/src/__tests__/unit/v2/prototype-discoverability.test.ts +1 -0
- package/src/__tests__/unit/v2/research.test.ts +1 -1
- package/src/__tests__/unit/v2/scrape-browser.unit.test.ts +8 -8
- package/src/__tests__/unit/v2/scrape.unit.test.ts +1 -1
- package/src/__tests__/unit/v2/scrapeAutoResume.test.ts +13 -13
- package/src/__tests__/unit/v2/search-country.unit.test.ts +2 -2
- package/src/__tests__/unit/v2/search-data-property.test.ts +3 -3
- package/src/__tests__/unit/v2/search-developer.unit.test.ts +2 -2
- package/src/__tests__/unit/v2/search-highlights.unit.test.ts +2 -2
- package/src/__tests__/unit/v2/threat-protection.unit.test.ts +3 -3
- package/src/__tests__/unit/v2/tools.unit.test.ts +3 -3
- package/src/__tests__/unit/v2/v1-aliases.test.ts +2 -2
- package/src/__tests__/unit/v2/validation.test.ts +1 -1
- package/src/__tests__/unit/v2/zodSchemaToJson.test.ts +1 -1
- package/src/v1/index.ts +1 -3
- package/src/v2/types.ts +8 -0
- package/vitest.config.ts +7 -0
- package/jest.config.js +0 -16
package/audit-ci.jsonc
CHANGED
|
@@ -1,19 +1,5 @@
|
|
|
1
1
|
{
|
|
2
2
|
"$schema": "https://github.com/IBM/audit-ci/raw/main/docs/schema.json",
|
|
3
3
|
"low": true,
|
|
4
|
-
"allowlist": [
|
|
5
|
-
// GHSA-vfj7-8cjw-p6xm (braces):
|
|
6
|
-
// Latest published braces is 3.0.3; advisory range is <=3.0.3 and
|
|
7
|
-
// no 3.0.4+ exists on npm. Same-major jest 30.x still resolves
|
|
8
|
-
// 3.0.3 via micromatch. Cannot override to a non-existent version.
|
|
9
|
-
// Owner: team-engineering-reviewers. Tracking: create internal ticket.
|
|
10
|
-
// Expires: 2026-10-17.
|
|
11
|
-
"GHSA-vfj7-8cjw-p6xm",
|
|
12
|
-
// GHSA-hp3w-g68c-fv3c (sprintf-js):
|
|
13
|
-
// Advisory asks for >=1.1.4; latest published sprintf-js is
|
|
14
|
-
// 1.1.3. Cannot override to a non-existent version.
|
|
15
|
-
// Owner: team-engineering-reviewers. Tracking: create internal ticket.
|
|
16
|
-
// Expires: 2026-10-20.
|
|
17
|
-
"GHSA-hp3w-g68c-fv3c"
|
|
18
|
-
]
|
|
4
|
+
"allowlist": []
|
|
19
5
|
}
|
|
@@ -12,7 +12,7 @@ var require_package = __commonJS({
|
|
|
12
12
|
"package.json"(exports, module) {
|
|
13
13
|
module.exports = {
|
|
14
14
|
name: "@mendable/firecrawl-js",
|
|
15
|
-
version: "4.
|
|
15
|
+
version: "4.46.0",
|
|
16
16
|
description: "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
|
|
17
17
|
main: "dist/index.js",
|
|
18
18
|
types: "dist/index.d.ts",
|
|
@@ -28,8 +28,8 @@ var require_package = __commonJS({
|
|
|
28
28
|
build: "tsup",
|
|
29
29
|
"build-and-publish": "pnpm run build && pnpm publish --access public",
|
|
30
30
|
"publish-beta": "pnpm run build && pnpm publish --access public --tag beta",
|
|
31
|
-
test: "
|
|
32
|
-
"test:unit": "
|
|
31
|
+
test: "vitest run --reporter=verbose src/__tests__/e2e/v2",
|
|
32
|
+
"test:unit": "vitest run --reporter=verbose src/__tests__/unit"
|
|
33
33
|
},
|
|
34
34
|
repository: {
|
|
35
35
|
type: "git",
|
|
@@ -49,18 +49,14 @@ var require_package = __commonJS({
|
|
|
49
49
|
},
|
|
50
50
|
homepage: "https://github.com/firecrawl/firecrawl#readme",
|
|
51
51
|
devDependencies: {
|
|
52
|
-
"@jest/globals": "^30.2.0",
|
|
53
52
|
"@types/dotenv": "^8.2.0",
|
|
54
|
-
"@types/jest": "^30.0.0",
|
|
55
|
-
"@types/mocha": "^10.0.6",
|
|
56
53
|
"@types/node": "^20.12.12",
|
|
57
54
|
"@types/uuid": "^9.0.8",
|
|
58
55
|
dotenv: "^16.4.5",
|
|
59
|
-
jest: "^30.2.0",
|
|
60
|
-
"ts-jest": "^29.4.5",
|
|
61
56
|
tsup: "^8.5.0",
|
|
62
57
|
typescript: "^5.4.5",
|
|
63
|
-
uuid: "^14.0.0"
|
|
58
|
+
uuid: "^14.0.0",
|
|
59
|
+
vitest: "^4.1.11"
|
|
64
60
|
},
|
|
65
61
|
keywords: [
|
|
66
62
|
"firecrawl",
|
package/dist/index.cjs
CHANGED
|
@@ -39,7 +39,7 @@ var require_package = __commonJS({
|
|
|
39
39
|
"package.json"(exports2, module2) {
|
|
40
40
|
module2.exports = {
|
|
41
41
|
name: "@mendable/firecrawl-js",
|
|
42
|
-
version: "4.
|
|
42
|
+
version: "4.46.0",
|
|
43
43
|
description: "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
|
|
44
44
|
main: "dist/index.js",
|
|
45
45
|
types: "dist/index.d.ts",
|
|
@@ -55,8 +55,8 @@ var require_package = __commonJS({
|
|
|
55
55
|
build: "tsup",
|
|
56
56
|
"build-and-publish": "pnpm run build && pnpm publish --access public",
|
|
57
57
|
"publish-beta": "pnpm run build && pnpm publish --access public --tag beta",
|
|
58
|
-
test: "
|
|
59
|
-
"test:unit": "
|
|
58
|
+
test: "vitest run --reporter=verbose src/__tests__/e2e/v2",
|
|
59
|
+
"test:unit": "vitest run --reporter=verbose src/__tests__/unit"
|
|
60
60
|
},
|
|
61
61
|
repository: {
|
|
62
62
|
type: "git",
|
|
@@ -76,18 +76,14 @@ var require_package = __commonJS({
|
|
|
76
76
|
},
|
|
77
77
|
homepage: "https://github.com/firecrawl/firecrawl#readme",
|
|
78
78
|
devDependencies: {
|
|
79
|
-
"@jest/globals": "^30.2.0",
|
|
80
79
|
"@types/dotenv": "^8.2.0",
|
|
81
|
-
"@types/jest": "^30.0.0",
|
|
82
|
-
"@types/mocha": "^10.0.6",
|
|
83
80
|
"@types/node": "^20.12.12",
|
|
84
81
|
"@types/uuid": "^9.0.8",
|
|
85
82
|
dotenv: "^16.4.5",
|
|
86
|
-
jest: "^30.2.0",
|
|
87
|
-
"ts-jest": "^29.4.5",
|
|
88
83
|
tsup: "^8.5.0",
|
|
89
84
|
typescript: "^5.4.5",
|
|
90
|
-
uuid: "^14.0.0"
|
|
85
|
+
uuid: "^14.0.0",
|
|
86
|
+
vitest: "^4.1.11"
|
|
91
87
|
},
|
|
92
88
|
keywords: [
|
|
93
89
|
"firecrawl",
|
|
@@ -2935,7 +2931,7 @@ var FirecrawlApp = class {
|
|
|
2935
2931
|
const packageJson = await Promise.resolve().then(() => __toESM(require_package(), 1));
|
|
2936
2932
|
return packageJson.default.version;
|
|
2937
2933
|
} catch (error) {
|
|
2938
|
-
const isTest = typeof process !== "undefined" &&
|
|
2934
|
+
const isTest = typeof process !== "undefined" && false;
|
|
2939
2935
|
if (!isTest) {
|
|
2940
2936
|
console.error("Error getting version:", error);
|
|
2941
2937
|
}
|
package/dist/index.d.cts
CHANGED
|
@@ -16,6 +16,7 @@ interface JsonFormat extends Format {
|
|
|
16
16
|
type: "json";
|
|
17
17
|
prompt?: string;
|
|
18
18
|
schema?: Record<string, unknown> | ZodTypeAny;
|
|
19
|
+
/** @deprecated Use the top-level `checkPromptInjection` scrape option. */
|
|
19
20
|
checkPromptInjection?: boolean;
|
|
20
21
|
}
|
|
21
22
|
interface ScreenshotFormat {
|
|
@@ -200,6 +201,13 @@ interface ScrapeOptions {
|
|
|
200
201
|
minAge?: number;
|
|
201
202
|
storeInCache?: boolean;
|
|
202
203
|
lockdown?: boolean;
|
|
204
|
+
/**
|
|
205
|
+
* Scans the page content for prompt injection with any format except
|
|
206
|
+
* rawBase64, before LLM-backed formats run. A detection fails the scrape
|
|
207
|
+
* with SCRAPE_PROMPT_INJECTION_DETECTED.
|
|
208
|
+
* Adds 4 credits when the check scans the whole page.
|
|
209
|
+
*/
|
|
210
|
+
checkPromptInjection?: boolean;
|
|
203
211
|
redactPII?: boolean | RedactPIIOptions;
|
|
204
212
|
threatProtection?: ThreatProtectionOptions;
|
|
205
213
|
auditMetadata?: AuditMetadata$1;
|
package/dist/index.d.ts
CHANGED
|
@@ -16,6 +16,7 @@ interface JsonFormat extends Format {
|
|
|
16
16
|
type: "json";
|
|
17
17
|
prompt?: string;
|
|
18
18
|
schema?: Record<string, unknown> | ZodTypeAny;
|
|
19
|
+
/** @deprecated Use the top-level `checkPromptInjection` scrape option. */
|
|
19
20
|
checkPromptInjection?: boolean;
|
|
20
21
|
}
|
|
21
22
|
interface ScreenshotFormat {
|
|
@@ -200,6 +201,13 @@ interface ScrapeOptions {
|
|
|
200
201
|
minAge?: number;
|
|
201
202
|
storeInCache?: boolean;
|
|
202
203
|
lockdown?: boolean;
|
|
204
|
+
/**
|
|
205
|
+
* Scans the page content for prompt injection with any format except
|
|
206
|
+
* rawBase64, before LLM-backed formats run. A detection fails the scrape
|
|
207
|
+
* with SCRAPE_PROMPT_INJECTION_DETECTED.
|
|
208
|
+
* Adds 4 credits when the check scans the whole page.
|
|
209
|
+
*/
|
|
210
|
+
checkPromptInjection?: boolean;
|
|
203
211
|
redactPII?: boolean | RedactPIIOptions;
|
|
204
212
|
threatProtection?: ThreatProtectionOptions;
|
|
205
213
|
auditMetadata?: AuditMetadata$1;
|
package/dist/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
require_package
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-R4XT7PME.js";
|
|
4
4
|
|
|
5
5
|
// src/v2/utils/httpClient.ts
|
|
6
6
|
import axios from "axios";
|
|
@@ -2800,10 +2800,10 @@ var FirecrawlApp = class {
|
|
|
2800
2800
|
if (typeof process !== "undefined" && process.env && process.env.npm_package_version) {
|
|
2801
2801
|
return process.env.npm_package_version;
|
|
2802
2802
|
}
|
|
2803
|
-
const packageJson = await import("./package-
|
|
2803
|
+
const packageJson = await import("./package-EI4Q33HD.js");
|
|
2804
2804
|
return packageJson.default.version;
|
|
2805
2805
|
} catch (error) {
|
|
2806
|
-
const isTest = typeof process !== "undefined" &&
|
|
2806
|
+
const isTest = typeof process !== "undefined" && false;
|
|
2807
2807
|
if (!isTest) {
|
|
2808
2808
|
console.error("Error getting version:", error);
|
|
2809
2809
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "firecrawl",
|
|
3
|
-
"version": "4.
|
|
3
|
+
"version": "4.46.0",
|
|
4
4
|
"description": "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -29,18 +29,14 @@
|
|
|
29
29
|
},
|
|
30
30
|
"homepage": "https://github.com/firecrawl/firecrawl#readme",
|
|
31
31
|
"devDependencies": {
|
|
32
|
-
"@jest/globals": "^30.2.0",
|
|
33
32
|
"@types/dotenv": "^8.2.0",
|
|
34
|
-
"@types/jest": "^30.0.0",
|
|
35
|
-
"@types/mocha": "^10.0.6",
|
|
36
33
|
"@types/node": "^20.12.12",
|
|
37
34
|
"@types/uuid": "^9.0.8",
|
|
38
35
|
"dotenv": "^16.4.5",
|
|
39
|
-
"jest": "^30.2.0",
|
|
40
|
-
"ts-jest": "^29.4.5",
|
|
41
36
|
"tsup": "^8.5.0",
|
|
42
37
|
"typescript": "^5.4.5",
|
|
43
|
-
"uuid": "^14.0.0"
|
|
38
|
+
"uuid": "^14.0.0",
|
|
39
|
+
"vitest": "^4.1.11"
|
|
44
40
|
},
|
|
45
41
|
"keywords": [
|
|
46
42
|
"firecrawl",
|
|
@@ -72,7 +68,7 @@
|
|
|
72
68
|
"build": "tsup",
|
|
73
69
|
"build-and-publish": "pnpm run build && pnpm publish --access public",
|
|
74
70
|
"publish-beta": "pnpm run build && pnpm publish --access public --tag beta",
|
|
75
|
-
"test": "
|
|
76
|
-
"test:unit": "
|
|
71
|
+
"test": "vitest run --reporter=verbose src/__tests__/e2e/v2",
|
|
72
|
+
"test:unit": "vitest run --reporter=verbose src/__tests__/unit"
|
|
77
73
|
}
|
|
78
74
|
}
|
package/pnpm-workspace.yaml
CHANGED
|
@@ -1,17 +1,13 @@
|
|
|
1
1
|
allowBuilds:
|
|
2
2
|
esbuild: true
|
|
3
|
-
unrs-resolver: true
|
|
4
3
|
overrides:
|
|
5
4
|
"@isaacs/brace-expansion@<=5.0.0": ">=5.0.1"
|
|
6
5
|
"minimatch@<10.2.3": ">=10.2.3"
|
|
7
6
|
"rollup@<4.59.0": ">=4.59.0"
|
|
8
7
|
"picomatch@<4.0.4": ">=4.0.4"
|
|
9
|
-
handlebars: ">=4.7.9"
|
|
10
8
|
# GHSA-q2hr-2g5m-vwhr / GHSA-qhr7-859c-m2p7 / GHSA-6j4f-fj2g-mc7p:
|
|
11
9
|
# pin the 5.x line only. Exact 5.0.12 cannot resolve to 6.x.
|
|
12
10
|
"brace-expansion@>=5.0.0 <5.0.12": "5.0.12"
|
|
13
|
-
"js-yaml@>=3.0.0 <3.15.2": "3.15.2"
|
|
14
|
-
"js-yaml@>=4.0.0 <4.3.2": "4.3.2"
|
|
15
11
|
# GHSA-vh66-26gq-q6x8 / GHSA-9fr6-4gfg-395g / GHSA-c29m-xwm3-cm6r /
|
|
16
12
|
# GHSA-mghh-pgcx-3jjj / GHSA-x97p-jq2g-jp4f / GHSA-3pq3-5fj3-cg6v /
|
|
17
13
|
# GHSA-542g-h47m-68v8 / GHSA-j8rh-479h-cp32 / GHSA-4hqw-qxg8-jxx2 /
|
|
@@ -20,6 +16,4 @@ overrides:
|
|
|
20
16
|
"axios@>=1.0.0 <1.20.0": "1.20.0"
|
|
21
17
|
"follow-redirects@<1.16.0": ">=1.16.0 <2.0.0"
|
|
22
18
|
"form-data@>=4.0.0 <4.0.6": "4.0.6"
|
|
23
|
-
"@babel/core@>=7.0.0 <7.29.6": "7.29.7"
|
|
24
19
|
"esbuild@>=0.17.0 <0.28.1": "0.28.1"
|
|
25
|
-
"browserslist@<=4.28.6": ">=4.28.7 <5.0.0"
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import FirecrawlApp, { type CrawlParams, type CrawlResponse, type CrawlStatusResponse, type MapResponse, type ScrapeResponse } from '../../../index';
|
|
2
2
|
import { v4 as uuidv4 } from 'uuid';
|
|
3
3
|
import dotenv from 'dotenv';
|
|
4
|
-
import { describe, test, expect } from '
|
|
4
|
+
import { describe, test, expect } from 'vitest';
|
|
5
5
|
|
|
6
6
|
dotenv.config();
|
|
7
7
|
|
|
@@ -5,7 +5,7 @@ import Firecrawl from "../../../index";
|
|
|
5
5
|
import { config } from "dotenv";
|
|
6
6
|
import { getIdentity, getApiUrl } from "./utils/idmux";
|
|
7
7
|
import { testTimeoutMs, waitForJob, withRateLimitRetry } from "./utils/rateLimit";
|
|
8
|
-
import { describe, test, expect, beforeAll } from "
|
|
8
|
+
import { describe, test, expect, beforeAll } from "vitest";
|
|
9
9
|
|
|
10
10
|
config();
|
|
11
11
|
|
|
@@ -5,7 +5,7 @@ import Firecrawl from "../../../index";
|
|
|
5
5
|
import { config } from "dotenv";
|
|
6
6
|
import { getIdentity, getApiUrl } from "./utils/idmux";
|
|
7
7
|
import { testTimeoutMs, waitForJob, withRateLimitRetry } from "./utils/rateLimit";
|
|
8
|
-
import { describe, test, expect, beforeAll } from "
|
|
8
|
+
import { describe, test, expect, beforeAll } from "vitest";
|
|
9
9
|
|
|
10
10
|
config();
|
|
11
11
|
|
|
@@ -5,7 +5,7 @@ import Firecrawl from "../../../index";
|
|
|
5
5
|
import { config } from "dotenv";
|
|
6
6
|
import { getIdentity, getApiUrl } from "./utils/idmux";
|
|
7
7
|
import { testTimeoutMs, waitForJob, withRateLimitRetry } from "./utils/rateLimit";
|
|
8
|
-
import { describe, test, expect, beforeAll } from "
|
|
8
|
+
import { describe, test, expect, beforeAll } from "vitest";
|
|
9
9
|
import { z } from "zod";
|
|
10
10
|
|
|
11
11
|
config();
|
|
@@ -5,7 +5,7 @@ import Firecrawl from "../../../index";
|
|
|
5
5
|
import { config } from "dotenv";
|
|
6
6
|
import { getIdentity, getApiUrl } from "./utils/idmux";
|
|
7
7
|
import { testTimeoutMs, withRateLimitRetry } from "./utils/rateLimit";
|
|
8
|
-
import { describe, test, expect, beforeAll } from "
|
|
8
|
+
import { describe, test, expect, beforeAll } from "vitest";
|
|
9
9
|
|
|
10
10
|
config();
|
|
11
11
|
|
|
@@ -2,7 +2,7 @@ import Firecrawl from "../../../index";
|
|
|
2
2
|
import { config } from "dotenv";
|
|
3
3
|
import { getIdentity, getApiUrl } from "./utils/idmux";
|
|
4
4
|
import { testTimeoutMs, withRateLimitRetry } from "./utils/rateLimit";
|
|
5
|
-
import { describe, test, expect, beforeAll } from "
|
|
5
|
+
import { describe, test, expect, beforeAll } from "vitest";
|
|
6
6
|
import { SdkError } from "../../../v2/types";
|
|
7
7
|
|
|
8
8
|
config();
|
|
@@ -6,7 +6,7 @@ import { z } from "zod";
|
|
|
6
6
|
import { config } from "dotenv";
|
|
7
7
|
import { getIdentity, getApiUrl } from "./utils/idmux";
|
|
8
8
|
import { testTimeoutMs, withRateLimitRetry } from "./utils/rateLimit";
|
|
9
|
-
import { describe, test, expect, beforeAll } from "
|
|
9
|
+
import { describe, test, expect, beforeAll } from "vitest";
|
|
10
10
|
|
|
11
11
|
config();
|
|
12
12
|
|
|
@@ -6,7 +6,7 @@ import type { Document, SearchResultWeb, SearchResultNews, SearchResultImages }
|
|
|
6
6
|
import { config } from "dotenv";
|
|
7
7
|
import { getIdentity, getApiUrl } from "./utils/idmux";
|
|
8
8
|
import { testTimeoutMs, withRateLimitRetry } from "./utils/rateLimit";
|
|
9
|
-
import { describe, test, expect, beforeAll } from "
|
|
9
|
+
import { describe, test, expect, beforeAll } from "vitest";
|
|
10
10
|
|
|
11
11
|
config();
|
|
12
12
|
|
|
@@ -5,7 +5,7 @@ import Firecrawl from "../../../index";
|
|
|
5
5
|
import { config } from "dotenv";
|
|
6
6
|
import { getIdentity, getApiUrl } from "./utils/idmux";
|
|
7
7
|
import { testTimeoutMs, withRateLimitRetry } from "./utils/rateLimit";
|
|
8
|
-
import { describe, test, expect, beforeAll } from "
|
|
8
|
+
import { describe, test, expect, beforeAll } from "vitest";
|
|
9
9
|
|
|
10
10
|
config();
|
|
11
11
|
|
|
@@ -52,7 +52,7 @@ export const RETRY_BUDGET_MS = (MAX_ATTEMPTS - 1) * MAX_WAIT_MS;
|
|
|
52
52
|
*
|
|
53
53
|
* Every caller now passes its own timeout, sized to the work it waits for.
|
|
54
54
|
* This value is only a backstop for a new caller that forgets one. Without a
|
|
55
|
-
* bound the poll loop runs until the
|
|
55
|
+
* bound the poll loop runs until the test timeout, which hides the reason.
|
|
56
56
|
*
|
|
57
57
|
* The value stays short because it cannot know the work. It fits the smallest
|
|
58
58
|
* base any suite passes to testTimeoutMs, which is 60_000 ms:
|
|
@@ -69,7 +69,7 @@ export const RETRY_BUDGET_MS = (MAX_ATTEMPTS - 1) * MAX_WAIT_MS;
|
|
|
69
69
|
export const DEFAULT_JOB_TIMEOUT_MS = 45_000;
|
|
70
70
|
|
|
71
71
|
/**
|
|
72
|
-
*
|
|
72
|
+
* Vitest timeout for a test that calls a wrapped client.
|
|
73
73
|
*
|
|
74
74
|
* Pass the base: the time the test needs when the API answers at once. The
|
|
75
75
|
* result adds one retry budget on top, so a test whose single call hits the
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import Firecrawl from "../../../index";
|
|
2
2
|
import { config } from "dotenv";
|
|
3
|
-
import { describe, test, expect, beforeAll } from "
|
|
3
|
+
import { describe, test, expect, beforeAll } from "vitest";
|
|
4
4
|
import { getIdentity } from "./utils/idmux";
|
|
5
5
|
import { testTimeoutMs, withRateLimitRetry } from "./utils/rateLimit";
|
|
6
6
|
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { describe, test, expect,
|
|
1
|
+
import { describe, test, expect, vi, afterEach, beforeEach } from "vitest";
|
|
2
2
|
import axios from "axios";
|
|
3
3
|
import FirecrawlApp from "../../../v1";
|
|
4
4
|
|
|
@@ -20,7 +20,7 @@ const lastPage = () => ({ success: true, status: "completed", data: [{ markdown:
|
|
|
20
20
|
function mockPoll(path: string, nextOrigin = API_URL) {
|
|
21
21
|
const sent: string[] = [];
|
|
22
22
|
let polls = 0;
|
|
23
|
-
|
|
23
|
+
vi.spyOn(axios, "get").mockImplementation(async (url: string) => {
|
|
24
24
|
sent.push(url);
|
|
25
25
|
if (new URL(url).searchParams.has("skip")) return { status: 200, data: lastPage() };
|
|
26
26
|
polls++;
|
|
@@ -33,18 +33,18 @@ describe("v1 batch scrape waits on the batch scrape status endpoint", () => {
|
|
|
33
33
|
const app = new FirecrawlApp({ apiKey: API_KEY, apiUrl: API_URL });
|
|
34
34
|
|
|
35
35
|
beforeEach(() => {
|
|
36
|
-
|
|
36
|
+
vi.spyOn(global, "setTimeout").mockImplementation(((fn: () => void) => {
|
|
37
37
|
fn();
|
|
38
38
|
return 0;
|
|
39
39
|
}) as any);
|
|
40
40
|
});
|
|
41
41
|
|
|
42
42
|
afterEach(() => {
|
|
43
|
-
|
|
43
|
+
vi.restoreAllMocks();
|
|
44
44
|
});
|
|
45
45
|
|
|
46
46
|
test("batchScrapeUrls polls /v1/batch/scrape/{id} and follows pinned next pages", async () => {
|
|
47
|
-
const post =
|
|
47
|
+
const post = vi.spyOn(axios, "post").mockResolvedValue({ status: 200, data: { success: true, id: "abc" } });
|
|
48
48
|
const path = "/v1/batch/scrape/abc";
|
|
49
49
|
const sent = mockPoll(path, "https://evil.example");
|
|
50
50
|
|
|
@@ -58,9 +58,9 @@ describe("v1 batch scrape waits on the batch scrape status endpoint", () => {
|
|
|
58
58
|
});
|
|
59
59
|
|
|
60
60
|
test("batchScrapeUrls reports batch scrape failures as batch scrape errors", async () => {
|
|
61
|
-
|
|
61
|
+
vi.spyOn(axios, "post").mockResolvedValue({ status: 200, data: { success: true, id: "abc" } });
|
|
62
62
|
const sent: string[] = [];
|
|
63
|
-
|
|
63
|
+
vi.spyOn(axios, "get").mockImplementation(async (url: string) => {
|
|
64
64
|
sent.push(url);
|
|
65
65
|
return { status: 200, data: { success: true, status: "failed", data: [] } };
|
|
66
66
|
});
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import FirecrawlApp from '../../../v1';
|
|
2
|
-
import { describe, test, expect,
|
|
2
|
+
import { describe, test, expect, vi, beforeEach, afterEach } from 'vitest';
|
|
3
3
|
|
|
4
4
|
const successResponse = {
|
|
5
5
|
status: 200,
|
|
@@ -21,17 +21,17 @@ describe('monitorJobStatus retry logic', () => {
|
|
|
21
21
|
beforeEach(() => {
|
|
22
22
|
app = new FirecrawlApp({ apiKey: 'test-key', apiUrl: 'https://test.com' });
|
|
23
23
|
delays = [];
|
|
24
|
-
|
|
24
|
+
vi.useFakeTimers();
|
|
25
25
|
const fakeSetTimeout = globalThis.setTimeout;
|
|
26
|
-
|
|
26
|
+
vi.spyOn(globalThis, 'setTimeout').mockImplementation(((fn: () => void, ms?: number) => {
|
|
27
27
|
delays.push(ms ?? 0);
|
|
28
28
|
return fakeSetTimeout(fn, ms);
|
|
29
29
|
}) as any);
|
|
30
30
|
});
|
|
31
31
|
|
|
32
32
|
afterEach(() => {
|
|
33
|
-
|
|
34
|
-
|
|
33
|
+
vi.restoreAllMocks();
|
|
34
|
+
vi.useRealTimers();
|
|
35
35
|
});
|
|
36
36
|
|
|
37
37
|
function failFirst(n: number, error: Error, statuses: string[] = []) {
|
|
@@ -48,7 +48,7 @@ describe('monitorJobStatus retry logic', () => {
|
|
|
48
48
|
async function monitor() {
|
|
49
49
|
const result = app.monitorJobStatus('test-id', {} as any, 1);
|
|
50
50
|
result.catch(() => {});
|
|
51
|
-
await
|
|
51
|
+
await vi.runAllTimersAsync();
|
|
52
52
|
return result;
|
|
53
53
|
}
|
|
54
54
|
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { describe, test, expect,
|
|
1
|
+
import { describe, test, expect, vi, afterEach } from "vitest";
|
|
2
2
|
import axios from "axios";
|
|
3
3
|
import FirecrawlApp from "../../../v1";
|
|
4
4
|
|
|
@@ -16,7 +16,7 @@ const origins: Array<[string, string]> = [
|
|
|
16
16
|
|
|
17
17
|
function mockGet(respond: (url: URL) => unknown) {
|
|
18
18
|
const sent: Array<{ url: string; authorization: unknown }> = [];
|
|
19
|
-
|
|
19
|
+
vi.spyOn(axios, "get").mockImplementation(async (url: string, config?: any) => {
|
|
20
20
|
sent.push({ url, authorization: config?.headers?.Authorization });
|
|
21
21
|
return { status: 200, data: respond(new URL(url)) };
|
|
22
22
|
});
|
|
@@ -45,7 +45,7 @@ const lastPage = () => ({ success: true, status: "completed", data: [{ markdown:
|
|
|
45
45
|
|
|
46
46
|
describe("v1 pins next URLs to the api_url origin", () => {
|
|
47
47
|
afterEach(() => {
|
|
48
|
-
|
|
48
|
+
vi.restoreAllMocks();
|
|
49
49
|
});
|
|
50
50
|
|
|
51
51
|
describe.each(origins)("%s next", (_label, origin) => {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { describe, test, expect,
|
|
1
|
+
import { describe, test, expect, vi, beforeEach, afterEach } from "vitest";
|
|
2
2
|
import axios from "axios";
|
|
3
3
|
import FirecrawlApp from "../../../v1";
|
|
4
4
|
|
|
@@ -20,11 +20,11 @@ describe("v1 methods work without params", () => {
|
|
|
20
20
|
beforeEach(() => {
|
|
21
21
|
app = new FirecrawlApp({ apiKey: "fc-test", apiUrl: API_URL });
|
|
22
22
|
posts = [];
|
|
23
|
-
|
|
23
|
+
vi.spyOn(axios, "post").mockImplementation(async (url: string, data?: any) => {
|
|
24
24
|
posts.push({ url, data });
|
|
25
25
|
return { status: 200, data: { success: true, id: "job-1", data: [], links: [] } };
|
|
26
26
|
});
|
|
27
|
-
|
|
27
|
+
vi.spyOn(axios, "get").mockImplementation(async () => ({
|
|
28
28
|
status: 200,
|
|
29
29
|
data: { success: true, status: "completed", data: [{ markdown: "ok" }] },
|
|
30
30
|
}));
|
|
@@ -32,7 +32,7 @@ describe("v1 methods work without params", () => {
|
|
|
32
32
|
});
|
|
33
33
|
|
|
34
34
|
afterEach(() => {
|
|
35
|
-
|
|
35
|
+
vi.restoreAllMocks();
|
|
36
36
|
(globalThis as any).WebSocket = originalWebSocket;
|
|
37
37
|
});
|
|
38
38
|
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { describe, expect,
|
|
1
|
+
import { describe, expect, vi, test } from "vitest";
|
|
2
2
|
import {
|
|
3
3
|
startAgent,
|
|
4
4
|
getAgentTrace,
|
|
@@ -8,7 +8,7 @@ import type { AgentTraceEvent } from "../../../v2/types";
|
|
|
8
8
|
|
|
9
9
|
describe("v2.agent unit", () => {
|
|
10
10
|
test("startAgent forwards effort in request payload", async () => {
|
|
11
|
-
const post =
|
|
11
|
+
const post = vi.fn().mockResolvedValue({
|
|
12
12
|
status: 200,
|
|
13
13
|
data: { success: true, id: "agent-job" },
|
|
14
14
|
});
|
|
@@ -25,7 +25,7 @@ describe("v2.agent unit", () => {
|
|
|
25
25
|
});
|
|
26
26
|
|
|
27
27
|
test("startAgent omits effort when unset", async () => {
|
|
28
|
-
const post =
|
|
28
|
+
const post = vi.fn().mockResolvedValue({
|
|
29
29
|
status: 200,
|
|
30
30
|
data: { success: true, id: "agent-job" },
|
|
31
31
|
});
|
|
@@ -70,7 +70,7 @@ describe("v2.agent unit", () => {
|
|
|
70
70
|
change: "partial",
|
|
71
71
|
},
|
|
72
72
|
};
|
|
73
|
-
const get =
|
|
73
|
+
const get = vi.fn().mockResolvedValue({
|
|
74
74
|
status: 200,
|
|
75
75
|
data: {
|
|
76
76
|
success: true,
|
|
@@ -95,7 +95,7 @@ describe("v2.agent unit", () => {
|
|
|
95
95
|
});
|
|
96
96
|
|
|
97
97
|
test("getAgentTrace appends liveView query param when requested", async () => {
|
|
98
|
-
const get =
|
|
98
|
+
const get = vi.fn().mockResolvedValue({
|
|
99
99
|
status: 200,
|
|
100
100
|
data: {
|
|
101
101
|
success: true,
|
|
@@ -122,7 +122,7 @@ describe("v2.agent unit", () => {
|
|
|
122
122
|
});
|
|
123
123
|
|
|
124
124
|
test("getAgentSnapshot hits the snapshot endpoint", async () => {
|
|
125
|
-
const get =
|
|
125
|
+
const get = vi.fn().mockResolvedValue({
|
|
126
126
|
status: 200,
|
|
127
127
|
data: {
|
|
128
128
|
success: true,
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { describe, expect, it,
|
|
1
|
+
import { describe, expect, it, vi } from "vitest";
|
|
2
2
|
import { search } from "../../../v2/methods/search";
|
|
3
3
|
import { scrape } from "../../../v2/methods/scrape";
|
|
4
4
|
import { parse } from "../../../v2/methods/parse";
|
|
@@ -14,8 +14,8 @@ const hints = [
|
|
|
14
14
|
];
|
|
15
15
|
function httpFor(body: object) {
|
|
16
16
|
return {
|
|
17
|
-
post:
|
|
18
|
-
postMultipart:
|
|
17
|
+
post: vi.fn<any>().mockResolvedValue({ status: 200, data: body }),
|
|
18
|
+
postMultipart: vi
|
|
19
19
|
.fn<any>()
|
|
20
20
|
.mockResolvedValue({ status: 200, data: body }),
|
|
21
21
|
} as any;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { describe, expect,
|
|
1
|
+
import { describe, expect, vi, test } from "vitest";
|
|
2
2
|
import { listAgents } from "../../../v2/methods/agent";
|
|
3
3
|
|
|
4
4
|
describe("v2.agent listAgents unit", () => {
|
|
@@ -17,7 +17,7 @@ describe("v2.agent listAgents unit", () => {
|
|
|
17
17
|
};
|
|
18
18
|
|
|
19
19
|
test("hits GET /v2/agent without query by default", async () => {
|
|
20
|
-
const get =
|
|
20
|
+
const get = vi.fn().mockResolvedValue({
|
|
21
21
|
status: 200,
|
|
22
22
|
data: { success: true, agents: [sampleAgent] },
|
|
23
23
|
});
|
|
@@ -33,7 +33,7 @@ describe("v2.agent listAgents unit", () => {
|
|
|
33
33
|
});
|
|
34
34
|
|
|
35
35
|
test("passes before as a query param", async () => {
|
|
36
|
-
const get =
|
|
36
|
+
const get = vi.fn().mockResolvedValue({
|
|
37
37
|
status: 200,
|
|
38
38
|
data: {
|
|
39
39
|
success: true,
|
|
@@ -51,7 +51,7 @@ describe("v2.agent listAgents unit", () => {
|
|
|
51
51
|
});
|
|
52
52
|
|
|
53
53
|
test("does not auto-paginate when next is present", async () => {
|
|
54
|
-
const get =
|
|
54
|
+
const get = vi.fn().mockResolvedValue({
|
|
55
55
|
status: 200,
|
|
56
56
|
data: {
|
|
57
57
|
success: true,
|