@mendable/firecrawl-js 4.45.0 → 4.47.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -8
- package/audit-ci.jsonc +1 -15
- package/dist/chunk-H75FJ6LW.js +114 -0
- package/dist/index.cjs +61 -27
- package/dist/index.d.cts +15 -3
- package/dist/index.d.ts +15 -3
- package/dist/index.js +58 -21
- package/dist/package-NZI5TDUV.js +40 -0
- package/package.json +5 -9
- package/pnpm-workspace.yaml +0 -6
- package/src/__tests__/e2e/v1/index.test.ts +1 -1
- package/src/__tests__/e2e/v2/batch.test.ts +1 -1
- package/src/__tests__/e2e/v2/crawl.test.ts +1 -1
- package/src/__tests__/e2e/v2/extract.test.ts +1 -1
- package/src/__tests__/e2e/v2/map.test.ts +1 -1
- package/src/__tests__/e2e/v2/parse.test.ts +1 -1
- package/src/__tests__/e2e/v2/scrape.test.ts +1 -1
- package/src/__tests__/e2e/v2/search.test.ts +1 -1
- package/src/__tests__/e2e/v2/usage.test.ts +1 -1
- package/src/__tests__/e2e/v2/utils/rateLimit.ts +2 -2
- package/src/__tests__/e2e/v2/watcher.test.ts +1 -1
- package/src/__tests__/unit/v1/batch-scrape-poll.test.ts +7 -7
- package/src/__tests__/unit/v1/monitor-job-status-retry.test.ts +6 -6
- package/src/__tests__/unit/v1/next-url-pinning.test.ts +3 -3
- package/src/__tests__/unit/v1/optional-params.test.ts +4 -4
- package/src/__tests__/unit/v2/agent-effort-trace.unit.test.ts +6 -6
- package/src/__tests__/unit/v2/agent-hints.test.ts +3 -3
- package/src/__tests__/unit/v2/agent-list.unit.test.ts +4 -4
- package/src/__tests__/unit/v2/agent-thread.unit.test.ts +7 -7
- package/src/__tests__/unit/v2/agent.test.ts +1 -1
- package/src/__tests__/unit/v2/audit-metadata.unit.test.ts +3 -3
- package/src/__tests__/unit/v2/branding.test.ts +2 -2
- package/src/__tests__/unit/v2/browser.unit.test.ts +28 -0
- package/src/__tests__/unit/v2/clientOptions.test.ts +1 -0
- package/src/__tests__/unit/v2/crawl-cancel-abort.unit.test.ts +1 -1
- package/src/__tests__/unit/v2/developer.test.ts +4 -4
- package/src/__tests__/unit/v2/e2eRateLimitRetry.test.ts +16 -16
- package/src/__tests__/unit/v2/errorHandler.test.ts +24 -1
- package/src/__tests__/unit/v2/extract.unit.test.ts +4 -4
- package/src/__tests__/unit/v2/gov.test.ts +4 -4
- package/src/__tests__/unit/v2/menu.test.ts +2 -2
- package/src/__tests__/unit/v2/monitor-search-target.test.ts +1 -0
- package/src/__tests__/unit/v2/next-url-pinning.test.ts +1 -1
- package/src/__tests__/unit/v2/pagination.test.ts +26 -6
- package/src/__tests__/unit/v2/parse-formats.unit.test.ts +1 -1
- package/src/__tests__/unit/v2/parse.unit.test.ts +1 -1
- package/src/__tests__/unit/v2/product.test.ts +2 -2
- package/src/__tests__/unit/v2/prototype-discoverability.test.ts +1 -0
- package/src/__tests__/unit/v2/research.test.ts +1 -1
- package/src/__tests__/unit/v2/scrape-browser.unit.test.ts +8 -8
- package/src/__tests__/unit/v2/scrape.unit.test.ts +1 -1
- package/src/__tests__/unit/v2/scrapeAutoResume.test.ts +13 -13
- package/src/__tests__/unit/v2/search-country.unit.test.ts +2 -2
- package/src/__tests__/unit/v2/search-data-property.test.ts +3 -3
- package/src/__tests__/unit/v2/search-developer.unit.test.ts +2 -2
- package/src/__tests__/unit/v2/search-highlights.unit.test.ts +2 -2
- package/src/__tests__/unit/v2/threat-protection.unit.test.ts +3 -3
- package/src/__tests__/unit/v2/tools.unit.test.ts +3 -3
- package/src/__tests__/unit/v2/v1-aliases.test.ts +2 -2
- package/src/__tests__/unit/v2/validation.test.ts +40 -2
- package/src/__tests__/unit/v2/zodSchemaToJson.test.ts +1 -1
- package/src/v1/index.ts +1 -3
- package/src/v2/client.ts +1 -1
- package/src/v2/methods/browser.ts +3 -0
- package/src/v2/methods/crawl.ts +2 -1
- package/src/v2/methods/search.ts +2 -1
- package/src/v2/types.ts +10 -2
- package/src/v2/utils/errorHandler.ts +23 -2
- package/src/v2/utils/getVersion.ts +4 -12
- package/src/v2/utils/pagination.ts +4 -1
- package/src/v2/utils/validation.ts +28 -0
- package/vitest.config.ts +7 -0
- package/dist/chunk-LKCOCWYW.js +0 -97
- package/dist/package-C6PK3CK7.js +0 -4
- package/jest.config.js +0 -16
package/dist/index.js
CHANGED
|
@@ -1,21 +1,13 @@
|
|
|
1
1
|
import {
|
|
2
|
-
|
|
3
|
-
} from "./chunk-
|
|
2
|
+
version
|
|
3
|
+
} from "./chunk-H75FJ6LW.js";
|
|
4
4
|
|
|
5
5
|
// src/v2/utils/httpClient.ts
|
|
6
6
|
import axios from "axios";
|
|
7
7
|
|
|
8
8
|
// src/v2/utils/getVersion.ts
|
|
9
9
|
function getVersion() {
|
|
10
|
-
|
|
11
|
-
if (typeof process !== "undefined" && process.env && process.env.npm_package_version) {
|
|
12
|
-
return process.env.npm_package_version;
|
|
13
|
-
}
|
|
14
|
-
const pkg = require_package();
|
|
15
|
-
return pkg?.version || "3.x.x";
|
|
16
|
-
} catch {
|
|
17
|
-
return "3.x.x";
|
|
18
|
-
}
|
|
10
|
+
return version;
|
|
19
11
|
}
|
|
20
12
|
|
|
21
13
|
// src/utils/apiOrigin.ts
|
|
@@ -69,7 +61,7 @@ var HttpClient = class {
|
|
|
69
61
|
return this.instance.defaults.timeout ?? 3e5;
|
|
70
62
|
}
|
|
71
63
|
async request(config) {
|
|
72
|
-
const
|
|
64
|
+
const version2 = getVersion();
|
|
73
65
|
config.headers = {
|
|
74
66
|
...config.headers || {}
|
|
75
67
|
};
|
|
@@ -84,7 +76,7 @@ var HttpClient = class {
|
|
|
84
76
|
const data = cfg.data ?? {};
|
|
85
77
|
cfg.data = {
|
|
86
78
|
...data,
|
|
87
|
-
origin: typeof data.origin === "string" && data.origin.includes("mcp") ? data.origin : `js-sdk@${
|
|
79
|
+
origin: typeof data.origin === "string" && data.origin.includes("mcp") ? data.origin : `js-sdk@${version2}`
|
|
88
80
|
};
|
|
89
81
|
}
|
|
90
82
|
if (isFormDataBody) {
|
|
@@ -323,6 +315,30 @@ function ensureValidFormats(formats) {
|
|
|
323
315
|
}
|
|
324
316
|
}
|
|
325
317
|
}
|
|
318
|
+
var V1_SCRAPE_OPTION_REPLACEMENTS = {
|
|
319
|
+
pageOptions: "use scrapeOptions",
|
|
320
|
+
includeLinks: "add 'links' to scrapeOptions.formats",
|
|
321
|
+
parsePDF: "use scrapeOptions.parsers"
|
|
322
|
+
};
|
|
323
|
+
var V1_OPTION_REPLACEMENTS = {
|
|
324
|
+
crawl: {
|
|
325
|
+
crawlerOptions: "move its fields to the top level",
|
|
326
|
+
maxDepth: "use maxDiscoveryDepth",
|
|
327
|
+
allowBackwardLinks: "use crawlEntireDomain",
|
|
328
|
+
ignoreSitemap: "use sitemap: 'skip'",
|
|
329
|
+
...V1_SCRAPE_OPTION_REPLACEMENTS
|
|
330
|
+
},
|
|
331
|
+
search: V1_SCRAPE_OPTION_REPLACEMENTS
|
|
332
|
+
};
|
|
333
|
+
function ensureNoV1Options(options, method) {
|
|
334
|
+
if (!options) return;
|
|
335
|
+
const replacements = V1_OPTION_REPLACEMENTS[method];
|
|
336
|
+
const found = Object.keys(replacements).filter((key) => options[key] != null);
|
|
337
|
+
if (found.length === 0) return;
|
|
338
|
+
throw new Error(
|
|
339
|
+
`${method}: ${found.map((key) => `${key} is a v1 option, ${replacements[key]}`).join("; ")}`
|
|
340
|
+
);
|
|
341
|
+
}
|
|
326
342
|
function ensureValidScrapeOptions(options) {
|
|
327
343
|
if (!options) return;
|
|
328
344
|
if (options.timeout != null && options.timeout <= 0) {
|
|
@@ -436,12 +452,28 @@ function ensureValidParseOptions(options) {
|
|
|
436
452
|
|
|
437
453
|
// src/v2/utils/errorHandler.ts
|
|
438
454
|
import "axios";
|
|
455
|
+
function describeValidationIssues(details) {
|
|
456
|
+
if (!Array.isArray(details)) return void 0;
|
|
457
|
+
const parts = details.slice(0, 3).flatMap((issue) => {
|
|
458
|
+
if (!issue || typeof issue !== "object") return [];
|
|
459
|
+
const { path, message, keys } = issue;
|
|
460
|
+
const what = Array.isArray(keys) && keys.length > 0 ? `unrecognized key(s) ${keys.join(", ")}` : typeof message === "string" ? message : void 0;
|
|
461
|
+
if (!what) return [];
|
|
462
|
+
return Array.isArray(path) && path.length > 0 ? [`${path.join(".")}: ${what}`] : [what];
|
|
463
|
+
});
|
|
464
|
+
return parts.length > 0 ? parts.join("; ") : void 0;
|
|
465
|
+
}
|
|
466
|
+
function withIssues(message, details) {
|
|
467
|
+
const text = typeof message === "string" ? message : JSON.stringify(message);
|
|
468
|
+
const issues = describeValidationIssues(details);
|
|
469
|
+
return issues && !text.includes(issues) ? `${text} (${issues})` : text;
|
|
470
|
+
}
|
|
439
471
|
function throwForBadResponse(resp, action) {
|
|
440
472
|
const status = resp.status;
|
|
441
473
|
const body = resp.data || {};
|
|
442
474
|
const msg = body?.error || body?.message || `Request failed (${status}) while trying to ${action}`;
|
|
443
475
|
throw Object.assign(new SdkError(
|
|
444
|
-
msg,
|
|
476
|
+
withIssues(msg, body?.details),
|
|
445
477
|
status,
|
|
446
478
|
body?.code,
|
|
447
479
|
body?.details,
|
|
@@ -456,7 +488,7 @@ function normalizeAxiosError(err, action) {
|
|
|
456
488
|
const message = body?.error || err.message || `Request failed${status ? ` (${status})` : ""} while trying to ${action}`;
|
|
457
489
|
const code = body?.code || err.code;
|
|
458
490
|
throw Object.assign(new SdkError(
|
|
459
|
-
message,
|
|
491
|
+
withIssues(message, body?.details),
|
|
460
492
|
status,
|
|
461
493
|
code,
|
|
462
494
|
body?.details ?? body,
|
|
@@ -627,10 +659,10 @@ async function parse(http, file, options) {
|
|
|
627
659
|
throw new Error("file data cannot be empty");
|
|
628
660
|
}
|
|
629
661
|
if (options) ensureValidParseOptions(options);
|
|
630
|
-
const
|
|
662
|
+
const version2 = getVersion();
|
|
631
663
|
const normalizedOptions = {
|
|
632
664
|
...options ?? {},
|
|
633
|
-
origin: typeof options?.origin === "string" && options.origin.includes("mcp") ? options.origin : options?.origin ?? `js-sdk@${
|
|
665
|
+
origin: typeof options?.origin === "string" && options.origin.includes("mcp") ? options.origin : options?.origin ?? `js-sdk@${version2}`
|
|
634
666
|
};
|
|
635
667
|
const formData = new FormData();
|
|
636
668
|
formData.append("options", JSON.stringify(normalizedOptions));
|
|
@@ -668,6 +700,7 @@ async function getParseFormats(http) {
|
|
|
668
700
|
// src/v2/methods/search.ts
|
|
669
701
|
function prepareSearchPayload(req) {
|
|
670
702
|
if (!req.query || !req.query.trim()) throw new Error("Query cannot be empty");
|
|
703
|
+
ensureNoV1Options(req, "search");
|
|
671
704
|
if (req.limit != null && req.limit <= 0)
|
|
672
705
|
throw new Error("limit must be positive");
|
|
673
706
|
if (req.timeout != null && req.timeout <= 0)
|
|
@@ -1035,7 +1068,9 @@ async function fetchAllPages(http, nextUrl, initial, pagination) {
|
|
|
1035
1068
|
docs.push(d);
|
|
1036
1069
|
}
|
|
1037
1070
|
if (maxResults != null && docs.length >= maxResults) break;
|
|
1038
|
-
|
|
1071
|
+
const next = payload.next ?? (Array.isArray(payload.data) ? null : payload.data?.next) ?? null;
|
|
1072
|
+
if (next === current) break;
|
|
1073
|
+
current = next;
|
|
1039
1074
|
pageCount += 1;
|
|
1040
1075
|
}
|
|
1041
1076
|
return docs;
|
|
@@ -1044,6 +1079,7 @@ async function fetchAllPages(http, nextUrl, initial, pagination) {
|
|
|
1044
1079
|
// src/v2/methods/crawl.ts
|
|
1045
1080
|
function prepareCrawlPayload(request) {
|
|
1046
1081
|
if (!request.url || !request.url.trim()) throw new Error("URL cannot be empty");
|
|
1082
|
+
ensureNoV1Options(request, "crawl");
|
|
1047
1083
|
const data = { url: request.url.trim() };
|
|
1048
1084
|
if (request.prompt) data.prompt = request.prompt;
|
|
1049
1085
|
if (request.excludePaths) data.excludePaths = request.excludePaths;
|
|
@@ -1595,6 +1631,7 @@ async function browser(http, args = {}) {
|
|
|
1595
1631
|
if (args.streamWebView != null) body.streamWebView = args.streamWebView;
|
|
1596
1632
|
if (args.blockAds != null) body.blockAds = args.blockAds;
|
|
1597
1633
|
if (args.profile != null) body.profile = args.profile;
|
|
1634
|
+
if (args.location != null) body.location = args.location;
|
|
1598
1635
|
if (args.integration != null) body.integration = args.integration;
|
|
1599
1636
|
if (args.origin) body.origin = args.origin;
|
|
1600
1637
|
try {
|
|
@@ -2661,7 +2698,7 @@ var FirecrawlClient = class {
|
|
|
2661
2698
|
// Browser
|
|
2662
2699
|
/**
|
|
2663
2700
|
* Create a new browser session.
|
|
2664
|
-
* @param args Session options (ttl, activityTtl, streamWebView, blockAds, profile).
|
|
2701
|
+
* @param args Session options (ttl, activityTtl, streamWebView, blockAds, profile, location).
|
|
2665
2702
|
* @returns Session id, CDP URL, live view URL, and expiration time.
|
|
2666
2703
|
*/
|
|
2667
2704
|
async browser(args = {}) {
|
|
@@ -2800,10 +2837,10 @@ var FirecrawlApp = class {
|
|
|
2800
2837
|
if (typeof process !== "undefined" && process.env && process.env.npm_package_version) {
|
|
2801
2838
|
return process.env.npm_package_version;
|
|
2802
2839
|
}
|
|
2803
|
-
const packageJson = await import("./package-
|
|
2840
|
+
const packageJson = await import("./package-NZI5TDUV.js");
|
|
2804
2841
|
return packageJson.default.version;
|
|
2805
2842
|
} catch (error) {
|
|
2806
|
-
const isTest = typeof process !== "undefined" &&
|
|
2843
|
+
const isTest = typeof process !== "undefined" && false;
|
|
2807
2844
|
if (!isTest) {
|
|
2808
2845
|
console.error("Error getting version:", error);
|
|
2809
2846
|
}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import {
|
|
2
|
+
author,
|
|
3
|
+
bugs,
|
|
4
|
+
dependencies,
|
|
5
|
+
description,
|
|
6
|
+
devDependencies,
|
|
7
|
+
engines,
|
|
8
|
+
exports,
|
|
9
|
+
homepage,
|
|
10
|
+
keywords,
|
|
11
|
+
license,
|
|
12
|
+
main,
|
|
13
|
+
name,
|
|
14
|
+
package_default,
|
|
15
|
+
repository,
|
|
16
|
+
scripts,
|
|
17
|
+
type,
|
|
18
|
+
types,
|
|
19
|
+
version
|
|
20
|
+
} from "./chunk-H75FJ6LW.js";
|
|
21
|
+
export {
|
|
22
|
+
author,
|
|
23
|
+
bugs,
|
|
24
|
+
package_default as default,
|
|
25
|
+
dependencies,
|
|
26
|
+
description,
|
|
27
|
+
devDependencies,
|
|
28
|
+
engines,
|
|
29
|
+
exports,
|
|
30
|
+
homepage,
|
|
31
|
+
keywords,
|
|
32
|
+
license,
|
|
33
|
+
main,
|
|
34
|
+
name,
|
|
35
|
+
repository,
|
|
36
|
+
scripts,
|
|
37
|
+
type,
|
|
38
|
+
types,
|
|
39
|
+
version
|
|
40
|
+
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mendable/firecrawl-js",
|
|
3
|
-
"version": "4.
|
|
3
|
+
"version": "4.47.0",
|
|
4
4
|
"description": "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -30,18 +30,14 @@
|
|
|
30
30
|
},
|
|
31
31
|
"homepage": "https://github.com/firecrawl/firecrawl#readme",
|
|
32
32
|
"devDependencies": {
|
|
33
|
-
"@jest/globals": "^30.2.0",
|
|
34
33
|
"@types/dotenv": "^8.2.0",
|
|
35
|
-
"@types/jest": "^30.0.0",
|
|
36
|
-
"@types/mocha": "^10.0.6",
|
|
37
34
|
"@types/node": "^20.12.12",
|
|
38
35
|
"@types/uuid": "^9.0.8",
|
|
39
36
|
"dotenv": "^16.4.5",
|
|
40
|
-
"jest": "^30.2.0",
|
|
41
|
-
"ts-jest": "^29.4.5",
|
|
42
37
|
"tsup": "^8.5.0",
|
|
43
38
|
"typescript": "^5.4.5",
|
|
44
|
-
"uuid": "^14.0.0"
|
|
39
|
+
"uuid": "^14.0.0",
|
|
40
|
+
"vitest": "^4.1.11"
|
|
45
41
|
},
|
|
46
42
|
"keywords": [
|
|
47
43
|
"firecrawl",
|
|
@@ -73,7 +69,7 @@
|
|
|
73
69
|
"build": "tsup",
|
|
74
70
|
"build-and-publish": "pnpm run build && pnpm publish --access public",
|
|
75
71
|
"publish-beta": "pnpm run build && pnpm publish --access public --tag beta",
|
|
76
|
-
"test": "
|
|
77
|
-
"test:unit": "
|
|
72
|
+
"test": "vitest run --reporter=verbose src/__tests__/e2e/v2",
|
|
73
|
+
"test:unit": "vitest run --reporter=verbose src/__tests__/unit"
|
|
78
74
|
}
|
|
79
75
|
}
|
package/pnpm-workspace.yaml
CHANGED
|
@@ -1,17 +1,13 @@
|
|
|
1
1
|
allowBuilds:
|
|
2
2
|
esbuild: true
|
|
3
|
-
unrs-resolver: true
|
|
4
3
|
overrides:
|
|
5
4
|
"@isaacs/brace-expansion@<=5.0.0": ">=5.0.1"
|
|
6
5
|
"minimatch@<10.2.3": ">=10.2.3"
|
|
7
6
|
"rollup@<4.59.0": ">=4.59.0"
|
|
8
7
|
"picomatch@<4.0.4": ">=4.0.4"
|
|
9
|
-
handlebars: ">=4.7.9"
|
|
10
8
|
# GHSA-q2hr-2g5m-vwhr / GHSA-qhr7-859c-m2p7 / GHSA-6j4f-fj2g-mc7p:
|
|
11
9
|
# pin the 5.x line only. Exact 5.0.12 cannot resolve to 6.x.
|
|
12
10
|
"brace-expansion@>=5.0.0 <5.0.12": "5.0.12"
|
|
13
|
-
"js-yaml@>=3.0.0 <3.15.2": "3.15.2"
|
|
14
|
-
"js-yaml@>=4.0.0 <4.3.2": "4.3.2"
|
|
15
11
|
# GHSA-vh66-26gq-q6x8 / GHSA-9fr6-4gfg-395g / GHSA-c29m-xwm3-cm6r /
|
|
16
12
|
# GHSA-mghh-pgcx-3jjj / GHSA-x97p-jq2g-jp4f / GHSA-3pq3-5fj3-cg6v /
|
|
17
13
|
# GHSA-542g-h47m-68v8 / GHSA-j8rh-479h-cp32 / GHSA-4hqw-qxg8-jxx2 /
|
|
@@ -20,6 +16,4 @@ overrides:
|
|
|
20
16
|
"axios@>=1.0.0 <1.20.0": "1.20.0"
|
|
21
17
|
"follow-redirects@<1.16.0": ">=1.16.0 <2.0.0"
|
|
22
18
|
"form-data@>=4.0.0 <4.0.6": "4.0.6"
|
|
23
|
-
"@babel/core@>=7.0.0 <7.29.6": "7.29.7"
|
|
24
19
|
"esbuild@>=0.17.0 <0.28.1": "0.28.1"
|
|
25
|
-
"browserslist@<=4.28.6": ">=4.28.7 <5.0.0"
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import FirecrawlApp, { type CrawlParams, type CrawlResponse, type CrawlStatusResponse, type MapResponse, type ScrapeResponse } from '../../../index';
|
|
2
2
|
import { v4 as uuidv4 } from 'uuid';
|
|
3
3
|
import dotenv from 'dotenv';
|
|
4
|
-
import { describe, test, expect } from '
|
|
4
|
+
import { describe, test, expect } from 'vitest';
|
|
5
5
|
|
|
6
6
|
dotenv.config();
|
|
7
7
|
|
|
@@ -5,7 +5,7 @@ import Firecrawl from "../../../index";
|
|
|
5
5
|
import { config } from "dotenv";
|
|
6
6
|
import { getIdentity, getApiUrl } from "./utils/idmux";
|
|
7
7
|
import { testTimeoutMs, waitForJob, withRateLimitRetry } from "./utils/rateLimit";
|
|
8
|
-
import { describe, test, expect, beforeAll } from "
|
|
8
|
+
import { describe, test, expect, beforeAll } from "vitest";
|
|
9
9
|
|
|
10
10
|
config();
|
|
11
11
|
|
|
@@ -5,7 +5,7 @@ import Firecrawl from "../../../index";
|
|
|
5
5
|
import { config } from "dotenv";
|
|
6
6
|
import { getIdentity, getApiUrl } from "./utils/idmux";
|
|
7
7
|
import { testTimeoutMs, waitForJob, withRateLimitRetry } from "./utils/rateLimit";
|
|
8
|
-
import { describe, test, expect, beforeAll } from "
|
|
8
|
+
import { describe, test, expect, beforeAll } from "vitest";
|
|
9
9
|
|
|
10
10
|
config();
|
|
11
11
|
|
|
@@ -5,7 +5,7 @@ import Firecrawl from "../../../index";
|
|
|
5
5
|
import { config } from "dotenv";
|
|
6
6
|
import { getIdentity, getApiUrl } from "./utils/idmux";
|
|
7
7
|
import { testTimeoutMs, waitForJob, withRateLimitRetry } from "./utils/rateLimit";
|
|
8
|
-
import { describe, test, expect, beforeAll } from "
|
|
8
|
+
import { describe, test, expect, beforeAll } from "vitest";
|
|
9
9
|
import { z } from "zod";
|
|
10
10
|
|
|
11
11
|
config();
|
|
@@ -5,7 +5,7 @@ import Firecrawl from "../../../index";
|
|
|
5
5
|
import { config } from "dotenv";
|
|
6
6
|
import { getIdentity, getApiUrl } from "./utils/idmux";
|
|
7
7
|
import { testTimeoutMs, withRateLimitRetry } from "./utils/rateLimit";
|
|
8
|
-
import { describe, test, expect, beforeAll } from "
|
|
8
|
+
import { describe, test, expect, beforeAll } from "vitest";
|
|
9
9
|
|
|
10
10
|
config();
|
|
11
11
|
|
|
@@ -2,7 +2,7 @@ import Firecrawl from "../../../index";
|
|
|
2
2
|
import { config } from "dotenv";
|
|
3
3
|
import { getIdentity, getApiUrl } from "./utils/idmux";
|
|
4
4
|
import { testTimeoutMs, withRateLimitRetry } from "./utils/rateLimit";
|
|
5
|
-
import { describe, test, expect, beforeAll } from "
|
|
5
|
+
import { describe, test, expect, beforeAll } from "vitest";
|
|
6
6
|
import { SdkError } from "../../../v2/types";
|
|
7
7
|
|
|
8
8
|
config();
|
|
@@ -6,7 +6,7 @@ import { z } from "zod";
|
|
|
6
6
|
import { config } from "dotenv";
|
|
7
7
|
import { getIdentity, getApiUrl } from "./utils/idmux";
|
|
8
8
|
import { testTimeoutMs, withRateLimitRetry } from "./utils/rateLimit";
|
|
9
|
-
import { describe, test, expect, beforeAll } from "
|
|
9
|
+
import { describe, test, expect, beforeAll } from "vitest";
|
|
10
10
|
|
|
11
11
|
config();
|
|
12
12
|
|
|
@@ -6,7 +6,7 @@ import type { Document, SearchResultWeb, SearchResultNews, SearchResultImages }
|
|
|
6
6
|
import { config } from "dotenv";
|
|
7
7
|
import { getIdentity, getApiUrl } from "./utils/idmux";
|
|
8
8
|
import { testTimeoutMs, withRateLimitRetry } from "./utils/rateLimit";
|
|
9
|
-
import { describe, test, expect, beforeAll } from "
|
|
9
|
+
import { describe, test, expect, beforeAll } from "vitest";
|
|
10
10
|
|
|
11
11
|
config();
|
|
12
12
|
|
|
@@ -5,7 +5,7 @@ import Firecrawl from "../../../index";
|
|
|
5
5
|
import { config } from "dotenv";
|
|
6
6
|
import { getIdentity, getApiUrl } from "./utils/idmux";
|
|
7
7
|
import { testTimeoutMs, withRateLimitRetry } from "./utils/rateLimit";
|
|
8
|
-
import { describe, test, expect, beforeAll } from "
|
|
8
|
+
import { describe, test, expect, beforeAll } from "vitest";
|
|
9
9
|
|
|
10
10
|
config();
|
|
11
11
|
|
|
@@ -52,7 +52,7 @@ export const RETRY_BUDGET_MS = (MAX_ATTEMPTS - 1) * MAX_WAIT_MS;
|
|
|
52
52
|
*
|
|
53
53
|
* Every caller now passes its own timeout, sized to the work it waits for.
|
|
54
54
|
* This value is only a backstop for a new caller that forgets one. Without a
|
|
55
|
-
* bound the poll loop runs until the
|
|
55
|
+
* bound the poll loop runs until the test timeout, which hides the reason.
|
|
56
56
|
*
|
|
57
57
|
* The value stays short because it cannot know the work. It fits the smallest
|
|
58
58
|
* base any suite passes to testTimeoutMs, which is 60_000 ms:
|
|
@@ -69,7 +69,7 @@ export const RETRY_BUDGET_MS = (MAX_ATTEMPTS - 1) * MAX_WAIT_MS;
|
|
|
69
69
|
export const DEFAULT_JOB_TIMEOUT_MS = 45_000;
|
|
70
70
|
|
|
71
71
|
/**
|
|
72
|
-
*
|
|
72
|
+
* Vitest timeout for a test that calls a wrapped client.
|
|
73
73
|
*
|
|
74
74
|
* Pass the base: the time the test needs when the API answers at once. The
|
|
75
75
|
* result adds one retry budget on top, so a test whose single call hits the
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import Firecrawl from "../../../index";
|
|
2
2
|
import { config } from "dotenv";
|
|
3
|
-
import { describe, test, expect, beforeAll } from "
|
|
3
|
+
import { describe, test, expect, beforeAll } from "vitest";
|
|
4
4
|
import { getIdentity } from "./utils/idmux";
|
|
5
5
|
import { testTimeoutMs, withRateLimitRetry } from "./utils/rateLimit";
|
|
6
6
|
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { describe, test, expect,
|
|
1
|
+
import { describe, test, expect, vi, afterEach, beforeEach } from "vitest";
|
|
2
2
|
import axios from "axios";
|
|
3
3
|
import FirecrawlApp from "../../../v1";
|
|
4
4
|
|
|
@@ -20,7 +20,7 @@ const lastPage = () => ({ success: true, status: "completed", data: [{ markdown:
|
|
|
20
20
|
function mockPoll(path: string, nextOrigin = API_URL) {
|
|
21
21
|
const sent: string[] = [];
|
|
22
22
|
let polls = 0;
|
|
23
|
-
|
|
23
|
+
vi.spyOn(axios, "get").mockImplementation(async (url: string) => {
|
|
24
24
|
sent.push(url);
|
|
25
25
|
if (new URL(url).searchParams.has("skip")) return { status: 200, data: lastPage() };
|
|
26
26
|
polls++;
|
|
@@ -33,18 +33,18 @@ describe("v1 batch scrape waits on the batch scrape status endpoint", () => {
|
|
|
33
33
|
const app = new FirecrawlApp({ apiKey: API_KEY, apiUrl: API_URL });
|
|
34
34
|
|
|
35
35
|
beforeEach(() => {
|
|
36
|
-
|
|
36
|
+
vi.spyOn(global, "setTimeout").mockImplementation(((fn: () => void) => {
|
|
37
37
|
fn();
|
|
38
38
|
return 0;
|
|
39
39
|
}) as any);
|
|
40
40
|
});
|
|
41
41
|
|
|
42
42
|
afterEach(() => {
|
|
43
|
-
|
|
43
|
+
vi.restoreAllMocks();
|
|
44
44
|
});
|
|
45
45
|
|
|
46
46
|
test("batchScrapeUrls polls /v1/batch/scrape/{id} and follows pinned next pages", async () => {
|
|
47
|
-
const post =
|
|
47
|
+
const post = vi.spyOn(axios, "post").mockResolvedValue({ status: 200, data: { success: true, id: "abc" } });
|
|
48
48
|
const path = "/v1/batch/scrape/abc";
|
|
49
49
|
const sent = mockPoll(path, "https://evil.example");
|
|
50
50
|
|
|
@@ -58,9 +58,9 @@ describe("v1 batch scrape waits on the batch scrape status endpoint", () => {
|
|
|
58
58
|
});
|
|
59
59
|
|
|
60
60
|
test("batchScrapeUrls reports batch scrape failures as batch scrape errors", async () => {
|
|
61
|
-
|
|
61
|
+
vi.spyOn(axios, "post").mockResolvedValue({ status: 200, data: { success: true, id: "abc" } });
|
|
62
62
|
const sent: string[] = [];
|
|
63
|
-
|
|
63
|
+
vi.spyOn(axios, "get").mockImplementation(async (url: string) => {
|
|
64
64
|
sent.push(url);
|
|
65
65
|
return { status: 200, data: { success: true, status: "failed", data: [] } };
|
|
66
66
|
});
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import FirecrawlApp from '../../../v1';
|
|
2
|
-
import { describe, test, expect,
|
|
2
|
+
import { describe, test, expect, vi, beforeEach, afterEach } from 'vitest';
|
|
3
3
|
|
|
4
4
|
const successResponse = {
|
|
5
5
|
status: 200,
|
|
@@ -21,17 +21,17 @@ describe('monitorJobStatus retry logic', () => {
|
|
|
21
21
|
beforeEach(() => {
|
|
22
22
|
app = new FirecrawlApp({ apiKey: 'test-key', apiUrl: 'https://test.com' });
|
|
23
23
|
delays = [];
|
|
24
|
-
|
|
24
|
+
vi.useFakeTimers();
|
|
25
25
|
const fakeSetTimeout = globalThis.setTimeout;
|
|
26
|
-
|
|
26
|
+
vi.spyOn(globalThis, 'setTimeout').mockImplementation(((fn: () => void, ms?: number) => {
|
|
27
27
|
delays.push(ms ?? 0);
|
|
28
28
|
return fakeSetTimeout(fn, ms);
|
|
29
29
|
}) as any);
|
|
30
30
|
});
|
|
31
31
|
|
|
32
32
|
afterEach(() => {
|
|
33
|
-
|
|
34
|
-
|
|
33
|
+
vi.restoreAllMocks();
|
|
34
|
+
vi.useRealTimers();
|
|
35
35
|
});
|
|
36
36
|
|
|
37
37
|
function failFirst(n: number, error: Error, statuses: string[] = []) {
|
|
@@ -48,7 +48,7 @@ describe('monitorJobStatus retry logic', () => {
|
|
|
48
48
|
async function monitor() {
|
|
49
49
|
const result = app.monitorJobStatus('test-id', {} as any, 1);
|
|
50
50
|
result.catch(() => {});
|
|
51
|
-
await
|
|
51
|
+
await vi.runAllTimersAsync();
|
|
52
52
|
return result;
|
|
53
53
|
}
|
|
54
54
|
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { describe, test, expect,
|
|
1
|
+
import { describe, test, expect, vi, afterEach } from "vitest";
|
|
2
2
|
import axios from "axios";
|
|
3
3
|
import FirecrawlApp from "../../../v1";
|
|
4
4
|
|
|
@@ -16,7 +16,7 @@ const origins: Array<[string, string]> = [
|
|
|
16
16
|
|
|
17
17
|
function mockGet(respond: (url: URL) => unknown) {
|
|
18
18
|
const sent: Array<{ url: string; authorization: unknown }> = [];
|
|
19
|
-
|
|
19
|
+
vi.spyOn(axios, "get").mockImplementation(async (url: string, config?: any) => {
|
|
20
20
|
sent.push({ url, authorization: config?.headers?.Authorization });
|
|
21
21
|
return { status: 200, data: respond(new URL(url)) };
|
|
22
22
|
});
|
|
@@ -45,7 +45,7 @@ const lastPage = () => ({ success: true, status: "completed", data: [{ markdown:
|
|
|
45
45
|
|
|
46
46
|
describe("v1 pins next URLs to the api_url origin", () => {
|
|
47
47
|
afterEach(() => {
|
|
48
|
-
|
|
48
|
+
vi.restoreAllMocks();
|
|
49
49
|
});
|
|
50
50
|
|
|
51
51
|
describe.each(origins)("%s next", (_label, origin) => {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { describe, test, expect,
|
|
1
|
+
import { describe, test, expect, vi, beforeEach, afterEach } from "vitest";
|
|
2
2
|
import axios from "axios";
|
|
3
3
|
import FirecrawlApp from "../../../v1";
|
|
4
4
|
|
|
@@ -20,11 +20,11 @@ describe("v1 methods work without params", () => {
|
|
|
20
20
|
beforeEach(() => {
|
|
21
21
|
app = new FirecrawlApp({ apiKey: "fc-test", apiUrl: API_URL });
|
|
22
22
|
posts = [];
|
|
23
|
-
|
|
23
|
+
vi.spyOn(axios, "post").mockImplementation(async (url: string, data?: any) => {
|
|
24
24
|
posts.push({ url, data });
|
|
25
25
|
return { status: 200, data: { success: true, id: "job-1", data: [], links: [] } };
|
|
26
26
|
});
|
|
27
|
-
|
|
27
|
+
vi.spyOn(axios, "get").mockImplementation(async () => ({
|
|
28
28
|
status: 200,
|
|
29
29
|
data: { success: true, status: "completed", data: [{ markdown: "ok" }] },
|
|
30
30
|
}));
|
|
@@ -32,7 +32,7 @@ describe("v1 methods work without params", () => {
|
|
|
32
32
|
});
|
|
33
33
|
|
|
34
34
|
afterEach(() => {
|
|
35
|
-
|
|
35
|
+
vi.restoreAllMocks();
|
|
36
36
|
(globalThis as any).WebSocket = originalWebSocket;
|
|
37
37
|
});
|
|
38
38
|
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { describe, expect,
|
|
1
|
+
import { describe, expect, vi, test } from "vitest";
|
|
2
2
|
import {
|
|
3
3
|
startAgent,
|
|
4
4
|
getAgentTrace,
|
|
@@ -8,7 +8,7 @@ import type { AgentTraceEvent } from "../../../v2/types";
|
|
|
8
8
|
|
|
9
9
|
describe("v2.agent unit", () => {
|
|
10
10
|
test("startAgent forwards effort in request payload", async () => {
|
|
11
|
-
const post =
|
|
11
|
+
const post = vi.fn().mockResolvedValue({
|
|
12
12
|
status: 200,
|
|
13
13
|
data: { success: true, id: "agent-job" },
|
|
14
14
|
});
|
|
@@ -25,7 +25,7 @@ describe("v2.agent unit", () => {
|
|
|
25
25
|
});
|
|
26
26
|
|
|
27
27
|
test("startAgent omits effort when unset", async () => {
|
|
28
|
-
const post =
|
|
28
|
+
const post = vi.fn().mockResolvedValue({
|
|
29
29
|
status: 200,
|
|
30
30
|
data: { success: true, id: "agent-job" },
|
|
31
31
|
});
|
|
@@ -70,7 +70,7 @@ describe("v2.agent unit", () => {
|
|
|
70
70
|
change: "partial",
|
|
71
71
|
},
|
|
72
72
|
};
|
|
73
|
-
const get =
|
|
73
|
+
const get = vi.fn().mockResolvedValue({
|
|
74
74
|
status: 200,
|
|
75
75
|
data: {
|
|
76
76
|
success: true,
|
|
@@ -95,7 +95,7 @@ describe("v2.agent unit", () => {
|
|
|
95
95
|
});
|
|
96
96
|
|
|
97
97
|
test("getAgentTrace appends liveView query param when requested", async () => {
|
|
98
|
-
const get =
|
|
98
|
+
const get = vi.fn().mockResolvedValue({
|
|
99
99
|
status: 200,
|
|
100
100
|
data: {
|
|
101
101
|
success: true,
|
|
@@ -122,7 +122,7 @@ describe("v2.agent unit", () => {
|
|
|
122
122
|
});
|
|
123
123
|
|
|
124
124
|
test("getAgentSnapshot hits the snapshot endpoint", async () => {
|
|
125
|
-
const get =
|
|
125
|
+
const get = vi.fn().mockResolvedValue({
|
|
126
126
|
status: 200,
|
|
127
127
|
data: {
|
|
128
128
|
success: true,
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { describe, expect, it,
|
|
1
|
+
import { describe, expect, it, vi } from "vitest";
|
|
2
2
|
import { search } from "../../../v2/methods/search";
|
|
3
3
|
import { scrape } from "../../../v2/methods/scrape";
|
|
4
4
|
import { parse } from "../../../v2/methods/parse";
|
|
@@ -14,8 +14,8 @@ const hints = [
|
|
|
14
14
|
];
|
|
15
15
|
function httpFor(body: object) {
|
|
16
16
|
return {
|
|
17
|
-
post:
|
|
18
|
-
postMultipart:
|
|
17
|
+
post: vi.fn<any>().mockResolvedValue({ status: 200, data: body }),
|
|
18
|
+
postMultipart: vi
|
|
19
19
|
.fn<any>()
|
|
20
20
|
.mockResolvedValue({ status: 200, data: body }),
|
|
21
21
|
} as any;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { describe, expect,
|
|
1
|
+
import { describe, expect, vi, test } from "vitest";
|
|
2
2
|
import { listAgents } from "../../../v2/methods/agent";
|
|
3
3
|
|
|
4
4
|
describe("v2.agent listAgents unit", () => {
|
|
@@ -17,7 +17,7 @@ describe("v2.agent listAgents unit", () => {
|
|
|
17
17
|
};
|
|
18
18
|
|
|
19
19
|
test("hits GET /v2/agent without query by default", async () => {
|
|
20
|
-
const get =
|
|
20
|
+
const get = vi.fn().mockResolvedValue({
|
|
21
21
|
status: 200,
|
|
22
22
|
data: { success: true, agents: [sampleAgent] },
|
|
23
23
|
});
|
|
@@ -33,7 +33,7 @@ describe("v2.agent listAgents unit", () => {
|
|
|
33
33
|
});
|
|
34
34
|
|
|
35
35
|
test("passes before as a query param", async () => {
|
|
36
|
-
const get =
|
|
36
|
+
const get = vi.fn().mockResolvedValue({
|
|
37
37
|
status: 200,
|
|
38
38
|
data: {
|
|
39
39
|
success: true,
|
|
@@ -51,7 +51,7 @@ describe("v2.agent listAgents unit", () => {
|
|
|
51
51
|
});
|
|
52
52
|
|
|
53
53
|
test("does not auto-paginate when next is present", async () => {
|
|
54
|
-
const get =
|
|
54
|
+
const get = vi.fn().mockResolvedValue({
|
|
55
55
|
status: 200,
|
|
56
56
|
data: {
|
|
57
57
|
success: true,
|