firecrawl 4.46.0 → 4.47.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -8
- package/dist/chunk-H75FJ6LW.js +114 -0
- package/dist/index.cjs +56 -18
- package/dist/index.d.cts +7 -3
- package/dist/index.d.ts +7 -3
- package/dist/index.js +57 -20
- package/dist/package-NZI5TDUV.js +40 -0
- package/package.json +1 -1
- package/src/__tests__/unit/v2/browser.unit.test.ts +28 -0
- package/src/__tests__/unit/v2/errorHandler.test.ts +23 -0
- package/src/__tests__/unit/v2/pagination.test.ts +20 -0
- package/src/__tests__/unit/v2/validation.test.ts +39 -1
- package/src/v2/client.ts +1 -1
- package/src/v2/methods/browser.ts +3 -0
- package/src/v2/methods/crawl.ts +2 -1
- package/src/v2/methods/search.ts +2 -1
- package/src/v2/types.ts +2 -2
- package/src/v2/utils/errorHandler.ts +23 -2
- package/src/v2/utils/getVersion.ts +4 -12
- package/src/v2/utils/pagination.ts +4 -1
- package/src/v2/utils/validation.ts +28 -0
- package/dist/chunk-R4XT7PME.js +0 -93
- package/dist/package-EI4Q33HD.js +0 -4
package/README.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Firecrawl Node SDK
|
|
2
2
|
|
|
3
|
-
The Firecrawl Node SDK is
|
|
3
|
+
The Firecrawl Node SDK is the JavaScript client for the Firecrawl API, the context API for AI agents. Search, scrape, parse, crawl, map, and interact with the web, and get back clean Markdown or structured data your agents can ship with.
|
|
4
4
|
|
|
5
5
|
## Installation
|
|
6
6
|
|
|
@@ -13,7 +13,7 @@ npm install firecrawl
|
|
|
13
13
|
## Usage
|
|
14
14
|
|
|
15
15
|
1. Get an API key from [firecrawl.dev](https://firecrawl.dev)
|
|
16
|
-
2. Set the API key as an environment variable named `FIRECRAWL_API_KEY` or pass it as
|
|
16
|
+
2. Set the API key as an environment variable named `FIRECRAWL_API_KEY` or pass it as the `apiKey` option to the `Firecrawl` class.
|
|
17
17
|
|
|
18
18
|
Here's an example of how to use the SDK with error handling:
|
|
19
19
|
|
|
@@ -126,7 +126,7 @@ const crawlResponse = await app.crawl('https://firecrawl.dev', {
|
|
|
126
126
|
To start an asynchronous crawl, use `startCrawl`. It returns a job ID you can poll with `getCrawlStatus`.
|
|
127
127
|
|
|
128
128
|
```js
|
|
129
|
-
const start = await app.startCrawl('https://
|
|
129
|
+
const start = await app.startCrawl('https://firecrawl.dev', {
|
|
130
130
|
excludePaths: ['blog/*'],
|
|
131
131
|
limit: 5,
|
|
132
132
|
});
|
|
@@ -137,7 +137,7 @@ const start = await app.startCrawl('https://mendable.ai', {
|
|
|
137
137
|
To check the status of a crawl job with error handling, use the `getCrawlStatus` method. It takes the job ID as a parameter and returns the current status.
|
|
138
138
|
|
|
139
139
|
```js
|
|
140
|
-
const status = await app.getCrawlStatus(id);
|
|
140
|
+
const status = await app.getCrawlStatus(start.id);
|
|
141
141
|
```
|
|
142
142
|
|
|
143
143
|
### Extracting structured data from URLs
|
|
@@ -323,7 +323,7 @@ await app.stopInteraction(scrapeJobId);
|
|
|
323
323
|
To receive real‑time updates, start a crawl and attach a watcher.
|
|
324
324
|
|
|
325
325
|
```js
|
|
326
|
-
const start = await app.startCrawl('https://
|
|
326
|
+
const start = await app.startCrawl('https://firecrawl.dev', { excludePaths: ['blog/*'], limit: 5 });
|
|
327
327
|
const watch = app.watcher(start.id, { kind: 'crawl', pollInterval: 2 });
|
|
328
328
|
|
|
329
329
|
watch.on('document', (doc) => {
|
|
@@ -346,7 +346,7 @@ await watch.start();
|
|
|
346
346
|
To batch scrape multiple URLs with error handling, use the `batchScrape` method.
|
|
347
347
|
|
|
348
348
|
```js
|
|
349
|
-
const batchScrapeResponse = await app.batchScrape(['https://firecrawl.dev', 'https://
|
|
349
|
+
const batchScrapeResponse = await app.batchScrape(['https://firecrawl.dev', 'https://docs.firecrawl.dev'], {
|
|
350
350
|
formats: ['markdown', 'html'],
|
|
351
351
|
});
|
|
352
352
|
```
|
|
@@ -357,7 +357,7 @@ const batchScrapeResponse = await app.batchScrape(['https://firecrawl.dev', 'htt
|
|
|
357
357
|
To start an asynchronous batch scrape, use `startBatchScrape` and poll with `getBatchScrapeStatus`.
|
|
358
358
|
|
|
359
359
|
```js
|
|
360
|
-
const asyncBatchScrapeResult = await app.startBatchScrape(['https://firecrawl.dev', 'https://
|
|
360
|
+
const asyncBatchScrapeResult = await app.startBatchScrape(['https://firecrawl.dev', 'https://docs.firecrawl.dev'], {
|
|
361
361
|
formats: ['markdown', 'html'],
|
|
362
362
|
});
|
|
363
363
|
```
|
|
@@ -367,7 +367,7 @@ const asyncBatchScrapeResult = await app.startBatchScrape(['https://firecrawl.de
|
|
|
367
367
|
To use batch scrape with real‑time updates, start the job and watch it using the watcher.
|
|
368
368
|
|
|
369
369
|
```js
|
|
370
|
-
const start = await app.startBatchScrape(['https://firecrawl.dev', 'https://
|
|
370
|
+
const start = await app.startBatchScrape(['https://firecrawl.dev', 'https://docs.firecrawl.dev'], { formats: ['markdown', 'html'] });
|
|
371
371
|
const watch = app.watcher(start.id, { kind: 'batch', pollInterval: 2 });
|
|
372
372
|
|
|
373
373
|
watch.on('document', (doc) => {
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
// package.json
|
|
2
|
+
var name = "@mendable/firecrawl-js";
|
|
3
|
+
var version = "4.47.0";
|
|
4
|
+
var description = "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts";
|
|
5
|
+
var main = "dist/index.js";
|
|
6
|
+
var types = "dist/index.d.ts";
|
|
7
|
+
var exports = {
|
|
8
|
+
"./package.json": "./package.json",
|
|
9
|
+
".": {
|
|
10
|
+
import: "./dist/index.js",
|
|
11
|
+
default: "./dist/index.cjs"
|
|
12
|
+
}
|
|
13
|
+
};
|
|
14
|
+
var type = "module";
|
|
15
|
+
var scripts = {
|
|
16
|
+
build: "tsup",
|
|
17
|
+
"build-and-publish": "pnpm run build && pnpm publish --access public",
|
|
18
|
+
"publish-beta": "pnpm run build && pnpm publish --access public --tag beta",
|
|
19
|
+
test: "vitest run --reporter=verbose src/__tests__/e2e/v2",
|
|
20
|
+
"test:unit": "vitest run --reporter=verbose src/__tests__/unit"
|
|
21
|
+
};
|
|
22
|
+
var repository = {
|
|
23
|
+
type: "git",
|
|
24
|
+
url: "git+https://github.com/firecrawl/firecrawl.git"
|
|
25
|
+
};
|
|
26
|
+
var author = "Mendable.ai";
|
|
27
|
+
var license = "MIT";
|
|
28
|
+
var dependencies = {
|
|
29
|
+
axios: "1.20.0",
|
|
30
|
+
firecrawl: "4.30.1",
|
|
31
|
+
"typescript-event-target": "^1.1.1",
|
|
32
|
+
zod: "^3.23.8",
|
|
33
|
+
"zod-to-json-schema": "^3.23.0"
|
|
34
|
+
};
|
|
35
|
+
var bugs = {
|
|
36
|
+
url: "https://github.com/firecrawl/firecrawl/issues"
|
|
37
|
+
};
|
|
38
|
+
var homepage = "https://github.com/firecrawl/firecrawl#readme";
|
|
39
|
+
var devDependencies = {
|
|
40
|
+
"@types/dotenv": "^8.2.0",
|
|
41
|
+
"@types/node": "^20.12.12",
|
|
42
|
+
"@types/uuid": "^9.0.8",
|
|
43
|
+
dotenv: "^16.4.5",
|
|
44
|
+
tsup: "^8.5.0",
|
|
45
|
+
typescript: "^5.4.5",
|
|
46
|
+
uuid: "^14.0.0",
|
|
47
|
+
vitest: "^4.1.11"
|
|
48
|
+
};
|
|
49
|
+
var keywords = [
|
|
50
|
+
"firecrawl",
|
|
51
|
+
"mendable",
|
|
52
|
+
"crawler",
|
|
53
|
+
"web",
|
|
54
|
+
"scraper",
|
|
55
|
+
"api",
|
|
56
|
+
"sdk",
|
|
57
|
+
"web-search",
|
|
58
|
+
"literature-search",
|
|
59
|
+
"research",
|
|
60
|
+
"scientific-papers",
|
|
61
|
+
"academic-search",
|
|
62
|
+
"biomedical",
|
|
63
|
+
"life-sciences",
|
|
64
|
+
"pubmed",
|
|
65
|
+
"biorxiv",
|
|
66
|
+
"medrxiv",
|
|
67
|
+
"arxiv",
|
|
68
|
+
"preprints",
|
|
69
|
+
"citations",
|
|
70
|
+
"bioinformatics"
|
|
71
|
+
];
|
|
72
|
+
var engines = {
|
|
73
|
+
node: ">=22.0.0"
|
|
74
|
+
};
|
|
75
|
+
var package_default = {
|
|
76
|
+
name,
|
|
77
|
+
version,
|
|
78
|
+
description,
|
|
79
|
+
main,
|
|
80
|
+
types,
|
|
81
|
+
exports,
|
|
82
|
+
type,
|
|
83
|
+
scripts,
|
|
84
|
+
repository,
|
|
85
|
+
author,
|
|
86
|
+
license,
|
|
87
|
+
dependencies,
|
|
88
|
+
bugs,
|
|
89
|
+
homepage,
|
|
90
|
+
devDependencies,
|
|
91
|
+
keywords,
|
|
92
|
+
engines
|
|
93
|
+
};
|
|
94
|
+
|
|
95
|
+
export {
|
|
96
|
+
name,
|
|
97
|
+
version,
|
|
98
|
+
description,
|
|
99
|
+
main,
|
|
100
|
+
types,
|
|
101
|
+
exports,
|
|
102
|
+
type,
|
|
103
|
+
scripts,
|
|
104
|
+
repository,
|
|
105
|
+
author,
|
|
106
|
+
license,
|
|
107
|
+
dependencies,
|
|
108
|
+
bugs,
|
|
109
|
+
homepage,
|
|
110
|
+
devDependencies,
|
|
111
|
+
keywords,
|
|
112
|
+
engines,
|
|
113
|
+
package_default
|
|
114
|
+
};
|
package/dist/index.cjs
CHANGED
|
@@ -39,7 +39,7 @@ var require_package = __commonJS({
|
|
|
39
39
|
"package.json"(exports2, module2) {
|
|
40
40
|
module2.exports = {
|
|
41
41
|
name: "@mendable/firecrawl-js",
|
|
42
|
-
version: "4.
|
|
42
|
+
version: "4.47.0",
|
|
43
43
|
description: "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
|
|
44
44
|
main: "dist/index.js",
|
|
45
45
|
types: "dist/index.d.ts",
|
|
@@ -134,16 +134,9 @@ module.exports = __toCommonJS(index_exports);
|
|
|
134
134
|
var import_axios = __toESM(require("axios"), 1);
|
|
135
135
|
|
|
136
136
|
// src/v2/utils/getVersion.ts
|
|
137
|
+
var import_package = __toESM(require_package(), 1);
|
|
137
138
|
function getVersion() {
|
|
138
|
-
|
|
139
|
-
if (typeof process !== "undefined" && process.env && process.env.npm_package_version) {
|
|
140
|
-
return process.env.npm_package_version;
|
|
141
|
-
}
|
|
142
|
-
const pkg = require_package();
|
|
143
|
-
return pkg?.version || "3.x.x";
|
|
144
|
-
} catch {
|
|
145
|
-
return "3.x.x";
|
|
146
|
-
}
|
|
139
|
+
return import_package.version;
|
|
147
140
|
}
|
|
148
141
|
|
|
149
142
|
// src/utils/apiOrigin.ts
|
|
@@ -197,7 +190,7 @@ var HttpClient = class {
|
|
|
197
190
|
return this.instance.defaults.timeout ?? 3e5;
|
|
198
191
|
}
|
|
199
192
|
async request(config) {
|
|
200
|
-
const
|
|
193
|
+
const version2 = getVersion();
|
|
201
194
|
config.headers = {
|
|
202
195
|
...config.headers || {}
|
|
203
196
|
};
|
|
@@ -212,7 +205,7 @@ var HttpClient = class {
|
|
|
212
205
|
const data = cfg.data ?? {};
|
|
213
206
|
cfg.data = {
|
|
214
207
|
...data,
|
|
215
|
-
origin: typeof data.origin === "string" && data.origin.includes("mcp") ? data.origin : `js-sdk@${
|
|
208
|
+
origin: typeof data.origin === "string" && data.origin.includes("mcp") ? data.origin : `js-sdk@${version2}`
|
|
216
209
|
};
|
|
217
210
|
}
|
|
218
211
|
if (isFormDataBody) {
|
|
@@ -451,6 +444,30 @@ function ensureValidFormats(formats) {
|
|
|
451
444
|
}
|
|
452
445
|
}
|
|
453
446
|
}
|
|
447
|
+
var V1_SCRAPE_OPTION_REPLACEMENTS = {
|
|
448
|
+
pageOptions: "use scrapeOptions",
|
|
449
|
+
includeLinks: "add 'links' to scrapeOptions.formats",
|
|
450
|
+
parsePDF: "use scrapeOptions.parsers"
|
|
451
|
+
};
|
|
452
|
+
var V1_OPTION_REPLACEMENTS = {
|
|
453
|
+
crawl: {
|
|
454
|
+
crawlerOptions: "move its fields to the top level",
|
|
455
|
+
maxDepth: "use maxDiscoveryDepth",
|
|
456
|
+
allowBackwardLinks: "use crawlEntireDomain",
|
|
457
|
+
ignoreSitemap: "use sitemap: 'skip'",
|
|
458
|
+
...V1_SCRAPE_OPTION_REPLACEMENTS
|
|
459
|
+
},
|
|
460
|
+
search: V1_SCRAPE_OPTION_REPLACEMENTS
|
|
461
|
+
};
|
|
462
|
+
function ensureNoV1Options(options, method) {
|
|
463
|
+
if (!options) return;
|
|
464
|
+
const replacements = V1_OPTION_REPLACEMENTS[method];
|
|
465
|
+
const found = Object.keys(replacements).filter((key) => options[key] != null);
|
|
466
|
+
if (found.length === 0) return;
|
|
467
|
+
throw new Error(
|
|
468
|
+
`${method}: ${found.map((key) => `${key} is a v1 option, ${replacements[key]}`).join("; ")}`
|
|
469
|
+
);
|
|
470
|
+
}
|
|
454
471
|
function ensureValidScrapeOptions(options) {
|
|
455
472
|
if (!options) return;
|
|
456
473
|
if (options.timeout != null && options.timeout <= 0) {
|
|
@@ -564,12 +581,28 @@ function ensureValidParseOptions(options) {
|
|
|
564
581
|
|
|
565
582
|
// src/v2/utils/errorHandler.ts
|
|
566
583
|
var import_axios2 = require("axios");
|
|
584
|
+
function describeValidationIssues(details) {
|
|
585
|
+
if (!Array.isArray(details)) return void 0;
|
|
586
|
+
const parts = details.slice(0, 3).flatMap((issue) => {
|
|
587
|
+
if (!issue || typeof issue !== "object") return [];
|
|
588
|
+
const { path, message, keys } = issue;
|
|
589
|
+
const what = Array.isArray(keys) && keys.length > 0 ? `unrecognized key(s) ${keys.join(", ")}` : typeof message === "string" ? message : void 0;
|
|
590
|
+
if (!what) return [];
|
|
591
|
+
return Array.isArray(path) && path.length > 0 ? [`${path.join(".")}: ${what}`] : [what];
|
|
592
|
+
});
|
|
593
|
+
return parts.length > 0 ? parts.join("; ") : void 0;
|
|
594
|
+
}
|
|
595
|
+
function withIssues(message, details) {
|
|
596
|
+
const text = typeof message === "string" ? message : JSON.stringify(message);
|
|
597
|
+
const issues = describeValidationIssues(details);
|
|
598
|
+
return issues && !text.includes(issues) ? `${text} (${issues})` : text;
|
|
599
|
+
}
|
|
567
600
|
function throwForBadResponse(resp, action) {
|
|
568
601
|
const status = resp.status;
|
|
569
602
|
const body = resp.data || {};
|
|
570
603
|
const msg = body?.error || body?.message || `Request failed (${status}) while trying to ${action}`;
|
|
571
604
|
throw Object.assign(new SdkError(
|
|
572
|
-
msg,
|
|
605
|
+
withIssues(msg, body?.details),
|
|
573
606
|
status,
|
|
574
607
|
body?.code,
|
|
575
608
|
body?.details,
|
|
@@ -584,7 +617,7 @@ function normalizeAxiosError(err, action) {
|
|
|
584
617
|
const message = body?.error || err.message || `Request failed${status ? ` (${status})` : ""} while trying to ${action}`;
|
|
585
618
|
const code = body?.code || err.code;
|
|
586
619
|
throw Object.assign(new SdkError(
|
|
587
|
-
message,
|
|
620
|
+
withIssues(message, body?.details),
|
|
588
621
|
status,
|
|
589
622
|
code,
|
|
590
623
|
body?.details ?? body,
|
|
@@ -755,10 +788,10 @@ async function parse(http, file, options) {
|
|
|
755
788
|
throw new Error("file data cannot be empty");
|
|
756
789
|
}
|
|
757
790
|
if (options) ensureValidParseOptions(options);
|
|
758
|
-
const
|
|
791
|
+
const version2 = getVersion();
|
|
759
792
|
const normalizedOptions = {
|
|
760
793
|
...options ?? {},
|
|
761
|
-
origin: typeof options?.origin === "string" && options.origin.includes("mcp") ? options.origin : options?.origin ?? `js-sdk@${
|
|
794
|
+
origin: typeof options?.origin === "string" && options.origin.includes("mcp") ? options.origin : options?.origin ?? `js-sdk@${version2}`
|
|
762
795
|
};
|
|
763
796
|
const formData = new FormData();
|
|
764
797
|
formData.append("options", JSON.stringify(normalizedOptions));
|
|
@@ -796,6 +829,7 @@ async function getParseFormats(http) {
|
|
|
796
829
|
// src/v2/methods/search.ts
|
|
797
830
|
function prepareSearchPayload(req) {
|
|
798
831
|
if (!req.query || !req.query.trim()) throw new Error("Query cannot be empty");
|
|
832
|
+
ensureNoV1Options(req, "search");
|
|
799
833
|
if (req.limit != null && req.limit <= 0)
|
|
800
834
|
throw new Error("limit must be positive");
|
|
801
835
|
if (req.timeout != null && req.timeout <= 0)
|
|
@@ -1163,7 +1197,9 @@ async function fetchAllPages(http, nextUrl, initial, pagination) {
|
|
|
1163
1197
|
docs.push(d);
|
|
1164
1198
|
}
|
|
1165
1199
|
if (maxResults != null && docs.length >= maxResults) break;
|
|
1166
|
-
|
|
1200
|
+
const next = payload.next ?? (Array.isArray(payload.data) ? null : payload.data?.next) ?? null;
|
|
1201
|
+
if (next === current) break;
|
|
1202
|
+
current = next;
|
|
1167
1203
|
pageCount += 1;
|
|
1168
1204
|
}
|
|
1169
1205
|
return docs;
|
|
@@ -1172,6 +1208,7 @@ async function fetchAllPages(http, nextUrl, initial, pagination) {
|
|
|
1172
1208
|
// src/v2/methods/crawl.ts
|
|
1173
1209
|
function prepareCrawlPayload(request) {
|
|
1174
1210
|
if (!request.url || !request.url.trim()) throw new Error("URL cannot be empty");
|
|
1211
|
+
ensureNoV1Options(request, "crawl");
|
|
1175
1212
|
const data = { url: request.url.trim() };
|
|
1176
1213
|
if (request.prompt) data.prompt = request.prompt;
|
|
1177
1214
|
if (request.excludePaths) data.excludePaths = request.excludePaths;
|
|
@@ -1723,6 +1760,7 @@ async function browser(http, args = {}) {
|
|
|
1723
1760
|
if (args.streamWebView != null) body.streamWebView = args.streamWebView;
|
|
1724
1761
|
if (args.blockAds != null) body.blockAds = args.blockAds;
|
|
1725
1762
|
if (args.profile != null) body.profile = args.profile;
|
|
1763
|
+
if (args.location != null) body.location = args.location;
|
|
1726
1764
|
if (args.integration != null) body.integration = args.integration;
|
|
1727
1765
|
if (args.origin) body.origin = args.origin;
|
|
1728
1766
|
try {
|
|
@@ -2789,7 +2827,7 @@ var FirecrawlClient = class {
|
|
|
2789
2827
|
// Browser
|
|
2790
2828
|
/**
|
|
2791
2829
|
* Create a new browser session.
|
|
2792
|
-
* @param args Session options (ttl, activityTtl, streamWebView, blockAds, profile).
|
|
2830
|
+
* @param args Session options (ttl, activityTtl, streamWebView, blockAds, profile, location).
|
|
2793
2831
|
* @returns Session id, CDP URL, live view URL, and expiration time.
|
|
2794
2832
|
*/
|
|
2795
2833
|
async browser(args = {}) {
|
package/dist/index.d.cts
CHANGED
|
@@ -765,10 +765,10 @@ interface FindToolsOptions {
|
|
|
765
765
|
offset?: number;
|
|
766
766
|
}
|
|
767
767
|
interface FindToolsData {
|
|
768
|
-
level: "providers" | "groups" | "tools";
|
|
768
|
+
level: "categories" | "providers" | "groups" | "tools";
|
|
769
769
|
items: Array<{
|
|
770
770
|
id: string;
|
|
771
|
-
name
|
|
771
|
+
name?: string;
|
|
772
772
|
next?: AlexandriaCall;
|
|
773
773
|
execute?: Pick<AlexandriaCall, "provider" | "capability">;
|
|
774
774
|
[key: string]: unknown;
|
|
@@ -2119,6 +2119,10 @@ declare function browser(http: HttpClient, args?: {
|
|
|
2119
2119
|
name: string;
|
|
2120
2120
|
saveChanges?: boolean;
|
|
2121
2121
|
};
|
|
2122
|
+
/** Country the session browses from, as an ISO 3166-1 alpha-2 code (default US). */
|
|
2123
|
+
location?: {
|
|
2124
|
+
country: string;
|
|
2125
|
+
};
|
|
2122
2126
|
integration?: string;
|
|
2123
2127
|
origin?: string;
|
|
2124
2128
|
}): Promise<BrowserCreateResponse>;
|
|
@@ -2585,7 +2589,7 @@ declare class FirecrawlClient {
|
|
|
2585
2589
|
}): Promise<AgentThreadResponse>;
|
|
2586
2590
|
/**
|
|
2587
2591
|
* Create a new browser session.
|
|
2588
|
-
* @param args Session options (ttl, activityTtl, streamWebView, blockAds, profile).
|
|
2592
|
+
* @param args Session options (ttl, activityTtl, streamWebView, blockAds, profile, location).
|
|
2589
2593
|
* @returns Session id, CDP URL, live view URL, and expiration time.
|
|
2590
2594
|
*/
|
|
2591
2595
|
browser(args?: Parameters<typeof browser>[1]): Promise<BrowserCreateResponse>;
|
package/dist/index.d.ts
CHANGED
|
@@ -765,10 +765,10 @@ interface FindToolsOptions {
|
|
|
765
765
|
offset?: number;
|
|
766
766
|
}
|
|
767
767
|
interface FindToolsData {
|
|
768
|
-
level: "providers" | "groups" | "tools";
|
|
768
|
+
level: "categories" | "providers" | "groups" | "tools";
|
|
769
769
|
items: Array<{
|
|
770
770
|
id: string;
|
|
771
|
-
name
|
|
771
|
+
name?: string;
|
|
772
772
|
next?: AlexandriaCall;
|
|
773
773
|
execute?: Pick<AlexandriaCall, "provider" | "capability">;
|
|
774
774
|
[key: string]: unknown;
|
|
@@ -2119,6 +2119,10 @@ declare function browser(http: HttpClient, args?: {
|
|
|
2119
2119
|
name: string;
|
|
2120
2120
|
saveChanges?: boolean;
|
|
2121
2121
|
};
|
|
2122
|
+
/** Country the session browses from, as an ISO 3166-1 alpha-2 code (default US). */
|
|
2123
|
+
location?: {
|
|
2124
|
+
country: string;
|
|
2125
|
+
};
|
|
2122
2126
|
integration?: string;
|
|
2123
2127
|
origin?: string;
|
|
2124
2128
|
}): Promise<BrowserCreateResponse>;
|
|
@@ -2585,7 +2589,7 @@ declare class FirecrawlClient {
|
|
|
2585
2589
|
}): Promise<AgentThreadResponse>;
|
|
2586
2590
|
/**
|
|
2587
2591
|
* Create a new browser session.
|
|
2588
|
-
* @param args Session options (ttl, activityTtl, streamWebView, blockAds, profile).
|
|
2592
|
+
* @param args Session options (ttl, activityTtl, streamWebView, blockAds, profile, location).
|
|
2589
2593
|
* @returns Session id, CDP URL, live view URL, and expiration time.
|
|
2590
2594
|
*/
|
|
2591
2595
|
browser(args?: Parameters<typeof browser>[1]): Promise<BrowserCreateResponse>;
|
package/dist/index.js
CHANGED
|
@@ -1,21 +1,13 @@
|
|
|
1
1
|
import {
|
|
2
|
-
|
|
3
|
-
} from "./chunk-
|
|
2
|
+
version
|
|
3
|
+
} from "./chunk-H75FJ6LW.js";
|
|
4
4
|
|
|
5
5
|
// src/v2/utils/httpClient.ts
|
|
6
6
|
import axios from "axios";
|
|
7
7
|
|
|
8
8
|
// src/v2/utils/getVersion.ts
|
|
9
9
|
function getVersion() {
|
|
10
|
-
|
|
11
|
-
if (typeof process !== "undefined" && process.env && process.env.npm_package_version) {
|
|
12
|
-
return process.env.npm_package_version;
|
|
13
|
-
}
|
|
14
|
-
const pkg = require_package();
|
|
15
|
-
return pkg?.version || "3.x.x";
|
|
16
|
-
} catch {
|
|
17
|
-
return "3.x.x";
|
|
18
|
-
}
|
|
10
|
+
return version;
|
|
19
11
|
}
|
|
20
12
|
|
|
21
13
|
// src/utils/apiOrigin.ts
|
|
@@ -69,7 +61,7 @@ var HttpClient = class {
|
|
|
69
61
|
return this.instance.defaults.timeout ?? 3e5;
|
|
70
62
|
}
|
|
71
63
|
async request(config) {
|
|
72
|
-
const
|
|
64
|
+
const version2 = getVersion();
|
|
73
65
|
config.headers = {
|
|
74
66
|
...config.headers || {}
|
|
75
67
|
};
|
|
@@ -84,7 +76,7 @@ var HttpClient = class {
|
|
|
84
76
|
const data = cfg.data ?? {};
|
|
85
77
|
cfg.data = {
|
|
86
78
|
...data,
|
|
87
|
-
origin: typeof data.origin === "string" && data.origin.includes("mcp") ? data.origin : `js-sdk@${
|
|
79
|
+
origin: typeof data.origin === "string" && data.origin.includes("mcp") ? data.origin : `js-sdk@${version2}`
|
|
88
80
|
};
|
|
89
81
|
}
|
|
90
82
|
if (isFormDataBody) {
|
|
@@ -323,6 +315,30 @@ function ensureValidFormats(formats) {
|
|
|
323
315
|
}
|
|
324
316
|
}
|
|
325
317
|
}
|
|
318
|
+
var V1_SCRAPE_OPTION_REPLACEMENTS = {
|
|
319
|
+
pageOptions: "use scrapeOptions",
|
|
320
|
+
includeLinks: "add 'links' to scrapeOptions.formats",
|
|
321
|
+
parsePDF: "use scrapeOptions.parsers"
|
|
322
|
+
};
|
|
323
|
+
var V1_OPTION_REPLACEMENTS = {
|
|
324
|
+
crawl: {
|
|
325
|
+
crawlerOptions: "move its fields to the top level",
|
|
326
|
+
maxDepth: "use maxDiscoveryDepth",
|
|
327
|
+
allowBackwardLinks: "use crawlEntireDomain",
|
|
328
|
+
ignoreSitemap: "use sitemap: 'skip'",
|
|
329
|
+
...V1_SCRAPE_OPTION_REPLACEMENTS
|
|
330
|
+
},
|
|
331
|
+
search: V1_SCRAPE_OPTION_REPLACEMENTS
|
|
332
|
+
};
|
|
333
|
+
function ensureNoV1Options(options, method) {
|
|
334
|
+
if (!options) return;
|
|
335
|
+
const replacements = V1_OPTION_REPLACEMENTS[method];
|
|
336
|
+
const found = Object.keys(replacements).filter((key) => options[key] != null);
|
|
337
|
+
if (found.length === 0) return;
|
|
338
|
+
throw new Error(
|
|
339
|
+
`${method}: ${found.map((key) => `${key} is a v1 option, ${replacements[key]}`).join("; ")}`
|
|
340
|
+
);
|
|
341
|
+
}
|
|
326
342
|
function ensureValidScrapeOptions(options) {
|
|
327
343
|
if (!options) return;
|
|
328
344
|
if (options.timeout != null && options.timeout <= 0) {
|
|
@@ -436,12 +452,28 @@ function ensureValidParseOptions(options) {
|
|
|
436
452
|
|
|
437
453
|
// src/v2/utils/errorHandler.ts
|
|
438
454
|
import "axios";
|
|
455
|
+
function describeValidationIssues(details) {
|
|
456
|
+
if (!Array.isArray(details)) return void 0;
|
|
457
|
+
const parts = details.slice(0, 3).flatMap((issue) => {
|
|
458
|
+
if (!issue || typeof issue !== "object") return [];
|
|
459
|
+
const { path, message, keys } = issue;
|
|
460
|
+
const what = Array.isArray(keys) && keys.length > 0 ? `unrecognized key(s) ${keys.join(", ")}` : typeof message === "string" ? message : void 0;
|
|
461
|
+
if (!what) return [];
|
|
462
|
+
return Array.isArray(path) && path.length > 0 ? [`${path.join(".")}: ${what}`] : [what];
|
|
463
|
+
});
|
|
464
|
+
return parts.length > 0 ? parts.join("; ") : void 0;
|
|
465
|
+
}
|
|
466
|
+
function withIssues(message, details) {
|
|
467
|
+
const text = typeof message === "string" ? message : JSON.stringify(message);
|
|
468
|
+
const issues = describeValidationIssues(details);
|
|
469
|
+
return issues && !text.includes(issues) ? `${text} (${issues})` : text;
|
|
470
|
+
}
|
|
439
471
|
function throwForBadResponse(resp, action) {
|
|
440
472
|
const status = resp.status;
|
|
441
473
|
const body = resp.data || {};
|
|
442
474
|
const msg = body?.error || body?.message || `Request failed (${status}) while trying to ${action}`;
|
|
443
475
|
throw Object.assign(new SdkError(
|
|
444
|
-
msg,
|
|
476
|
+
withIssues(msg, body?.details),
|
|
445
477
|
status,
|
|
446
478
|
body?.code,
|
|
447
479
|
body?.details,
|
|
@@ -456,7 +488,7 @@ function normalizeAxiosError(err, action) {
|
|
|
456
488
|
const message = body?.error || err.message || `Request failed${status ? ` (${status})` : ""} while trying to ${action}`;
|
|
457
489
|
const code = body?.code || err.code;
|
|
458
490
|
throw Object.assign(new SdkError(
|
|
459
|
-
message,
|
|
491
|
+
withIssues(message, body?.details),
|
|
460
492
|
status,
|
|
461
493
|
code,
|
|
462
494
|
body?.details ?? body,
|
|
@@ -627,10 +659,10 @@ async function parse(http, file, options) {
|
|
|
627
659
|
throw new Error("file data cannot be empty");
|
|
628
660
|
}
|
|
629
661
|
if (options) ensureValidParseOptions(options);
|
|
630
|
-
const
|
|
662
|
+
const version2 = getVersion();
|
|
631
663
|
const normalizedOptions = {
|
|
632
664
|
...options ?? {},
|
|
633
|
-
origin: typeof options?.origin === "string" && options.origin.includes("mcp") ? options.origin : options?.origin ?? `js-sdk@${
|
|
665
|
+
origin: typeof options?.origin === "string" && options.origin.includes("mcp") ? options.origin : options?.origin ?? `js-sdk@${version2}`
|
|
634
666
|
};
|
|
635
667
|
const formData = new FormData();
|
|
636
668
|
formData.append("options", JSON.stringify(normalizedOptions));
|
|
@@ -668,6 +700,7 @@ async function getParseFormats(http) {
|
|
|
668
700
|
// src/v2/methods/search.ts
|
|
669
701
|
function prepareSearchPayload(req) {
|
|
670
702
|
if (!req.query || !req.query.trim()) throw new Error("Query cannot be empty");
|
|
703
|
+
ensureNoV1Options(req, "search");
|
|
671
704
|
if (req.limit != null && req.limit <= 0)
|
|
672
705
|
throw new Error("limit must be positive");
|
|
673
706
|
if (req.timeout != null && req.timeout <= 0)
|
|
@@ -1035,7 +1068,9 @@ async function fetchAllPages(http, nextUrl, initial, pagination) {
|
|
|
1035
1068
|
docs.push(d);
|
|
1036
1069
|
}
|
|
1037
1070
|
if (maxResults != null && docs.length >= maxResults) break;
|
|
1038
|
-
|
|
1071
|
+
const next = payload.next ?? (Array.isArray(payload.data) ? null : payload.data?.next) ?? null;
|
|
1072
|
+
if (next === current) break;
|
|
1073
|
+
current = next;
|
|
1039
1074
|
pageCount += 1;
|
|
1040
1075
|
}
|
|
1041
1076
|
return docs;
|
|
@@ -1044,6 +1079,7 @@ async function fetchAllPages(http, nextUrl, initial, pagination) {
|
|
|
1044
1079
|
// src/v2/methods/crawl.ts
|
|
1045
1080
|
function prepareCrawlPayload(request) {
|
|
1046
1081
|
if (!request.url || !request.url.trim()) throw new Error("URL cannot be empty");
|
|
1082
|
+
ensureNoV1Options(request, "crawl");
|
|
1047
1083
|
const data = { url: request.url.trim() };
|
|
1048
1084
|
if (request.prompt) data.prompt = request.prompt;
|
|
1049
1085
|
if (request.excludePaths) data.excludePaths = request.excludePaths;
|
|
@@ -1595,6 +1631,7 @@ async function browser(http, args = {}) {
|
|
|
1595
1631
|
if (args.streamWebView != null) body.streamWebView = args.streamWebView;
|
|
1596
1632
|
if (args.blockAds != null) body.blockAds = args.blockAds;
|
|
1597
1633
|
if (args.profile != null) body.profile = args.profile;
|
|
1634
|
+
if (args.location != null) body.location = args.location;
|
|
1598
1635
|
if (args.integration != null) body.integration = args.integration;
|
|
1599
1636
|
if (args.origin) body.origin = args.origin;
|
|
1600
1637
|
try {
|
|
@@ -2661,7 +2698,7 @@ var FirecrawlClient = class {
|
|
|
2661
2698
|
// Browser
|
|
2662
2699
|
/**
|
|
2663
2700
|
* Create a new browser session.
|
|
2664
|
-
* @param args Session options (ttl, activityTtl, streamWebView, blockAds, profile).
|
|
2701
|
+
* @param args Session options (ttl, activityTtl, streamWebView, blockAds, profile, location).
|
|
2665
2702
|
* @returns Session id, CDP URL, live view URL, and expiration time.
|
|
2666
2703
|
*/
|
|
2667
2704
|
async browser(args = {}) {
|
|
@@ -2800,7 +2837,7 @@ var FirecrawlApp = class {
|
|
|
2800
2837
|
if (typeof process !== "undefined" && process.env && process.env.npm_package_version) {
|
|
2801
2838
|
return process.env.npm_package_version;
|
|
2802
2839
|
}
|
|
2803
|
-
const packageJson = await import("./package-
|
|
2840
|
+
const packageJson = await import("./package-NZI5TDUV.js");
|
|
2804
2841
|
return packageJson.default.version;
|
|
2805
2842
|
} catch (error) {
|
|
2806
2843
|
const isTest = typeof process !== "undefined" && false;
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import {
|
|
2
|
+
author,
|
|
3
|
+
bugs,
|
|
4
|
+
dependencies,
|
|
5
|
+
description,
|
|
6
|
+
devDependencies,
|
|
7
|
+
engines,
|
|
8
|
+
exports,
|
|
9
|
+
homepage,
|
|
10
|
+
keywords,
|
|
11
|
+
license,
|
|
12
|
+
main,
|
|
13
|
+
name,
|
|
14
|
+
package_default,
|
|
15
|
+
repository,
|
|
16
|
+
scripts,
|
|
17
|
+
type,
|
|
18
|
+
types,
|
|
19
|
+
version
|
|
20
|
+
} from "./chunk-H75FJ6LW.js";
|
|
21
|
+
export {
|
|
22
|
+
author,
|
|
23
|
+
bugs,
|
|
24
|
+
package_default as default,
|
|
25
|
+
dependencies,
|
|
26
|
+
description,
|
|
27
|
+
devDependencies,
|
|
28
|
+
engines,
|
|
29
|
+
exports,
|
|
30
|
+
homepage,
|
|
31
|
+
keywords,
|
|
32
|
+
license,
|
|
33
|
+
main,
|
|
34
|
+
name,
|
|
35
|
+
repository,
|
|
36
|
+
scripts,
|
|
37
|
+
type,
|
|
38
|
+
types,
|
|
39
|
+
version
|
|
40
|
+
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "firecrawl",
|
|
3
|
-
"version": "4.
|
|
3
|
+
"version": "4.47.0",
|
|
4
4
|
"description": "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import { describe, test, expect, vi } from "vitest";
|
|
2
|
+
import { browser } from "../../../v2/methods/browser";
|
|
3
|
+
|
|
4
|
+
describe("JS SDK v2 browser", () => {
|
|
5
|
+
test("sends location when creating a session", async () => {
|
|
6
|
+
const post = vi.fn(async () => ({
|
|
7
|
+
status: 200,
|
|
8
|
+
data: { success: true, id: "session-1" },
|
|
9
|
+
}));
|
|
10
|
+
|
|
11
|
+
await browser({ post } as any, { location: { country: "GB" } });
|
|
12
|
+
|
|
13
|
+
expect(post).toHaveBeenCalledWith("/v2/browser", {
|
|
14
|
+
location: { country: "GB" },
|
|
15
|
+
});
|
|
16
|
+
});
|
|
17
|
+
|
|
18
|
+
test("omits location by default", async () => {
|
|
19
|
+
const post = vi.fn(async () => ({
|
|
20
|
+
status: 200,
|
|
21
|
+
data: { success: true, id: "session-1" },
|
|
22
|
+
}));
|
|
23
|
+
|
|
24
|
+
await browser({ post } as any);
|
|
25
|
+
|
|
26
|
+
expect(post).toHaveBeenCalledWith("/v2/browser", {});
|
|
27
|
+
});
|
|
28
|
+
});
|
|
@@ -15,5 +15,28 @@ describe("v2 utils: errorHandler", () => {
|
|
|
15
15
|
};
|
|
16
16
|
expect(() => normalizeAxiosError(err, "action")).toThrow(/payment required/);
|
|
17
17
|
});
|
|
18
|
+
|
|
19
|
+
test("normalizeAxiosError: names the fields a 400 validation error rejected", () => {
|
|
20
|
+
const err: any = {
|
|
21
|
+
isAxiosError: true,
|
|
22
|
+
response: {
|
|
23
|
+
status: 400,
|
|
24
|
+
data: {
|
|
25
|
+
error: "Unrecognized key in body -- please review the v2 API documentation for request body changes",
|
|
26
|
+
details: [{ code: "unrecognized_keys", keys: ["parsePDF"], path: [], message: "Unrecognized key(s) in object: 'parsePDF'" }],
|
|
27
|
+
},
|
|
28
|
+
},
|
|
29
|
+
message: "Request failed with status code 400",
|
|
30
|
+
};
|
|
31
|
+
expect(() => normalizeAxiosError(err, "scrape")).toThrow(/unrecognized key\(s\) parsePDF/);
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
test("throwForBadResponse: appends the path and message of each issue", () => {
|
|
35
|
+
const resp: any = {
|
|
36
|
+
status: 400,
|
|
37
|
+
data: { error: "Bad Request", details: [{ code: "invalid_union", path: ["formats", 0], message: "Invalid input" }] },
|
|
38
|
+
};
|
|
39
|
+
expect(() => throwForBadResponse(resp, "scrape")).toThrow("Bad Request (formats.0: Invalid input)");
|
|
40
|
+
});
|
|
18
41
|
});
|
|
19
42
|
|
|
@@ -107,6 +107,26 @@ describe("JS SDK v2 pagination", () => {
|
|
|
107
107
|
}
|
|
108
108
|
});
|
|
109
109
|
|
|
110
|
+
test("crawl: stops when a running job's cursor does not advance", async () => {
|
|
111
|
+
const next = "https://api/v2/crawl/jobR?skip=2";
|
|
112
|
+
const first = { status: 200, data: { success: true, status: "scraping", completed: 2, total: 2, next, data: [{ markdown: "a" }, { markdown: "b" }] } };
|
|
113
|
+
const stalled = { status: 200, data: { success: true, status: "scraping", next, data: [] } };
|
|
114
|
+
const http = makeHttp((url) => (url === next ? stalled : first));
|
|
115
|
+
const res = await getCrawlStatus(http, "jobR");
|
|
116
|
+
expect(res.data.length).toBe(2);
|
|
117
|
+
expect((http.get as Mock).mock.calls.length).toBe(2);
|
|
118
|
+
});
|
|
119
|
+
|
|
120
|
+
test("batch: stops when a running job's cursor does not advance", async () => {
|
|
121
|
+
const next = "https://api/v2/batch/scrape/jobR?skip=2";
|
|
122
|
+
const first = { status: 200, data: { success: true, status: "scraping", completed: 2, total: 2, next, data: [{ markdown: "a" }, { markdown: "b" }] } };
|
|
123
|
+
const stalled = { status: 200, data: { success: true, status: "scraping", next, data: [] } };
|
|
124
|
+
const http = makeHttp((url) => (url === next ? stalled : first));
|
|
125
|
+
const res = await getBatchScrapeStatus(http, "jobR");
|
|
126
|
+
expect(res.data.length).toBe(2);
|
|
127
|
+
expect((http.get as Mock).mock.calls.length).toBe(2);
|
|
128
|
+
});
|
|
129
|
+
|
|
110
130
|
test("batch: maxWaitTime stops pagination after first page", async () => {
|
|
111
131
|
const first = { status: 200, data: { success: true, status: "completed", completed: 1, total: 5, next: "https://api/b1", data: [{ markdown: "a" }] } };
|
|
112
132
|
const p1 = { status: 200, data: { success: true, next: "https://api/b2", data: [{ markdown: "b" }] } };
|
|
@@ -1,9 +1,47 @@
|
|
|
1
1
|
import { describe, test, expect } from "vitest";
|
|
2
|
-
import { ensureValidFormats, ensureValidScrapeOptions } from "../../../v2/utils/validation";
|
|
2
|
+
import { ensureNoV1Options, ensureValidFormats, ensureValidScrapeOptions } from "../../../v2/utils/validation";
|
|
3
3
|
import type { FormatOption } from "../../../v2/types";
|
|
4
4
|
import { z } from "zod";
|
|
5
5
|
|
|
6
6
|
describe("v2 utils: validation", () => {
|
|
7
|
+
const scrapeLevel = {
|
|
8
|
+
pageOptions: "use scrapeOptions",
|
|
9
|
+
includeLinks: "add 'links' to scrapeOptions.formats",
|
|
10
|
+
parsePDF: "use scrapeOptions.parsers",
|
|
11
|
+
};
|
|
12
|
+
const crawlOnly = {
|
|
13
|
+
crawlerOptions: "move its fields to the top level",
|
|
14
|
+
maxDepth: "use maxDiscoveryDepth",
|
|
15
|
+
allowBackwardLinks: "use crawlEntireDomain",
|
|
16
|
+
ignoreSitemap: "use sitemap: 'skip'",
|
|
17
|
+
};
|
|
18
|
+
|
|
19
|
+
test.each(Object.entries({ ...crawlOnly, ...scrapeLevel }))("ensureNoV1Options: crawl names %s and its replacement", (option, replacement) => {
|
|
20
|
+
expect(() => ensureNoV1Options({ url: "https://a.dev", [option]: true }, "crawl")).toThrow(
|
|
21
|
+
`crawl: ${option} is a v1 option, ${replacement}`,
|
|
22
|
+
);
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
test.each(Object.entries(scrapeLevel))("ensureNoV1Options: search names %s and its replacement", (option, replacement) => {
|
|
26
|
+
expect(() => ensureNoV1Options({ query: "q", [option]: true }, "search")).toThrow(
|
|
27
|
+
`search: ${option} is a v1 option, ${replacement}`,
|
|
28
|
+
);
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
test("ensureNoV1Options: search does not apply crawl-only hints", () => {
|
|
32
|
+
expect(() => ensureNoV1Options({ query: "q", maxDepth: 2, crawlerOptions: {} }, "search")).not.toThrow();
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
test("ensureNoV1Options: lists every v1 option found", () => {
|
|
36
|
+
expect(() => ensureNoV1Options({ maxDepth: 2, allowBackwardLinks: true }, "crawl")).toThrow(/maxDepth.*maxDiscoveryDepth.*allowBackwardLinks.*crawlEntireDomain/);
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
test("ensureNoV1Options: accepts v2 options and unset v1 keys", () => {
|
|
40
|
+
expect(() => ensureNoV1Options({ url: "https://a.dev", limit: 2, maxDiscoveryDepth: 1 }, "crawl")).not.toThrow();
|
|
41
|
+
expect(() => ensureNoV1Options({ url: "https://a.dev", maxDepth: undefined }, "crawl")).not.toThrow();
|
|
42
|
+
expect(() => ensureNoV1Options(undefined, "search")).not.toThrow();
|
|
43
|
+
});
|
|
44
|
+
|
|
7
45
|
test("ensureValidFormats: plain 'json' string is invalid", () => {
|
|
8
46
|
const formats: FormatOption[] = ["markdown", "json"] as unknown as FormatOption[];
|
|
9
47
|
expect(() => ensureValidFormats(formats)).toThrow(/json format must be an object/i);
|
package/src/v2/client.ts
CHANGED
|
@@ -644,7 +644,7 @@ export class FirecrawlClient {
|
|
|
644
644
|
// Browser
|
|
645
645
|
/**
|
|
646
646
|
* Create a new browser session.
|
|
647
|
-
* @param args Session options (ttl, activityTtl, streamWebView, blockAds, profile).
|
|
647
|
+
* @param args Session options (ttl, activityTtl, streamWebView, blockAds, profile, location).
|
|
648
648
|
* @returns Session id, CDP URL, live view URL, and expiration time.
|
|
649
649
|
*/
|
|
650
650
|
async browser(
|
|
@@ -22,6 +22,8 @@ export async function browser(
|
|
|
22
22
|
name: string;
|
|
23
23
|
saveChanges?: boolean;
|
|
24
24
|
};
|
|
25
|
+
/** Country the session browses from, as an ISO 3166-1 alpha-2 code (default US). */
|
|
26
|
+
location?: { country: string };
|
|
25
27
|
integration?: string;
|
|
26
28
|
origin?: string;
|
|
27
29
|
} = {},
|
|
@@ -32,6 +34,7 @@ export async function browser(
|
|
|
32
34
|
if (args.streamWebView != null) body.streamWebView = args.streamWebView;
|
|
33
35
|
if (args.blockAds != null) body.blockAds = args.blockAds;
|
|
34
36
|
if (args.profile != null) body.profile = args.profile;
|
|
37
|
+
if (args.location != null) body.location = args.location;
|
|
35
38
|
if (args.integration != null) body.integration = args.integration;
|
|
36
39
|
if (args.origin) body.origin = args.origin;
|
|
37
40
|
|
package/src/v2/methods/crawl.ts
CHANGED
|
@@ -10,7 +10,7 @@ import {
|
|
|
10
10
|
SdkError,
|
|
11
11
|
} from "../types";
|
|
12
12
|
import { HttpClient } from "../utils/httpClient";
|
|
13
|
-
import { ensureValidScrapeOptions } from "../utils/validation";
|
|
13
|
+
import { ensureNoV1Options, ensureValidScrapeOptions } from "../utils/validation";
|
|
14
14
|
import { normalizeAxiosError, throwForBadResponse, isRetryableError } from "../utils/errorHandler";
|
|
15
15
|
import type { HttpClient as _Http } from "../utils/httpClient";
|
|
16
16
|
import { fetchAllPages } from "../utils/pagination";
|
|
@@ -21,6 +21,7 @@ export type CrawlRequest = CrawlOptions & {
|
|
|
21
21
|
|
|
22
22
|
function prepareCrawlPayload(request: CrawlRequest): Record<string, unknown> {
|
|
23
23
|
if (!request.url || !request.url.trim()) throw new Error("URL cannot be empty");
|
|
24
|
+
ensureNoV1Options(request, "crawl");
|
|
24
25
|
const data: Record<string, unknown> = { url: request.url.trim() };
|
|
25
26
|
if (request.prompt) data.prompt = request.prompt;
|
|
26
27
|
if (request.excludePaths) data.excludePaths = request.excludePaths;
|
package/src/v2/methods/search.ts
CHANGED
|
@@ -9,7 +9,7 @@ import {
|
|
|
9
9
|
} from "../types";
|
|
10
10
|
import { HttpClient } from "../utils/httpClient";
|
|
11
11
|
import { agentHintMetadata } from "../utils/agentHints";
|
|
12
|
-
import { ensureValidScrapeOptions } from "../utils/validation";
|
|
12
|
+
import { ensureNoV1Options, ensureValidScrapeOptions } from "../utils/validation";
|
|
13
13
|
import {
|
|
14
14
|
throwForBadResponse,
|
|
15
15
|
normalizeAxiosError,
|
|
@@ -17,6 +17,7 @@ import {
|
|
|
17
17
|
|
|
18
18
|
function prepareSearchPayload(req: SearchRequest): Record<string, unknown> {
|
|
19
19
|
if (!req.query || !req.query.trim()) throw new Error("Query cannot be empty");
|
|
20
|
+
ensureNoV1Options(req, "search");
|
|
20
21
|
if (req.limit != null && req.limit <= 0)
|
|
21
22
|
throw new Error("limit must be positive");
|
|
22
23
|
if (req.timeout != null && req.timeout <= 0)
|
package/src/v2/types.ts
CHANGED
|
@@ -915,10 +915,10 @@ export interface FindToolsOptions {
|
|
|
915
915
|
}
|
|
916
916
|
|
|
917
917
|
export interface FindToolsData {
|
|
918
|
-
level: "providers" | "groups" | "tools";
|
|
918
|
+
level: "categories" | "providers" | "groups" | "tools";
|
|
919
919
|
items: Array<{
|
|
920
920
|
id: string;
|
|
921
|
-
name
|
|
921
|
+
name?: string;
|
|
922
922
|
next?: AlexandriaCall;
|
|
923
923
|
execute?: Pick<AlexandriaCall, "provider" | "capability">;
|
|
924
924
|
[key: string]: unknown;
|
|
@@ -2,12 +2,33 @@ import { type AxiosError, type AxiosResponse } from "axios";
|
|
|
2
2
|
import { SdkError, JobTimeoutError, parseRequiresAction } from "../types";
|
|
3
3
|
import { agentHintMetadata } from "./agentHints";
|
|
4
4
|
|
|
5
|
+
/** Names the fields a validation error rejected, so a bare "Bad Request" says what to change. */
|
|
6
|
+
function describeValidationIssues(details: unknown): string | undefined {
|
|
7
|
+
if (!Array.isArray(details)) return undefined;
|
|
8
|
+
const parts = details.slice(0, 3).flatMap((issue) => {
|
|
9
|
+
if (!issue || typeof issue !== "object") return [];
|
|
10
|
+
const { path, message, keys } = issue as { path?: unknown; message?: unknown; keys?: unknown };
|
|
11
|
+
const what = Array.isArray(keys) && keys.length > 0
|
|
12
|
+
? `unrecognized key(s) ${keys.join(", ")}`
|
|
13
|
+
: typeof message === "string" ? message : undefined;
|
|
14
|
+
if (!what) return [];
|
|
15
|
+
return Array.isArray(path) && path.length > 0 ? [`${path.join(".")}: ${what}`] : [what];
|
|
16
|
+
});
|
|
17
|
+
return parts.length > 0 ? parts.join("; ") : undefined;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
function withIssues(message: unknown, details: unknown): string {
|
|
21
|
+
const text = typeof message === "string" ? message : JSON.stringify(message);
|
|
22
|
+
const issues = describeValidationIssues(details);
|
|
23
|
+
return issues && !text.includes(issues) ? `${text} (${issues})` : text;
|
|
24
|
+
}
|
|
25
|
+
|
|
5
26
|
export function throwForBadResponse(resp: AxiosResponse, action: string): never {
|
|
6
27
|
const status = resp.status;
|
|
7
28
|
const body = resp.data || {};
|
|
8
29
|
const msg = body?.error || body?.message || `Request failed (${status}) while trying to ${action}`;
|
|
9
30
|
throw Object.assign(new SdkError(
|
|
10
|
-
msg,
|
|
31
|
+
withIssues(msg, body?.details),
|
|
11
32
|
status,
|
|
12
33
|
body?.code,
|
|
13
34
|
body?.details,
|
|
@@ -23,7 +44,7 @@ export function normalizeAxiosError(err: AxiosError, action: string): never {
|
|
|
23
44
|
const message = body?.error || err.message || `Request failed${status ? ` (${status})` : ""} while trying to ${action}`;
|
|
24
45
|
const code = (body?.code as string) || err.code;
|
|
25
46
|
throw Object.assign(new SdkError(
|
|
26
|
-
message,
|
|
47
|
+
withIssues(message, body?.details),
|
|
27
48
|
status,
|
|
28
49
|
code,
|
|
29
50
|
body?.details ?? body,
|
|
@@ -1,14 +1,6 @@
|
|
|
1
|
-
|
|
2
|
-
try {
|
|
3
|
-
if (typeof process !== "undefined" && process.env && process.env.npm_package_version) {
|
|
4
|
-
return process.env.npm_package_version as string;
|
|
5
|
-
}
|
|
1
|
+
import { version } from "../../../package.json";
|
|
6
2
|
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
} catch {
|
|
11
|
-
return "3.x.x";
|
|
12
|
-
}
|
|
3
|
+
/** The SDK's own version, inlined at build time so a host app's npm_package_version never replaces it. */
|
|
4
|
+
export function getVersion(): string {
|
|
5
|
+
return version;
|
|
13
6
|
}
|
|
14
|
-
|
|
@@ -39,7 +39,10 @@ export async function fetchAllPages<T = Document>(
|
|
|
39
39
|
docs.push(d as T);
|
|
40
40
|
}
|
|
41
41
|
if (maxResults != null && docs.length >= maxResults) break;
|
|
42
|
-
|
|
42
|
+
const next = (payload.next ?? (Array.isArray(payload.data) ? null : payload.data?.next) ?? null) as string | null;
|
|
43
|
+
// A running job's cursor can point back at itself until new documents land; re-requesting it here only spins.
|
|
44
|
+
if (next === current) break;
|
|
45
|
+
current = next;
|
|
43
46
|
pageCount += 1;
|
|
44
47
|
}
|
|
45
48
|
return docs;
|
|
@@ -86,6 +86,34 @@ export function ensureValidFormats(formats?: FormatOption[]): void {
|
|
|
86
86
|
}
|
|
87
87
|
}
|
|
88
88
|
|
|
89
|
+
const V1_SCRAPE_OPTION_REPLACEMENTS: Record<string, string> = {
|
|
90
|
+
pageOptions: "use scrapeOptions",
|
|
91
|
+
includeLinks: "add 'links' to scrapeOptions.formats",
|
|
92
|
+
parsePDF: "use scrapeOptions.parsers",
|
|
93
|
+
};
|
|
94
|
+
|
|
95
|
+
const V1_OPTION_REPLACEMENTS: Record<"crawl" | "search", Record<string, string>> = {
|
|
96
|
+
crawl: {
|
|
97
|
+
crawlerOptions: "move its fields to the top level",
|
|
98
|
+
maxDepth: "use maxDiscoveryDepth",
|
|
99
|
+
allowBackwardLinks: "use crawlEntireDomain",
|
|
100
|
+
ignoreSitemap: "use sitemap: 'skip'",
|
|
101
|
+
...V1_SCRAPE_OPTION_REPLACEMENTS,
|
|
102
|
+
},
|
|
103
|
+
search: V1_SCRAPE_OPTION_REPLACEMENTS,
|
|
104
|
+
};
|
|
105
|
+
|
|
106
|
+
/** Throws on v1 option names, which the v2 payload builders would otherwise drop without a word. */
|
|
107
|
+
export function ensureNoV1Options(options: object | undefined, method: "crawl" | "search"): void {
|
|
108
|
+
if (!options) return;
|
|
109
|
+
const replacements = V1_OPTION_REPLACEMENTS[method];
|
|
110
|
+
const found = Object.keys(replacements).filter((key) => (options as Record<string, unknown>)[key] != null);
|
|
111
|
+
if (found.length === 0) return;
|
|
112
|
+
throw new Error(
|
|
113
|
+
`${method}: ${found.map((key) => `${key} is a v1 option, ${replacements[key]}`).join("; ")}`,
|
|
114
|
+
);
|
|
115
|
+
}
|
|
116
|
+
|
|
89
117
|
export function ensureValidScrapeOptions(options?: ScrapeOptions): void {
|
|
90
118
|
if (!options) return;
|
|
91
119
|
if (options.timeout != null && options.timeout <= 0) {
|
package/dist/chunk-R4XT7PME.js
DELETED
|
@@ -1,93 +0,0 @@
|
|
|
1
|
-
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
2
|
-
var __commonJS = (cb, mod) => function __require() {
|
|
3
|
-
try {
|
|
4
|
-
return mod || (0, cb[__getOwnPropNames(cb)[0]])((mod = { exports: {} }).exports, mod), mod.exports;
|
|
5
|
-
} catch (e) {
|
|
6
|
-
throw mod = 0, e;
|
|
7
|
-
}
|
|
8
|
-
};
|
|
9
|
-
|
|
10
|
-
// package.json
|
|
11
|
-
var require_package = __commonJS({
|
|
12
|
-
"package.json"(exports, module) {
|
|
13
|
-
module.exports = {
|
|
14
|
-
name: "@mendable/firecrawl-js",
|
|
15
|
-
version: "4.46.0",
|
|
16
|
-
description: "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
|
|
17
|
-
main: "dist/index.js",
|
|
18
|
-
types: "dist/index.d.ts",
|
|
19
|
-
exports: {
|
|
20
|
-
"./package.json": "./package.json",
|
|
21
|
-
".": {
|
|
22
|
-
import: "./dist/index.js",
|
|
23
|
-
default: "./dist/index.cjs"
|
|
24
|
-
}
|
|
25
|
-
},
|
|
26
|
-
type: "module",
|
|
27
|
-
scripts: {
|
|
28
|
-
build: "tsup",
|
|
29
|
-
"build-and-publish": "pnpm run build && pnpm publish --access public",
|
|
30
|
-
"publish-beta": "pnpm run build && pnpm publish --access public --tag beta",
|
|
31
|
-
test: "vitest run --reporter=verbose src/__tests__/e2e/v2",
|
|
32
|
-
"test:unit": "vitest run --reporter=verbose src/__tests__/unit"
|
|
33
|
-
},
|
|
34
|
-
repository: {
|
|
35
|
-
type: "git",
|
|
36
|
-
url: "git+https://github.com/firecrawl/firecrawl.git"
|
|
37
|
-
},
|
|
38
|
-
author: "Mendable.ai",
|
|
39
|
-
license: "MIT",
|
|
40
|
-
dependencies: {
|
|
41
|
-
axios: "1.20.0",
|
|
42
|
-
firecrawl: "4.30.1",
|
|
43
|
-
"typescript-event-target": "^1.1.1",
|
|
44
|
-
zod: "^3.23.8",
|
|
45
|
-
"zod-to-json-schema": "^3.23.0"
|
|
46
|
-
},
|
|
47
|
-
bugs: {
|
|
48
|
-
url: "https://github.com/firecrawl/firecrawl/issues"
|
|
49
|
-
},
|
|
50
|
-
homepage: "https://github.com/firecrawl/firecrawl#readme",
|
|
51
|
-
devDependencies: {
|
|
52
|
-
"@types/dotenv": "^8.2.0",
|
|
53
|
-
"@types/node": "^20.12.12",
|
|
54
|
-
"@types/uuid": "^9.0.8",
|
|
55
|
-
dotenv: "^16.4.5",
|
|
56
|
-
tsup: "^8.5.0",
|
|
57
|
-
typescript: "^5.4.5",
|
|
58
|
-
uuid: "^14.0.0",
|
|
59
|
-
vitest: "^4.1.11"
|
|
60
|
-
},
|
|
61
|
-
keywords: [
|
|
62
|
-
"firecrawl",
|
|
63
|
-
"mendable",
|
|
64
|
-
"crawler",
|
|
65
|
-
"web",
|
|
66
|
-
"scraper",
|
|
67
|
-
"api",
|
|
68
|
-
"sdk",
|
|
69
|
-
"web-search",
|
|
70
|
-
"literature-search",
|
|
71
|
-
"research",
|
|
72
|
-
"scientific-papers",
|
|
73
|
-
"academic-search",
|
|
74
|
-
"biomedical",
|
|
75
|
-
"life-sciences",
|
|
76
|
-
"pubmed",
|
|
77
|
-
"biorxiv",
|
|
78
|
-
"medrxiv",
|
|
79
|
-
"arxiv",
|
|
80
|
-
"preprints",
|
|
81
|
-
"citations",
|
|
82
|
-
"bioinformatics"
|
|
83
|
-
],
|
|
84
|
-
engines: {
|
|
85
|
-
node: ">=22.0.0"
|
|
86
|
-
}
|
|
87
|
-
};
|
|
88
|
-
}
|
|
89
|
-
});
|
|
90
|
-
|
|
91
|
-
export {
|
|
92
|
-
require_package
|
|
93
|
-
};
|
package/dist/package-EI4Q33HD.js
DELETED