@mendable/firecrawl-js 4.46.0 → 4.48.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -8
- package/dist/chunk-GD463K4D.js +114 -0
- package/dist/index.cjs +163 -39
- package/dist/index.d.cts +24 -5
- package/dist/index.d.ts +24 -5
- package/dist/index.js +164 -41
- package/dist/package-WONXBIJS.js +40 -0
- package/package.json +1 -1
- package/src/__tests__/unit/v2/agent-hints.test.ts +1 -0
- package/src/__tests__/unit/v2/browser.unit.test.ts +28 -0
- package/src/__tests__/unit/v2/errorHandler.test.ts +23 -0
- package/src/__tests__/unit/v2/origin.test.ts +132 -0
- package/src/__tests__/unit/v2/pagination.test.ts +20 -0
- package/src/__tests__/unit/v2/research.test.ts +1 -0
- package/src/__tests__/unit/v2/validation.test.ts +39 -1
- package/src/index.ts +1 -0
- package/src/utils/callerOrigin.ts +74 -0
- package/src/v1/index.ts +28 -13
- package/src/v2/client.ts +11 -2
- package/src/v2/methods/browser.ts +3 -0
- package/src/v2/methods/crawl.ts +2 -1
- package/src/v2/methods/parse.ts +1 -6
- package/src/v2/methods/research.ts +20 -7
- package/src/v2/methods/search.ts +2 -1
- package/src/v2/types.ts +2 -2
- package/src/v2/utils/errorHandler.ts +23 -2
- package/src/v2/utils/getVersion.ts +4 -12
- package/src/v2/utils/httpClient.ts +10 -3
- package/src/v2/utils/pagination.ts +4 -1
- package/src/v2/utils/validation.ts +28 -0
- package/dist/chunk-R4XT7PME.js +0 -93
- package/dist/package-EI4Q33HD.js +0 -4
package/README.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Firecrawl Node SDK
|
|
2
2
|
|
|
3
|
-
The Firecrawl Node SDK is
|
|
3
|
+
The Firecrawl Node SDK is the JavaScript client for the Firecrawl API, the context API for AI agents. Search, scrape, parse, crawl, map, and interact with the web, and get back clean Markdown or structured data your agents can ship with.
|
|
4
4
|
|
|
5
5
|
## Installation
|
|
6
6
|
|
|
@@ -13,7 +13,7 @@ npm install firecrawl
|
|
|
13
13
|
## Usage
|
|
14
14
|
|
|
15
15
|
1. Get an API key from [firecrawl.dev](https://firecrawl.dev)
|
|
16
|
-
2. Set the API key as an environment variable named `FIRECRAWL_API_KEY` or pass it as
|
|
16
|
+
2. Set the API key as an environment variable named `FIRECRAWL_API_KEY` or pass it as the `apiKey` option to the `Firecrawl` class.
|
|
17
17
|
|
|
18
18
|
Here's an example of how to use the SDK with error handling:
|
|
19
19
|
|
|
@@ -37,6 +37,16 @@ const crawlResponse = await app.crawl('https://firecrawl.dev', {
|
|
|
37
37
|
console.log(crawlResponse);
|
|
38
38
|
```
|
|
39
39
|
|
|
40
|
+
### Request origin
|
|
41
|
+
|
|
42
|
+
Requests carry an `origin` tag that identifies the client. Pass the `origin` option to set it. Set `origin` on a single call to override it for that call, except for `app.research` methods, which always use the client origin.
|
|
43
|
+
|
|
44
|
+
```js
|
|
45
|
+
const app = new Firecrawl({ apiKey: 'fc-YOUR_API_KEY', origin: 'js-sdk.my-wrapper@1.0.0' });
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
By default the tag is `js-sdk@<version>`. When the client is created by code inside an installed package (a wrapper library), the default is `js-sdk.<package>@<version>` instead, for example `js-sdk.scope-name@<version>`. Application code, REPLs and bundled files without `node_modules` in their path keep the plain default. Detection runs once when the client is created and sends only the package name, lowercased and sanitized into a slug. Set `FIRECRAWL_DISABLE_CALLER_ORIGIN=1` to always use the plain default.
|
|
49
|
+
|
|
40
50
|
### Scraping a URL
|
|
41
51
|
|
|
42
52
|
To scrape a single URL with error handling, use the `scrape` method. It takes the URL as a parameter and returns the scraped data.
|
|
@@ -126,7 +136,7 @@ const crawlResponse = await app.crawl('https://firecrawl.dev', {
|
|
|
126
136
|
To start an asynchronous crawl, use `startCrawl`. It returns a job ID you can poll with `getCrawlStatus`.
|
|
127
137
|
|
|
128
138
|
```js
|
|
129
|
-
const start = await app.startCrawl('https://
|
|
139
|
+
const start = await app.startCrawl('https://firecrawl.dev', {
|
|
130
140
|
excludePaths: ['blog/*'],
|
|
131
141
|
limit: 5,
|
|
132
142
|
});
|
|
@@ -137,7 +147,7 @@ const start = await app.startCrawl('https://mendable.ai', {
|
|
|
137
147
|
To check the status of a crawl job with error handling, use the `getCrawlStatus` method. It takes the job ID as a parameter and returns the current status.
|
|
138
148
|
|
|
139
149
|
```js
|
|
140
|
-
const status = await app.getCrawlStatus(id);
|
|
150
|
+
const status = await app.getCrawlStatus(start.id);
|
|
141
151
|
```
|
|
142
152
|
|
|
143
153
|
### Extracting structured data from URLs
|
|
@@ -323,7 +333,7 @@ await app.stopInteraction(scrapeJobId);
|
|
|
323
333
|
To receive real‑time updates, start a crawl and attach a watcher.
|
|
324
334
|
|
|
325
335
|
```js
|
|
326
|
-
const start = await app.startCrawl('https://
|
|
336
|
+
const start = await app.startCrawl('https://firecrawl.dev', { excludePaths: ['blog/*'], limit: 5 });
|
|
327
337
|
const watch = app.watcher(start.id, { kind: 'crawl', pollInterval: 2 });
|
|
328
338
|
|
|
329
339
|
watch.on('document', (doc) => {
|
|
@@ -346,7 +356,7 @@ await watch.start();
|
|
|
346
356
|
To batch scrape multiple URLs with error handling, use the `batchScrape` method.
|
|
347
357
|
|
|
348
358
|
```js
|
|
349
|
-
const batchScrapeResponse = await app.batchScrape(['https://firecrawl.dev', 'https://
|
|
359
|
+
const batchScrapeResponse = await app.batchScrape(['https://firecrawl.dev', 'https://docs.firecrawl.dev'], {
|
|
350
360
|
formats: ['markdown', 'html'],
|
|
351
361
|
});
|
|
352
362
|
```
|
|
@@ -357,7 +367,7 @@ const batchScrapeResponse = await app.batchScrape(['https://firecrawl.dev', 'htt
|
|
|
357
367
|
To start an asynchronous batch scrape, use `startBatchScrape` and poll with `getBatchScrapeStatus`.
|
|
358
368
|
|
|
359
369
|
```js
|
|
360
|
-
const asyncBatchScrapeResult = await app.startBatchScrape(['https://firecrawl.dev', 'https://
|
|
370
|
+
const asyncBatchScrapeResult = await app.startBatchScrape(['https://firecrawl.dev', 'https://docs.firecrawl.dev'], {
|
|
361
371
|
formats: ['markdown', 'html'],
|
|
362
372
|
});
|
|
363
373
|
```
|
|
@@ -367,7 +377,7 @@ const asyncBatchScrapeResult = await app.startBatchScrape(['https://firecrawl.de
|
|
|
367
377
|
To use batch scrape with real‑time updates, start the job and watch it using the watcher.
|
|
368
378
|
|
|
369
379
|
```js
|
|
370
|
-
const start = await app.startBatchScrape(['https://firecrawl.dev', 'https://
|
|
380
|
+
const start = await app.startBatchScrape(['https://firecrawl.dev', 'https://docs.firecrawl.dev'], { formats: ['markdown', 'html'] });
|
|
371
381
|
const watch = app.watcher(start.id, { kind: 'batch', pollInterval: 2 });
|
|
372
382
|
|
|
373
383
|
watch.on('document', (doc) => {
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
// package.json
|
|
2
|
+
var name = "@mendable/firecrawl-js";
|
|
3
|
+
var version = "4.48.0";
|
|
4
|
+
var description = "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts";
|
|
5
|
+
var main = "dist/index.js";
|
|
6
|
+
var types = "dist/index.d.ts";
|
|
7
|
+
var exports = {
|
|
8
|
+
"./package.json": "./package.json",
|
|
9
|
+
".": {
|
|
10
|
+
import: "./dist/index.js",
|
|
11
|
+
default: "./dist/index.cjs"
|
|
12
|
+
}
|
|
13
|
+
};
|
|
14
|
+
var type = "module";
|
|
15
|
+
var scripts = {
|
|
16
|
+
build: "tsup",
|
|
17
|
+
"build-and-publish": "pnpm run build && pnpm publish --access public",
|
|
18
|
+
"publish-beta": "pnpm run build && pnpm publish --access public --tag beta",
|
|
19
|
+
test: "vitest run --reporter=verbose src/__tests__/e2e/v2",
|
|
20
|
+
"test:unit": "vitest run --reporter=verbose src/__tests__/unit"
|
|
21
|
+
};
|
|
22
|
+
var repository = {
|
|
23
|
+
type: "git",
|
|
24
|
+
url: "git+https://github.com/firecrawl/firecrawl.git"
|
|
25
|
+
};
|
|
26
|
+
var author = "Mendable.ai";
|
|
27
|
+
var license = "MIT";
|
|
28
|
+
var dependencies = {
|
|
29
|
+
axios: "1.20.0",
|
|
30
|
+
firecrawl: "4.30.1",
|
|
31
|
+
"typescript-event-target": "^1.1.1",
|
|
32
|
+
zod: "^3.23.8",
|
|
33
|
+
"zod-to-json-schema": "^3.23.0"
|
|
34
|
+
};
|
|
35
|
+
var bugs = {
|
|
36
|
+
url: "https://github.com/firecrawl/firecrawl/issues"
|
|
37
|
+
};
|
|
38
|
+
var homepage = "https://github.com/firecrawl/firecrawl#readme";
|
|
39
|
+
var devDependencies = {
|
|
40
|
+
"@types/dotenv": "^8.2.0",
|
|
41
|
+
"@types/node": "^20.12.12",
|
|
42
|
+
"@types/uuid": "^9.0.8",
|
|
43
|
+
dotenv: "^16.4.5",
|
|
44
|
+
tsup: "^8.5.0",
|
|
45
|
+
typescript: "^5.4.5",
|
|
46
|
+
uuid: "^14.0.0",
|
|
47
|
+
vitest: "^4.1.11"
|
|
48
|
+
};
|
|
49
|
+
var keywords = [
|
|
50
|
+
"firecrawl",
|
|
51
|
+
"mendable",
|
|
52
|
+
"crawler",
|
|
53
|
+
"web",
|
|
54
|
+
"scraper",
|
|
55
|
+
"api",
|
|
56
|
+
"sdk",
|
|
57
|
+
"web-search",
|
|
58
|
+
"literature-search",
|
|
59
|
+
"research",
|
|
60
|
+
"scientific-papers",
|
|
61
|
+
"academic-search",
|
|
62
|
+
"biomedical",
|
|
63
|
+
"life-sciences",
|
|
64
|
+
"pubmed",
|
|
65
|
+
"biorxiv",
|
|
66
|
+
"medrxiv",
|
|
67
|
+
"arxiv",
|
|
68
|
+
"preprints",
|
|
69
|
+
"citations",
|
|
70
|
+
"bioinformatics"
|
|
71
|
+
];
|
|
72
|
+
var engines = {
|
|
73
|
+
node: ">=22.0.0"
|
|
74
|
+
};
|
|
75
|
+
var package_default = {
|
|
76
|
+
name,
|
|
77
|
+
version,
|
|
78
|
+
description,
|
|
79
|
+
main,
|
|
80
|
+
types,
|
|
81
|
+
exports,
|
|
82
|
+
type,
|
|
83
|
+
scripts,
|
|
84
|
+
repository,
|
|
85
|
+
author,
|
|
86
|
+
license,
|
|
87
|
+
dependencies,
|
|
88
|
+
bugs,
|
|
89
|
+
homepage,
|
|
90
|
+
devDependencies,
|
|
91
|
+
keywords,
|
|
92
|
+
engines
|
|
93
|
+
};
|
|
94
|
+
|
|
95
|
+
export {
|
|
96
|
+
name,
|
|
97
|
+
version,
|
|
98
|
+
description,
|
|
99
|
+
main,
|
|
100
|
+
types,
|
|
101
|
+
exports,
|
|
102
|
+
type,
|
|
103
|
+
scripts,
|
|
104
|
+
repository,
|
|
105
|
+
author,
|
|
106
|
+
license,
|
|
107
|
+
dependencies,
|
|
108
|
+
bugs,
|
|
109
|
+
homepage,
|
|
110
|
+
devDependencies,
|
|
111
|
+
keywords,
|
|
112
|
+
engines,
|
|
113
|
+
package_default
|
|
114
|
+
};
|
package/dist/index.cjs
CHANGED
|
@@ -39,7 +39,7 @@ var require_package = __commonJS({
|
|
|
39
39
|
"package.json"(exports2, module2) {
|
|
40
40
|
module2.exports = {
|
|
41
41
|
name: "@mendable/firecrawl-js",
|
|
42
|
-
version: "4.
|
|
42
|
+
version: "4.48.0",
|
|
43
43
|
description: "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
|
|
44
44
|
main: "dist/index.js",
|
|
45
45
|
types: "dist/index.d.ts",
|
|
@@ -134,16 +134,9 @@ module.exports = __toCommonJS(index_exports);
|
|
|
134
134
|
var import_axios = __toESM(require("axios"), 1);
|
|
135
135
|
|
|
136
136
|
// src/v2/utils/getVersion.ts
|
|
137
|
+
var import_package = __toESM(require_package(), 1);
|
|
137
138
|
function getVersion() {
|
|
138
|
-
|
|
139
|
-
if (typeof process !== "undefined" && process.env && process.env.npm_package_version) {
|
|
140
|
-
return process.env.npm_package_version;
|
|
141
|
-
}
|
|
142
|
-
const pkg = require_package();
|
|
143
|
-
return pkg?.version || "3.x.x";
|
|
144
|
-
} catch {
|
|
145
|
-
return "3.x.x";
|
|
146
|
-
}
|
|
139
|
+
return import_package.version;
|
|
147
140
|
}
|
|
148
141
|
|
|
149
142
|
// src/utils/apiOrigin.ts
|
|
@@ -168,11 +161,13 @@ var HttpClient = class {
|
|
|
168
161
|
apiUrl;
|
|
169
162
|
maxRetries;
|
|
170
163
|
backoffFactor;
|
|
164
|
+
origin;
|
|
171
165
|
constructor(options) {
|
|
172
166
|
this.apiKey = options.apiKey;
|
|
173
167
|
this.apiUrl = options.apiUrl.replace(/\/$/, "");
|
|
174
168
|
this.maxRetries = options.maxRetries ?? 3;
|
|
175
169
|
this.backoffFactor = options.backoffFactor ?? 0.5;
|
|
170
|
+
this.origin = options.origin || `js-sdk@${getVersion()}`;
|
|
176
171
|
this.instance = import_axios.default.create({
|
|
177
172
|
baseURL: this.apiUrl,
|
|
178
173
|
timeout: options.timeoutMs ?? 3e5,
|
|
@@ -191,13 +186,16 @@ var HttpClient = class {
|
|
|
191
186
|
getApiKey() {
|
|
192
187
|
return this.apiKey;
|
|
193
188
|
}
|
|
189
|
+
/** The attribution origin stamped into requests that do not set their own. */
|
|
190
|
+
getOrigin() {
|
|
191
|
+
return this.origin;
|
|
192
|
+
}
|
|
194
193
|
/** The client-wide default request timeout, for callers that need to
|
|
195
194
|
* extend a specific request beyond it (e.g. scrape auto-resume). */
|
|
196
195
|
getTimeoutMs() {
|
|
197
196
|
return this.instance.defaults.timeout ?? 3e5;
|
|
198
197
|
}
|
|
199
198
|
async request(config) {
|
|
200
|
-
const version = getVersion();
|
|
201
199
|
config.headers = {
|
|
202
200
|
...config.headers || {}
|
|
203
201
|
};
|
|
@@ -212,7 +210,7 @@ var HttpClient = class {
|
|
|
212
210
|
const data = cfg.data ?? {};
|
|
213
211
|
cfg.data = {
|
|
214
212
|
...data,
|
|
215
|
-
origin: typeof data.origin === "string" && data.origin
|
|
213
|
+
origin: typeof data.origin === "string" && data.origin ? data.origin : this.origin
|
|
216
214
|
};
|
|
217
215
|
}
|
|
218
216
|
if (isFormDataBody) {
|
|
@@ -451,6 +449,30 @@ function ensureValidFormats(formats) {
|
|
|
451
449
|
}
|
|
452
450
|
}
|
|
453
451
|
}
|
|
452
|
+
var V1_SCRAPE_OPTION_REPLACEMENTS = {
|
|
453
|
+
pageOptions: "use scrapeOptions",
|
|
454
|
+
includeLinks: "add 'links' to scrapeOptions.formats",
|
|
455
|
+
parsePDF: "use scrapeOptions.parsers"
|
|
456
|
+
};
|
|
457
|
+
var V1_OPTION_REPLACEMENTS = {
|
|
458
|
+
crawl: {
|
|
459
|
+
crawlerOptions: "move its fields to the top level",
|
|
460
|
+
maxDepth: "use maxDiscoveryDepth",
|
|
461
|
+
allowBackwardLinks: "use crawlEntireDomain",
|
|
462
|
+
ignoreSitemap: "use sitemap: 'skip'",
|
|
463
|
+
...V1_SCRAPE_OPTION_REPLACEMENTS
|
|
464
|
+
},
|
|
465
|
+
search: V1_SCRAPE_OPTION_REPLACEMENTS
|
|
466
|
+
};
|
|
467
|
+
function ensureNoV1Options(options, method) {
|
|
468
|
+
if (!options) return;
|
|
469
|
+
const replacements = V1_OPTION_REPLACEMENTS[method];
|
|
470
|
+
const found = Object.keys(replacements).filter((key) => options[key] != null);
|
|
471
|
+
if (found.length === 0) return;
|
|
472
|
+
throw new Error(
|
|
473
|
+
`${method}: ${found.map((key) => `${key} is a v1 option, ${replacements[key]}`).join("; ")}`
|
|
474
|
+
);
|
|
475
|
+
}
|
|
454
476
|
function ensureValidScrapeOptions(options) {
|
|
455
477
|
if (!options) return;
|
|
456
478
|
if (options.timeout != null && options.timeout <= 0) {
|
|
@@ -564,12 +586,28 @@ function ensureValidParseOptions(options) {
|
|
|
564
586
|
|
|
565
587
|
// src/v2/utils/errorHandler.ts
|
|
566
588
|
var import_axios2 = require("axios");
|
|
589
|
+
function describeValidationIssues(details) {
|
|
590
|
+
if (!Array.isArray(details)) return void 0;
|
|
591
|
+
const parts = details.slice(0, 3).flatMap((issue) => {
|
|
592
|
+
if (!issue || typeof issue !== "object") return [];
|
|
593
|
+
const { path, message, keys } = issue;
|
|
594
|
+
const what = Array.isArray(keys) && keys.length > 0 ? `unrecognized key(s) ${keys.join(", ")}` : typeof message === "string" ? message : void 0;
|
|
595
|
+
if (!what) return [];
|
|
596
|
+
return Array.isArray(path) && path.length > 0 ? [`${path.join(".")}: ${what}`] : [what];
|
|
597
|
+
});
|
|
598
|
+
return parts.length > 0 ? parts.join("; ") : void 0;
|
|
599
|
+
}
|
|
600
|
+
function withIssues(message, details) {
|
|
601
|
+
const text = typeof message === "string" ? message : JSON.stringify(message);
|
|
602
|
+
const issues = describeValidationIssues(details);
|
|
603
|
+
return issues && !text.includes(issues) ? `${text} (${issues})` : text;
|
|
604
|
+
}
|
|
567
605
|
function throwForBadResponse(resp, action) {
|
|
568
606
|
const status = resp.status;
|
|
569
607
|
const body = resp.data || {};
|
|
570
608
|
const msg = body?.error || body?.message || `Request failed (${status}) while trying to ${action}`;
|
|
571
609
|
throw Object.assign(new SdkError(
|
|
572
|
-
msg,
|
|
610
|
+
withIssues(msg, body?.details),
|
|
573
611
|
status,
|
|
574
612
|
body?.code,
|
|
575
613
|
body?.details,
|
|
@@ -584,7 +622,7 @@ function normalizeAxiosError(err, action) {
|
|
|
584
622
|
const message = body?.error || err.message || `Request failed${status ? ` (${status})` : ""} while trying to ${action}`;
|
|
585
623
|
const code = body?.code || err.code;
|
|
586
624
|
throw Object.assign(new SdkError(
|
|
587
|
-
message,
|
|
625
|
+
withIssues(message, body?.details),
|
|
588
626
|
status,
|
|
589
627
|
code,
|
|
590
628
|
body?.details ?? body,
|
|
@@ -755,10 +793,9 @@ async function parse(http, file, options) {
|
|
|
755
793
|
throw new Error("file data cannot be empty");
|
|
756
794
|
}
|
|
757
795
|
if (options) ensureValidParseOptions(options);
|
|
758
|
-
const version = getVersion();
|
|
759
796
|
const normalizedOptions = {
|
|
760
797
|
...options ?? {},
|
|
761
|
-
origin:
|
|
798
|
+
origin: options?.origin || http.getOrigin()
|
|
762
799
|
};
|
|
763
800
|
const formData = new FormData();
|
|
764
801
|
formData.append("options", JSON.stringify(normalizedOptions));
|
|
@@ -796,6 +833,7 @@ async function getParseFormats(http) {
|
|
|
796
833
|
// src/v2/methods/search.ts
|
|
797
834
|
function prepareSearchPayload(req) {
|
|
798
835
|
if (!req.query || !req.query.trim()) throw new Error("Query cannot be empty");
|
|
836
|
+
ensureNoV1Options(req, "search");
|
|
799
837
|
if (req.limit != null && req.limit <= 0)
|
|
800
838
|
throw new Error("limit must be positive");
|
|
801
839
|
if (req.timeout != null && req.timeout <= 0)
|
|
@@ -1163,7 +1201,9 @@ async function fetchAllPages(http, nextUrl, initial, pagination) {
|
|
|
1163
1201
|
docs.push(d);
|
|
1164
1202
|
}
|
|
1165
1203
|
if (maxResults != null && docs.length >= maxResults) break;
|
|
1166
|
-
|
|
1204
|
+
const next = payload.next ?? (Array.isArray(payload.data) ? null : payload.data?.next) ?? null;
|
|
1205
|
+
if (next === current) break;
|
|
1206
|
+
current = next;
|
|
1167
1207
|
pageCount += 1;
|
|
1168
1208
|
}
|
|
1169
1209
|
return docs;
|
|
@@ -1172,6 +1212,7 @@ async function fetchAllPages(http, nextUrl, initial, pagination) {
|
|
|
1172
1212
|
// src/v2/methods/crawl.ts
|
|
1173
1213
|
function prepareCrawlPayload(request) {
|
|
1174
1214
|
if (!request.url || !request.url.trim()) throw new Error("URL cannot be empty");
|
|
1215
|
+
ensureNoV1Options(request, "crawl");
|
|
1175
1216
|
const data = { url: request.url.trim() };
|
|
1176
1217
|
if (request.prompt) data.prompt = request.prompt;
|
|
1177
1218
|
if (request.excludePaths) data.excludePaths = request.excludePaths;
|
|
@@ -1723,6 +1764,7 @@ async function browser(http, args = {}) {
|
|
|
1723
1764
|
if (args.streamWebView != null) body.streamWebView = args.streamWebView;
|
|
1724
1765
|
if (args.blockAds != null) body.blockAds = args.blockAds;
|
|
1725
1766
|
if (args.profile != null) body.profile = args.profile;
|
|
1767
|
+
if (args.location != null) body.location = args.location;
|
|
1726
1768
|
if (args.integration != null) body.integration = args.integration;
|
|
1727
1769
|
if (args.origin) body.origin = args.origin;
|
|
1728
1770
|
try {
|
|
@@ -1871,8 +1913,8 @@ function appendParam(params, key, value) {
|
|
|
1871
1913
|
params.append(key, String(value));
|
|
1872
1914
|
}
|
|
1873
1915
|
}
|
|
1874
|
-
function withQuery(path, params) {
|
|
1875
|
-
params.append("origin",
|
|
1916
|
+
function withQuery(origin, path, params) {
|
|
1917
|
+
params.append("origin", origin);
|
|
1876
1918
|
const qs = params.toString();
|
|
1877
1919
|
return qs ? `${path}?${qs}` : path;
|
|
1878
1920
|
}
|
|
@@ -1927,7 +1969,7 @@ var ResearchClient = class {
|
|
|
1927
1969
|
appendParam(params, "to", options.to);
|
|
1928
1970
|
try {
|
|
1929
1971
|
const res = await this.http.get(
|
|
1930
|
-
withQuery(`${BASE}/papers`, params)
|
|
1972
|
+
withQuery(this.http.getOrigin(), `${BASE}/papers`, params)
|
|
1931
1973
|
);
|
|
1932
1974
|
if (res.status !== 200) throwForBadResponse(res, "search papers");
|
|
1933
1975
|
return res.data;
|
|
@@ -1945,7 +1987,13 @@ var ResearchClient = class {
|
|
|
1945
1987
|
appendParam(params, "query", options.query);
|
|
1946
1988
|
appendParam(params, "k", options.k);
|
|
1947
1989
|
try {
|
|
1948
|
-
const res = await this.http.get(
|
|
1990
|
+
const res = await this.http.get(
|
|
1991
|
+
withQuery(
|
|
1992
|
+
this.http.getOrigin(),
|
|
1993
|
+
`${BASE}/papers/${encodeURIComponent(id)}`,
|
|
1994
|
+
params
|
|
1995
|
+
)
|
|
1996
|
+
);
|
|
1949
1997
|
if (res.status !== 200) throwForBadResponse(res, "get paper");
|
|
1950
1998
|
return res.data;
|
|
1951
1999
|
} catch (err) {
|
|
@@ -1971,7 +2019,11 @@ var ResearchClient = class {
|
|
|
1971
2019
|
appendParam(params, "anchor", options.anchor);
|
|
1972
2020
|
try {
|
|
1973
2021
|
const res = await this.http.get(
|
|
1974
|
-
withQuery(
|
|
2022
|
+
withQuery(
|
|
2023
|
+
this.http.getOrigin(),
|
|
2024
|
+
`${BASE}/papers/${encodeURIComponent(id)}/similar`,
|
|
2025
|
+
params
|
|
2026
|
+
)
|
|
1975
2027
|
);
|
|
1976
2028
|
if (res.status !== 200) throwForBadResponse(res, "find similar papers");
|
|
1977
2029
|
return res.data;
|
|
@@ -2002,7 +2054,7 @@ var ResearchClient = class {
|
|
|
2002
2054
|
appendParam(params, "k", options.k);
|
|
2003
2055
|
try {
|
|
2004
2056
|
const res = await this.http.get(
|
|
2005
|
-
withQuery(`${BASE}/github`, params)
|
|
2057
|
+
withQuery(this.http.getOrigin(), `${BASE}/github`, params)
|
|
2006
2058
|
);
|
|
2007
2059
|
if (res.status !== 200) throwForBadResponse(res, "search github");
|
|
2008
2060
|
return res.data;
|
|
@@ -2132,6 +2184,67 @@ async function getMonitorCheck(http, monitorId, checkId, options) {
|
|
|
2132
2184
|
}
|
|
2133
2185
|
}
|
|
2134
2186
|
|
|
2187
|
+
// src/utils/callerOrigin.ts
|
|
2188
|
+
var NODE_MODULES = "/node_modules/";
|
|
2189
|
+
function packageOf(file) {
|
|
2190
|
+
let path = file.replace(/^file:\/\//, "").replace(/\\/g, "/");
|
|
2191
|
+
try {
|
|
2192
|
+
path = decodeURIComponent(path);
|
|
2193
|
+
} catch {
|
|
2194
|
+
}
|
|
2195
|
+
const at = path.lastIndexOf(NODE_MODULES);
|
|
2196
|
+
if (at < 0) return null;
|
|
2197
|
+
const [first, second] = path.slice(at + NODE_MODULES.length).split("/");
|
|
2198
|
+
const name = first?.startsWith("@") ? second ? `${first}/${second}` : "" : first;
|
|
2199
|
+
return name && !name.startsWith(".") ? name : null;
|
|
2200
|
+
}
|
|
2201
|
+
function frameFile(line) {
|
|
2202
|
+
const text = line.trim().replace(/^at /, "");
|
|
2203
|
+
const open = text.indexOf(" (");
|
|
2204
|
+
const location = open >= 0 && text.endsWith(")") ? text.slice(open + 2, -1) : text;
|
|
2205
|
+
const match = /^(.*):\d+:\d+$/.exec(location);
|
|
2206
|
+
return match && !match[1].startsWith("node:") ? match[1] : null;
|
|
2207
|
+
}
|
|
2208
|
+
function slugify(name) {
|
|
2209
|
+
return name.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 48).replace(/-+$/, "");
|
|
2210
|
+
}
|
|
2211
|
+
function detectCallerPackage(stack) {
|
|
2212
|
+
try {
|
|
2213
|
+
if (process.env.FIRECRAWL_DISABLE_CALLER_ORIGIN === "1") return null;
|
|
2214
|
+
if (stack === void 0) {
|
|
2215
|
+
const limit = Error.stackTraceLimit;
|
|
2216
|
+
Error.stackTraceLimit = 50;
|
|
2217
|
+
try {
|
|
2218
|
+
stack = new Error().stack ?? "";
|
|
2219
|
+
} finally {
|
|
2220
|
+
Error.stackTraceLimit = limit;
|
|
2221
|
+
}
|
|
2222
|
+
}
|
|
2223
|
+
let ownFile = null;
|
|
2224
|
+
let ownPackage = null;
|
|
2225
|
+
for (const line of stack.split("\n")) {
|
|
2226
|
+
if (!/^\s*at /.test(line)) continue;
|
|
2227
|
+
const file = frameFile(line);
|
|
2228
|
+
if (!file) continue;
|
|
2229
|
+
if (ownFile === null) {
|
|
2230
|
+
ownFile = file;
|
|
2231
|
+
ownPackage = packageOf(file);
|
|
2232
|
+
}
|
|
2233
|
+
if (file === ownFile) continue;
|
|
2234
|
+
const name = packageOf(file);
|
|
2235
|
+
if (name && name === ownPackage) continue;
|
|
2236
|
+
const slug = name ? slugify(name) : "";
|
|
2237
|
+
return slug && !slug.includes("playground") ? slug : null;
|
|
2238
|
+
}
|
|
2239
|
+
} catch {
|
|
2240
|
+
}
|
|
2241
|
+
return null;
|
|
2242
|
+
}
|
|
2243
|
+
function defaultOrigin(version2) {
|
|
2244
|
+
const caller = detectCallerPackage();
|
|
2245
|
+
return caller ? `js-sdk.${caller}@${version2}` : `js-sdk@${version2}`;
|
|
2246
|
+
}
|
|
2247
|
+
|
|
2135
2248
|
// src/v2/watcher.ts
|
|
2136
2249
|
var import_events = require("events");
|
|
2137
2250
|
var hasGlobalWebSocket = () => {
|
|
@@ -2394,7 +2507,8 @@ var FirecrawlClient = class {
|
|
|
2394
2507
|
apiUrl,
|
|
2395
2508
|
timeoutMs: opts.timeoutMs,
|
|
2396
2509
|
maxRetries: opts.maxRetries,
|
|
2397
|
-
backoffFactor: opts.backoffFactor
|
|
2510
|
+
backoffFactor: opts.backoffFactor,
|
|
2511
|
+
origin: opts.origin || defaultOrigin(getVersion())
|
|
2398
2512
|
});
|
|
2399
2513
|
}
|
|
2400
2514
|
async scrape(url, options) {
|
|
@@ -2789,7 +2903,7 @@ var FirecrawlClient = class {
|
|
|
2789
2903
|
// Browser
|
|
2790
2904
|
/**
|
|
2791
2905
|
* Create a new browser session.
|
|
2792
|
-
* @param args Session options (ttl, activityTtl, streamWebView, blockAds, profile).
|
|
2906
|
+
* @param args Session options (ttl, activityTtl, streamWebView, blockAds, profile, location).
|
|
2793
2907
|
* @returns Session id, CDP URL, live view URL, and expiration time.
|
|
2794
2908
|
*/
|
|
2795
2909
|
async browser(args = {}) {
|
|
@@ -2920,6 +3034,8 @@ var FirecrawlApp = class {
|
|
|
2920
3034
|
apiKey;
|
|
2921
3035
|
apiUrl;
|
|
2922
3036
|
version = "1.25.1";
|
|
3037
|
+
clientOrigin;
|
|
3038
|
+
callerPackage;
|
|
2923
3039
|
isCloudService(url) {
|
|
2924
3040
|
return url.includes("api.firecrawl.dev");
|
|
2925
3041
|
}
|
|
@@ -2945,15 +3061,22 @@ var FirecrawlApp = class {
|
|
|
2945
3061
|
* Initializes a new instance of the FirecrawlApp class.
|
|
2946
3062
|
* @param config - Configuration options for the FirecrawlApp instance.
|
|
2947
3063
|
*/
|
|
2948
|
-
constructor({ apiKey = null, apiUrl = null }) {
|
|
3064
|
+
constructor({ apiKey = null, apiUrl = null, origin }) {
|
|
2949
3065
|
const baseUrl = apiUrl || "https://api.firecrawl.dev";
|
|
2950
3066
|
if (this.isCloudService(baseUrl) && typeof apiKey !== "string") {
|
|
2951
3067
|
throw new FirecrawlError("No API key provided", 401);
|
|
2952
3068
|
}
|
|
2953
3069
|
this.apiKey = apiKey || "";
|
|
2954
3070
|
this.apiUrl = baseUrl;
|
|
3071
|
+
this.clientOrigin = origin || void 0;
|
|
3072
|
+
this.callerPackage = this.clientOrigin ? null : detectCallerPackage();
|
|
2955
3073
|
this.init();
|
|
2956
3074
|
}
|
|
3075
|
+
resolveOrigin(params) {
|
|
3076
|
+
if (typeof params?.origin === "string" && params.origin) return params.origin;
|
|
3077
|
+
if (this.clientOrigin) return this.clientOrigin;
|
|
3078
|
+
return this.callerPackage ? `js-sdk.${this.callerPackage}@${this.version}` : `js-sdk@${this.version}`;
|
|
3079
|
+
}
|
|
2957
3080
|
/**
|
|
2958
3081
|
* Scrapes a URL using the Firecrawl API.
|
|
2959
3082
|
* @param url - The URL to scrape.
|
|
@@ -2965,7 +3088,7 @@ var FirecrawlApp = class {
|
|
|
2965
3088
|
"Content-Type": "application/json",
|
|
2966
3089
|
Authorization: `Bearer ${this.apiKey}`
|
|
2967
3090
|
};
|
|
2968
|
-
let jsonData = { url, ...params, origin:
|
|
3091
|
+
let jsonData = { url, ...params, origin: this.resolveOrigin(params) };
|
|
2969
3092
|
if (jsonData?.extract?.schema) {
|
|
2970
3093
|
jsonData = {
|
|
2971
3094
|
...jsonData,
|
|
@@ -3029,7 +3152,7 @@ var FirecrawlApp = class {
|
|
|
3029
3152
|
lang: params?.lang ?? "en",
|
|
3030
3153
|
country: params?.country ?? "us",
|
|
3031
3154
|
location: params?.location,
|
|
3032
|
-
origin:
|
|
3155
|
+
origin: this.resolveOrigin(params),
|
|
3033
3156
|
timeout: params?.timeout ?? 6e4,
|
|
3034
3157
|
scrapeOptions: params?.scrapeOptions ?? { formats: [] }
|
|
3035
3158
|
};
|
|
@@ -3084,7 +3207,7 @@ var FirecrawlApp = class {
|
|
|
3084
3207
|
*/
|
|
3085
3208
|
async crawlUrl(url, params, pollInterval = 2, idempotencyKey) {
|
|
3086
3209
|
const headers = this.prepareHeaders(idempotencyKey);
|
|
3087
|
-
let jsonData = { url, ...params, origin:
|
|
3210
|
+
let jsonData = { url, ...params, origin: this.resolveOrigin(params) };
|
|
3088
3211
|
try {
|
|
3089
3212
|
const response = await this.postRequest(
|
|
3090
3213
|
this.apiUrl + `/v1/crawl`,
|
|
@@ -3108,7 +3231,7 @@ var FirecrawlApp = class {
|
|
|
3108
3231
|
}
|
|
3109
3232
|
async asyncCrawlUrl(url, params, idempotencyKey) {
|
|
3110
3233
|
const headers = this.prepareHeaders(idempotencyKey);
|
|
3111
|
-
let jsonData = { url, ...params, origin:
|
|
3234
|
+
let jsonData = { url, ...params, origin: this.resolveOrigin(params) };
|
|
3112
3235
|
try {
|
|
3113
3236
|
const response = await this.postRequest(
|
|
3114
3237
|
this.apiUrl + `/v1/crawl`,
|
|
@@ -3267,7 +3390,7 @@ var FirecrawlApp = class {
|
|
|
3267
3390
|
*/
|
|
3268
3391
|
async mapUrl(url, params) {
|
|
3269
3392
|
const headers = this.prepareHeaders();
|
|
3270
|
-
let jsonData = { url, ...params, origin:
|
|
3393
|
+
let jsonData = { url, ...params, origin: this.resolveOrigin(params) };
|
|
3271
3394
|
try {
|
|
3272
3395
|
const response = await this.postRequest(
|
|
3273
3396
|
this.apiUrl + `/v1/map`,
|
|
@@ -3296,7 +3419,7 @@ var FirecrawlApp = class {
|
|
|
3296
3419
|
*/
|
|
3297
3420
|
async batchScrapeUrls(urls, params, pollInterval = 2, idempotencyKey, webhook, ignoreInvalidURLs, maxConcurrency) {
|
|
3298
3421
|
const headers = this.prepareHeaders(idempotencyKey);
|
|
3299
|
-
let jsonData = { urls, webhook, ignoreInvalidURLs, maxConcurrency, ...params, origin:
|
|
3422
|
+
let jsonData = { urls, webhook, ignoreInvalidURLs, maxConcurrency, ...params, origin: this.resolveOrigin(params) };
|
|
3300
3423
|
if (jsonData?.extract?.schema) {
|
|
3301
3424
|
jsonData = {
|
|
3302
3425
|
...jsonData,
|
|
@@ -3338,7 +3461,7 @@ var FirecrawlApp = class {
|
|
|
3338
3461
|
}
|
|
3339
3462
|
async asyncBatchScrapeUrls(urls, params, idempotencyKey, webhook, ignoreInvalidURLs) {
|
|
3340
3463
|
const headers = this.prepareHeaders(idempotencyKey);
|
|
3341
|
-
let jsonData = { urls, webhook, ignoreInvalidURLs, ...params, origin:
|
|
3464
|
+
let jsonData = { urls, webhook, ignoreInvalidURLs, ...params, origin: this.resolveOrigin(params) };
|
|
3342
3465
|
try {
|
|
3343
3466
|
const response = await this.postRequest(
|
|
3344
3467
|
this.apiUrl + `/v1/batch/scrape`,
|
|
@@ -3482,7 +3605,7 @@ var FirecrawlApp = class {
|
|
|
3482
3605
|
try {
|
|
3483
3606
|
const response = await this.postRequest(
|
|
3484
3607
|
this.apiUrl + `/v1/extract`,
|
|
3485
|
-
{ ...jsonData, schema: jsonSchema, origin:
|
|
3608
|
+
{ ...jsonData, schema: jsonSchema, origin: this.resolveOrigin(params) },
|
|
3486
3609
|
headers
|
|
3487
3610
|
);
|
|
3488
3611
|
if (response.status === 200) {
|
|
@@ -3535,7 +3658,7 @@ var FirecrawlApp = class {
|
|
|
3535
3658
|
try {
|
|
3536
3659
|
const response = await this.postRequest(
|
|
3537
3660
|
this.apiUrl + `/v1/extract`,
|
|
3538
|
-
{ ...jsonData, schema: jsonSchema, origin:
|
|
3661
|
+
{ ...jsonData, schema: jsonSchema, origin: this.resolveOrigin(params) },
|
|
3539
3662
|
headers
|
|
3540
3663
|
);
|
|
3541
3664
|
if (response.status === 200) {
|
|
@@ -3817,7 +3940,7 @@ var FirecrawlApp = class {
|
|
|
3817
3940
|
*/
|
|
3818
3941
|
async asyncDeepResearch(query, params) {
|
|
3819
3942
|
const headers = this.prepareHeaders();
|
|
3820
|
-
let jsonData = { query, ...params, origin:
|
|
3943
|
+
let jsonData = { query, ...params, origin: this.resolveOrigin(params) };
|
|
3821
3944
|
if (jsonData?.jsonOptions?.schema) {
|
|
3822
3945
|
jsonData = {
|
|
3823
3946
|
...jsonData,
|
|
@@ -3936,7 +4059,7 @@ var FirecrawlApp = class {
|
|
|
3936
4059
|
async __asyncDeepResearch(topic, params) {
|
|
3937
4060
|
const headers = this.prepareHeaders();
|
|
3938
4061
|
try {
|
|
3939
|
-
let jsonData = { topic, ...params, origin:
|
|
4062
|
+
let jsonData = { topic, ...params, origin: this.resolveOrigin(params) };
|
|
3940
4063
|
const response = await this.postRequest(
|
|
3941
4064
|
`${this.apiUrl}/v1/deep-research`,
|
|
3942
4065
|
jsonData,
|
|
@@ -4036,7 +4159,7 @@ var FirecrawlApp = class {
|
|
|
4036
4159
|
*/
|
|
4037
4160
|
async asyncGenerateLLMsText(url, params) {
|
|
4038
4161
|
const headers = this.prepareHeaders();
|
|
4039
|
-
let jsonData = { url, ...params, origin:
|
|
4162
|
+
let jsonData = { url, ...params, origin: this.resolveOrigin(params) };
|
|
4040
4163
|
try {
|
|
4041
4164
|
const response = await this.postRequest(
|
|
4042
4165
|
`${this.apiUrl}/v1/llmstxt`,
|
|
@@ -4304,7 +4427,8 @@ var Firecrawl = class extends FirecrawlClient {
|
|
|
4304
4427
|
super(resolved);
|
|
4305
4428
|
this._v1Opts = {
|
|
4306
4429
|
apiKey: resolved.apiKey,
|
|
4307
|
-
apiUrl: resolved.apiUrl
|
|
4430
|
+
apiUrl: resolved.apiUrl,
|
|
4431
|
+
origin: this.http.getOrigin()
|
|
4308
4432
|
};
|
|
4309
4433
|
}
|
|
4310
4434
|
/** Access the legacy v1 client (instantiated on first access). */
|