@mendable/firecrawl-js 4.47.0 → 4.49.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -1
- package/dist/{chunk-H75FJ6LW.js → chunk-JNLB66PW.js} +1 -1
- package/dist/index.cjs +158 -46
- package/dist/index.d.cts +49 -23
- package/dist/index.d.ts +49 -23
- package/dist/index.js +159 -47
- package/dist/{package-NZI5TDUV.js → package-7E5DZETG.js} +1 -1
- package/package.json +1 -1
- package/src/__tests__/unit/v2/agent-hints.test.ts +1 -0
- package/src/__tests__/unit/v2/origin.test.ts +132 -0
- package/src/__tests__/unit/v2/research.test.ts +1 -0
- package/src/__tests__/unit/v2/v1-aliases.test.ts +89 -4
- package/src/index.ts +1 -0
- package/src/utils/callerOrigin.ts +74 -0
- package/src/v1/index.ts +28 -13
- package/src/v2/client.ts +41 -31
- package/src/v2/methods/parse.ts +1 -6
- package/src/v2/methods/research.ts +20 -7
- package/src/v2/utils/httpClient.ts +10 -3
- package/src/v2/utils/v1Compat.ts +38 -0
package/README.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Firecrawl Node SDK
|
|
2
2
|
|
|
3
|
-
The Firecrawl Node SDK is the JavaScript client for the Firecrawl API, the context API for AI agents. Search, scrape, parse, crawl, map, and interact with the web, and get back clean Markdown or structured data your agents can ship with.
|
|
3
|
+
The Firecrawl Node SDK is the JavaScript client for the Firecrawl API, the context API for AI agents. Search, scrape, parse, crawl, map, and interact with the web, run Agent and Monitor jobs, and get back clean Markdown or structured data your agents can ship with.
|
|
4
4
|
|
|
5
5
|
## Installation
|
|
6
6
|
|
|
@@ -37,6 +37,16 @@ const crawlResponse = await app.crawl('https://firecrawl.dev', {
|
|
|
37
37
|
console.log(crawlResponse);
|
|
38
38
|
```
|
|
39
39
|
|
|
40
|
+
### Request origin
|
|
41
|
+
|
|
42
|
+
Requests carry an `origin` tag that identifies the client. Pass the `origin` option to set it. Set `origin` on a single call to override it for that call, except for `app.research` methods, which always use the client origin.
|
|
43
|
+
|
|
44
|
+
```js
|
|
45
|
+
const app = new Firecrawl({ apiKey: 'fc-YOUR_API_KEY', origin: 'js-sdk.my-wrapper@1.0.0' });
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
By default the tag is `js-sdk@<version>`. When the client is created by code inside an installed package (a wrapper library), the default is `js-sdk.<package>@<version>` instead, for example `js-sdk.scope-name@<version>`. Application code, REPLs and bundled files without `node_modules` in their path keep the plain default. Detection runs once when the client is created and sends only the package name, lowercased and sanitized into a slug. Set `FIRECRAWL_DISABLE_CALLER_ORIGIN=1` to always use the plain default.
|
|
49
|
+
|
|
40
50
|
### Scraping a URL
|
|
41
51
|
|
|
42
52
|
To scrape a single URL with error handling, use the `scrape` method. It takes the URL as a parameter and returns the scraped data.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// package.json
|
|
2
2
|
var name = "@mendable/firecrawl-js";
|
|
3
|
-
var version = "4.
|
|
3
|
+
var version = "4.49.0";
|
|
4
4
|
var description = "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts";
|
|
5
5
|
var main = "dist/index.js";
|
|
6
6
|
var types = "dist/index.d.ts";
|
package/dist/index.cjs
CHANGED
|
@@ -39,7 +39,7 @@ var require_package = __commonJS({
|
|
|
39
39
|
"package.json"(exports2, module2) {
|
|
40
40
|
module2.exports = {
|
|
41
41
|
name: "@mendable/firecrawl-js",
|
|
42
|
-
version: "4.
|
|
42
|
+
version: "4.49.0",
|
|
43
43
|
description: "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
|
|
44
44
|
main: "dist/index.js",
|
|
45
45
|
types: "dist/index.d.ts",
|
|
@@ -161,11 +161,13 @@ var HttpClient = class {
|
|
|
161
161
|
apiUrl;
|
|
162
162
|
maxRetries;
|
|
163
163
|
backoffFactor;
|
|
164
|
+
origin;
|
|
164
165
|
constructor(options) {
|
|
165
166
|
this.apiKey = options.apiKey;
|
|
166
167
|
this.apiUrl = options.apiUrl.replace(/\/$/, "");
|
|
167
168
|
this.maxRetries = options.maxRetries ?? 3;
|
|
168
169
|
this.backoffFactor = options.backoffFactor ?? 0.5;
|
|
170
|
+
this.origin = options.origin || `js-sdk@${getVersion()}`;
|
|
169
171
|
this.instance = import_axios.default.create({
|
|
170
172
|
baseURL: this.apiUrl,
|
|
171
173
|
timeout: options.timeoutMs ?? 3e5,
|
|
@@ -184,13 +186,16 @@ var HttpClient = class {
|
|
|
184
186
|
getApiKey() {
|
|
185
187
|
return this.apiKey;
|
|
186
188
|
}
|
|
189
|
+
/** The attribution origin stamped into requests that do not set their own. */
|
|
190
|
+
getOrigin() {
|
|
191
|
+
return this.origin;
|
|
192
|
+
}
|
|
187
193
|
/** The client-wide default request timeout, for callers that need to
|
|
188
194
|
* extend a specific request beyond it (e.g. scrape auto-resume). */
|
|
189
195
|
getTimeoutMs() {
|
|
190
196
|
return this.instance.defaults.timeout ?? 3e5;
|
|
191
197
|
}
|
|
192
198
|
async request(config) {
|
|
193
|
-
const version2 = getVersion();
|
|
194
199
|
config.headers = {
|
|
195
200
|
...config.headers || {}
|
|
196
201
|
};
|
|
@@ -205,7 +210,7 @@ var HttpClient = class {
|
|
|
205
210
|
const data = cfg.data ?? {};
|
|
206
211
|
cfg.data = {
|
|
207
212
|
...data,
|
|
208
|
-
origin: typeof data.origin === "string" && data.origin
|
|
213
|
+
origin: typeof data.origin === "string" && data.origin ? data.origin : this.origin
|
|
209
214
|
};
|
|
210
215
|
}
|
|
211
216
|
if (isFormDataBody) {
|
|
@@ -788,10 +793,9 @@ async function parse(http, file, options) {
|
|
|
788
793
|
throw new Error("file data cannot be empty");
|
|
789
794
|
}
|
|
790
795
|
if (options) ensureValidParseOptions(options);
|
|
791
|
-
const version2 = getVersion();
|
|
792
796
|
const normalizedOptions = {
|
|
793
797
|
...options ?? {},
|
|
794
|
-
origin:
|
|
798
|
+
origin: options?.origin || http.getOrigin()
|
|
795
799
|
};
|
|
796
800
|
const formData = new FormData();
|
|
797
801
|
formData.append("options", JSON.stringify(normalizedOptions));
|
|
@@ -1909,8 +1913,8 @@ function appendParam(params, key, value) {
|
|
|
1909
1913
|
params.append(key, String(value));
|
|
1910
1914
|
}
|
|
1911
1915
|
}
|
|
1912
|
-
function withQuery(path, params) {
|
|
1913
|
-
params.append("origin",
|
|
1916
|
+
function withQuery(origin, path, params) {
|
|
1917
|
+
params.append("origin", origin);
|
|
1914
1918
|
const qs = params.toString();
|
|
1915
1919
|
return qs ? `${path}?${qs}` : path;
|
|
1916
1920
|
}
|
|
@@ -1965,7 +1969,7 @@ var ResearchClient = class {
|
|
|
1965
1969
|
appendParam(params, "to", options.to);
|
|
1966
1970
|
try {
|
|
1967
1971
|
const res = await this.http.get(
|
|
1968
|
-
withQuery(`${BASE}/papers`, params)
|
|
1972
|
+
withQuery(this.http.getOrigin(), `${BASE}/papers`, params)
|
|
1969
1973
|
);
|
|
1970
1974
|
if (res.status !== 200) throwForBadResponse(res, "search papers");
|
|
1971
1975
|
return res.data;
|
|
@@ -1983,7 +1987,13 @@ var ResearchClient = class {
|
|
|
1983
1987
|
appendParam(params, "query", options.query);
|
|
1984
1988
|
appendParam(params, "k", options.k);
|
|
1985
1989
|
try {
|
|
1986
|
-
const res = await this.http.get(
|
|
1990
|
+
const res = await this.http.get(
|
|
1991
|
+
withQuery(
|
|
1992
|
+
this.http.getOrigin(),
|
|
1993
|
+
`${BASE}/papers/${encodeURIComponent(id)}`,
|
|
1994
|
+
params
|
|
1995
|
+
)
|
|
1996
|
+
);
|
|
1987
1997
|
if (res.status !== 200) throwForBadResponse(res, "get paper");
|
|
1988
1998
|
return res.data;
|
|
1989
1999
|
} catch (err) {
|
|
@@ -2009,7 +2019,11 @@ var ResearchClient = class {
|
|
|
2009
2019
|
appendParam(params, "anchor", options.anchor);
|
|
2010
2020
|
try {
|
|
2011
2021
|
const res = await this.http.get(
|
|
2012
|
-
withQuery(
|
|
2022
|
+
withQuery(
|
|
2023
|
+
this.http.getOrigin(),
|
|
2024
|
+
`${BASE}/papers/${encodeURIComponent(id)}/similar`,
|
|
2025
|
+
params
|
|
2026
|
+
)
|
|
2013
2027
|
);
|
|
2014
2028
|
if (res.status !== 200) throwForBadResponse(res, "find similar papers");
|
|
2015
2029
|
return res.data;
|
|
@@ -2040,7 +2054,7 @@ var ResearchClient = class {
|
|
|
2040
2054
|
appendParam(params, "k", options.k);
|
|
2041
2055
|
try {
|
|
2042
2056
|
const res = await this.http.get(
|
|
2043
|
-
withQuery(`${BASE}/github`, params)
|
|
2057
|
+
withQuery(this.http.getOrigin(), `${BASE}/github`, params)
|
|
2044
2058
|
);
|
|
2045
2059
|
if (res.status !== 200) throwForBadResponse(res, "search github");
|
|
2046
2060
|
return res.data;
|
|
@@ -2170,6 +2184,67 @@ async function getMonitorCheck(http, monitorId, checkId, options) {
|
|
|
2170
2184
|
}
|
|
2171
2185
|
}
|
|
2172
2186
|
|
|
2187
|
+
// src/utils/callerOrigin.ts
|
|
2188
|
+
var NODE_MODULES = "/node_modules/";
|
|
2189
|
+
function packageOf(file) {
|
|
2190
|
+
let path = file.replace(/^file:\/\//, "").replace(/\\/g, "/");
|
|
2191
|
+
try {
|
|
2192
|
+
path = decodeURIComponent(path);
|
|
2193
|
+
} catch {
|
|
2194
|
+
}
|
|
2195
|
+
const at = path.lastIndexOf(NODE_MODULES);
|
|
2196
|
+
if (at < 0) return null;
|
|
2197
|
+
const [first, second] = path.slice(at + NODE_MODULES.length).split("/");
|
|
2198
|
+
const name = first?.startsWith("@") ? second ? `${first}/${second}` : "" : first;
|
|
2199
|
+
return name && !name.startsWith(".") ? name : null;
|
|
2200
|
+
}
|
|
2201
|
+
function frameFile(line) {
|
|
2202
|
+
const text = line.trim().replace(/^at /, "");
|
|
2203
|
+
const open = text.indexOf(" (");
|
|
2204
|
+
const location = open >= 0 && text.endsWith(")") ? text.slice(open + 2, -1) : text;
|
|
2205
|
+
const match = /^(.*):\d+:\d+$/.exec(location);
|
|
2206
|
+
return match && !match[1].startsWith("node:") ? match[1] : null;
|
|
2207
|
+
}
|
|
2208
|
+
function slugify(name) {
|
|
2209
|
+
return name.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 48).replace(/-+$/, "");
|
|
2210
|
+
}
|
|
2211
|
+
function detectCallerPackage(stack) {
|
|
2212
|
+
try {
|
|
2213
|
+
if (process.env.FIRECRAWL_DISABLE_CALLER_ORIGIN === "1") return null;
|
|
2214
|
+
if (stack === void 0) {
|
|
2215
|
+
const limit = Error.stackTraceLimit;
|
|
2216
|
+
Error.stackTraceLimit = 50;
|
|
2217
|
+
try {
|
|
2218
|
+
stack = new Error().stack ?? "";
|
|
2219
|
+
} finally {
|
|
2220
|
+
Error.stackTraceLimit = limit;
|
|
2221
|
+
}
|
|
2222
|
+
}
|
|
2223
|
+
let ownFile = null;
|
|
2224
|
+
let ownPackage = null;
|
|
2225
|
+
for (const line of stack.split("\n")) {
|
|
2226
|
+
if (!/^\s*at /.test(line)) continue;
|
|
2227
|
+
const file = frameFile(line);
|
|
2228
|
+
if (!file) continue;
|
|
2229
|
+
if (ownFile === null) {
|
|
2230
|
+
ownFile = file;
|
|
2231
|
+
ownPackage = packageOf(file);
|
|
2232
|
+
}
|
|
2233
|
+
if (file === ownFile) continue;
|
|
2234
|
+
const name = packageOf(file);
|
|
2235
|
+
if (name && name === ownPackage) continue;
|
|
2236
|
+
const slug = name ? slugify(name) : "";
|
|
2237
|
+
return slug && !slug.includes("playground") ? slug : null;
|
|
2238
|
+
}
|
|
2239
|
+
} catch {
|
|
2240
|
+
}
|
|
2241
|
+
return null;
|
|
2242
|
+
}
|
|
2243
|
+
function defaultOrigin(version2) {
|
|
2244
|
+
const caller = detectCallerPackage();
|
|
2245
|
+
return caller ? `js-sdk.${caller}@${version2}` : `js-sdk@${version2}`;
|
|
2246
|
+
}
|
|
2247
|
+
|
|
2173
2248
|
// src/v2/watcher.ts
|
|
2174
2249
|
var import_events = require("events");
|
|
2175
2250
|
var hasGlobalWebSocket = () => {
|
|
@@ -2411,6 +2486,32 @@ var Watcher = class extends import_events.EventEmitter {
|
|
|
2411
2486
|
}
|
|
2412
2487
|
};
|
|
2413
2488
|
|
|
2489
|
+
// src/v2/utils/v1Compat.ts
|
|
2490
|
+
var warned = false;
|
|
2491
|
+
function warnV1Usage() {
|
|
2492
|
+
if (warned) return;
|
|
2493
|
+
warned = true;
|
|
2494
|
+
const message = "firecrawl: v1-style methods (scrapeUrl, crawlUrl, batchScrapeUrls, ...) are deprecated and mapped to the v2 API. Use scrape(), crawl(), batchScrape() and the other v2 methods instead.";
|
|
2495
|
+
if (typeof process !== "undefined" && typeof process.emitWarning === "function") {
|
|
2496
|
+
process.emitWarning(message, { type: "DeprecationWarning", code: "FIRECRAWL_V1_METHOD" });
|
|
2497
|
+
} else {
|
|
2498
|
+
console.warn(message);
|
|
2499
|
+
}
|
|
2500
|
+
}
|
|
2501
|
+
function toV2CrawlOptions(req) {
|
|
2502
|
+
const { maxDepth, allowBackwardLinks, ignoreSitemap, ...rest } = req;
|
|
2503
|
+
return {
|
|
2504
|
+
...maxDepth != null && { maxDiscoveryDepth: maxDepth },
|
|
2505
|
+
...allowBackwardLinks != null && { crawlEntireDomain: allowBackwardLinks },
|
|
2506
|
+
...ignoreSitemap != null && { sitemap: ignoreSitemap ? "skip" : "include" },
|
|
2507
|
+
...rest
|
|
2508
|
+
};
|
|
2509
|
+
}
|
|
2510
|
+
async function callV1(call) {
|
|
2511
|
+
warnV1Usage();
|
|
2512
|
+
return { ...await call(), success: true };
|
|
2513
|
+
}
|
|
2514
|
+
|
|
2414
2515
|
// src/v2/client.ts
|
|
2415
2516
|
var zt = require("zod");
|
|
2416
2517
|
var FirecrawlClient = class {
|
|
@@ -2432,7 +2533,8 @@ var FirecrawlClient = class {
|
|
|
2432
2533
|
apiUrl,
|
|
2433
2534
|
timeoutMs: opts.timeoutMs,
|
|
2434
2535
|
maxRetries: opts.maxRetries,
|
|
2435
|
-
backoffFactor: opts.backoffFactor
|
|
2536
|
+
backoffFactor: opts.backoffFactor,
|
|
2537
|
+
origin: opts.origin || defaultOrigin(getVersion())
|
|
2436
2538
|
});
|
|
2437
2539
|
}
|
|
2438
2540
|
async scrape(url, options) {
|
|
@@ -2891,45 +2993,45 @@ var FirecrawlClient = class {
|
|
|
2891
2993
|
watcher(jobId, opts = {}) {
|
|
2892
2994
|
return new Watcher(this.http, jobId, opts);
|
|
2893
2995
|
}
|
|
2894
|
-
/** @deprecated
|
|
2996
|
+
/** @deprecated v1 alias for scrape(); its result also carries v1's `success: true`. */
|
|
2895
2997
|
async scrapeUrl(url, options) {
|
|
2896
|
-
return this.scrape(url, options);
|
|
2998
|
+
return callV1(() => this.scrape(url, options));
|
|
2897
2999
|
}
|
|
2898
|
-
/** @deprecated
|
|
3000
|
+
/** @deprecated v1 alias for crawl(); its result also carries v1's `success: true`. */
|
|
2899
3001
|
async crawlUrl(url, req = {}) {
|
|
2900
|
-
return this.crawl(url, req);
|
|
3002
|
+
return callV1(() => this.crawl(url, toV2CrawlOptions(req)));
|
|
2901
3003
|
}
|
|
2902
|
-
/** @deprecated
|
|
3004
|
+
/** @deprecated v1 alias for startCrawl(); its result also carries v1's `success: true`. */
|
|
2903
3005
|
async asyncCrawlUrl(url, req = {}) {
|
|
2904
|
-
return this.startCrawl(url, req);
|
|
3006
|
+
return callV1(() => this.startCrawl(url, toV2CrawlOptions(req)));
|
|
2905
3007
|
}
|
|
2906
|
-
/** @deprecated
|
|
3008
|
+
/** @deprecated v1 alias for getCrawlStatus(); its result also carries v1's `success: true`. */
|
|
2907
3009
|
async checkCrawlStatus(jobId, pagination) {
|
|
2908
|
-
return this.getCrawlStatus(jobId, pagination);
|
|
3010
|
+
return callV1(() => this.getCrawlStatus(jobId, pagination));
|
|
2909
3011
|
}
|
|
2910
|
-
/** @deprecated
|
|
3012
|
+
/** @deprecated v1 alias for getCrawlErrors(); its result also carries v1's `success: true`. */
|
|
2911
3013
|
async checkCrawlErrors(crawlId) {
|
|
2912
|
-
return this.getCrawlErrors(crawlId);
|
|
3014
|
+
return callV1(() => this.getCrawlErrors(crawlId));
|
|
2913
3015
|
}
|
|
2914
|
-
/** @deprecated
|
|
3016
|
+
/** @deprecated v1 alias for map(); its result also carries v1's `success: true`. */
|
|
2915
3017
|
async mapUrl(url, options) {
|
|
2916
|
-
return this.map(url, options);
|
|
3018
|
+
return callV1(() => this.map(url, options));
|
|
2917
3019
|
}
|
|
2918
|
-
/** @deprecated
|
|
3020
|
+
/** @deprecated v1 alias for batchScrape(); its result also carries v1's `success: true`. */
|
|
2919
3021
|
async batchScrapeUrls(urls, opts) {
|
|
2920
|
-
return this.batchScrape(urls, opts);
|
|
3022
|
+
return callV1(() => this.batchScrape(urls, opts));
|
|
2921
3023
|
}
|
|
2922
|
-
/** @deprecated
|
|
3024
|
+
/** @deprecated v1 alias for startBatchScrape(); its result also carries v1's `success: true`. */
|
|
2923
3025
|
async asyncBatchScrapeUrls(urls, opts) {
|
|
2924
|
-
return this.startBatchScrape(urls, opts);
|
|
3026
|
+
return callV1(() => this.startBatchScrape(urls, opts));
|
|
2925
3027
|
}
|
|
2926
|
-
/** @deprecated
|
|
3028
|
+
/** @deprecated v1 alias for getBatchScrapeStatus(); its result also carries v1's `success: true`. */
|
|
2927
3029
|
async checkBatchScrapeStatus(jobId, pagination) {
|
|
2928
|
-
return this.getBatchScrapeStatus(jobId, pagination);
|
|
3030
|
+
return callV1(() => this.getBatchScrapeStatus(jobId, pagination));
|
|
2929
3031
|
}
|
|
2930
|
-
/** @deprecated
|
|
3032
|
+
/** @deprecated v1 alias for getBatchScrapeErrors(); its result also carries v1's `success: true`. */
|
|
2931
3033
|
async checkBatchScrapeErrors(jobId) {
|
|
2932
|
-
return this.getBatchScrapeErrors(jobId);
|
|
3034
|
+
return callV1(() => this.getBatchScrapeErrors(jobId));
|
|
2933
3035
|
}
|
|
2934
3036
|
};
|
|
2935
3037
|
|
|
@@ -2958,6 +3060,8 @@ var FirecrawlApp = class {
|
|
|
2958
3060
|
apiKey;
|
|
2959
3061
|
apiUrl;
|
|
2960
3062
|
version = "1.25.1";
|
|
3063
|
+
clientOrigin;
|
|
3064
|
+
callerPackage;
|
|
2961
3065
|
isCloudService(url) {
|
|
2962
3066
|
return url.includes("api.firecrawl.dev");
|
|
2963
3067
|
}
|
|
@@ -2983,15 +3087,22 @@ var FirecrawlApp = class {
|
|
|
2983
3087
|
* Initializes a new instance of the FirecrawlApp class.
|
|
2984
3088
|
* @param config - Configuration options for the FirecrawlApp instance.
|
|
2985
3089
|
*/
|
|
2986
|
-
constructor({ apiKey = null, apiUrl = null }) {
|
|
3090
|
+
constructor({ apiKey = null, apiUrl = null, origin }) {
|
|
2987
3091
|
const baseUrl = apiUrl || "https://api.firecrawl.dev";
|
|
2988
3092
|
if (this.isCloudService(baseUrl) && typeof apiKey !== "string") {
|
|
2989
3093
|
throw new FirecrawlError("No API key provided", 401);
|
|
2990
3094
|
}
|
|
2991
3095
|
this.apiKey = apiKey || "";
|
|
2992
3096
|
this.apiUrl = baseUrl;
|
|
3097
|
+
this.clientOrigin = origin || void 0;
|
|
3098
|
+
this.callerPackage = this.clientOrigin ? null : detectCallerPackage();
|
|
2993
3099
|
this.init();
|
|
2994
3100
|
}
|
|
3101
|
+
resolveOrigin(params) {
|
|
3102
|
+
if (typeof params?.origin === "string" && params.origin) return params.origin;
|
|
3103
|
+
if (this.clientOrigin) return this.clientOrigin;
|
|
3104
|
+
return this.callerPackage ? `js-sdk.${this.callerPackage}@${this.version}` : `js-sdk@${this.version}`;
|
|
3105
|
+
}
|
|
2995
3106
|
/**
|
|
2996
3107
|
* Scrapes a URL using the Firecrawl API.
|
|
2997
3108
|
* @param url - The URL to scrape.
|
|
@@ -3003,7 +3114,7 @@ var FirecrawlApp = class {
|
|
|
3003
3114
|
"Content-Type": "application/json",
|
|
3004
3115
|
Authorization: `Bearer ${this.apiKey}`
|
|
3005
3116
|
};
|
|
3006
|
-
let jsonData = { url, ...params, origin:
|
|
3117
|
+
let jsonData = { url, ...params, origin: this.resolveOrigin(params) };
|
|
3007
3118
|
if (jsonData?.extract?.schema) {
|
|
3008
3119
|
jsonData = {
|
|
3009
3120
|
...jsonData,
|
|
@@ -3067,7 +3178,7 @@ var FirecrawlApp = class {
|
|
|
3067
3178
|
lang: params?.lang ?? "en",
|
|
3068
3179
|
country: params?.country ?? "us",
|
|
3069
3180
|
location: params?.location,
|
|
3070
|
-
origin:
|
|
3181
|
+
origin: this.resolveOrigin(params),
|
|
3071
3182
|
timeout: params?.timeout ?? 6e4,
|
|
3072
3183
|
scrapeOptions: params?.scrapeOptions ?? { formats: [] }
|
|
3073
3184
|
};
|
|
@@ -3122,7 +3233,7 @@ var FirecrawlApp = class {
|
|
|
3122
3233
|
*/
|
|
3123
3234
|
async crawlUrl(url, params, pollInterval = 2, idempotencyKey) {
|
|
3124
3235
|
const headers = this.prepareHeaders(idempotencyKey);
|
|
3125
|
-
let jsonData = { url, ...params, origin:
|
|
3236
|
+
let jsonData = { url, ...params, origin: this.resolveOrigin(params) };
|
|
3126
3237
|
try {
|
|
3127
3238
|
const response = await this.postRequest(
|
|
3128
3239
|
this.apiUrl + `/v1/crawl`,
|
|
@@ -3146,7 +3257,7 @@ var FirecrawlApp = class {
|
|
|
3146
3257
|
}
|
|
3147
3258
|
async asyncCrawlUrl(url, params, idempotencyKey) {
|
|
3148
3259
|
const headers = this.prepareHeaders(idempotencyKey);
|
|
3149
|
-
let jsonData = { url, ...params, origin:
|
|
3260
|
+
let jsonData = { url, ...params, origin: this.resolveOrigin(params) };
|
|
3150
3261
|
try {
|
|
3151
3262
|
const response = await this.postRequest(
|
|
3152
3263
|
this.apiUrl + `/v1/crawl`,
|
|
@@ -3305,7 +3416,7 @@ var FirecrawlApp = class {
|
|
|
3305
3416
|
*/
|
|
3306
3417
|
async mapUrl(url, params) {
|
|
3307
3418
|
const headers = this.prepareHeaders();
|
|
3308
|
-
let jsonData = { url, ...params, origin:
|
|
3419
|
+
let jsonData = { url, ...params, origin: this.resolveOrigin(params) };
|
|
3309
3420
|
try {
|
|
3310
3421
|
const response = await this.postRequest(
|
|
3311
3422
|
this.apiUrl + `/v1/map`,
|
|
@@ -3334,7 +3445,7 @@ var FirecrawlApp = class {
|
|
|
3334
3445
|
*/
|
|
3335
3446
|
async batchScrapeUrls(urls, params, pollInterval = 2, idempotencyKey, webhook, ignoreInvalidURLs, maxConcurrency) {
|
|
3336
3447
|
const headers = this.prepareHeaders(idempotencyKey);
|
|
3337
|
-
let jsonData = { urls, webhook, ignoreInvalidURLs, maxConcurrency, ...params, origin:
|
|
3448
|
+
let jsonData = { urls, webhook, ignoreInvalidURLs, maxConcurrency, ...params, origin: this.resolveOrigin(params) };
|
|
3338
3449
|
if (jsonData?.extract?.schema) {
|
|
3339
3450
|
jsonData = {
|
|
3340
3451
|
...jsonData,
|
|
@@ -3376,7 +3487,7 @@ var FirecrawlApp = class {
|
|
|
3376
3487
|
}
|
|
3377
3488
|
async asyncBatchScrapeUrls(urls, params, idempotencyKey, webhook, ignoreInvalidURLs) {
|
|
3378
3489
|
const headers = this.prepareHeaders(idempotencyKey);
|
|
3379
|
-
let jsonData = { urls, webhook, ignoreInvalidURLs, ...params, origin:
|
|
3490
|
+
let jsonData = { urls, webhook, ignoreInvalidURLs, ...params, origin: this.resolveOrigin(params) };
|
|
3380
3491
|
try {
|
|
3381
3492
|
const response = await this.postRequest(
|
|
3382
3493
|
this.apiUrl + `/v1/batch/scrape`,
|
|
@@ -3520,7 +3631,7 @@ var FirecrawlApp = class {
|
|
|
3520
3631
|
try {
|
|
3521
3632
|
const response = await this.postRequest(
|
|
3522
3633
|
this.apiUrl + `/v1/extract`,
|
|
3523
|
-
{ ...jsonData, schema: jsonSchema, origin:
|
|
3634
|
+
{ ...jsonData, schema: jsonSchema, origin: this.resolveOrigin(params) },
|
|
3524
3635
|
headers
|
|
3525
3636
|
);
|
|
3526
3637
|
if (response.status === 200) {
|
|
@@ -3573,7 +3684,7 @@ var FirecrawlApp = class {
|
|
|
3573
3684
|
try {
|
|
3574
3685
|
const response = await this.postRequest(
|
|
3575
3686
|
this.apiUrl + `/v1/extract`,
|
|
3576
|
-
{ ...jsonData, schema: jsonSchema, origin:
|
|
3687
|
+
{ ...jsonData, schema: jsonSchema, origin: this.resolveOrigin(params) },
|
|
3577
3688
|
headers
|
|
3578
3689
|
);
|
|
3579
3690
|
if (response.status === 200) {
|
|
@@ -3855,7 +3966,7 @@ var FirecrawlApp = class {
|
|
|
3855
3966
|
*/
|
|
3856
3967
|
async asyncDeepResearch(query, params) {
|
|
3857
3968
|
const headers = this.prepareHeaders();
|
|
3858
|
-
let jsonData = { query, ...params, origin:
|
|
3969
|
+
let jsonData = { query, ...params, origin: this.resolveOrigin(params) };
|
|
3859
3970
|
if (jsonData?.jsonOptions?.schema) {
|
|
3860
3971
|
jsonData = {
|
|
3861
3972
|
...jsonData,
|
|
@@ -3974,7 +4085,7 @@ var FirecrawlApp = class {
|
|
|
3974
4085
|
async __asyncDeepResearch(topic, params) {
|
|
3975
4086
|
const headers = this.prepareHeaders();
|
|
3976
4087
|
try {
|
|
3977
|
-
let jsonData = { topic, ...params, origin:
|
|
4088
|
+
let jsonData = { topic, ...params, origin: this.resolveOrigin(params) };
|
|
3978
4089
|
const response = await this.postRequest(
|
|
3979
4090
|
`${this.apiUrl}/v1/deep-research`,
|
|
3980
4091
|
jsonData,
|
|
@@ -4074,7 +4185,7 @@ var FirecrawlApp = class {
|
|
|
4074
4185
|
*/
|
|
4075
4186
|
async asyncGenerateLLMsText(url, params) {
|
|
4076
4187
|
const headers = this.prepareHeaders();
|
|
4077
|
-
let jsonData = { url, ...params, origin:
|
|
4188
|
+
let jsonData = { url, ...params, origin: this.resolveOrigin(params) };
|
|
4078
4189
|
try {
|
|
4079
4190
|
const response = await this.postRequest(
|
|
4080
4191
|
`${this.apiUrl}/v1/llmstxt`,
|
|
@@ -4342,7 +4453,8 @@ var Firecrawl = class extends FirecrawlClient {
|
|
|
4342
4453
|
super(resolved);
|
|
4343
4454
|
this._v1Opts = {
|
|
4344
4455
|
apiKey: resolved.apiKey,
|
|
4345
|
-
apiUrl: resolved.apiUrl
|
|
4456
|
+
apiUrl: resolved.apiUrl,
|
|
4457
|
+
origin: this.http.getOrigin()
|
|
4346
4458
|
};
|
|
4347
4459
|
}
|
|
4348
4460
|
/** Access the legacy v1 client (instantiated on first access). */
|
package/dist/index.d.cts
CHANGED
|
@@ -2019,6 +2019,7 @@ interface HttpClientOptions {
|
|
|
2019
2019
|
timeoutMs?: number;
|
|
2020
2020
|
maxRetries?: number;
|
|
2021
2021
|
backoffFactor?: number;
|
|
2022
|
+
origin?: string;
|
|
2022
2023
|
}
|
|
2023
2024
|
interface RequestOptions {
|
|
2024
2025
|
headers?: Record<string, string>;
|
|
@@ -2030,9 +2031,12 @@ declare class HttpClient {
|
|
|
2030
2031
|
private readonly apiUrl;
|
|
2031
2032
|
private readonly maxRetries;
|
|
2032
2033
|
private readonly backoffFactor;
|
|
2034
|
+
private readonly origin;
|
|
2033
2035
|
constructor(options: HttpClientOptions);
|
|
2034
2036
|
getApiUrl(): string;
|
|
2035
2037
|
getApiKey(): string;
|
|
2038
|
+
/** The attribution origin stamped into requests that do not set their own. */
|
|
2039
|
+
getOrigin(): string;
|
|
2036
2040
|
/** The client-wide default request timeout, for callers that need to
|
|
2037
2041
|
* extend a specific request beyond it (e.g. scrape auto-resume). */
|
|
2038
2042
|
getTimeoutMs(): number;
|
|
@@ -2244,6 +2248,17 @@ declare class Watcher extends EventEmitter {
|
|
|
2244
2248
|
close(): void;
|
|
2245
2249
|
}
|
|
2246
2250
|
|
|
2251
|
+
/** v1 crawl options that have a v2 equivalent. */
|
|
2252
|
+
type V1CrawlOptions = {
|
|
2253
|
+
maxDepth?: number;
|
|
2254
|
+
allowBackwardLinks?: boolean;
|
|
2255
|
+
ignoreSitemap?: boolean;
|
|
2256
|
+
};
|
|
2257
|
+
/** A v2 result with the `success` flag v1 results carried. */
|
|
2258
|
+
type V1Result<T> = T & {
|
|
2259
|
+
success: true;
|
|
2260
|
+
};
|
|
2261
|
+
|
|
2247
2262
|
type ExtractJsonSchemaFromFormats<Formats> = Formats extends readonly any[] ? Extract<Formats[number], {
|
|
2248
2263
|
type: "json";
|
|
2249
2264
|
schema?: unknown;
|
|
@@ -2265,6 +2280,12 @@ interface FirecrawlClientOptions {
|
|
|
2265
2280
|
maxRetries?: number;
|
|
2266
2281
|
/** Exponential backoff factor for retries (optional). */
|
|
2267
2282
|
backoffFactor?: number;
|
|
2283
|
+
/**
|
|
2284
|
+
* Attribution tag stamped into request bodies. Defaults to `js-sdk@<version>`,
|
|
2285
|
+
* or `js-sdk.<package>@<version>` when an installed package creates the client
|
|
2286
|
+
* (set FIRECRAWL_DISABLE_CALLER_ORIGIN=1 to keep the plain default).
|
|
2287
|
+
*/
|
|
2288
|
+
origin?: string;
|
|
2268
2289
|
}
|
|
2269
2290
|
/** Accepts a plain API key string or a full options object. */
|
|
2270
2291
|
type FirecrawlClientInput = FirecrawlClientOptions | string;
|
|
@@ -2272,7 +2293,7 @@ type FirecrawlClientInput = FirecrawlClientOptions | string;
|
|
|
2272
2293
|
* Firecrawl v2 client. Provides typed access to all v2 endpoints and utilities.
|
|
2273
2294
|
*/
|
|
2274
2295
|
declare class FirecrawlClient {
|
|
2275
|
-
|
|
2296
|
+
protected readonly http: HttpClient;
|
|
2276
2297
|
private _research?;
|
|
2277
2298
|
private isCloudService;
|
|
2278
2299
|
/**
|
|
@@ -2629,43 +2650,45 @@ declare class FirecrawlClient {
|
|
|
2629
2650
|
* @param opts Watcher options (kind, pollInterval, timeout seconds).
|
|
2630
2651
|
*/
|
|
2631
2652
|
watcher(jobId: string, opts?: WatcherOptions): Watcher;
|
|
2632
|
-
/** @deprecated
|
|
2633
|
-
scrapeUrl(url: string, options?: ScrapeOptions): Promise<Document
|
|
2634
|
-
/** @deprecated
|
|
2635
|
-
crawlUrl(url: string, req?: CrawlOptions & {
|
|
2653
|
+
/** @deprecated v1 alias for scrape(); its result also carries v1's `success: true`. */
|
|
2654
|
+
scrapeUrl(url: string, options?: ScrapeOptions): Promise<V1Result<Document>>;
|
|
2655
|
+
/** @deprecated v1 alias for crawl(); its result also carries v1's `success: true`. */
|
|
2656
|
+
crawlUrl(url: string, req?: CrawlOptions & V1CrawlOptions & {
|
|
2636
2657
|
pollInterval?: number;
|
|
2637
2658
|
timeout?: number;
|
|
2638
2659
|
signal?: AbortSignal;
|
|
2639
|
-
}): Promise<CrawlJob
|
|
2640
|
-
/** @deprecated
|
|
2641
|
-
asyncCrawlUrl(url: string, req?: CrawlOptions): Promise<CrawlResponse$1
|
|
2642
|
-
/** @deprecated
|
|
2643
|
-
checkCrawlStatus(jobId: string, pagination?: PaginationConfig): Promise<CrawlJob
|
|
2644
|
-
/** @deprecated
|
|
2645
|
-
checkCrawlErrors(crawlId: string): Promise<CrawlErrorsResponse$1
|
|
2646
|
-
/** @deprecated
|
|
2647
|
-
mapUrl(url: string, options?: MapOptions): Promise<MapData
|
|
2648
|
-
/** @deprecated
|
|
2660
|
+
}): Promise<V1Result<CrawlJob>>;
|
|
2661
|
+
/** @deprecated v1 alias for startCrawl(); its result also carries v1's `success: true`. */
|
|
2662
|
+
asyncCrawlUrl(url: string, req?: CrawlOptions & V1CrawlOptions): Promise<V1Result<CrawlResponse$1>>;
|
|
2663
|
+
/** @deprecated v1 alias for getCrawlStatus(); its result also carries v1's `success: true`. */
|
|
2664
|
+
checkCrawlStatus(jobId: string, pagination?: PaginationConfig): Promise<V1Result<CrawlJob>>;
|
|
2665
|
+
/** @deprecated v1 alias for getCrawlErrors(); its result also carries v1's `success: true`. */
|
|
2666
|
+
checkCrawlErrors(crawlId: string): Promise<V1Result<CrawlErrorsResponse$1>>;
|
|
2667
|
+
/** @deprecated v1 alias for map(); its result also carries v1's `success: true`. */
|
|
2668
|
+
mapUrl(url: string, options?: MapOptions): Promise<V1Result<MapData>>;
|
|
2669
|
+
/** @deprecated v1 alias for batchScrape(); its result also carries v1's `success: true`. */
|
|
2649
2670
|
batchScrapeUrls(urls: string[], opts?: BatchScrapeOptions & {
|
|
2650
2671
|
pollInterval?: number;
|
|
2651
2672
|
timeout?: number;
|
|
2652
|
-
}): Promise<BatchScrapeJob
|
|
2653
|
-
/** @deprecated
|
|
2654
|
-
asyncBatchScrapeUrls(urls: string[], opts?: BatchScrapeOptions): Promise<BatchScrapeResponse$1
|
|
2655
|
-
/** @deprecated
|
|
2656
|
-
checkBatchScrapeStatus(jobId: string, pagination?: PaginationConfig): Promise<BatchScrapeJob
|
|
2657
|
-
/** @deprecated
|
|
2658
|
-
checkBatchScrapeErrors(jobId: string): Promise<CrawlErrorsResponse$1
|
|
2673
|
+
}): Promise<V1Result<BatchScrapeJob>>;
|
|
2674
|
+
/** @deprecated v1 alias for startBatchScrape(); its result also carries v1's `success: true`. */
|
|
2675
|
+
asyncBatchScrapeUrls(urls: string[], opts?: BatchScrapeOptions): Promise<V1Result<BatchScrapeResponse$1>>;
|
|
2676
|
+
/** @deprecated v1 alias for getBatchScrapeStatus(); its result also carries v1's `success: true`. */
|
|
2677
|
+
checkBatchScrapeStatus(jobId: string, pagination?: PaginationConfig): Promise<V1Result<BatchScrapeJob>>;
|
|
2678
|
+
/** @deprecated v1 alias for getBatchScrapeErrors(); its result also carries v1's `success: true`. */
|
|
2679
|
+
checkBatchScrapeErrors(jobId: string): Promise<V1Result<CrawlErrorsResponse$1>>;
|
|
2659
2680
|
}
|
|
2660
2681
|
|
|
2661
2682
|
/**
|
|
2662
2683
|
* Configuration interface for FirecrawlApp.
|
|
2663
2684
|
* @param apiKey - Optional API key for authentication.
|
|
2664
2685
|
* @param apiUrl - Optional base URL of the API; defaults to 'https://api.firecrawl.dev'.
|
|
2686
|
+
* @param origin - Optional attribution tag stamped into request bodies.
|
|
2665
2687
|
*/
|
|
2666
2688
|
interface FirecrawlAppConfig {
|
|
2667
2689
|
apiKey?: string | null;
|
|
2668
2690
|
apiUrl?: string | null;
|
|
2691
|
+
origin?: string;
|
|
2669
2692
|
}
|
|
2670
2693
|
/**
|
|
2671
2694
|
* Metadata for a Firecrawl document.
|
|
@@ -3262,6 +3285,8 @@ declare class FirecrawlApp {
|
|
|
3262
3285
|
apiKey: string;
|
|
3263
3286
|
apiUrl: string;
|
|
3264
3287
|
version: string;
|
|
3288
|
+
private readonly clientOrigin?;
|
|
3289
|
+
private readonly callerPackage;
|
|
3265
3290
|
private isCloudService;
|
|
3266
3291
|
private getVersion;
|
|
3267
3292
|
private init;
|
|
@@ -3269,7 +3294,8 @@ declare class FirecrawlApp {
|
|
|
3269
3294
|
* Initializes a new instance of the FirecrawlApp class.
|
|
3270
3295
|
* @param config - Configuration options for the FirecrawlApp instance.
|
|
3271
3296
|
*/
|
|
3272
|
-
constructor({ apiKey, apiUrl }: FirecrawlAppConfig);
|
|
3297
|
+
constructor({ apiKey, apiUrl, origin }: FirecrawlAppConfig);
|
|
3298
|
+
private resolveOrigin;
|
|
3273
3299
|
/**
|
|
3274
3300
|
* Scrapes a URL using the Firecrawl API.
|
|
3275
3301
|
* @param url - The URL to scrape.
|