firecrawl 4.47.0 → 4.48.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -0
- package/dist/{chunk-H75FJ6LW.js → chunk-GD463K4D.js} +1 -1
- package/dist/index.cjs +112 -26
- package/dist/index.d.cts +17 -2
- package/dist/index.d.ts +17 -2
- package/dist/index.js +113 -27
- package/dist/{package-NZI5TDUV.js → package-WONXBIJS.js} +1 -1
- package/package.json +1 -1
- package/src/__tests__/unit/v2/agent-hints.test.ts +1 -0
- package/src/__tests__/unit/v2/origin.test.ts +132 -0
- package/src/__tests__/unit/v2/research.test.ts +1 -0
- package/src/index.ts +1 -0
- package/src/utils/callerOrigin.ts +74 -0
- package/src/v1/index.ts +28 -13
- package/src/v2/client.ts +10 -1
- package/src/v2/methods/parse.ts +1 -6
- package/src/v2/methods/research.ts +20 -7
- package/src/v2/utils/httpClient.ts +10 -3
package/dist/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
version
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-GD463K4D.js";
|
|
4
4
|
|
|
5
5
|
// src/v2/utils/httpClient.ts
|
|
6
6
|
import axios from "axios";
|
|
@@ -32,11 +32,13 @@ var HttpClient = class {
|
|
|
32
32
|
apiUrl;
|
|
33
33
|
maxRetries;
|
|
34
34
|
backoffFactor;
|
|
35
|
+
origin;
|
|
35
36
|
constructor(options) {
|
|
36
37
|
this.apiKey = options.apiKey;
|
|
37
38
|
this.apiUrl = options.apiUrl.replace(/\/$/, "");
|
|
38
39
|
this.maxRetries = options.maxRetries ?? 3;
|
|
39
40
|
this.backoffFactor = options.backoffFactor ?? 0.5;
|
|
41
|
+
this.origin = options.origin || `js-sdk@${getVersion()}`;
|
|
40
42
|
this.instance = axios.create({
|
|
41
43
|
baseURL: this.apiUrl,
|
|
42
44
|
timeout: options.timeoutMs ?? 3e5,
|
|
@@ -55,13 +57,16 @@ var HttpClient = class {
|
|
|
55
57
|
getApiKey() {
|
|
56
58
|
return this.apiKey;
|
|
57
59
|
}
|
|
60
|
+
/** The attribution origin stamped into requests that do not set their own. */
|
|
61
|
+
getOrigin() {
|
|
62
|
+
return this.origin;
|
|
63
|
+
}
|
|
58
64
|
/** The client-wide default request timeout, for callers that need to
|
|
59
65
|
* extend a specific request beyond it (e.g. scrape auto-resume). */
|
|
60
66
|
getTimeoutMs() {
|
|
61
67
|
return this.instance.defaults.timeout ?? 3e5;
|
|
62
68
|
}
|
|
63
69
|
async request(config) {
|
|
64
|
-
const version2 = getVersion();
|
|
65
70
|
config.headers = {
|
|
66
71
|
...config.headers || {}
|
|
67
72
|
};
|
|
@@ -76,7 +81,7 @@ var HttpClient = class {
|
|
|
76
81
|
const data = cfg.data ?? {};
|
|
77
82
|
cfg.data = {
|
|
78
83
|
...data,
|
|
79
|
-
origin: typeof data.origin === "string" && data.origin
|
|
84
|
+
origin: typeof data.origin === "string" && data.origin ? data.origin : this.origin
|
|
80
85
|
};
|
|
81
86
|
}
|
|
82
87
|
if (isFormDataBody) {
|
|
@@ -659,10 +664,9 @@ async function parse(http, file, options) {
|
|
|
659
664
|
throw new Error("file data cannot be empty");
|
|
660
665
|
}
|
|
661
666
|
if (options) ensureValidParseOptions(options);
|
|
662
|
-
const version2 = getVersion();
|
|
663
667
|
const normalizedOptions = {
|
|
664
668
|
...options ?? {},
|
|
665
|
-
origin:
|
|
669
|
+
origin: options?.origin || http.getOrigin()
|
|
666
670
|
};
|
|
667
671
|
const formData = new FormData();
|
|
668
672
|
formData.append("options", JSON.stringify(normalizedOptions));
|
|
@@ -1780,8 +1784,8 @@ function appendParam(params, key, value) {
|
|
|
1780
1784
|
params.append(key, String(value));
|
|
1781
1785
|
}
|
|
1782
1786
|
}
|
|
1783
|
-
function withQuery(path, params) {
|
|
1784
|
-
params.append("origin",
|
|
1787
|
+
function withQuery(origin, path, params) {
|
|
1788
|
+
params.append("origin", origin);
|
|
1785
1789
|
const qs = params.toString();
|
|
1786
1790
|
return qs ? `${path}?${qs}` : path;
|
|
1787
1791
|
}
|
|
@@ -1836,7 +1840,7 @@ var ResearchClient = class {
|
|
|
1836
1840
|
appendParam(params, "to", options.to);
|
|
1837
1841
|
try {
|
|
1838
1842
|
const res = await this.http.get(
|
|
1839
|
-
withQuery(`${BASE}/papers`, params)
|
|
1843
|
+
withQuery(this.http.getOrigin(), `${BASE}/papers`, params)
|
|
1840
1844
|
);
|
|
1841
1845
|
if (res.status !== 200) throwForBadResponse(res, "search papers");
|
|
1842
1846
|
return res.data;
|
|
@@ -1854,7 +1858,13 @@ var ResearchClient = class {
|
|
|
1854
1858
|
appendParam(params, "query", options.query);
|
|
1855
1859
|
appendParam(params, "k", options.k);
|
|
1856
1860
|
try {
|
|
1857
|
-
const res = await this.http.get(
|
|
1861
|
+
const res = await this.http.get(
|
|
1862
|
+
withQuery(
|
|
1863
|
+
this.http.getOrigin(),
|
|
1864
|
+
`${BASE}/papers/${encodeURIComponent(id)}`,
|
|
1865
|
+
params
|
|
1866
|
+
)
|
|
1867
|
+
);
|
|
1858
1868
|
if (res.status !== 200) throwForBadResponse(res, "get paper");
|
|
1859
1869
|
return res.data;
|
|
1860
1870
|
} catch (err) {
|
|
@@ -1880,7 +1890,11 @@ var ResearchClient = class {
|
|
|
1880
1890
|
appendParam(params, "anchor", options.anchor);
|
|
1881
1891
|
try {
|
|
1882
1892
|
const res = await this.http.get(
|
|
1883
|
-
withQuery(
|
|
1893
|
+
withQuery(
|
|
1894
|
+
this.http.getOrigin(),
|
|
1895
|
+
`${BASE}/papers/${encodeURIComponent(id)}/similar`,
|
|
1896
|
+
params
|
|
1897
|
+
)
|
|
1884
1898
|
);
|
|
1885
1899
|
if (res.status !== 200) throwForBadResponse(res, "find similar papers");
|
|
1886
1900
|
return res.data;
|
|
@@ -1911,7 +1925,7 @@ var ResearchClient = class {
|
|
|
1911
1925
|
appendParam(params, "k", options.k);
|
|
1912
1926
|
try {
|
|
1913
1927
|
const res = await this.http.get(
|
|
1914
|
-
withQuery(`${BASE}/github`, params)
|
|
1928
|
+
withQuery(this.http.getOrigin(), `${BASE}/github`, params)
|
|
1915
1929
|
);
|
|
1916
1930
|
if (res.status !== 200) throwForBadResponse(res, "search github");
|
|
1917
1931
|
return res.data;
|
|
@@ -2041,6 +2055,67 @@ async function getMonitorCheck(http, monitorId, checkId, options) {
|
|
|
2041
2055
|
}
|
|
2042
2056
|
}
|
|
2043
2057
|
|
|
2058
|
+
// src/utils/callerOrigin.ts
|
|
2059
|
+
var NODE_MODULES = "/node_modules/";
|
|
2060
|
+
function packageOf(file) {
|
|
2061
|
+
let path = file.replace(/^file:\/\//, "").replace(/\\/g, "/");
|
|
2062
|
+
try {
|
|
2063
|
+
path = decodeURIComponent(path);
|
|
2064
|
+
} catch {
|
|
2065
|
+
}
|
|
2066
|
+
const at = path.lastIndexOf(NODE_MODULES);
|
|
2067
|
+
if (at < 0) return null;
|
|
2068
|
+
const [first, second] = path.slice(at + NODE_MODULES.length).split("/");
|
|
2069
|
+
const name = first?.startsWith("@") ? second ? `${first}/${second}` : "" : first;
|
|
2070
|
+
return name && !name.startsWith(".") ? name : null;
|
|
2071
|
+
}
|
|
2072
|
+
function frameFile(line) {
|
|
2073
|
+
const text = line.trim().replace(/^at /, "");
|
|
2074
|
+
const open = text.indexOf(" (");
|
|
2075
|
+
const location = open >= 0 && text.endsWith(")") ? text.slice(open + 2, -1) : text;
|
|
2076
|
+
const match = /^(.*):\d+:\d+$/.exec(location);
|
|
2077
|
+
return match && !match[1].startsWith("node:") ? match[1] : null;
|
|
2078
|
+
}
|
|
2079
|
+
function slugify(name) {
|
|
2080
|
+
return name.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 48).replace(/-+$/, "");
|
|
2081
|
+
}
|
|
2082
|
+
function detectCallerPackage(stack) {
|
|
2083
|
+
try {
|
|
2084
|
+
if (process.env.FIRECRAWL_DISABLE_CALLER_ORIGIN === "1") return null;
|
|
2085
|
+
if (stack === void 0) {
|
|
2086
|
+
const limit = Error.stackTraceLimit;
|
|
2087
|
+
Error.stackTraceLimit = 50;
|
|
2088
|
+
try {
|
|
2089
|
+
stack = new Error().stack ?? "";
|
|
2090
|
+
} finally {
|
|
2091
|
+
Error.stackTraceLimit = limit;
|
|
2092
|
+
}
|
|
2093
|
+
}
|
|
2094
|
+
let ownFile = null;
|
|
2095
|
+
let ownPackage = null;
|
|
2096
|
+
for (const line of stack.split("\n")) {
|
|
2097
|
+
if (!/^\s*at /.test(line)) continue;
|
|
2098
|
+
const file = frameFile(line);
|
|
2099
|
+
if (!file) continue;
|
|
2100
|
+
if (ownFile === null) {
|
|
2101
|
+
ownFile = file;
|
|
2102
|
+
ownPackage = packageOf(file);
|
|
2103
|
+
}
|
|
2104
|
+
if (file === ownFile) continue;
|
|
2105
|
+
const name = packageOf(file);
|
|
2106
|
+
if (name && name === ownPackage) continue;
|
|
2107
|
+
const slug = name ? slugify(name) : "";
|
|
2108
|
+
return slug && !slug.includes("playground") ? slug : null;
|
|
2109
|
+
}
|
|
2110
|
+
} catch {
|
|
2111
|
+
}
|
|
2112
|
+
return null;
|
|
2113
|
+
}
|
|
2114
|
+
function defaultOrigin(version2) {
|
|
2115
|
+
const caller = detectCallerPackage();
|
|
2116
|
+
return caller ? `js-sdk.${caller}@${version2}` : `js-sdk@${version2}`;
|
|
2117
|
+
}
|
|
2118
|
+
|
|
2044
2119
|
// src/v2/watcher.ts
|
|
2045
2120
|
import { EventEmitter } from "events";
|
|
2046
2121
|
var hasGlobalWebSocket = () => {
|
|
@@ -2303,7 +2378,8 @@ var FirecrawlClient = class {
|
|
|
2303
2378
|
apiUrl,
|
|
2304
2379
|
timeoutMs: opts.timeoutMs,
|
|
2305
2380
|
maxRetries: opts.maxRetries,
|
|
2306
|
-
backoffFactor: opts.backoffFactor
|
|
2381
|
+
backoffFactor: opts.backoffFactor,
|
|
2382
|
+
origin: opts.origin || defaultOrigin(getVersion())
|
|
2307
2383
|
});
|
|
2308
2384
|
}
|
|
2309
2385
|
async scrape(url, options) {
|
|
@@ -2829,6 +2905,8 @@ var FirecrawlApp = class {
|
|
|
2829
2905
|
apiKey;
|
|
2830
2906
|
apiUrl;
|
|
2831
2907
|
version = "1.25.1";
|
|
2908
|
+
clientOrigin;
|
|
2909
|
+
callerPackage;
|
|
2832
2910
|
isCloudService(url) {
|
|
2833
2911
|
return url.includes("api.firecrawl.dev");
|
|
2834
2912
|
}
|
|
@@ -2837,7 +2915,7 @@ var FirecrawlApp = class {
|
|
|
2837
2915
|
if (typeof process !== "undefined" && process.env && process.env.npm_package_version) {
|
|
2838
2916
|
return process.env.npm_package_version;
|
|
2839
2917
|
}
|
|
2840
|
-
const packageJson = await import("./package-
|
|
2918
|
+
const packageJson = await import("./package-WONXBIJS.js");
|
|
2841
2919
|
return packageJson.default.version;
|
|
2842
2920
|
} catch (error) {
|
|
2843
2921
|
const isTest = typeof process !== "undefined" && false;
|
|
@@ -2854,15 +2932,22 @@ var FirecrawlApp = class {
|
|
|
2854
2932
|
* Initializes a new instance of the FirecrawlApp class.
|
|
2855
2933
|
* @param config - Configuration options for the FirecrawlApp instance.
|
|
2856
2934
|
*/
|
|
2857
|
-
constructor({ apiKey = null, apiUrl = null }) {
|
|
2935
|
+
constructor({ apiKey = null, apiUrl = null, origin }) {
|
|
2858
2936
|
const baseUrl = apiUrl || "https://api.firecrawl.dev";
|
|
2859
2937
|
if (this.isCloudService(baseUrl) && typeof apiKey !== "string") {
|
|
2860
2938
|
throw new FirecrawlError("No API key provided", 401);
|
|
2861
2939
|
}
|
|
2862
2940
|
this.apiKey = apiKey || "";
|
|
2863
2941
|
this.apiUrl = baseUrl;
|
|
2942
|
+
this.clientOrigin = origin || void 0;
|
|
2943
|
+
this.callerPackage = this.clientOrigin ? null : detectCallerPackage();
|
|
2864
2944
|
this.init();
|
|
2865
2945
|
}
|
|
2946
|
+
resolveOrigin(params) {
|
|
2947
|
+
if (typeof params?.origin === "string" && params.origin) return params.origin;
|
|
2948
|
+
if (this.clientOrigin) return this.clientOrigin;
|
|
2949
|
+
return this.callerPackage ? `js-sdk.${this.callerPackage}@${this.version}` : `js-sdk@${this.version}`;
|
|
2950
|
+
}
|
|
2866
2951
|
/**
|
|
2867
2952
|
* Scrapes a URL using the Firecrawl API.
|
|
2868
2953
|
* @param url - The URL to scrape.
|
|
@@ -2874,7 +2959,7 @@ var FirecrawlApp = class {
|
|
|
2874
2959
|
"Content-Type": "application/json",
|
|
2875
2960
|
Authorization: `Bearer ${this.apiKey}`
|
|
2876
2961
|
};
|
|
2877
|
-
let jsonData = { url, ...params, origin:
|
|
2962
|
+
let jsonData = { url, ...params, origin: this.resolveOrigin(params) };
|
|
2878
2963
|
if (jsonData?.extract?.schema) {
|
|
2879
2964
|
jsonData = {
|
|
2880
2965
|
...jsonData,
|
|
@@ -2938,7 +3023,7 @@ var FirecrawlApp = class {
|
|
|
2938
3023
|
lang: params?.lang ?? "en",
|
|
2939
3024
|
country: params?.country ?? "us",
|
|
2940
3025
|
location: params?.location,
|
|
2941
|
-
origin:
|
|
3026
|
+
origin: this.resolveOrigin(params),
|
|
2942
3027
|
timeout: params?.timeout ?? 6e4,
|
|
2943
3028
|
scrapeOptions: params?.scrapeOptions ?? { formats: [] }
|
|
2944
3029
|
};
|
|
@@ -2993,7 +3078,7 @@ var FirecrawlApp = class {
|
|
|
2993
3078
|
*/
|
|
2994
3079
|
async crawlUrl(url, params, pollInterval = 2, idempotencyKey) {
|
|
2995
3080
|
const headers = this.prepareHeaders(idempotencyKey);
|
|
2996
|
-
let jsonData = { url, ...params, origin:
|
|
3081
|
+
let jsonData = { url, ...params, origin: this.resolveOrigin(params) };
|
|
2997
3082
|
try {
|
|
2998
3083
|
const response = await this.postRequest(
|
|
2999
3084
|
this.apiUrl + `/v1/crawl`,
|
|
@@ -3017,7 +3102,7 @@ var FirecrawlApp = class {
|
|
|
3017
3102
|
}
|
|
3018
3103
|
async asyncCrawlUrl(url, params, idempotencyKey) {
|
|
3019
3104
|
const headers = this.prepareHeaders(idempotencyKey);
|
|
3020
|
-
let jsonData = { url, ...params, origin:
|
|
3105
|
+
let jsonData = { url, ...params, origin: this.resolveOrigin(params) };
|
|
3021
3106
|
try {
|
|
3022
3107
|
const response = await this.postRequest(
|
|
3023
3108
|
this.apiUrl + `/v1/crawl`,
|
|
@@ -3176,7 +3261,7 @@ var FirecrawlApp = class {
|
|
|
3176
3261
|
*/
|
|
3177
3262
|
async mapUrl(url, params) {
|
|
3178
3263
|
const headers = this.prepareHeaders();
|
|
3179
|
-
let jsonData = { url, ...params, origin:
|
|
3264
|
+
let jsonData = { url, ...params, origin: this.resolveOrigin(params) };
|
|
3180
3265
|
try {
|
|
3181
3266
|
const response = await this.postRequest(
|
|
3182
3267
|
this.apiUrl + `/v1/map`,
|
|
@@ -3205,7 +3290,7 @@ var FirecrawlApp = class {
|
|
|
3205
3290
|
*/
|
|
3206
3291
|
async batchScrapeUrls(urls, params, pollInterval = 2, idempotencyKey, webhook, ignoreInvalidURLs, maxConcurrency) {
|
|
3207
3292
|
const headers = this.prepareHeaders(idempotencyKey);
|
|
3208
|
-
let jsonData = { urls, webhook, ignoreInvalidURLs, maxConcurrency, ...params, origin:
|
|
3293
|
+
let jsonData = { urls, webhook, ignoreInvalidURLs, maxConcurrency, ...params, origin: this.resolveOrigin(params) };
|
|
3209
3294
|
if (jsonData?.extract?.schema) {
|
|
3210
3295
|
jsonData = {
|
|
3211
3296
|
...jsonData,
|
|
@@ -3247,7 +3332,7 @@ var FirecrawlApp = class {
|
|
|
3247
3332
|
}
|
|
3248
3333
|
async asyncBatchScrapeUrls(urls, params, idempotencyKey, webhook, ignoreInvalidURLs) {
|
|
3249
3334
|
const headers = this.prepareHeaders(idempotencyKey);
|
|
3250
|
-
let jsonData = { urls, webhook, ignoreInvalidURLs, ...params, origin:
|
|
3335
|
+
let jsonData = { urls, webhook, ignoreInvalidURLs, ...params, origin: this.resolveOrigin(params) };
|
|
3251
3336
|
try {
|
|
3252
3337
|
const response = await this.postRequest(
|
|
3253
3338
|
this.apiUrl + `/v1/batch/scrape`,
|
|
@@ -3391,7 +3476,7 @@ var FirecrawlApp = class {
|
|
|
3391
3476
|
try {
|
|
3392
3477
|
const response = await this.postRequest(
|
|
3393
3478
|
this.apiUrl + `/v1/extract`,
|
|
3394
|
-
{ ...jsonData, schema: jsonSchema, origin:
|
|
3479
|
+
{ ...jsonData, schema: jsonSchema, origin: this.resolveOrigin(params) },
|
|
3395
3480
|
headers
|
|
3396
3481
|
);
|
|
3397
3482
|
if (response.status === 200) {
|
|
@@ -3444,7 +3529,7 @@ var FirecrawlApp = class {
|
|
|
3444
3529
|
try {
|
|
3445
3530
|
const response = await this.postRequest(
|
|
3446
3531
|
this.apiUrl + `/v1/extract`,
|
|
3447
|
-
{ ...jsonData, schema: jsonSchema, origin:
|
|
3532
|
+
{ ...jsonData, schema: jsonSchema, origin: this.resolveOrigin(params) },
|
|
3448
3533
|
headers
|
|
3449
3534
|
);
|
|
3450
3535
|
if (response.status === 200) {
|
|
@@ -3726,7 +3811,7 @@ var FirecrawlApp = class {
|
|
|
3726
3811
|
*/
|
|
3727
3812
|
async asyncDeepResearch(query, params) {
|
|
3728
3813
|
const headers = this.prepareHeaders();
|
|
3729
|
-
let jsonData = { query, ...params, origin:
|
|
3814
|
+
let jsonData = { query, ...params, origin: this.resolveOrigin(params) };
|
|
3730
3815
|
if (jsonData?.jsonOptions?.schema) {
|
|
3731
3816
|
jsonData = {
|
|
3732
3817
|
...jsonData,
|
|
@@ -3845,7 +3930,7 @@ var FirecrawlApp = class {
|
|
|
3845
3930
|
async __asyncDeepResearch(topic, params) {
|
|
3846
3931
|
const headers = this.prepareHeaders();
|
|
3847
3932
|
try {
|
|
3848
|
-
let jsonData = { topic, ...params, origin:
|
|
3933
|
+
let jsonData = { topic, ...params, origin: this.resolveOrigin(params) };
|
|
3849
3934
|
const response = await this.postRequest(
|
|
3850
3935
|
`${this.apiUrl}/v1/deep-research`,
|
|
3851
3936
|
jsonData,
|
|
@@ -3945,7 +4030,7 @@ var FirecrawlApp = class {
|
|
|
3945
4030
|
*/
|
|
3946
4031
|
async asyncGenerateLLMsText(url, params) {
|
|
3947
4032
|
const headers = this.prepareHeaders();
|
|
3948
|
-
let jsonData = { url, ...params, origin:
|
|
4033
|
+
let jsonData = { url, ...params, origin: this.resolveOrigin(params) };
|
|
3949
4034
|
try {
|
|
3950
4035
|
const response = await this.postRequest(
|
|
3951
4036
|
`${this.apiUrl}/v1/llmstxt`,
|
|
@@ -4213,7 +4298,8 @@ var Firecrawl = class extends FirecrawlClient {
|
|
|
4213
4298
|
super(resolved);
|
|
4214
4299
|
this._v1Opts = {
|
|
4215
4300
|
apiKey: resolved.apiKey,
|
|
4216
|
-
apiUrl: resolved.apiUrl
|
|
4301
|
+
apiUrl: resolved.apiUrl,
|
|
4302
|
+
origin: this.http.getOrigin()
|
|
4217
4303
|
};
|
|
4218
4304
|
}
|
|
4219
4305
|
/** Access the legacy v1 client (instantiated on first access). */
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "firecrawl",
|
|
3
|
-
"version": "4.
|
|
3
|
+
"version": "4.48.0",
|
|
4
4
|
"description": "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
import { afterEach, beforeEach, describe, test, expect } from "vitest";
|
|
2
|
+
import type { AxiosAdapter } from "axios";
|
|
3
|
+
import { FirecrawlClient } from "../../../v2/client";
|
|
4
|
+
import { Firecrawl } from "../../../index";
|
|
5
|
+
import { getVersion } from "../../../v2/utils/getVersion";
|
|
6
|
+
import { detectCallerPackage } from "../../../utils/callerOrigin";
|
|
7
|
+
|
|
8
|
+
function withCapture<T extends { http: any }>(client: T) {
|
|
9
|
+
const bodies: Array<Record<string, unknown>> = [];
|
|
10
|
+
const adapter: AxiosAdapter = async config => {
|
|
11
|
+
bodies.push(JSON.parse(config.data));
|
|
12
|
+
return {
|
|
13
|
+
data: { success: true, data: { markdown: "# hello" } },
|
|
14
|
+
status: 200,
|
|
15
|
+
statusText: "OK",
|
|
16
|
+
headers: {},
|
|
17
|
+
config,
|
|
18
|
+
};
|
|
19
|
+
};
|
|
20
|
+
client.http.instance.defaults.adapter = adapter;
|
|
21
|
+
return { client, bodies };
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
describe("v2 request origin", () => {
|
|
25
|
+
const previousOptOut = process.env.FIRECRAWL_DISABLE_CALLER_ORIGIN;
|
|
26
|
+
afterEach(() => {
|
|
27
|
+
if (previousOptOut === undefined) {
|
|
28
|
+
delete process.env.FIRECRAWL_DISABLE_CALLER_ORIGIN;
|
|
29
|
+
} else {
|
|
30
|
+
process.env.FIRECRAWL_DISABLE_CALLER_ORIGIN = previousOptOut;
|
|
31
|
+
}
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
test("defaults to the plain SDK origin when application code creates the client", async () => {
|
|
35
|
+
const { client, bodies } = withCapture(new FirecrawlClient({ apiKey: "fc-test" }));
|
|
36
|
+
await client.scrape("https://example.com");
|
|
37
|
+
expect(bodies[0]!.origin).toBe(`js-sdk@${getVersion()}`);
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
test("the origin client option is stamped into request bodies", async () => {
|
|
41
|
+
const { client, bodies } = withCapture(
|
|
42
|
+
new FirecrawlClient({ apiKey: "fc-test", origin: "js-sdk.example@1.0.0" }),
|
|
43
|
+
);
|
|
44
|
+
await client.scrape("https://example.com");
|
|
45
|
+
expect(bodies[0]!.origin).toBe("js-sdk.example@1.0.0");
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
test("a per-call origin wins over the client origin", async () => {
|
|
49
|
+
const { client, bodies } = withCapture(
|
|
50
|
+
new FirecrawlClient({ apiKey: "fc-test", origin: "js-sdk.example@1.0.0" }),
|
|
51
|
+
);
|
|
52
|
+
await client.scrape("https://example.com", { origin: "per-call" } as any);
|
|
53
|
+
await client.scrape("https://example.com", { origin: "mcp-server" } as any);
|
|
54
|
+
expect(bodies.map(b => b.origin)).toEqual(["per-call", "mcp-server"]);
|
|
55
|
+
});
|
|
56
|
+
|
|
57
|
+
test("the unified client forwards the origin option to v1", () => {
|
|
58
|
+
const app = new Firecrawl({ apiKey: "fc-test", origin: "js-sdk.example@1.0.0" });
|
|
59
|
+
expect((app.v1 as any).clientOrigin).toBe("js-sdk.example@1.0.0");
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
describe("detectCallerPackage", () => {
|
|
65
|
+
const sdkFrames = [
|
|
66
|
+
"Error",
|
|
67
|
+
" at detectCallerPackage (file:///srv/app/node_modules/@mendable/firecrawl-js/dist/index.js:10:5)",
|
|
68
|
+
" at new FirecrawlClient (file:///srv/app/node_modules/@mendable/firecrawl-js/dist/index.js:20:9)",
|
|
69
|
+
];
|
|
70
|
+
const stackFrom = (...callers: string[]) =>
|
|
71
|
+
[...sdkFrames, ...callers.map(c => ` at ${c}`)].join("\n");
|
|
72
|
+
const previousOptOut = process.env.FIRECRAWL_DISABLE_CALLER_ORIGIN;
|
|
73
|
+
|
|
74
|
+
beforeEach(() => {
|
|
75
|
+
delete process.env.FIRECRAWL_DISABLE_CALLER_ORIGIN;
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
afterEach(() => {
|
|
79
|
+
if (previousOptOut === undefined) {
|
|
80
|
+
delete process.env.FIRECRAWL_DISABLE_CALLER_ORIGIN;
|
|
81
|
+
} else {
|
|
82
|
+
process.env.FIRECRAWL_DISABLE_CALLER_ORIGIN = previousOptOut;
|
|
83
|
+
}
|
|
84
|
+
});
|
|
85
|
+
|
|
86
|
+
test("names the scoped package that created the client", () => {
|
|
87
|
+
const stack = stackFrom(
|
|
88
|
+
"make (file:///srv/app/node_modules/@Scope/Some_Agent/dist/index.js:3:12)",
|
|
89
|
+
"main (file:///srv/app/src/main.js:1:1)",
|
|
90
|
+
);
|
|
91
|
+
expect(detectCallerPackage(stack)).toBe("scope-some-agent");
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
test("uses the last node_modules segment of a nested install", () => {
|
|
95
|
+
const stack = stackFrom(
|
|
96
|
+
"make (/srv/app/node_modules/.pnpm/outer@1.0.0/node_modules/inner-pkg/lib/index.cjs:3:12)",
|
|
97
|
+
);
|
|
98
|
+
expect(detectCallerPackage(stack)).toBe("inner-pkg");
|
|
99
|
+
});
|
|
100
|
+
|
|
101
|
+
test("handles windows paths with parentheses and anonymous frames", () => {
|
|
102
|
+
const stack = stackFrom(
|
|
103
|
+
"Array.map (<anonymous>)",
|
|
104
|
+
"C:\\Program Files (x86)\\app\\node_modules\\wrapper\\index.js:4:2",
|
|
105
|
+
);
|
|
106
|
+
expect(detectCallerPackage(stack)).toBe("wrapper");
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
test("returns null when application code created the client", () => {
|
|
110
|
+
const stack = stackFrom(
|
|
111
|
+
"main (file:///srv/app/src/main.js:1:1)",
|
|
112
|
+
"make (file:///srv/app/node_modules/wrapper/index.js:3:12)",
|
|
113
|
+
);
|
|
114
|
+
expect(detectCallerPackage(stack)).toBeNull();
|
|
115
|
+
});
|
|
116
|
+
|
|
117
|
+
test("returns null for a package whose slug would read as another traffic class", () => {
|
|
118
|
+
const stack = stackFrom("make (/srv/app/node_modules/my-playground-kit/index.js:3:12)");
|
|
119
|
+
expect(detectCallerPackage(stack)).toBeNull();
|
|
120
|
+
});
|
|
121
|
+
|
|
122
|
+
test("returns null for a bundle without node_modules in its path", () => {
|
|
123
|
+
expect(detectCallerPackage(stackFrom("make (/srv/app/dist/bundle.js:1:1)"))).toBeNull();
|
|
124
|
+
});
|
|
125
|
+
|
|
126
|
+
test("FIRECRAWL_DISABLE_CALLER_ORIGIN turns detection off", () => {
|
|
127
|
+
const stack = stackFrom("make (/srv/app/node_modules/wrapper/index.js:3:12)");
|
|
128
|
+
expect(detectCallerPackage(stack)).toBe("wrapper");
|
|
129
|
+
process.env.FIRECRAWL_DISABLE_CALLER_ORIGIN = "1";
|
|
130
|
+
expect(detectCallerPackage(stack)).toBeNull();
|
|
131
|
+
});
|
|
132
|
+
});
|
package/src/index.ts
CHANGED
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
const NODE_MODULES = "/node_modules/";
|
|
2
|
+
|
|
3
|
+
function packageOf(file: string): string | null {
|
|
4
|
+
let path = file.replace(/^file:\/\//, "").replace(/\\/g, "/");
|
|
5
|
+
try {
|
|
6
|
+
path = decodeURIComponent(path);
|
|
7
|
+
} catch {}
|
|
8
|
+
const at = path.lastIndexOf(NODE_MODULES);
|
|
9
|
+
if (at < 0) return null;
|
|
10
|
+
const [first, second] = path.slice(at + NODE_MODULES.length).split("/");
|
|
11
|
+
const name = first?.startsWith("@") ? (second ? `${first}/${second}` : "") : first;
|
|
12
|
+
return name && !name.startsWith(".") ? name : null;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
/** File of a V8 stack line, or null for frames without one (native, anonymous, runtime internals). */
|
|
16
|
+
function frameFile(line: string): string | null {
|
|
17
|
+
const text = line.trim().replace(/^at /, "");
|
|
18
|
+
const open = text.indexOf(" (");
|
|
19
|
+
const location = open >= 0 && text.endsWith(")") ? text.slice(open + 2, -1) : text;
|
|
20
|
+
const match = /^(.*):\d+:\d+$/.exec(location);
|
|
21
|
+
return match && !match[1]!.startsWith("node:") ? match[1]! : null;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
// Downstream surface classification matches "playground" anywhere in an origin.
|
|
25
|
+
function slugify(name: string): string {
|
|
26
|
+
return name
|
|
27
|
+
.toLowerCase()
|
|
28
|
+
.replace(/[^a-z0-9]+/g, "-")
|
|
29
|
+
.replace(/^-+|-+$/g, "")
|
|
30
|
+
.slice(0, 48)
|
|
31
|
+
.replace(/-+$/, "");
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Slug of the installed package that created the client, or null when the
|
|
36
|
+
* caller is application code or detection is disabled. Never throws.
|
|
37
|
+
*/
|
|
38
|
+
export function detectCallerPackage(stack?: string): string | null {
|
|
39
|
+
try {
|
|
40
|
+
if (process.env.FIRECRAWL_DISABLE_CALLER_ORIGIN === "1") return null;
|
|
41
|
+
if (stack === undefined) {
|
|
42
|
+
const limit = Error.stackTraceLimit;
|
|
43
|
+
Error.stackTraceLimit = 50;
|
|
44
|
+
try {
|
|
45
|
+
stack = new Error().stack ?? "";
|
|
46
|
+
} finally {
|
|
47
|
+
Error.stackTraceLimit = limit;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
let ownFile: string | null = null;
|
|
51
|
+
let ownPackage: string | null = null;
|
|
52
|
+
for (const line of stack.split("\n")) {
|
|
53
|
+
if (!/^\s*at /.test(line)) continue;
|
|
54
|
+
const file = frameFile(line);
|
|
55
|
+
if (!file) continue;
|
|
56
|
+
if (ownFile === null) {
|
|
57
|
+
ownFile = file;
|
|
58
|
+
ownPackage = packageOf(file);
|
|
59
|
+
}
|
|
60
|
+
if (file === ownFile) continue;
|
|
61
|
+
const name = packageOf(file);
|
|
62
|
+
if (name && name === ownPackage) continue;
|
|
63
|
+
const slug = name ? slugify(name) : "";
|
|
64
|
+
return slug && !slug.includes("playground") ? slug : null;
|
|
65
|
+
}
|
|
66
|
+
} catch {}
|
|
67
|
+
return null;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/** Origin sent with requests: `js-sdk@<version>`, or `js-sdk.<package>@<version>` for wrapper libraries. */
|
|
71
|
+
export function defaultOrigin(version: string): string {
|
|
72
|
+
const caller = detectCallerPackage();
|
|
73
|
+
return caller ? `js-sdk.${caller}@${version}` : `js-sdk@${version}`;
|
|
74
|
+
}
|