firecrawl 4.47.0 → 4.49.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -1
- package/dist/{chunk-H75FJ6LW.js → chunk-JNLB66PW.js} +1 -1
- package/dist/index.cjs +158 -46
- package/dist/index.d.cts +49 -23
- package/dist/index.d.ts +49 -23
- package/dist/index.js +159 -47
- package/dist/{package-NZI5TDUV.js → package-7E5DZETG.js} +1 -1
- package/package.json +1 -1
- package/src/__tests__/unit/v2/agent-hints.test.ts +1 -0
- package/src/__tests__/unit/v2/origin.test.ts +132 -0
- package/src/__tests__/unit/v2/research.test.ts +1 -0
- package/src/__tests__/unit/v2/v1-aliases.test.ts +89 -4
- package/src/index.ts +1 -0
- package/src/utils/callerOrigin.ts +74 -0
- package/src/v1/index.ts +28 -13
- package/src/v2/client.ts +41 -31
- package/src/v2/methods/parse.ts +1 -6
- package/src/v2/methods/research.ts +20 -7
- package/src/v2/utils/httpClient.ts +10 -3
- package/src/v2/utils/v1Compat.ts +38 -0
package/dist/index.d.ts
CHANGED
|
@@ -2019,6 +2019,7 @@ interface HttpClientOptions {
|
|
|
2019
2019
|
timeoutMs?: number;
|
|
2020
2020
|
maxRetries?: number;
|
|
2021
2021
|
backoffFactor?: number;
|
|
2022
|
+
origin?: string;
|
|
2022
2023
|
}
|
|
2023
2024
|
interface RequestOptions {
|
|
2024
2025
|
headers?: Record<string, string>;
|
|
@@ -2030,9 +2031,12 @@ declare class HttpClient {
|
|
|
2030
2031
|
private readonly apiUrl;
|
|
2031
2032
|
private readonly maxRetries;
|
|
2032
2033
|
private readonly backoffFactor;
|
|
2034
|
+
private readonly origin;
|
|
2033
2035
|
constructor(options: HttpClientOptions);
|
|
2034
2036
|
getApiUrl(): string;
|
|
2035
2037
|
getApiKey(): string;
|
|
2038
|
+
/** The attribution origin stamped into requests that do not set their own. */
|
|
2039
|
+
getOrigin(): string;
|
|
2036
2040
|
/** The client-wide default request timeout, for callers that need to
|
|
2037
2041
|
* extend a specific request beyond it (e.g. scrape auto-resume). */
|
|
2038
2042
|
getTimeoutMs(): number;
|
|
@@ -2244,6 +2248,17 @@ declare class Watcher extends EventEmitter {
|
|
|
2244
2248
|
close(): void;
|
|
2245
2249
|
}
|
|
2246
2250
|
|
|
2251
|
+
/** v1 crawl options that have a v2 equivalent. */
|
|
2252
|
+
type V1CrawlOptions = {
|
|
2253
|
+
maxDepth?: number;
|
|
2254
|
+
allowBackwardLinks?: boolean;
|
|
2255
|
+
ignoreSitemap?: boolean;
|
|
2256
|
+
};
|
|
2257
|
+
/** A v2 result with the `success` flag v1 results carried. */
|
|
2258
|
+
type V1Result<T> = T & {
|
|
2259
|
+
success: true;
|
|
2260
|
+
};
|
|
2261
|
+
|
|
2247
2262
|
type ExtractJsonSchemaFromFormats<Formats> = Formats extends readonly any[] ? Extract<Formats[number], {
|
|
2248
2263
|
type: "json";
|
|
2249
2264
|
schema?: unknown;
|
|
@@ -2265,6 +2280,12 @@ interface FirecrawlClientOptions {
|
|
|
2265
2280
|
maxRetries?: number;
|
|
2266
2281
|
/** Exponential backoff factor for retries (optional). */
|
|
2267
2282
|
backoffFactor?: number;
|
|
2283
|
+
/**
|
|
2284
|
+
* Attribution tag stamped into request bodies. Defaults to `js-sdk@<version>`,
|
|
2285
|
+
* or `js-sdk.<package>@<version>` when an installed package creates the client
|
|
2286
|
+
* (set FIRECRAWL_DISABLE_CALLER_ORIGIN=1 to keep the plain default).
|
|
2287
|
+
*/
|
|
2288
|
+
origin?: string;
|
|
2268
2289
|
}
|
|
2269
2290
|
/** Accepts a plain API key string or a full options object. */
|
|
2270
2291
|
type FirecrawlClientInput = FirecrawlClientOptions | string;
|
|
@@ -2272,7 +2293,7 @@ type FirecrawlClientInput = FirecrawlClientOptions | string;
|
|
|
2272
2293
|
* Firecrawl v2 client. Provides typed access to all v2 endpoints and utilities.
|
|
2273
2294
|
*/
|
|
2274
2295
|
declare class FirecrawlClient {
|
|
2275
|
-
|
|
2296
|
+
protected readonly http: HttpClient;
|
|
2276
2297
|
private _research?;
|
|
2277
2298
|
private isCloudService;
|
|
2278
2299
|
/**
|
|
@@ -2629,43 +2650,45 @@ declare class FirecrawlClient {
|
|
|
2629
2650
|
* @param opts Watcher options (kind, pollInterval, timeout seconds).
|
|
2630
2651
|
*/
|
|
2631
2652
|
watcher(jobId: string, opts?: WatcherOptions): Watcher;
|
|
2632
|
-
/** @deprecated
|
|
2633
|
-
scrapeUrl(url: string, options?: ScrapeOptions): Promise<Document
|
|
2634
|
-
/** @deprecated
|
|
2635
|
-
crawlUrl(url: string, req?: CrawlOptions & {
|
|
2653
|
+
/** @deprecated v1 alias for scrape(); its result also carries v1's `success: true`. */
|
|
2654
|
+
scrapeUrl(url: string, options?: ScrapeOptions): Promise<V1Result<Document>>;
|
|
2655
|
+
/** @deprecated v1 alias for crawl(); its result also carries v1's `success: true`. */
|
|
2656
|
+
crawlUrl(url: string, req?: CrawlOptions & V1CrawlOptions & {
|
|
2636
2657
|
pollInterval?: number;
|
|
2637
2658
|
timeout?: number;
|
|
2638
2659
|
signal?: AbortSignal;
|
|
2639
|
-
}): Promise<CrawlJob
|
|
2640
|
-
/** @deprecated
|
|
2641
|
-
asyncCrawlUrl(url: string, req?: CrawlOptions): Promise<CrawlResponse$1
|
|
2642
|
-
/** @deprecated
|
|
2643
|
-
checkCrawlStatus(jobId: string, pagination?: PaginationConfig): Promise<CrawlJob
|
|
2644
|
-
/** @deprecated
|
|
2645
|
-
checkCrawlErrors(crawlId: string): Promise<CrawlErrorsResponse$1
|
|
2646
|
-
/** @deprecated
|
|
2647
|
-
mapUrl(url: string, options?: MapOptions): Promise<MapData
|
|
2648
|
-
/** @deprecated
|
|
2660
|
+
}): Promise<V1Result<CrawlJob>>;
|
|
2661
|
+
/** @deprecated v1 alias for startCrawl(); its result also carries v1's `success: true`. */
|
|
2662
|
+
asyncCrawlUrl(url: string, req?: CrawlOptions & V1CrawlOptions): Promise<V1Result<CrawlResponse$1>>;
|
|
2663
|
+
/** @deprecated v1 alias for getCrawlStatus(); its result also carries v1's `success: true`. */
|
|
2664
|
+
checkCrawlStatus(jobId: string, pagination?: PaginationConfig): Promise<V1Result<CrawlJob>>;
|
|
2665
|
+
/** @deprecated v1 alias for getCrawlErrors(); its result also carries v1's `success: true`. */
|
|
2666
|
+
checkCrawlErrors(crawlId: string): Promise<V1Result<CrawlErrorsResponse$1>>;
|
|
2667
|
+
/** @deprecated v1 alias for map(); its result also carries v1's `success: true`. */
|
|
2668
|
+
mapUrl(url: string, options?: MapOptions): Promise<V1Result<MapData>>;
|
|
2669
|
+
/** @deprecated v1 alias for batchScrape(); its result also carries v1's `success: true`. */
|
|
2649
2670
|
batchScrapeUrls(urls: string[], opts?: BatchScrapeOptions & {
|
|
2650
2671
|
pollInterval?: number;
|
|
2651
2672
|
timeout?: number;
|
|
2652
|
-
}): Promise<BatchScrapeJob
|
|
2653
|
-
/** @deprecated
|
|
2654
|
-
asyncBatchScrapeUrls(urls: string[], opts?: BatchScrapeOptions): Promise<BatchScrapeResponse$1
|
|
2655
|
-
/** @deprecated
|
|
2656
|
-
checkBatchScrapeStatus(jobId: string, pagination?: PaginationConfig): Promise<BatchScrapeJob
|
|
2657
|
-
/** @deprecated
|
|
2658
|
-
checkBatchScrapeErrors(jobId: string): Promise<CrawlErrorsResponse$1
|
|
2673
|
+
}): Promise<V1Result<BatchScrapeJob>>;
|
|
2674
|
+
/** @deprecated v1 alias for startBatchScrape(); its result also carries v1's `success: true`. */
|
|
2675
|
+
asyncBatchScrapeUrls(urls: string[], opts?: BatchScrapeOptions): Promise<V1Result<BatchScrapeResponse$1>>;
|
|
2676
|
+
/** @deprecated v1 alias for getBatchScrapeStatus(); its result also carries v1's `success: true`. */
|
|
2677
|
+
checkBatchScrapeStatus(jobId: string, pagination?: PaginationConfig): Promise<V1Result<BatchScrapeJob>>;
|
|
2678
|
+
/** @deprecated v1 alias for getBatchScrapeErrors(); its result also carries v1's `success: true`. */
|
|
2679
|
+
checkBatchScrapeErrors(jobId: string): Promise<V1Result<CrawlErrorsResponse$1>>;
|
|
2659
2680
|
}
|
|
2660
2681
|
|
|
2661
2682
|
/**
|
|
2662
2683
|
* Configuration interface for FirecrawlApp.
|
|
2663
2684
|
* @param apiKey - Optional API key for authentication.
|
|
2664
2685
|
* @param apiUrl - Optional base URL of the API; defaults to 'https://api.firecrawl.dev'.
|
|
2686
|
+
* @param origin - Optional attribution tag stamped into request bodies.
|
|
2665
2687
|
*/
|
|
2666
2688
|
interface FirecrawlAppConfig {
|
|
2667
2689
|
apiKey?: string | null;
|
|
2668
2690
|
apiUrl?: string | null;
|
|
2691
|
+
origin?: string;
|
|
2669
2692
|
}
|
|
2670
2693
|
/**
|
|
2671
2694
|
* Metadata for a Firecrawl document.
|
|
@@ -3262,6 +3285,8 @@ declare class FirecrawlApp {
|
|
|
3262
3285
|
apiKey: string;
|
|
3263
3286
|
apiUrl: string;
|
|
3264
3287
|
version: string;
|
|
3288
|
+
private readonly clientOrigin?;
|
|
3289
|
+
private readonly callerPackage;
|
|
3265
3290
|
private isCloudService;
|
|
3266
3291
|
private getVersion;
|
|
3267
3292
|
private init;
|
|
@@ -3269,7 +3294,8 @@ declare class FirecrawlApp {
|
|
|
3269
3294
|
* Initializes a new instance of the FirecrawlApp class.
|
|
3270
3295
|
* @param config - Configuration options for the FirecrawlApp instance.
|
|
3271
3296
|
*/
|
|
3272
|
-
constructor({ apiKey, apiUrl }: FirecrawlAppConfig);
|
|
3297
|
+
constructor({ apiKey, apiUrl, origin }: FirecrawlAppConfig);
|
|
3298
|
+
private resolveOrigin;
|
|
3273
3299
|
/**
|
|
3274
3300
|
* Scrapes a URL using the Firecrawl API.
|
|
3275
3301
|
* @param url - The URL to scrape.
|
package/dist/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
version
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-JNLB66PW.js";
|
|
4
4
|
|
|
5
5
|
// src/v2/utils/httpClient.ts
|
|
6
6
|
import axios from "axios";
|
|
@@ -32,11 +32,13 @@ var HttpClient = class {
|
|
|
32
32
|
apiUrl;
|
|
33
33
|
maxRetries;
|
|
34
34
|
backoffFactor;
|
|
35
|
+
origin;
|
|
35
36
|
constructor(options) {
|
|
36
37
|
this.apiKey = options.apiKey;
|
|
37
38
|
this.apiUrl = options.apiUrl.replace(/\/$/, "");
|
|
38
39
|
this.maxRetries = options.maxRetries ?? 3;
|
|
39
40
|
this.backoffFactor = options.backoffFactor ?? 0.5;
|
|
41
|
+
this.origin = options.origin || `js-sdk@${getVersion()}`;
|
|
40
42
|
this.instance = axios.create({
|
|
41
43
|
baseURL: this.apiUrl,
|
|
42
44
|
timeout: options.timeoutMs ?? 3e5,
|
|
@@ -55,13 +57,16 @@ var HttpClient = class {
|
|
|
55
57
|
getApiKey() {
|
|
56
58
|
return this.apiKey;
|
|
57
59
|
}
|
|
60
|
+
/** The attribution origin stamped into requests that do not set their own. */
|
|
61
|
+
getOrigin() {
|
|
62
|
+
return this.origin;
|
|
63
|
+
}
|
|
58
64
|
/** The client-wide default request timeout, for callers that need to
|
|
59
65
|
* extend a specific request beyond it (e.g. scrape auto-resume). */
|
|
60
66
|
getTimeoutMs() {
|
|
61
67
|
return this.instance.defaults.timeout ?? 3e5;
|
|
62
68
|
}
|
|
63
69
|
async request(config) {
|
|
64
|
-
const version2 = getVersion();
|
|
65
70
|
config.headers = {
|
|
66
71
|
...config.headers || {}
|
|
67
72
|
};
|
|
@@ -76,7 +81,7 @@ var HttpClient = class {
|
|
|
76
81
|
const data = cfg.data ?? {};
|
|
77
82
|
cfg.data = {
|
|
78
83
|
...data,
|
|
79
|
-
origin: typeof data.origin === "string" && data.origin
|
|
84
|
+
origin: typeof data.origin === "string" && data.origin ? data.origin : this.origin
|
|
80
85
|
};
|
|
81
86
|
}
|
|
82
87
|
if (isFormDataBody) {
|
|
@@ -659,10 +664,9 @@ async function parse(http, file, options) {
|
|
|
659
664
|
throw new Error("file data cannot be empty");
|
|
660
665
|
}
|
|
661
666
|
if (options) ensureValidParseOptions(options);
|
|
662
|
-
const version2 = getVersion();
|
|
663
667
|
const normalizedOptions = {
|
|
664
668
|
...options ?? {},
|
|
665
|
-
origin:
|
|
669
|
+
origin: options?.origin || http.getOrigin()
|
|
666
670
|
};
|
|
667
671
|
const formData = new FormData();
|
|
668
672
|
formData.append("options", JSON.stringify(normalizedOptions));
|
|
@@ -1780,8 +1784,8 @@ function appendParam(params, key, value) {
|
|
|
1780
1784
|
params.append(key, String(value));
|
|
1781
1785
|
}
|
|
1782
1786
|
}
|
|
1783
|
-
function withQuery(path, params) {
|
|
1784
|
-
params.append("origin",
|
|
1787
|
+
function withQuery(origin, path, params) {
|
|
1788
|
+
params.append("origin", origin);
|
|
1785
1789
|
const qs = params.toString();
|
|
1786
1790
|
return qs ? `${path}?${qs}` : path;
|
|
1787
1791
|
}
|
|
@@ -1836,7 +1840,7 @@ var ResearchClient = class {
|
|
|
1836
1840
|
appendParam(params, "to", options.to);
|
|
1837
1841
|
try {
|
|
1838
1842
|
const res = await this.http.get(
|
|
1839
|
-
withQuery(`${BASE}/papers`, params)
|
|
1843
|
+
withQuery(this.http.getOrigin(), `${BASE}/papers`, params)
|
|
1840
1844
|
);
|
|
1841
1845
|
if (res.status !== 200) throwForBadResponse(res, "search papers");
|
|
1842
1846
|
return res.data;
|
|
@@ -1854,7 +1858,13 @@ var ResearchClient = class {
|
|
|
1854
1858
|
appendParam(params, "query", options.query);
|
|
1855
1859
|
appendParam(params, "k", options.k);
|
|
1856
1860
|
try {
|
|
1857
|
-
const res = await this.http.get(
|
|
1861
|
+
const res = await this.http.get(
|
|
1862
|
+
withQuery(
|
|
1863
|
+
this.http.getOrigin(),
|
|
1864
|
+
`${BASE}/papers/${encodeURIComponent(id)}`,
|
|
1865
|
+
params
|
|
1866
|
+
)
|
|
1867
|
+
);
|
|
1858
1868
|
if (res.status !== 200) throwForBadResponse(res, "get paper");
|
|
1859
1869
|
return res.data;
|
|
1860
1870
|
} catch (err) {
|
|
@@ -1880,7 +1890,11 @@ var ResearchClient = class {
|
|
|
1880
1890
|
appendParam(params, "anchor", options.anchor);
|
|
1881
1891
|
try {
|
|
1882
1892
|
const res = await this.http.get(
|
|
1883
|
-
withQuery(
|
|
1893
|
+
withQuery(
|
|
1894
|
+
this.http.getOrigin(),
|
|
1895
|
+
`${BASE}/papers/${encodeURIComponent(id)}/similar`,
|
|
1896
|
+
params
|
|
1897
|
+
)
|
|
1884
1898
|
);
|
|
1885
1899
|
if (res.status !== 200) throwForBadResponse(res, "find similar papers");
|
|
1886
1900
|
return res.data;
|
|
@@ -1911,7 +1925,7 @@ var ResearchClient = class {
|
|
|
1911
1925
|
appendParam(params, "k", options.k);
|
|
1912
1926
|
try {
|
|
1913
1927
|
const res = await this.http.get(
|
|
1914
|
-
withQuery(`${BASE}/github`, params)
|
|
1928
|
+
withQuery(this.http.getOrigin(), `${BASE}/github`, params)
|
|
1915
1929
|
);
|
|
1916
1930
|
if (res.status !== 200) throwForBadResponse(res, "search github");
|
|
1917
1931
|
return res.data;
|
|
@@ -2041,6 +2055,67 @@ async function getMonitorCheck(http, monitorId, checkId, options) {
|
|
|
2041
2055
|
}
|
|
2042
2056
|
}
|
|
2043
2057
|
|
|
2058
|
+
// src/utils/callerOrigin.ts
|
|
2059
|
+
var NODE_MODULES = "/node_modules/";
|
|
2060
|
+
function packageOf(file) {
|
|
2061
|
+
let path = file.replace(/^file:\/\//, "").replace(/\\/g, "/");
|
|
2062
|
+
try {
|
|
2063
|
+
path = decodeURIComponent(path);
|
|
2064
|
+
} catch {
|
|
2065
|
+
}
|
|
2066
|
+
const at = path.lastIndexOf(NODE_MODULES);
|
|
2067
|
+
if (at < 0) return null;
|
|
2068
|
+
const [first, second] = path.slice(at + NODE_MODULES.length).split("/");
|
|
2069
|
+
const name = first?.startsWith("@") ? second ? `${first}/${second}` : "" : first;
|
|
2070
|
+
return name && !name.startsWith(".") ? name : null;
|
|
2071
|
+
}
|
|
2072
|
+
function frameFile(line) {
|
|
2073
|
+
const text = line.trim().replace(/^at /, "");
|
|
2074
|
+
const open = text.indexOf(" (");
|
|
2075
|
+
const location = open >= 0 && text.endsWith(")") ? text.slice(open + 2, -1) : text;
|
|
2076
|
+
const match = /^(.*):\d+:\d+$/.exec(location);
|
|
2077
|
+
return match && !match[1].startsWith("node:") ? match[1] : null;
|
|
2078
|
+
}
|
|
2079
|
+
function slugify(name) {
|
|
2080
|
+
return name.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 48).replace(/-+$/, "");
|
|
2081
|
+
}
|
|
2082
|
+
function detectCallerPackage(stack) {
|
|
2083
|
+
try {
|
|
2084
|
+
if (process.env.FIRECRAWL_DISABLE_CALLER_ORIGIN === "1") return null;
|
|
2085
|
+
if (stack === void 0) {
|
|
2086
|
+
const limit = Error.stackTraceLimit;
|
|
2087
|
+
Error.stackTraceLimit = 50;
|
|
2088
|
+
try {
|
|
2089
|
+
stack = new Error().stack ?? "";
|
|
2090
|
+
} finally {
|
|
2091
|
+
Error.stackTraceLimit = limit;
|
|
2092
|
+
}
|
|
2093
|
+
}
|
|
2094
|
+
let ownFile = null;
|
|
2095
|
+
let ownPackage = null;
|
|
2096
|
+
for (const line of stack.split("\n")) {
|
|
2097
|
+
if (!/^\s*at /.test(line)) continue;
|
|
2098
|
+
const file = frameFile(line);
|
|
2099
|
+
if (!file) continue;
|
|
2100
|
+
if (ownFile === null) {
|
|
2101
|
+
ownFile = file;
|
|
2102
|
+
ownPackage = packageOf(file);
|
|
2103
|
+
}
|
|
2104
|
+
if (file === ownFile) continue;
|
|
2105
|
+
const name = packageOf(file);
|
|
2106
|
+
if (name && name === ownPackage) continue;
|
|
2107
|
+
const slug = name ? slugify(name) : "";
|
|
2108
|
+
return slug && !slug.includes("playground") ? slug : null;
|
|
2109
|
+
}
|
|
2110
|
+
} catch {
|
|
2111
|
+
}
|
|
2112
|
+
return null;
|
|
2113
|
+
}
|
|
2114
|
+
function defaultOrigin(version2) {
|
|
2115
|
+
const caller = detectCallerPackage();
|
|
2116
|
+
return caller ? `js-sdk.${caller}@${version2}` : `js-sdk@${version2}`;
|
|
2117
|
+
}
|
|
2118
|
+
|
|
2044
2119
|
// src/v2/watcher.ts
|
|
2045
2120
|
import { EventEmitter } from "events";
|
|
2046
2121
|
var hasGlobalWebSocket = () => {
|
|
@@ -2282,6 +2357,32 @@ var Watcher = class extends EventEmitter {
|
|
|
2282
2357
|
}
|
|
2283
2358
|
};
|
|
2284
2359
|
|
|
2360
|
+
// src/v2/utils/v1Compat.ts
|
|
2361
|
+
var warned = false;
|
|
2362
|
+
function warnV1Usage() {
|
|
2363
|
+
if (warned) return;
|
|
2364
|
+
warned = true;
|
|
2365
|
+
const message = "firecrawl: v1-style methods (scrapeUrl, crawlUrl, batchScrapeUrls, ...) are deprecated and mapped to the v2 API. Use scrape(), crawl(), batchScrape() and the other v2 methods instead.";
|
|
2366
|
+
if (typeof process !== "undefined" && typeof process.emitWarning === "function") {
|
|
2367
|
+
process.emitWarning(message, { type: "DeprecationWarning", code: "FIRECRAWL_V1_METHOD" });
|
|
2368
|
+
} else {
|
|
2369
|
+
console.warn(message);
|
|
2370
|
+
}
|
|
2371
|
+
}
|
|
2372
|
+
function toV2CrawlOptions(req) {
|
|
2373
|
+
const { maxDepth, allowBackwardLinks, ignoreSitemap, ...rest } = req;
|
|
2374
|
+
return {
|
|
2375
|
+
...maxDepth != null && { maxDiscoveryDepth: maxDepth },
|
|
2376
|
+
...allowBackwardLinks != null && { crawlEntireDomain: allowBackwardLinks },
|
|
2377
|
+
...ignoreSitemap != null && { sitemap: ignoreSitemap ? "skip" : "include" },
|
|
2378
|
+
...rest
|
|
2379
|
+
};
|
|
2380
|
+
}
|
|
2381
|
+
async function callV1(call) {
|
|
2382
|
+
warnV1Usage();
|
|
2383
|
+
return { ...await call(), success: true };
|
|
2384
|
+
}
|
|
2385
|
+
|
|
2285
2386
|
// src/v2/client.ts
|
|
2286
2387
|
import "zod";
|
|
2287
2388
|
var FirecrawlClient = class {
|
|
@@ -2303,7 +2404,8 @@ var FirecrawlClient = class {
|
|
|
2303
2404
|
apiUrl,
|
|
2304
2405
|
timeoutMs: opts.timeoutMs,
|
|
2305
2406
|
maxRetries: opts.maxRetries,
|
|
2306
|
-
backoffFactor: opts.backoffFactor
|
|
2407
|
+
backoffFactor: opts.backoffFactor,
|
|
2408
|
+
origin: opts.origin || defaultOrigin(getVersion())
|
|
2307
2409
|
});
|
|
2308
2410
|
}
|
|
2309
2411
|
async scrape(url, options) {
|
|
@@ -2762,45 +2864,45 @@ var FirecrawlClient = class {
|
|
|
2762
2864
|
watcher(jobId, opts = {}) {
|
|
2763
2865
|
return new Watcher(this.http, jobId, opts);
|
|
2764
2866
|
}
|
|
2765
|
-
/** @deprecated
|
|
2867
|
+
/** @deprecated v1 alias for scrape(); its result also carries v1's `success: true`. */
|
|
2766
2868
|
async scrapeUrl(url, options) {
|
|
2767
|
-
return this.scrape(url, options);
|
|
2869
|
+
return callV1(() => this.scrape(url, options));
|
|
2768
2870
|
}
|
|
2769
|
-
/** @deprecated
|
|
2871
|
+
/** @deprecated v1 alias for crawl(); its result also carries v1's `success: true`. */
|
|
2770
2872
|
async crawlUrl(url, req = {}) {
|
|
2771
|
-
return this.crawl(url, req);
|
|
2873
|
+
return callV1(() => this.crawl(url, toV2CrawlOptions(req)));
|
|
2772
2874
|
}
|
|
2773
|
-
/** @deprecated
|
|
2875
|
+
/** @deprecated v1 alias for startCrawl(); its result also carries v1's `success: true`. */
|
|
2774
2876
|
async asyncCrawlUrl(url, req = {}) {
|
|
2775
|
-
return this.startCrawl(url, req);
|
|
2877
|
+
return callV1(() => this.startCrawl(url, toV2CrawlOptions(req)));
|
|
2776
2878
|
}
|
|
2777
|
-
/** @deprecated
|
|
2879
|
+
/** @deprecated v1 alias for getCrawlStatus(); its result also carries v1's `success: true`. */
|
|
2778
2880
|
async checkCrawlStatus(jobId, pagination) {
|
|
2779
|
-
return this.getCrawlStatus(jobId, pagination);
|
|
2881
|
+
return callV1(() => this.getCrawlStatus(jobId, pagination));
|
|
2780
2882
|
}
|
|
2781
|
-
/** @deprecated
|
|
2883
|
+
/** @deprecated v1 alias for getCrawlErrors(); its result also carries v1's `success: true`. */
|
|
2782
2884
|
async checkCrawlErrors(crawlId) {
|
|
2783
|
-
return this.getCrawlErrors(crawlId);
|
|
2885
|
+
return callV1(() => this.getCrawlErrors(crawlId));
|
|
2784
2886
|
}
|
|
2785
|
-
/** @deprecated
|
|
2887
|
+
/** @deprecated v1 alias for map(); its result also carries v1's `success: true`. */
|
|
2786
2888
|
async mapUrl(url, options) {
|
|
2787
|
-
return this.map(url, options);
|
|
2889
|
+
return callV1(() => this.map(url, options));
|
|
2788
2890
|
}
|
|
2789
|
-
/** @deprecated
|
|
2891
|
+
/** @deprecated v1 alias for batchScrape(); its result also carries v1's `success: true`. */
|
|
2790
2892
|
async batchScrapeUrls(urls, opts) {
|
|
2791
|
-
return this.batchScrape(urls, opts);
|
|
2893
|
+
return callV1(() => this.batchScrape(urls, opts));
|
|
2792
2894
|
}
|
|
2793
|
-
/** @deprecated
|
|
2895
|
+
/** @deprecated v1 alias for startBatchScrape(); its result also carries v1's `success: true`. */
|
|
2794
2896
|
async asyncBatchScrapeUrls(urls, opts) {
|
|
2795
|
-
return this.startBatchScrape(urls, opts);
|
|
2897
|
+
return callV1(() => this.startBatchScrape(urls, opts));
|
|
2796
2898
|
}
|
|
2797
|
-
/** @deprecated
|
|
2899
|
+
/** @deprecated v1 alias for getBatchScrapeStatus(); its result also carries v1's `success: true`. */
|
|
2798
2900
|
async checkBatchScrapeStatus(jobId, pagination) {
|
|
2799
|
-
return this.getBatchScrapeStatus(jobId, pagination);
|
|
2901
|
+
return callV1(() => this.getBatchScrapeStatus(jobId, pagination));
|
|
2800
2902
|
}
|
|
2801
|
-
/** @deprecated
|
|
2903
|
+
/** @deprecated v1 alias for getBatchScrapeErrors(); its result also carries v1's `success: true`. */
|
|
2802
2904
|
async checkBatchScrapeErrors(jobId) {
|
|
2803
|
-
return this.getBatchScrapeErrors(jobId);
|
|
2905
|
+
return callV1(() => this.getBatchScrapeErrors(jobId));
|
|
2804
2906
|
}
|
|
2805
2907
|
};
|
|
2806
2908
|
|
|
@@ -2829,6 +2931,8 @@ var FirecrawlApp = class {
|
|
|
2829
2931
|
apiKey;
|
|
2830
2932
|
apiUrl;
|
|
2831
2933
|
version = "1.25.1";
|
|
2934
|
+
clientOrigin;
|
|
2935
|
+
callerPackage;
|
|
2832
2936
|
isCloudService(url) {
|
|
2833
2937
|
return url.includes("api.firecrawl.dev");
|
|
2834
2938
|
}
|
|
@@ -2837,7 +2941,7 @@ var FirecrawlApp = class {
|
|
|
2837
2941
|
if (typeof process !== "undefined" && process.env && process.env.npm_package_version) {
|
|
2838
2942
|
return process.env.npm_package_version;
|
|
2839
2943
|
}
|
|
2840
|
-
const packageJson = await import("./package-
|
|
2944
|
+
const packageJson = await import("./package-7E5DZETG.js");
|
|
2841
2945
|
return packageJson.default.version;
|
|
2842
2946
|
} catch (error) {
|
|
2843
2947
|
const isTest = typeof process !== "undefined" && false;
|
|
@@ -2854,15 +2958,22 @@ var FirecrawlApp = class {
|
|
|
2854
2958
|
* Initializes a new instance of the FirecrawlApp class.
|
|
2855
2959
|
* @param config - Configuration options for the FirecrawlApp instance.
|
|
2856
2960
|
*/
|
|
2857
|
-
constructor({ apiKey = null, apiUrl = null }) {
|
|
2961
|
+
constructor({ apiKey = null, apiUrl = null, origin }) {
|
|
2858
2962
|
const baseUrl = apiUrl || "https://api.firecrawl.dev";
|
|
2859
2963
|
if (this.isCloudService(baseUrl) && typeof apiKey !== "string") {
|
|
2860
2964
|
throw new FirecrawlError("No API key provided", 401);
|
|
2861
2965
|
}
|
|
2862
2966
|
this.apiKey = apiKey || "";
|
|
2863
2967
|
this.apiUrl = baseUrl;
|
|
2968
|
+
this.clientOrigin = origin || void 0;
|
|
2969
|
+
this.callerPackage = this.clientOrigin ? null : detectCallerPackage();
|
|
2864
2970
|
this.init();
|
|
2865
2971
|
}
|
|
2972
|
+
resolveOrigin(params) {
|
|
2973
|
+
if (typeof params?.origin === "string" && params.origin) return params.origin;
|
|
2974
|
+
if (this.clientOrigin) return this.clientOrigin;
|
|
2975
|
+
return this.callerPackage ? `js-sdk.${this.callerPackage}@${this.version}` : `js-sdk@${this.version}`;
|
|
2976
|
+
}
|
|
2866
2977
|
/**
|
|
2867
2978
|
* Scrapes a URL using the Firecrawl API.
|
|
2868
2979
|
* @param url - The URL to scrape.
|
|
@@ -2874,7 +2985,7 @@ var FirecrawlApp = class {
|
|
|
2874
2985
|
"Content-Type": "application/json",
|
|
2875
2986
|
Authorization: `Bearer ${this.apiKey}`
|
|
2876
2987
|
};
|
|
2877
|
-
let jsonData = { url, ...params, origin:
|
|
2988
|
+
let jsonData = { url, ...params, origin: this.resolveOrigin(params) };
|
|
2878
2989
|
if (jsonData?.extract?.schema) {
|
|
2879
2990
|
jsonData = {
|
|
2880
2991
|
...jsonData,
|
|
@@ -2938,7 +3049,7 @@ var FirecrawlApp = class {
|
|
|
2938
3049
|
lang: params?.lang ?? "en",
|
|
2939
3050
|
country: params?.country ?? "us",
|
|
2940
3051
|
location: params?.location,
|
|
2941
|
-
origin:
|
|
3052
|
+
origin: this.resolveOrigin(params),
|
|
2942
3053
|
timeout: params?.timeout ?? 6e4,
|
|
2943
3054
|
scrapeOptions: params?.scrapeOptions ?? { formats: [] }
|
|
2944
3055
|
};
|
|
@@ -2993,7 +3104,7 @@ var FirecrawlApp = class {
|
|
|
2993
3104
|
*/
|
|
2994
3105
|
async crawlUrl(url, params, pollInterval = 2, idempotencyKey) {
|
|
2995
3106
|
const headers = this.prepareHeaders(idempotencyKey);
|
|
2996
|
-
let jsonData = { url, ...params, origin:
|
|
3107
|
+
let jsonData = { url, ...params, origin: this.resolveOrigin(params) };
|
|
2997
3108
|
try {
|
|
2998
3109
|
const response = await this.postRequest(
|
|
2999
3110
|
this.apiUrl + `/v1/crawl`,
|
|
@@ -3017,7 +3128,7 @@ var FirecrawlApp = class {
|
|
|
3017
3128
|
}
|
|
3018
3129
|
async asyncCrawlUrl(url, params, idempotencyKey) {
|
|
3019
3130
|
const headers = this.prepareHeaders(idempotencyKey);
|
|
3020
|
-
let jsonData = { url, ...params, origin:
|
|
3131
|
+
let jsonData = { url, ...params, origin: this.resolveOrigin(params) };
|
|
3021
3132
|
try {
|
|
3022
3133
|
const response = await this.postRequest(
|
|
3023
3134
|
this.apiUrl + `/v1/crawl`,
|
|
@@ -3176,7 +3287,7 @@ var FirecrawlApp = class {
|
|
|
3176
3287
|
*/
|
|
3177
3288
|
async mapUrl(url, params) {
|
|
3178
3289
|
const headers = this.prepareHeaders();
|
|
3179
|
-
let jsonData = { url, ...params, origin:
|
|
3290
|
+
let jsonData = { url, ...params, origin: this.resolveOrigin(params) };
|
|
3180
3291
|
try {
|
|
3181
3292
|
const response = await this.postRequest(
|
|
3182
3293
|
this.apiUrl + `/v1/map`,
|
|
@@ -3205,7 +3316,7 @@ var FirecrawlApp = class {
|
|
|
3205
3316
|
*/
|
|
3206
3317
|
async batchScrapeUrls(urls, params, pollInterval = 2, idempotencyKey, webhook, ignoreInvalidURLs, maxConcurrency) {
|
|
3207
3318
|
const headers = this.prepareHeaders(idempotencyKey);
|
|
3208
|
-
let jsonData = { urls, webhook, ignoreInvalidURLs, maxConcurrency, ...params, origin:
|
|
3319
|
+
let jsonData = { urls, webhook, ignoreInvalidURLs, maxConcurrency, ...params, origin: this.resolveOrigin(params) };
|
|
3209
3320
|
if (jsonData?.extract?.schema) {
|
|
3210
3321
|
jsonData = {
|
|
3211
3322
|
...jsonData,
|
|
@@ -3247,7 +3358,7 @@ var FirecrawlApp = class {
|
|
|
3247
3358
|
}
|
|
3248
3359
|
async asyncBatchScrapeUrls(urls, params, idempotencyKey, webhook, ignoreInvalidURLs) {
|
|
3249
3360
|
const headers = this.prepareHeaders(idempotencyKey);
|
|
3250
|
-
let jsonData = { urls, webhook, ignoreInvalidURLs, ...params, origin:
|
|
3361
|
+
let jsonData = { urls, webhook, ignoreInvalidURLs, ...params, origin: this.resolveOrigin(params) };
|
|
3251
3362
|
try {
|
|
3252
3363
|
const response = await this.postRequest(
|
|
3253
3364
|
this.apiUrl + `/v1/batch/scrape`,
|
|
@@ -3391,7 +3502,7 @@ var FirecrawlApp = class {
|
|
|
3391
3502
|
try {
|
|
3392
3503
|
const response = await this.postRequest(
|
|
3393
3504
|
this.apiUrl + `/v1/extract`,
|
|
3394
|
-
{ ...jsonData, schema: jsonSchema, origin:
|
|
3505
|
+
{ ...jsonData, schema: jsonSchema, origin: this.resolveOrigin(params) },
|
|
3395
3506
|
headers
|
|
3396
3507
|
);
|
|
3397
3508
|
if (response.status === 200) {
|
|
@@ -3444,7 +3555,7 @@ var FirecrawlApp = class {
|
|
|
3444
3555
|
try {
|
|
3445
3556
|
const response = await this.postRequest(
|
|
3446
3557
|
this.apiUrl + `/v1/extract`,
|
|
3447
|
-
{ ...jsonData, schema: jsonSchema, origin:
|
|
3558
|
+
{ ...jsonData, schema: jsonSchema, origin: this.resolveOrigin(params) },
|
|
3448
3559
|
headers
|
|
3449
3560
|
);
|
|
3450
3561
|
if (response.status === 200) {
|
|
@@ -3726,7 +3837,7 @@ var FirecrawlApp = class {
|
|
|
3726
3837
|
*/
|
|
3727
3838
|
async asyncDeepResearch(query, params) {
|
|
3728
3839
|
const headers = this.prepareHeaders();
|
|
3729
|
-
let jsonData = { query, ...params, origin:
|
|
3840
|
+
let jsonData = { query, ...params, origin: this.resolveOrigin(params) };
|
|
3730
3841
|
if (jsonData?.jsonOptions?.schema) {
|
|
3731
3842
|
jsonData = {
|
|
3732
3843
|
...jsonData,
|
|
@@ -3845,7 +3956,7 @@ var FirecrawlApp = class {
|
|
|
3845
3956
|
async __asyncDeepResearch(topic, params) {
|
|
3846
3957
|
const headers = this.prepareHeaders();
|
|
3847
3958
|
try {
|
|
3848
|
-
let jsonData = { topic, ...params, origin:
|
|
3959
|
+
let jsonData = { topic, ...params, origin: this.resolveOrigin(params) };
|
|
3849
3960
|
const response = await this.postRequest(
|
|
3850
3961
|
`${this.apiUrl}/v1/deep-research`,
|
|
3851
3962
|
jsonData,
|
|
@@ -3945,7 +4056,7 @@ var FirecrawlApp = class {
|
|
|
3945
4056
|
*/
|
|
3946
4057
|
async asyncGenerateLLMsText(url, params) {
|
|
3947
4058
|
const headers = this.prepareHeaders();
|
|
3948
|
-
let jsonData = { url, ...params, origin:
|
|
4059
|
+
let jsonData = { url, ...params, origin: this.resolveOrigin(params) };
|
|
3949
4060
|
try {
|
|
3950
4061
|
const response = await this.postRequest(
|
|
3951
4062
|
`${this.apiUrl}/v1/llmstxt`,
|
|
@@ -4213,7 +4324,8 @@ var Firecrawl = class extends FirecrawlClient {
|
|
|
4213
4324
|
super(resolved);
|
|
4214
4325
|
this._v1Opts = {
|
|
4215
4326
|
apiKey: resolved.apiKey,
|
|
4216
|
-
apiUrl: resolved.apiUrl
|
|
4327
|
+
apiUrl: resolved.apiUrl,
|
|
4328
|
+
origin: this.http.getOrigin()
|
|
4217
4329
|
};
|
|
4218
4330
|
}
|
|
4219
4331
|
/** Access the legacy v1 client (instantiated on first access). */
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "firecrawl",
|
|
3
|
-
"version": "4.
|
|
3
|
+
"version": "4.49.0",
|
|
4
4
|
"description": "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|