@mendable/firecrawl-js 4.46.0 → 4.48.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -8
- package/dist/chunk-GD463K4D.js +114 -0
- package/dist/index.cjs +163 -39
- package/dist/index.d.cts +24 -5
- package/dist/index.d.ts +24 -5
- package/dist/index.js +164 -41
- package/dist/package-WONXBIJS.js +40 -0
- package/package.json +1 -1
- package/src/__tests__/unit/v2/agent-hints.test.ts +1 -0
- package/src/__tests__/unit/v2/browser.unit.test.ts +28 -0
- package/src/__tests__/unit/v2/errorHandler.test.ts +23 -0
- package/src/__tests__/unit/v2/origin.test.ts +132 -0
- package/src/__tests__/unit/v2/pagination.test.ts +20 -0
- package/src/__tests__/unit/v2/research.test.ts +1 -0
- package/src/__tests__/unit/v2/validation.test.ts +39 -1
- package/src/index.ts +1 -0
- package/src/utils/callerOrigin.ts +74 -0
- package/src/v1/index.ts +28 -13
- package/src/v2/client.ts +11 -2
- package/src/v2/methods/browser.ts +3 -0
- package/src/v2/methods/crawl.ts +2 -1
- package/src/v2/methods/parse.ts +1 -6
- package/src/v2/methods/research.ts +20 -7
- package/src/v2/methods/search.ts +2 -1
- package/src/v2/types.ts +2 -2
- package/src/v2/utils/errorHandler.ts +23 -2
- package/src/v2/utils/getVersion.ts +4 -12
- package/src/v2/utils/httpClient.ts +10 -3
- package/src/v2/utils/pagination.ts +4 -1
- package/src/v2/utils/validation.ts +28 -0
- package/dist/chunk-R4XT7PME.js +0 -93
- package/dist/package-EI4Q33HD.js +0 -4
package/dist/index.d.cts
CHANGED
|
@@ -765,10 +765,10 @@ interface FindToolsOptions {
|
|
|
765
765
|
offset?: number;
|
|
766
766
|
}
|
|
767
767
|
interface FindToolsData {
|
|
768
|
-
level: "providers" | "groups" | "tools";
|
|
768
|
+
level: "categories" | "providers" | "groups" | "tools";
|
|
769
769
|
items: Array<{
|
|
770
770
|
id: string;
|
|
771
|
-
name
|
|
771
|
+
name?: string;
|
|
772
772
|
next?: AlexandriaCall;
|
|
773
773
|
execute?: Pick<AlexandriaCall, "provider" | "capability">;
|
|
774
774
|
[key: string]: unknown;
|
|
@@ -2019,6 +2019,7 @@ interface HttpClientOptions {
|
|
|
2019
2019
|
timeoutMs?: number;
|
|
2020
2020
|
maxRetries?: number;
|
|
2021
2021
|
backoffFactor?: number;
|
|
2022
|
+
origin?: string;
|
|
2022
2023
|
}
|
|
2023
2024
|
interface RequestOptions {
|
|
2024
2025
|
headers?: Record<string, string>;
|
|
@@ -2030,9 +2031,12 @@ declare class HttpClient {
|
|
|
2030
2031
|
private readonly apiUrl;
|
|
2031
2032
|
private readonly maxRetries;
|
|
2032
2033
|
private readonly backoffFactor;
|
|
2034
|
+
private readonly origin;
|
|
2033
2035
|
constructor(options: HttpClientOptions);
|
|
2034
2036
|
getApiUrl(): string;
|
|
2035
2037
|
getApiKey(): string;
|
|
2038
|
+
/** The attribution origin stamped into requests that do not set their own. */
|
|
2039
|
+
getOrigin(): string;
|
|
2036
2040
|
/** The client-wide default request timeout, for callers that need to
|
|
2037
2041
|
* extend a specific request beyond it (e.g. scrape auto-resume). */
|
|
2038
2042
|
getTimeoutMs(): number;
|
|
@@ -2119,6 +2123,10 @@ declare function browser(http: HttpClient, args?: {
|
|
|
2119
2123
|
name: string;
|
|
2120
2124
|
saveChanges?: boolean;
|
|
2121
2125
|
};
|
|
2126
|
+
/** Country the session browses from, as an ISO 3166-1 alpha-2 code (default US). */
|
|
2127
|
+
location?: {
|
|
2128
|
+
country: string;
|
|
2129
|
+
};
|
|
2122
2130
|
integration?: string;
|
|
2123
2131
|
origin?: string;
|
|
2124
2132
|
}): Promise<BrowserCreateResponse>;
|
|
@@ -2261,6 +2269,12 @@ interface FirecrawlClientOptions {
|
|
|
2261
2269
|
maxRetries?: number;
|
|
2262
2270
|
/** Exponential backoff factor for retries (optional). */
|
|
2263
2271
|
backoffFactor?: number;
|
|
2272
|
+
/**
|
|
2273
|
+
* Attribution tag stamped into request bodies. Defaults to `js-sdk@<version>`,
|
|
2274
|
+
* or `js-sdk.<package>@<version>` when an installed package creates the client
|
|
2275
|
+
* (set FIRECRAWL_DISABLE_CALLER_ORIGIN=1 to keep the plain default).
|
|
2276
|
+
*/
|
|
2277
|
+
origin?: string;
|
|
2264
2278
|
}
|
|
2265
2279
|
/** Accepts a plain API key string or a full options object. */
|
|
2266
2280
|
type FirecrawlClientInput = FirecrawlClientOptions | string;
|
|
@@ -2268,7 +2282,7 @@ type FirecrawlClientInput = FirecrawlClientOptions | string;
|
|
|
2268
2282
|
* Firecrawl v2 client. Provides typed access to all v2 endpoints and utilities.
|
|
2269
2283
|
*/
|
|
2270
2284
|
declare class FirecrawlClient {
|
|
2271
|
-
|
|
2285
|
+
protected readonly http: HttpClient;
|
|
2272
2286
|
private _research?;
|
|
2273
2287
|
private isCloudService;
|
|
2274
2288
|
/**
|
|
@@ -2585,7 +2599,7 @@ declare class FirecrawlClient {
|
|
|
2585
2599
|
}): Promise<AgentThreadResponse>;
|
|
2586
2600
|
/**
|
|
2587
2601
|
* Create a new browser session.
|
|
2588
|
-
* @param args Session options (ttl, activityTtl, streamWebView, blockAds, profile).
|
|
2602
|
+
* @param args Session options (ttl, activityTtl, streamWebView, blockAds, profile, location).
|
|
2589
2603
|
* @returns Session id, CDP URL, live view URL, and expiration time.
|
|
2590
2604
|
*/
|
|
2591
2605
|
browser(args?: Parameters<typeof browser>[1]): Promise<BrowserCreateResponse>;
|
|
@@ -2658,10 +2672,12 @@ declare class FirecrawlClient {
|
|
|
2658
2672
|
* Configuration interface for FirecrawlApp.
|
|
2659
2673
|
* @param apiKey - Optional API key for authentication.
|
|
2660
2674
|
* @param apiUrl - Optional base URL of the API; defaults to 'https://api.firecrawl.dev'.
|
|
2675
|
+
* @param origin - Optional attribution tag stamped into request bodies.
|
|
2661
2676
|
*/
|
|
2662
2677
|
interface FirecrawlAppConfig {
|
|
2663
2678
|
apiKey?: string | null;
|
|
2664
2679
|
apiUrl?: string | null;
|
|
2680
|
+
origin?: string;
|
|
2665
2681
|
}
|
|
2666
2682
|
/**
|
|
2667
2683
|
* Metadata for a Firecrawl document.
|
|
@@ -3258,6 +3274,8 @@ declare class FirecrawlApp {
|
|
|
3258
3274
|
apiKey: string;
|
|
3259
3275
|
apiUrl: string;
|
|
3260
3276
|
version: string;
|
|
3277
|
+
private readonly clientOrigin?;
|
|
3278
|
+
private readonly callerPackage;
|
|
3261
3279
|
private isCloudService;
|
|
3262
3280
|
private getVersion;
|
|
3263
3281
|
private init;
|
|
@@ -3265,7 +3283,8 @@ declare class FirecrawlApp {
|
|
|
3265
3283
|
* Initializes a new instance of the FirecrawlApp class.
|
|
3266
3284
|
* @param config - Configuration options for the FirecrawlApp instance.
|
|
3267
3285
|
*/
|
|
3268
|
-
constructor({ apiKey, apiUrl }: FirecrawlAppConfig);
|
|
3286
|
+
constructor({ apiKey, apiUrl, origin }: FirecrawlAppConfig);
|
|
3287
|
+
private resolveOrigin;
|
|
3269
3288
|
/**
|
|
3270
3289
|
* Scrapes a URL using the Firecrawl API.
|
|
3271
3290
|
* @param url - The URL to scrape.
|
package/dist/index.d.ts
CHANGED
|
@@ -765,10 +765,10 @@ interface FindToolsOptions {
|
|
|
765
765
|
offset?: number;
|
|
766
766
|
}
|
|
767
767
|
interface FindToolsData {
|
|
768
|
-
level: "providers" | "groups" | "tools";
|
|
768
|
+
level: "categories" | "providers" | "groups" | "tools";
|
|
769
769
|
items: Array<{
|
|
770
770
|
id: string;
|
|
771
|
-
name
|
|
771
|
+
name?: string;
|
|
772
772
|
next?: AlexandriaCall;
|
|
773
773
|
execute?: Pick<AlexandriaCall, "provider" | "capability">;
|
|
774
774
|
[key: string]: unknown;
|
|
@@ -2019,6 +2019,7 @@ interface HttpClientOptions {
|
|
|
2019
2019
|
timeoutMs?: number;
|
|
2020
2020
|
maxRetries?: number;
|
|
2021
2021
|
backoffFactor?: number;
|
|
2022
|
+
origin?: string;
|
|
2022
2023
|
}
|
|
2023
2024
|
interface RequestOptions {
|
|
2024
2025
|
headers?: Record<string, string>;
|
|
@@ -2030,9 +2031,12 @@ declare class HttpClient {
|
|
|
2030
2031
|
private readonly apiUrl;
|
|
2031
2032
|
private readonly maxRetries;
|
|
2032
2033
|
private readonly backoffFactor;
|
|
2034
|
+
private readonly origin;
|
|
2033
2035
|
constructor(options: HttpClientOptions);
|
|
2034
2036
|
getApiUrl(): string;
|
|
2035
2037
|
getApiKey(): string;
|
|
2038
|
+
/** The attribution origin stamped into requests that do not set their own. */
|
|
2039
|
+
getOrigin(): string;
|
|
2036
2040
|
/** The client-wide default request timeout, for callers that need to
|
|
2037
2041
|
* extend a specific request beyond it (e.g. scrape auto-resume). */
|
|
2038
2042
|
getTimeoutMs(): number;
|
|
@@ -2119,6 +2123,10 @@ declare function browser(http: HttpClient, args?: {
|
|
|
2119
2123
|
name: string;
|
|
2120
2124
|
saveChanges?: boolean;
|
|
2121
2125
|
};
|
|
2126
|
+
/** Country the session browses from, as an ISO 3166-1 alpha-2 code (default US). */
|
|
2127
|
+
location?: {
|
|
2128
|
+
country: string;
|
|
2129
|
+
};
|
|
2122
2130
|
integration?: string;
|
|
2123
2131
|
origin?: string;
|
|
2124
2132
|
}): Promise<BrowserCreateResponse>;
|
|
@@ -2261,6 +2269,12 @@ interface FirecrawlClientOptions {
|
|
|
2261
2269
|
maxRetries?: number;
|
|
2262
2270
|
/** Exponential backoff factor for retries (optional). */
|
|
2263
2271
|
backoffFactor?: number;
|
|
2272
|
+
/**
|
|
2273
|
+
* Attribution tag stamped into request bodies. Defaults to `js-sdk@<version>`,
|
|
2274
|
+
* or `js-sdk.<package>@<version>` when an installed package creates the client
|
|
2275
|
+
* (set FIRECRAWL_DISABLE_CALLER_ORIGIN=1 to keep the plain default).
|
|
2276
|
+
*/
|
|
2277
|
+
origin?: string;
|
|
2264
2278
|
}
|
|
2265
2279
|
/** Accepts a plain API key string or a full options object. */
|
|
2266
2280
|
type FirecrawlClientInput = FirecrawlClientOptions | string;
|
|
@@ -2268,7 +2282,7 @@ type FirecrawlClientInput = FirecrawlClientOptions | string;
|
|
|
2268
2282
|
* Firecrawl v2 client. Provides typed access to all v2 endpoints and utilities.
|
|
2269
2283
|
*/
|
|
2270
2284
|
declare class FirecrawlClient {
|
|
2271
|
-
|
|
2285
|
+
protected readonly http: HttpClient;
|
|
2272
2286
|
private _research?;
|
|
2273
2287
|
private isCloudService;
|
|
2274
2288
|
/**
|
|
@@ -2585,7 +2599,7 @@ declare class FirecrawlClient {
|
|
|
2585
2599
|
}): Promise<AgentThreadResponse>;
|
|
2586
2600
|
/**
|
|
2587
2601
|
* Create a new browser session.
|
|
2588
|
-
* @param args Session options (ttl, activityTtl, streamWebView, blockAds, profile).
|
|
2602
|
+
* @param args Session options (ttl, activityTtl, streamWebView, blockAds, profile, location).
|
|
2589
2603
|
* @returns Session id, CDP URL, live view URL, and expiration time.
|
|
2590
2604
|
*/
|
|
2591
2605
|
browser(args?: Parameters<typeof browser>[1]): Promise<BrowserCreateResponse>;
|
|
@@ -2658,10 +2672,12 @@ declare class FirecrawlClient {
|
|
|
2658
2672
|
* Configuration interface for FirecrawlApp.
|
|
2659
2673
|
* @param apiKey - Optional API key for authentication.
|
|
2660
2674
|
* @param apiUrl - Optional base URL of the API; defaults to 'https://api.firecrawl.dev'.
|
|
2675
|
+
* @param origin - Optional attribution tag stamped into request bodies.
|
|
2661
2676
|
*/
|
|
2662
2677
|
interface FirecrawlAppConfig {
|
|
2663
2678
|
apiKey?: string | null;
|
|
2664
2679
|
apiUrl?: string | null;
|
|
2680
|
+
origin?: string;
|
|
2665
2681
|
}
|
|
2666
2682
|
/**
|
|
2667
2683
|
* Metadata for a Firecrawl document.
|
|
@@ -3258,6 +3274,8 @@ declare class FirecrawlApp {
|
|
|
3258
3274
|
apiKey: string;
|
|
3259
3275
|
apiUrl: string;
|
|
3260
3276
|
version: string;
|
|
3277
|
+
private readonly clientOrigin?;
|
|
3278
|
+
private readonly callerPackage;
|
|
3261
3279
|
private isCloudService;
|
|
3262
3280
|
private getVersion;
|
|
3263
3281
|
private init;
|
|
@@ -3265,7 +3283,8 @@ declare class FirecrawlApp {
|
|
|
3265
3283
|
* Initializes a new instance of the FirecrawlApp class.
|
|
3266
3284
|
* @param config - Configuration options for the FirecrawlApp instance.
|
|
3267
3285
|
*/
|
|
3268
|
-
constructor({ apiKey, apiUrl }: FirecrawlAppConfig);
|
|
3286
|
+
constructor({ apiKey, apiUrl, origin }: FirecrawlAppConfig);
|
|
3287
|
+
private resolveOrigin;
|
|
3269
3288
|
/**
|
|
3270
3289
|
* Scrapes a URL using the Firecrawl API.
|
|
3271
3290
|
* @param url - The URL to scrape.
|
package/dist/index.js
CHANGED
|
@@ -1,21 +1,13 @@
|
|
|
1
1
|
import {
|
|
2
|
-
|
|
3
|
-
} from "./chunk-
|
|
2
|
+
version
|
|
3
|
+
} from "./chunk-GD463K4D.js";
|
|
4
4
|
|
|
5
5
|
// src/v2/utils/httpClient.ts
|
|
6
6
|
import axios from "axios";
|
|
7
7
|
|
|
8
8
|
// src/v2/utils/getVersion.ts
|
|
9
9
|
function getVersion() {
|
|
10
|
-
|
|
11
|
-
if (typeof process !== "undefined" && process.env && process.env.npm_package_version) {
|
|
12
|
-
return process.env.npm_package_version;
|
|
13
|
-
}
|
|
14
|
-
const pkg = require_package();
|
|
15
|
-
return pkg?.version || "3.x.x";
|
|
16
|
-
} catch {
|
|
17
|
-
return "3.x.x";
|
|
18
|
-
}
|
|
10
|
+
return version;
|
|
19
11
|
}
|
|
20
12
|
|
|
21
13
|
// src/utils/apiOrigin.ts
|
|
@@ -40,11 +32,13 @@ var HttpClient = class {
|
|
|
40
32
|
apiUrl;
|
|
41
33
|
maxRetries;
|
|
42
34
|
backoffFactor;
|
|
35
|
+
origin;
|
|
43
36
|
constructor(options) {
|
|
44
37
|
this.apiKey = options.apiKey;
|
|
45
38
|
this.apiUrl = options.apiUrl.replace(/\/$/, "");
|
|
46
39
|
this.maxRetries = options.maxRetries ?? 3;
|
|
47
40
|
this.backoffFactor = options.backoffFactor ?? 0.5;
|
|
41
|
+
this.origin = options.origin || `js-sdk@${getVersion()}`;
|
|
48
42
|
this.instance = axios.create({
|
|
49
43
|
baseURL: this.apiUrl,
|
|
50
44
|
timeout: options.timeoutMs ?? 3e5,
|
|
@@ -63,13 +57,16 @@ var HttpClient = class {
|
|
|
63
57
|
getApiKey() {
|
|
64
58
|
return this.apiKey;
|
|
65
59
|
}
|
|
60
|
+
/** The attribution origin stamped into requests that do not set their own. */
|
|
61
|
+
getOrigin() {
|
|
62
|
+
return this.origin;
|
|
63
|
+
}
|
|
66
64
|
/** The client-wide default request timeout, for callers that need to
|
|
67
65
|
* extend a specific request beyond it (e.g. scrape auto-resume). */
|
|
68
66
|
getTimeoutMs() {
|
|
69
67
|
return this.instance.defaults.timeout ?? 3e5;
|
|
70
68
|
}
|
|
71
69
|
async request(config) {
|
|
72
|
-
const version = getVersion();
|
|
73
70
|
config.headers = {
|
|
74
71
|
...config.headers || {}
|
|
75
72
|
};
|
|
@@ -84,7 +81,7 @@ var HttpClient = class {
|
|
|
84
81
|
const data = cfg.data ?? {};
|
|
85
82
|
cfg.data = {
|
|
86
83
|
...data,
|
|
87
|
-
origin: typeof data.origin === "string" && data.origin
|
|
84
|
+
origin: typeof data.origin === "string" && data.origin ? data.origin : this.origin
|
|
88
85
|
};
|
|
89
86
|
}
|
|
90
87
|
if (isFormDataBody) {
|
|
@@ -323,6 +320,30 @@ function ensureValidFormats(formats) {
|
|
|
323
320
|
}
|
|
324
321
|
}
|
|
325
322
|
}
|
|
323
|
+
var V1_SCRAPE_OPTION_REPLACEMENTS = {
|
|
324
|
+
pageOptions: "use scrapeOptions",
|
|
325
|
+
includeLinks: "add 'links' to scrapeOptions.formats",
|
|
326
|
+
parsePDF: "use scrapeOptions.parsers"
|
|
327
|
+
};
|
|
328
|
+
var V1_OPTION_REPLACEMENTS = {
|
|
329
|
+
crawl: {
|
|
330
|
+
crawlerOptions: "move its fields to the top level",
|
|
331
|
+
maxDepth: "use maxDiscoveryDepth",
|
|
332
|
+
allowBackwardLinks: "use crawlEntireDomain",
|
|
333
|
+
ignoreSitemap: "use sitemap: 'skip'",
|
|
334
|
+
...V1_SCRAPE_OPTION_REPLACEMENTS
|
|
335
|
+
},
|
|
336
|
+
search: V1_SCRAPE_OPTION_REPLACEMENTS
|
|
337
|
+
};
|
|
338
|
+
function ensureNoV1Options(options, method) {
|
|
339
|
+
if (!options) return;
|
|
340
|
+
const replacements = V1_OPTION_REPLACEMENTS[method];
|
|
341
|
+
const found = Object.keys(replacements).filter((key) => options[key] != null);
|
|
342
|
+
if (found.length === 0) return;
|
|
343
|
+
throw new Error(
|
|
344
|
+
`${method}: ${found.map((key) => `${key} is a v1 option, ${replacements[key]}`).join("; ")}`
|
|
345
|
+
);
|
|
346
|
+
}
|
|
326
347
|
function ensureValidScrapeOptions(options) {
|
|
327
348
|
if (!options) return;
|
|
328
349
|
if (options.timeout != null && options.timeout <= 0) {
|
|
@@ -436,12 +457,28 @@ function ensureValidParseOptions(options) {
|
|
|
436
457
|
|
|
437
458
|
// src/v2/utils/errorHandler.ts
|
|
438
459
|
import "axios";
|
|
460
|
+
function describeValidationIssues(details) {
|
|
461
|
+
if (!Array.isArray(details)) return void 0;
|
|
462
|
+
const parts = details.slice(0, 3).flatMap((issue) => {
|
|
463
|
+
if (!issue || typeof issue !== "object") return [];
|
|
464
|
+
const { path, message, keys } = issue;
|
|
465
|
+
const what = Array.isArray(keys) && keys.length > 0 ? `unrecognized key(s) ${keys.join(", ")}` : typeof message === "string" ? message : void 0;
|
|
466
|
+
if (!what) return [];
|
|
467
|
+
return Array.isArray(path) && path.length > 0 ? [`${path.join(".")}: ${what}`] : [what];
|
|
468
|
+
});
|
|
469
|
+
return parts.length > 0 ? parts.join("; ") : void 0;
|
|
470
|
+
}
|
|
471
|
+
function withIssues(message, details) {
|
|
472
|
+
const text = typeof message === "string" ? message : JSON.stringify(message);
|
|
473
|
+
const issues = describeValidationIssues(details);
|
|
474
|
+
return issues && !text.includes(issues) ? `${text} (${issues})` : text;
|
|
475
|
+
}
|
|
439
476
|
function throwForBadResponse(resp, action) {
|
|
440
477
|
const status = resp.status;
|
|
441
478
|
const body = resp.data || {};
|
|
442
479
|
const msg = body?.error || body?.message || `Request failed (${status}) while trying to ${action}`;
|
|
443
480
|
throw Object.assign(new SdkError(
|
|
444
|
-
msg,
|
|
481
|
+
withIssues(msg, body?.details),
|
|
445
482
|
status,
|
|
446
483
|
body?.code,
|
|
447
484
|
body?.details,
|
|
@@ -456,7 +493,7 @@ function normalizeAxiosError(err, action) {
|
|
|
456
493
|
const message = body?.error || err.message || `Request failed${status ? ` (${status})` : ""} while trying to ${action}`;
|
|
457
494
|
const code = body?.code || err.code;
|
|
458
495
|
throw Object.assign(new SdkError(
|
|
459
|
-
message,
|
|
496
|
+
withIssues(message, body?.details),
|
|
460
497
|
status,
|
|
461
498
|
code,
|
|
462
499
|
body?.details ?? body,
|
|
@@ -627,10 +664,9 @@ async function parse(http, file, options) {
|
|
|
627
664
|
throw new Error("file data cannot be empty");
|
|
628
665
|
}
|
|
629
666
|
if (options) ensureValidParseOptions(options);
|
|
630
|
-
const version = getVersion();
|
|
631
667
|
const normalizedOptions = {
|
|
632
668
|
...options ?? {},
|
|
633
|
-
origin:
|
|
669
|
+
origin: options?.origin || http.getOrigin()
|
|
634
670
|
};
|
|
635
671
|
const formData = new FormData();
|
|
636
672
|
formData.append("options", JSON.stringify(normalizedOptions));
|
|
@@ -668,6 +704,7 @@ async function getParseFormats(http) {
|
|
|
668
704
|
// src/v2/methods/search.ts
|
|
669
705
|
function prepareSearchPayload(req) {
|
|
670
706
|
if (!req.query || !req.query.trim()) throw new Error("Query cannot be empty");
|
|
707
|
+
ensureNoV1Options(req, "search");
|
|
671
708
|
if (req.limit != null && req.limit <= 0)
|
|
672
709
|
throw new Error("limit must be positive");
|
|
673
710
|
if (req.timeout != null && req.timeout <= 0)
|
|
@@ -1035,7 +1072,9 @@ async function fetchAllPages(http, nextUrl, initial, pagination) {
|
|
|
1035
1072
|
docs.push(d);
|
|
1036
1073
|
}
|
|
1037
1074
|
if (maxResults != null && docs.length >= maxResults) break;
|
|
1038
|
-
|
|
1075
|
+
const next = payload.next ?? (Array.isArray(payload.data) ? null : payload.data?.next) ?? null;
|
|
1076
|
+
if (next === current) break;
|
|
1077
|
+
current = next;
|
|
1039
1078
|
pageCount += 1;
|
|
1040
1079
|
}
|
|
1041
1080
|
return docs;
|
|
@@ -1044,6 +1083,7 @@ async function fetchAllPages(http, nextUrl, initial, pagination) {
|
|
|
1044
1083
|
// src/v2/methods/crawl.ts
|
|
1045
1084
|
function prepareCrawlPayload(request) {
|
|
1046
1085
|
if (!request.url || !request.url.trim()) throw new Error("URL cannot be empty");
|
|
1086
|
+
ensureNoV1Options(request, "crawl");
|
|
1047
1087
|
const data = { url: request.url.trim() };
|
|
1048
1088
|
if (request.prompt) data.prompt = request.prompt;
|
|
1049
1089
|
if (request.excludePaths) data.excludePaths = request.excludePaths;
|
|
@@ -1595,6 +1635,7 @@ async function browser(http, args = {}) {
|
|
|
1595
1635
|
if (args.streamWebView != null) body.streamWebView = args.streamWebView;
|
|
1596
1636
|
if (args.blockAds != null) body.blockAds = args.blockAds;
|
|
1597
1637
|
if (args.profile != null) body.profile = args.profile;
|
|
1638
|
+
if (args.location != null) body.location = args.location;
|
|
1598
1639
|
if (args.integration != null) body.integration = args.integration;
|
|
1599
1640
|
if (args.origin) body.origin = args.origin;
|
|
1600
1641
|
try {
|
|
@@ -1743,8 +1784,8 @@ function appendParam(params, key, value) {
|
|
|
1743
1784
|
params.append(key, String(value));
|
|
1744
1785
|
}
|
|
1745
1786
|
}
|
|
1746
|
-
function withQuery(path, params) {
|
|
1747
|
-
params.append("origin",
|
|
1787
|
+
function withQuery(origin, path, params) {
|
|
1788
|
+
params.append("origin", origin);
|
|
1748
1789
|
const qs = params.toString();
|
|
1749
1790
|
return qs ? `${path}?${qs}` : path;
|
|
1750
1791
|
}
|
|
@@ -1799,7 +1840,7 @@ var ResearchClient = class {
|
|
|
1799
1840
|
appendParam(params, "to", options.to);
|
|
1800
1841
|
try {
|
|
1801
1842
|
const res = await this.http.get(
|
|
1802
|
-
withQuery(`${BASE}/papers`, params)
|
|
1843
|
+
withQuery(this.http.getOrigin(), `${BASE}/papers`, params)
|
|
1803
1844
|
);
|
|
1804
1845
|
if (res.status !== 200) throwForBadResponse(res, "search papers");
|
|
1805
1846
|
return res.data;
|
|
@@ -1817,7 +1858,13 @@ var ResearchClient = class {
|
|
|
1817
1858
|
appendParam(params, "query", options.query);
|
|
1818
1859
|
appendParam(params, "k", options.k);
|
|
1819
1860
|
try {
|
|
1820
|
-
const res = await this.http.get(
|
|
1861
|
+
const res = await this.http.get(
|
|
1862
|
+
withQuery(
|
|
1863
|
+
this.http.getOrigin(),
|
|
1864
|
+
`${BASE}/papers/${encodeURIComponent(id)}`,
|
|
1865
|
+
params
|
|
1866
|
+
)
|
|
1867
|
+
);
|
|
1821
1868
|
if (res.status !== 200) throwForBadResponse(res, "get paper");
|
|
1822
1869
|
return res.data;
|
|
1823
1870
|
} catch (err) {
|
|
@@ -1843,7 +1890,11 @@ var ResearchClient = class {
|
|
|
1843
1890
|
appendParam(params, "anchor", options.anchor);
|
|
1844
1891
|
try {
|
|
1845
1892
|
const res = await this.http.get(
|
|
1846
|
-
withQuery(
|
|
1893
|
+
withQuery(
|
|
1894
|
+
this.http.getOrigin(),
|
|
1895
|
+
`${BASE}/papers/${encodeURIComponent(id)}/similar`,
|
|
1896
|
+
params
|
|
1897
|
+
)
|
|
1847
1898
|
);
|
|
1848
1899
|
if (res.status !== 200) throwForBadResponse(res, "find similar papers");
|
|
1849
1900
|
return res.data;
|
|
@@ -1874,7 +1925,7 @@ var ResearchClient = class {
|
|
|
1874
1925
|
appendParam(params, "k", options.k);
|
|
1875
1926
|
try {
|
|
1876
1927
|
const res = await this.http.get(
|
|
1877
|
-
withQuery(`${BASE}/github`, params)
|
|
1928
|
+
withQuery(this.http.getOrigin(), `${BASE}/github`, params)
|
|
1878
1929
|
);
|
|
1879
1930
|
if (res.status !== 200) throwForBadResponse(res, "search github");
|
|
1880
1931
|
return res.data;
|
|
@@ -2004,6 +2055,67 @@ async function getMonitorCheck(http, monitorId, checkId, options) {
|
|
|
2004
2055
|
}
|
|
2005
2056
|
}
|
|
2006
2057
|
|
|
2058
|
+
// src/utils/callerOrigin.ts
|
|
2059
|
+
var NODE_MODULES = "/node_modules/";
|
|
2060
|
+
function packageOf(file) {
|
|
2061
|
+
let path = file.replace(/^file:\/\//, "").replace(/\\/g, "/");
|
|
2062
|
+
try {
|
|
2063
|
+
path = decodeURIComponent(path);
|
|
2064
|
+
} catch {
|
|
2065
|
+
}
|
|
2066
|
+
const at = path.lastIndexOf(NODE_MODULES);
|
|
2067
|
+
if (at < 0) return null;
|
|
2068
|
+
const [first, second] = path.slice(at + NODE_MODULES.length).split("/");
|
|
2069
|
+
const name = first?.startsWith("@") ? second ? `${first}/${second}` : "" : first;
|
|
2070
|
+
return name && !name.startsWith(".") ? name : null;
|
|
2071
|
+
}
|
|
2072
|
+
function frameFile(line) {
|
|
2073
|
+
const text = line.trim().replace(/^at /, "");
|
|
2074
|
+
const open = text.indexOf(" (");
|
|
2075
|
+
const location = open >= 0 && text.endsWith(")") ? text.slice(open + 2, -1) : text;
|
|
2076
|
+
const match = /^(.*):\d+:\d+$/.exec(location);
|
|
2077
|
+
return match && !match[1].startsWith("node:") ? match[1] : null;
|
|
2078
|
+
}
|
|
2079
|
+
function slugify(name) {
|
|
2080
|
+
return name.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 48).replace(/-+$/, "");
|
|
2081
|
+
}
|
|
2082
|
+
function detectCallerPackage(stack) {
|
|
2083
|
+
try {
|
|
2084
|
+
if (process.env.FIRECRAWL_DISABLE_CALLER_ORIGIN === "1") return null;
|
|
2085
|
+
if (stack === void 0) {
|
|
2086
|
+
const limit = Error.stackTraceLimit;
|
|
2087
|
+
Error.stackTraceLimit = 50;
|
|
2088
|
+
try {
|
|
2089
|
+
stack = new Error().stack ?? "";
|
|
2090
|
+
} finally {
|
|
2091
|
+
Error.stackTraceLimit = limit;
|
|
2092
|
+
}
|
|
2093
|
+
}
|
|
2094
|
+
let ownFile = null;
|
|
2095
|
+
let ownPackage = null;
|
|
2096
|
+
for (const line of stack.split("\n")) {
|
|
2097
|
+
if (!/^\s*at /.test(line)) continue;
|
|
2098
|
+
const file = frameFile(line);
|
|
2099
|
+
if (!file) continue;
|
|
2100
|
+
if (ownFile === null) {
|
|
2101
|
+
ownFile = file;
|
|
2102
|
+
ownPackage = packageOf(file);
|
|
2103
|
+
}
|
|
2104
|
+
if (file === ownFile) continue;
|
|
2105
|
+
const name = packageOf(file);
|
|
2106
|
+
if (name && name === ownPackage) continue;
|
|
2107
|
+
const slug = name ? slugify(name) : "";
|
|
2108
|
+
return slug && !slug.includes("playground") ? slug : null;
|
|
2109
|
+
}
|
|
2110
|
+
} catch {
|
|
2111
|
+
}
|
|
2112
|
+
return null;
|
|
2113
|
+
}
|
|
2114
|
+
function defaultOrigin(version2) {
|
|
2115
|
+
const caller = detectCallerPackage();
|
|
2116
|
+
return caller ? `js-sdk.${caller}@${version2}` : `js-sdk@${version2}`;
|
|
2117
|
+
}
|
|
2118
|
+
|
|
2007
2119
|
// src/v2/watcher.ts
|
|
2008
2120
|
import { EventEmitter } from "events";
|
|
2009
2121
|
var hasGlobalWebSocket = () => {
|
|
@@ -2266,7 +2378,8 @@ var FirecrawlClient = class {
|
|
|
2266
2378
|
apiUrl,
|
|
2267
2379
|
timeoutMs: opts.timeoutMs,
|
|
2268
2380
|
maxRetries: opts.maxRetries,
|
|
2269
|
-
backoffFactor: opts.backoffFactor
|
|
2381
|
+
backoffFactor: opts.backoffFactor,
|
|
2382
|
+
origin: opts.origin || defaultOrigin(getVersion())
|
|
2270
2383
|
});
|
|
2271
2384
|
}
|
|
2272
2385
|
async scrape(url, options) {
|
|
@@ -2661,7 +2774,7 @@ var FirecrawlClient = class {
|
|
|
2661
2774
|
// Browser
|
|
2662
2775
|
/**
|
|
2663
2776
|
* Create a new browser session.
|
|
2664
|
-
* @param args Session options (ttl, activityTtl, streamWebView, blockAds, profile).
|
|
2777
|
+
* @param args Session options (ttl, activityTtl, streamWebView, blockAds, profile, location).
|
|
2665
2778
|
* @returns Session id, CDP URL, live view URL, and expiration time.
|
|
2666
2779
|
*/
|
|
2667
2780
|
async browser(args = {}) {
|
|
@@ -2792,6 +2905,8 @@ var FirecrawlApp = class {
|
|
|
2792
2905
|
apiKey;
|
|
2793
2906
|
apiUrl;
|
|
2794
2907
|
version = "1.25.1";
|
|
2908
|
+
clientOrigin;
|
|
2909
|
+
callerPackage;
|
|
2795
2910
|
isCloudService(url) {
|
|
2796
2911
|
return url.includes("api.firecrawl.dev");
|
|
2797
2912
|
}
|
|
@@ -2800,7 +2915,7 @@ var FirecrawlApp = class {
|
|
|
2800
2915
|
if (typeof process !== "undefined" && process.env && process.env.npm_package_version) {
|
|
2801
2916
|
return process.env.npm_package_version;
|
|
2802
2917
|
}
|
|
2803
|
-
const packageJson = await import("./package-
|
|
2918
|
+
const packageJson = await import("./package-WONXBIJS.js");
|
|
2804
2919
|
return packageJson.default.version;
|
|
2805
2920
|
} catch (error) {
|
|
2806
2921
|
const isTest = typeof process !== "undefined" && false;
|
|
@@ -2817,15 +2932,22 @@ var FirecrawlApp = class {
|
|
|
2817
2932
|
* Initializes a new instance of the FirecrawlApp class.
|
|
2818
2933
|
* @param config - Configuration options for the FirecrawlApp instance.
|
|
2819
2934
|
*/
|
|
2820
|
-
constructor({ apiKey = null, apiUrl = null }) {
|
|
2935
|
+
constructor({ apiKey = null, apiUrl = null, origin }) {
|
|
2821
2936
|
const baseUrl = apiUrl || "https://api.firecrawl.dev";
|
|
2822
2937
|
if (this.isCloudService(baseUrl) && typeof apiKey !== "string") {
|
|
2823
2938
|
throw new FirecrawlError("No API key provided", 401);
|
|
2824
2939
|
}
|
|
2825
2940
|
this.apiKey = apiKey || "";
|
|
2826
2941
|
this.apiUrl = baseUrl;
|
|
2942
|
+
this.clientOrigin = origin || void 0;
|
|
2943
|
+
this.callerPackage = this.clientOrigin ? null : detectCallerPackage();
|
|
2827
2944
|
this.init();
|
|
2828
2945
|
}
|
|
2946
|
+
resolveOrigin(params) {
|
|
2947
|
+
if (typeof params?.origin === "string" && params.origin) return params.origin;
|
|
2948
|
+
if (this.clientOrigin) return this.clientOrigin;
|
|
2949
|
+
return this.callerPackage ? `js-sdk.${this.callerPackage}@${this.version}` : `js-sdk@${this.version}`;
|
|
2950
|
+
}
|
|
2829
2951
|
/**
|
|
2830
2952
|
* Scrapes a URL using the Firecrawl API.
|
|
2831
2953
|
* @param url - The URL to scrape.
|
|
@@ -2837,7 +2959,7 @@ var FirecrawlApp = class {
|
|
|
2837
2959
|
"Content-Type": "application/json",
|
|
2838
2960
|
Authorization: `Bearer ${this.apiKey}`
|
|
2839
2961
|
};
|
|
2840
|
-
let jsonData = { url, ...params, origin:
|
|
2962
|
+
let jsonData = { url, ...params, origin: this.resolveOrigin(params) };
|
|
2841
2963
|
if (jsonData?.extract?.schema) {
|
|
2842
2964
|
jsonData = {
|
|
2843
2965
|
...jsonData,
|
|
@@ -2901,7 +3023,7 @@ var FirecrawlApp = class {
|
|
|
2901
3023
|
lang: params?.lang ?? "en",
|
|
2902
3024
|
country: params?.country ?? "us",
|
|
2903
3025
|
location: params?.location,
|
|
2904
|
-
origin:
|
|
3026
|
+
origin: this.resolveOrigin(params),
|
|
2905
3027
|
timeout: params?.timeout ?? 6e4,
|
|
2906
3028
|
scrapeOptions: params?.scrapeOptions ?? { formats: [] }
|
|
2907
3029
|
};
|
|
@@ -2956,7 +3078,7 @@ var FirecrawlApp = class {
|
|
|
2956
3078
|
*/
|
|
2957
3079
|
async crawlUrl(url, params, pollInterval = 2, idempotencyKey) {
|
|
2958
3080
|
const headers = this.prepareHeaders(idempotencyKey);
|
|
2959
|
-
let jsonData = { url, ...params, origin:
|
|
3081
|
+
let jsonData = { url, ...params, origin: this.resolveOrigin(params) };
|
|
2960
3082
|
try {
|
|
2961
3083
|
const response = await this.postRequest(
|
|
2962
3084
|
this.apiUrl + `/v1/crawl`,
|
|
@@ -2980,7 +3102,7 @@ var FirecrawlApp = class {
|
|
|
2980
3102
|
}
|
|
2981
3103
|
async asyncCrawlUrl(url, params, idempotencyKey) {
|
|
2982
3104
|
const headers = this.prepareHeaders(idempotencyKey);
|
|
2983
|
-
let jsonData = { url, ...params, origin:
|
|
3105
|
+
let jsonData = { url, ...params, origin: this.resolveOrigin(params) };
|
|
2984
3106
|
try {
|
|
2985
3107
|
const response = await this.postRequest(
|
|
2986
3108
|
this.apiUrl + `/v1/crawl`,
|
|
@@ -3139,7 +3261,7 @@ var FirecrawlApp = class {
|
|
|
3139
3261
|
*/
|
|
3140
3262
|
async mapUrl(url, params) {
|
|
3141
3263
|
const headers = this.prepareHeaders();
|
|
3142
|
-
let jsonData = { url, ...params, origin:
|
|
3264
|
+
let jsonData = { url, ...params, origin: this.resolveOrigin(params) };
|
|
3143
3265
|
try {
|
|
3144
3266
|
const response = await this.postRequest(
|
|
3145
3267
|
this.apiUrl + `/v1/map`,
|
|
@@ -3168,7 +3290,7 @@ var FirecrawlApp = class {
|
|
|
3168
3290
|
*/
|
|
3169
3291
|
async batchScrapeUrls(urls, params, pollInterval = 2, idempotencyKey, webhook, ignoreInvalidURLs, maxConcurrency) {
|
|
3170
3292
|
const headers = this.prepareHeaders(idempotencyKey);
|
|
3171
|
-
let jsonData = { urls, webhook, ignoreInvalidURLs, maxConcurrency, ...params, origin:
|
|
3293
|
+
let jsonData = { urls, webhook, ignoreInvalidURLs, maxConcurrency, ...params, origin: this.resolveOrigin(params) };
|
|
3172
3294
|
if (jsonData?.extract?.schema) {
|
|
3173
3295
|
jsonData = {
|
|
3174
3296
|
...jsonData,
|
|
@@ -3210,7 +3332,7 @@ var FirecrawlApp = class {
|
|
|
3210
3332
|
}
|
|
3211
3333
|
async asyncBatchScrapeUrls(urls, params, idempotencyKey, webhook, ignoreInvalidURLs) {
|
|
3212
3334
|
const headers = this.prepareHeaders(idempotencyKey);
|
|
3213
|
-
let jsonData = { urls, webhook, ignoreInvalidURLs, ...params, origin:
|
|
3335
|
+
let jsonData = { urls, webhook, ignoreInvalidURLs, ...params, origin: this.resolveOrigin(params) };
|
|
3214
3336
|
try {
|
|
3215
3337
|
const response = await this.postRequest(
|
|
3216
3338
|
this.apiUrl + `/v1/batch/scrape`,
|
|
@@ -3354,7 +3476,7 @@ var FirecrawlApp = class {
|
|
|
3354
3476
|
try {
|
|
3355
3477
|
const response = await this.postRequest(
|
|
3356
3478
|
this.apiUrl + `/v1/extract`,
|
|
3357
|
-
{ ...jsonData, schema: jsonSchema, origin:
|
|
3479
|
+
{ ...jsonData, schema: jsonSchema, origin: this.resolveOrigin(params) },
|
|
3358
3480
|
headers
|
|
3359
3481
|
);
|
|
3360
3482
|
if (response.status === 200) {
|
|
@@ -3407,7 +3529,7 @@ var FirecrawlApp = class {
|
|
|
3407
3529
|
try {
|
|
3408
3530
|
const response = await this.postRequest(
|
|
3409
3531
|
this.apiUrl + `/v1/extract`,
|
|
3410
|
-
{ ...jsonData, schema: jsonSchema, origin:
|
|
3532
|
+
{ ...jsonData, schema: jsonSchema, origin: this.resolveOrigin(params) },
|
|
3411
3533
|
headers
|
|
3412
3534
|
);
|
|
3413
3535
|
if (response.status === 200) {
|
|
@@ -3689,7 +3811,7 @@ var FirecrawlApp = class {
|
|
|
3689
3811
|
*/
|
|
3690
3812
|
async asyncDeepResearch(query, params) {
|
|
3691
3813
|
const headers = this.prepareHeaders();
|
|
3692
|
-
let jsonData = { query, ...params, origin:
|
|
3814
|
+
let jsonData = { query, ...params, origin: this.resolveOrigin(params) };
|
|
3693
3815
|
if (jsonData?.jsonOptions?.schema) {
|
|
3694
3816
|
jsonData = {
|
|
3695
3817
|
...jsonData,
|
|
@@ -3808,7 +3930,7 @@ var FirecrawlApp = class {
|
|
|
3808
3930
|
async __asyncDeepResearch(topic, params) {
|
|
3809
3931
|
const headers = this.prepareHeaders();
|
|
3810
3932
|
try {
|
|
3811
|
-
let jsonData = { topic, ...params, origin:
|
|
3933
|
+
let jsonData = { topic, ...params, origin: this.resolveOrigin(params) };
|
|
3812
3934
|
const response = await this.postRequest(
|
|
3813
3935
|
`${this.apiUrl}/v1/deep-research`,
|
|
3814
3936
|
jsonData,
|
|
@@ -3908,7 +4030,7 @@ var FirecrawlApp = class {
|
|
|
3908
4030
|
*/
|
|
3909
4031
|
async asyncGenerateLLMsText(url, params) {
|
|
3910
4032
|
const headers = this.prepareHeaders();
|
|
3911
|
-
let jsonData = { url, ...params, origin:
|
|
4033
|
+
let jsonData = { url, ...params, origin: this.resolveOrigin(params) };
|
|
3912
4034
|
try {
|
|
3913
4035
|
const response = await this.postRequest(
|
|
3914
4036
|
`${this.apiUrl}/v1/llmstxt`,
|
|
@@ -4176,7 +4298,8 @@ var Firecrawl = class extends FirecrawlClient {
|
|
|
4176
4298
|
super(resolved);
|
|
4177
4299
|
this._v1Opts = {
|
|
4178
4300
|
apiKey: resolved.apiKey,
|
|
4179
|
-
apiUrl: resolved.apiUrl
|
|
4301
|
+
apiUrl: resolved.apiUrl,
|
|
4302
|
+
origin: this.http.getOrigin()
|
|
4180
4303
|
};
|
|
4181
4304
|
}
|
|
4182
4305
|
/** Access the legacy v1 client (instantiated on first access). */
|