@mendable/firecrawl-js 4.42.2 → 4.42.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-XZ5XI5DK.js → chunk-TALYWLKJ.js} +2 -2
- package/dist/index.cjs +25 -23
- package/dist/index.d.cts +4 -4
- package/dist/index.d.ts +4 -4
- package/dist/index.js +25 -23
- package/dist/{package-JVVNUZPT.js → package-NTEZDZT4.js} +1 -1
- package/package.json +2 -2
- package/src/__tests__/unit/v1/batch-scrape-poll.test.ts +91 -0
- package/src/__tests__/unit/v1/monitor-job-status-retry.test.ts +84 -131
- package/src/__tests__/unit/v1/optional-params.test.ts +70 -0
- package/src/__tests__/unit/v2/scrape.unit.test.ts +55 -6
- package/src/v1/index.ts +24 -21
|
@@ -12,7 +12,7 @@ var require_package = __commonJS({
|
|
|
12
12
|
"package.json"(exports, module) {
|
|
13
13
|
module.exports = {
|
|
14
14
|
name: "@mendable/firecrawl-js",
|
|
15
|
-
version: "4.42.
|
|
15
|
+
version: "4.42.4",
|
|
16
16
|
description: "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
|
|
17
17
|
main: "dist/index.js",
|
|
18
18
|
types: "dist/index.d.ts",
|
|
@@ -29,7 +29,7 @@ var require_package = __commonJS({
|
|
|
29
29
|
"build-and-publish": "pnpm run build && pnpm publish --access public",
|
|
30
30
|
"publish-beta": "pnpm run build && pnpm publish --access public --tag beta",
|
|
31
31
|
test: "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/e2e/v2/*.test.ts --detectOpenHandles",
|
|
32
|
-
"test:unit": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/unit
|
|
32
|
+
"test:unit": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/unit"
|
|
33
33
|
},
|
|
34
34
|
repository: {
|
|
35
35
|
type: "git",
|
package/dist/index.cjs
CHANGED
|
@@ -39,7 +39,7 @@ var require_package = __commonJS({
|
|
|
39
39
|
"package.json"(exports2, module2) {
|
|
40
40
|
module2.exports = {
|
|
41
41
|
name: "@mendable/firecrawl-js",
|
|
42
|
-
version: "4.42.
|
|
42
|
+
version: "4.42.4",
|
|
43
43
|
description: "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
|
|
44
44
|
main: "dist/index.js",
|
|
45
45
|
types: "dist/index.d.ts",
|
|
@@ -56,7 +56,7 @@ var require_package = __commonJS({
|
|
|
56
56
|
"build-and-publish": "pnpm run build && pnpm publish --access public",
|
|
57
57
|
"publish-beta": "pnpm run build && pnpm publish --access public --tag beta",
|
|
58
58
|
test: "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/e2e/v2/*.test.ts --detectOpenHandles",
|
|
59
|
-
"test:unit": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/unit
|
|
59
|
+
"test:unit": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/unit"
|
|
60
60
|
},
|
|
61
61
|
repository: {
|
|
62
62
|
type: "git",
|
|
@@ -2875,7 +2875,7 @@ var FirecrawlApp = class {
|
|
|
2875
2875
|
"Content-Type": "application/json",
|
|
2876
2876
|
Authorization: `Bearer ${this.apiKey}`
|
|
2877
2877
|
};
|
|
2878
|
-
let jsonData = { url, ...params, origin: typeof params
|
|
2878
|
+
let jsonData = { url, ...params, origin: typeof params?.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` };
|
|
2879
2879
|
if (jsonData?.extract?.schema) {
|
|
2880
2880
|
jsonData = {
|
|
2881
2881
|
...jsonData,
|
|
@@ -2939,7 +2939,7 @@ var FirecrawlApp = class {
|
|
|
2939
2939
|
lang: params?.lang ?? "en",
|
|
2940
2940
|
country: params?.country ?? "us",
|
|
2941
2941
|
location: params?.location,
|
|
2942
|
-
origin: typeof params
|
|
2942
|
+
origin: typeof params?.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}`,
|
|
2943
2943
|
timeout: params?.timeout ?? 6e4,
|
|
2944
2944
|
scrapeOptions: params?.scrapeOptions ?? { formats: [] }
|
|
2945
2945
|
};
|
|
@@ -2994,7 +2994,7 @@ var FirecrawlApp = class {
|
|
|
2994
2994
|
*/
|
|
2995
2995
|
async crawlUrl(url, params, pollInterval = 2, idempotencyKey) {
|
|
2996
2996
|
const headers = this.prepareHeaders(idempotencyKey);
|
|
2997
|
-
let jsonData = { url, ...params, origin: typeof params
|
|
2997
|
+
let jsonData = { url, ...params, origin: typeof params?.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` };
|
|
2998
2998
|
try {
|
|
2999
2999
|
const response = await this.postRequest(
|
|
3000
3000
|
this.apiUrl + `/v1/crawl`,
|
|
@@ -3018,7 +3018,7 @@ var FirecrawlApp = class {
|
|
|
3018
3018
|
}
|
|
3019
3019
|
async asyncCrawlUrl(url, params, idempotencyKey) {
|
|
3020
3020
|
const headers = this.prepareHeaders(idempotencyKey);
|
|
3021
|
-
let jsonData = { url, ...params, origin: typeof params
|
|
3021
|
+
let jsonData = { url, ...params, origin: typeof params?.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` };
|
|
3022
3022
|
try {
|
|
3023
3023
|
const response = await this.postRequest(
|
|
3024
3024
|
this.apiUrl + `/v1/crawl`,
|
|
@@ -3177,7 +3177,7 @@ var FirecrawlApp = class {
|
|
|
3177
3177
|
*/
|
|
3178
3178
|
async mapUrl(url, params) {
|
|
3179
3179
|
const headers = this.prepareHeaders();
|
|
3180
|
-
let jsonData = { url, ...params, origin: typeof params
|
|
3180
|
+
let jsonData = { url, ...params, origin: typeof params?.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` };
|
|
3181
3181
|
try {
|
|
3182
3182
|
const response = await this.postRequest(
|
|
3183
3183
|
this.apiUrl + `/v1/map`,
|
|
@@ -3206,7 +3206,7 @@ var FirecrawlApp = class {
|
|
|
3206
3206
|
*/
|
|
3207
3207
|
async batchScrapeUrls(urls, params, pollInterval = 2, idempotencyKey, webhook, ignoreInvalidURLs, maxConcurrency) {
|
|
3208
3208
|
const headers = this.prepareHeaders(idempotencyKey);
|
|
3209
|
-
let jsonData = { urls, webhook, ignoreInvalidURLs, maxConcurrency, ...params, origin: typeof params
|
|
3209
|
+
let jsonData = { urls, webhook, ignoreInvalidURLs, maxConcurrency, ...params, origin: typeof params?.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` };
|
|
3210
3210
|
if (jsonData?.extract?.schema) {
|
|
3211
3211
|
jsonData = {
|
|
3212
3212
|
...jsonData,
|
|
@@ -3233,7 +3233,7 @@ var FirecrawlApp = class {
|
|
|
3233
3233
|
);
|
|
3234
3234
|
if (response.status === 200) {
|
|
3235
3235
|
const id = response.data.id;
|
|
3236
|
-
return this.monitorJobStatus(id, headers, pollInterval);
|
|
3236
|
+
return this.monitorJobStatus(id, headers, pollInterval, "batch");
|
|
3237
3237
|
} else {
|
|
3238
3238
|
this.handleError(response, "start batch scrape job");
|
|
3239
3239
|
}
|
|
@@ -3248,7 +3248,7 @@ var FirecrawlApp = class {
|
|
|
3248
3248
|
}
|
|
3249
3249
|
async asyncBatchScrapeUrls(urls, params, idempotencyKey, webhook, ignoreInvalidURLs) {
|
|
3250
3250
|
const headers = this.prepareHeaders(idempotencyKey);
|
|
3251
|
-
let jsonData = { urls, webhook, ignoreInvalidURLs, ...params, origin: typeof params
|
|
3251
|
+
let jsonData = { urls, webhook, ignoreInvalidURLs, ...params, origin: typeof params?.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` };
|
|
3252
3252
|
try {
|
|
3253
3253
|
const response = await this.postRequest(
|
|
3254
3254
|
this.apiUrl + `/v1/batch/scrape`,
|
|
@@ -3392,7 +3392,7 @@ var FirecrawlApp = class {
|
|
|
3392
3392
|
try {
|
|
3393
3393
|
const response = await this.postRequest(
|
|
3394
3394
|
this.apiUrl + `/v1/extract`,
|
|
3395
|
-
{ ...jsonData, schema: jsonSchema, origin: typeof params
|
|
3395
|
+
{ ...jsonData, schema: jsonSchema, origin: typeof params?.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` },
|
|
3396
3396
|
headers
|
|
3397
3397
|
);
|
|
3398
3398
|
if (response.status === 200) {
|
|
@@ -3445,7 +3445,7 @@ var FirecrawlApp = class {
|
|
|
3445
3445
|
try {
|
|
3446
3446
|
const response = await this.postRequest(
|
|
3447
3447
|
this.apiUrl + `/v1/extract`,
|
|
3448
|
-
{ ...jsonData, schema: jsonSchema, origin: typeof params
|
|
3448
|
+
{ ...jsonData, schema: jsonSchema, origin: typeof params?.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` },
|
|
3449
3449
|
headers
|
|
3450
3450
|
);
|
|
3451
3451
|
if (response.status === 200) {
|
|
@@ -3537,21 +3537,23 @@ var FirecrawlApp = class {
|
|
|
3537
3537
|
}
|
|
3538
3538
|
}
|
|
3539
3539
|
/**
|
|
3540
|
-
* Monitors the status of a crawl job until completion or failure.
|
|
3541
|
-
* @param id - The ID of the crawl operation.
|
|
3540
|
+
* Monitors the status of a crawl or batch scrape job until completion or failure.
|
|
3541
|
+
* @param id - The ID of the crawl or batch scrape operation.
|
|
3542
3542
|
* @param headers - The headers for the request.
|
|
3543
3543
|
* @param checkInterval - Interval in seconds for job status checks.
|
|
3544
|
-
* @param
|
|
3544
|
+
* @param jobType - Which status endpoint to poll. Defaults to `"crawl"`.
|
|
3545
3545
|
* @returns The final job status or data.
|
|
3546
3546
|
*/
|
|
3547
|
-
async monitorJobStatus(id, headers, checkInterval) {
|
|
3547
|
+
async monitorJobStatus(id, headers, checkInterval, jobType = "crawl") {
|
|
3548
|
+
const statusPath = jobType === "batch" ? "batch/scrape" : "crawl";
|
|
3549
|
+
const jobLabel = jobType === "batch" ? "Batch scrape" : "Crawl";
|
|
3548
3550
|
let failedTries = 0;
|
|
3549
3551
|
let networkRetries = 0;
|
|
3550
3552
|
const maxNetworkRetries = 3;
|
|
3551
3553
|
while (true) {
|
|
3552
3554
|
try {
|
|
3553
3555
|
let statusResponse = await this.getRequest(
|
|
3554
|
-
`${this.apiUrl}/v1
|
|
3556
|
+
`${this.apiUrl}/v1/${statusPath}/${id}`,
|
|
3555
3557
|
headers
|
|
3556
3558
|
);
|
|
3557
3559
|
if (statusResponse.status === 200) {
|
|
@@ -3572,7 +3574,7 @@ var FirecrawlApp = class {
|
|
|
3572
3574
|
statusData.data = data;
|
|
3573
3575
|
return statusData;
|
|
3574
3576
|
} else {
|
|
3575
|
-
throw new FirecrawlError(
|
|
3577
|
+
throw new FirecrawlError(`${jobLabel} job completed but no data was returned`, 500);
|
|
3576
3578
|
}
|
|
3577
3579
|
} else if (["active", "paused", "pending", "queued", "waiting", "scraping"].includes(statusData.status)) {
|
|
3578
3580
|
checkInterval = Math.max(checkInterval, 2);
|
|
@@ -3581,14 +3583,14 @@ var FirecrawlApp = class {
|
|
|
3581
3583
|
);
|
|
3582
3584
|
} else {
|
|
3583
3585
|
throw new FirecrawlError(
|
|
3584
|
-
|
|
3586
|
+
`${jobLabel} job failed or was stopped. Status: ${statusData.status}`,
|
|
3585
3587
|
500
|
|
3586
3588
|
);
|
|
3587
3589
|
}
|
|
3588
3590
|
} else {
|
|
3589
3591
|
failedTries++;
|
|
3590
3592
|
if (failedTries >= 3) {
|
|
3591
|
-
this.handleError(statusResponse,
|
|
3593
|
+
this.handleError(statusResponse, `check ${jobLabel.toLowerCase()} status`);
|
|
3592
3594
|
}
|
|
3593
3595
|
}
|
|
3594
3596
|
} catch (error) {
|
|
@@ -3725,7 +3727,7 @@ var FirecrawlApp = class {
|
|
|
3725
3727
|
*/
|
|
3726
3728
|
async asyncDeepResearch(query, params) {
|
|
3727
3729
|
const headers = this.prepareHeaders();
|
|
3728
|
-
let jsonData = { query, ...params, origin: typeof params
|
|
3730
|
+
let jsonData = { query, ...params, origin: typeof params?.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` };
|
|
3729
3731
|
if (jsonData?.jsonOptions?.schema) {
|
|
3730
3732
|
jsonData = {
|
|
3731
3733
|
...jsonData,
|
|
@@ -3844,7 +3846,7 @@ var FirecrawlApp = class {
|
|
|
3844
3846
|
async __asyncDeepResearch(topic, params) {
|
|
3845
3847
|
const headers = this.prepareHeaders();
|
|
3846
3848
|
try {
|
|
3847
|
-
let jsonData = { topic, ...params, origin: typeof params
|
|
3849
|
+
let jsonData = { topic, ...params, origin: typeof params?.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` };
|
|
3848
3850
|
const response = await this.postRequest(
|
|
3849
3851
|
`${this.apiUrl}/v1/deep-research`,
|
|
3850
3852
|
jsonData,
|
|
@@ -3944,7 +3946,7 @@ var FirecrawlApp = class {
|
|
|
3944
3946
|
*/
|
|
3945
3947
|
async asyncGenerateLLMsText(url, params) {
|
|
3946
3948
|
const headers = this.prepareHeaders();
|
|
3947
|
-
let jsonData = { url, ...params, origin: typeof params
|
|
3949
|
+
let jsonData = { url, ...params, origin: typeof params?.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` };
|
|
3948
3950
|
try {
|
|
3949
3951
|
const response = await this.postRequest(
|
|
3950
3952
|
`${this.apiUrl}/v1/llmstxt`,
|
package/dist/index.d.cts
CHANGED
|
@@ -3388,14 +3388,14 @@ declare class FirecrawlApp {
|
|
|
3388
3388
|
*/
|
|
3389
3389
|
deleteRequest(url: string, headers: AxiosRequestHeaders): Promise<AxiosResponse>;
|
|
3390
3390
|
/**
|
|
3391
|
-
* Monitors the status of a crawl job until completion or failure.
|
|
3392
|
-
* @param id - The ID of the crawl operation.
|
|
3391
|
+
* Monitors the status of a crawl or batch scrape job until completion or failure.
|
|
3392
|
+
* @param id - The ID of the crawl or batch scrape operation.
|
|
3393
3393
|
* @param headers - The headers for the request.
|
|
3394
3394
|
* @param checkInterval - Interval in seconds for job status checks.
|
|
3395
|
-
* @param
|
|
3395
|
+
* @param jobType - Which status endpoint to poll. Defaults to `"crawl"`.
|
|
3396
3396
|
* @returns The final job status or data.
|
|
3397
3397
|
*/
|
|
3398
|
-
monitorJobStatus(id: string, headers: AxiosRequestHeaders, checkInterval: number): Promise<CrawlStatusResponse | ErrorResponse>;
|
|
3398
|
+
monitorJobStatus(id: string, headers: AxiosRequestHeaders, checkInterval: number, jobType?: "crawl" | "batch"): Promise<CrawlStatusResponse | ErrorResponse>;
|
|
3399
3399
|
/**
|
|
3400
3400
|
* Determines if an error is retryable (transient network error)
|
|
3401
3401
|
* @param error - The error to check
|
package/dist/index.d.ts
CHANGED
|
@@ -3388,14 +3388,14 @@ declare class FirecrawlApp {
|
|
|
3388
3388
|
*/
|
|
3389
3389
|
deleteRequest(url: string, headers: AxiosRequestHeaders): Promise<AxiosResponse>;
|
|
3390
3390
|
/**
|
|
3391
|
-
* Monitors the status of a crawl job until completion or failure.
|
|
3392
|
-
* @param id - The ID of the crawl operation.
|
|
3391
|
+
* Monitors the status of a crawl or batch scrape job until completion or failure.
|
|
3392
|
+
* @param id - The ID of the crawl or batch scrape operation.
|
|
3393
3393
|
* @param headers - The headers for the request.
|
|
3394
3394
|
* @param checkInterval - Interval in seconds for job status checks.
|
|
3395
|
-
* @param
|
|
3395
|
+
* @param jobType - Which status endpoint to poll. Defaults to `"crawl"`.
|
|
3396
3396
|
* @returns The final job status or data.
|
|
3397
3397
|
*/
|
|
3398
|
-
monitorJobStatus(id: string, headers: AxiosRequestHeaders, checkInterval: number): Promise<CrawlStatusResponse | ErrorResponse>;
|
|
3398
|
+
monitorJobStatus(id: string, headers: AxiosRequestHeaders, checkInterval: number, jobType?: "crawl" | "batch"): Promise<CrawlStatusResponse | ErrorResponse>;
|
|
3399
3399
|
/**
|
|
3400
3400
|
* Determines if an error is retryable (transient network error)
|
|
3401
3401
|
* @param error - The error to check
|
package/dist/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
require_package
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-TALYWLKJ.js";
|
|
4
4
|
|
|
5
5
|
// src/v2/utils/httpClient.ts
|
|
6
6
|
import axios from "axios";
|
|
@@ -2706,7 +2706,7 @@ var FirecrawlApp = class {
|
|
|
2706
2706
|
if (typeof process !== "undefined" && process.env && process.env.npm_package_version) {
|
|
2707
2707
|
return process.env.npm_package_version;
|
|
2708
2708
|
}
|
|
2709
|
-
const packageJson = await import("./package-
|
|
2709
|
+
const packageJson = await import("./package-NTEZDZT4.js");
|
|
2710
2710
|
return packageJson.default.version;
|
|
2711
2711
|
} catch (error) {
|
|
2712
2712
|
const isTest = typeof process !== "undefined" && (process.env.JEST_WORKER_ID != null || false);
|
|
@@ -2743,7 +2743,7 @@ var FirecrawlApp = class {
|
|
|
2743
2743
|
"Content-Type": "application/json",
|
|
2744
2744
|
Authorization: `Bearer ${this.apiKey}`
|
|
2745
2745
|
};
|
|
2746
|
-
let jsonData = { url, ...params, origin: typeof params
|
|
2746
|
+
let jsonData = { url, ...params, origin: typeof params?.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` };
|
|
2747
2747
|
if (jsonData?.extract?.schema) {
|
|
2748
2748
|
jsonData = {
|
|
2749
2749
|
...jsonData,
|
|
@@ -2807,7 +2807,7 @@ var FirecrawlApp = class {
|
|
|
2807
2807
|
lang: params?.lang ?? "en",
|
|
2808
2808
|
country: params?.country ?? "us",
|
|
2809
2809
|
location: params?.location,
|
|
2810
|
-
origin: typeof params
|
|
2810
|
+
origin: typeof params?.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}`,
|
|
2811
2811
|
timeout: params?.timeout ?? 6e4,
|
|
2812
2812
|
scrapeOptions: params?.scrapeOptions ?? { formats: [] }
|
|
2813
2813
|
};
|
|
@@ -2862,7 +2862,7 @@ var FirecrawlApp = class {
|
|
|
2862
2862
|
*/
|
|
2863
2863
|
async crawlUrl(url, params, pollInterval = 2, idempotencyKey) {
|
|
2864
2864
|
const headers = this.prepareHeaders(idempotencyKey);
|
|
2865
|
-
let jsonData = { url, ...params, origin: typeof params
|
|
2865
|
+
let jsonData = { url, ...params, origin: typeof params?.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` };
|
|
2866
2866
|
try {
|
|
2867
2867
|
const response = await this.postRequest(
|
|
2868
2868
|
this.apiUrl + `/v1/crawl`,
|
|
@@ -2886,7 +2886,7 @@ var FirecrawlApp = class {
|
|
|
2886
2886
|
}
|
|
2887
2887
|
async asyncCrawlUrl(url, params, idempotencyKey) {
|
|
2888
2888
|
const headers = this.prepareHeaders(idempotencyKey);
|
|
2889
|
-
let jsonData = { url, ...params, origin: typeof params
|
|
2889
|
+
let jsonData = { url, ...params, origin: typeof params?.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` };
|
|
2890
2890
|
try {
|
|
2891
2891
|
const response = await this.postRequest(
|
|
2892
2892
|
this.apiUrl + `/v1/crawl`,
|
|
@@ -3045,7 +3045,7 @@ var FirecrawlApp = class {
|
|
|
3045
3045
|
*/
|
|
3046
3046
|
async mapUrl(url, params) {
|
|
3047
3047
|
const headers = this.prepareHeaders();
|
|
3048
|
-
let jsonData = { url, ...params, origin: typeof params
|
|
3048
|
+
let jsonData = { url, ...params, origin: typeof params?.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` };
|
|
3049
3049
|
try {
|
|
3050
3050
|
const response = await this.postRequest(
|
|
3051
3051
|
this.apiUrl + `/v1/map`,
|
|
@@ -3074,7 +3074,7 @@ var FirecrawlApp = class {
|
|
|
3074
3074
|
*/
|
|
3075
3075
|
async batchScrapeUrls(urls, params, pollInterval = 2, idempotencyKey, webhook, ignoreInvalidURLs, maxConcurrency) {
|
|
3076
3076
|
const headers = this.prepareHeaders(idempotencyKey);
|
|
3077
|
-
let jsonData = { urls, webhook, ignoreInvalidURLs, maxConcurrency, ...params, origin: typeof params
|
|
3077
|
+
let jsonData = { urls, webhook, ignoreInvalidURLs, maxConcurrency, ...params, origin: typeof params?.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` };
|
|
3078
3078
|
if (jsonData?.extract?.schema) {
|
|
3079
3079
|
jsonData = {
|
|
3080
3080
|
...jsonData,
|
|
@@ -3101,7 +3101,7 @@ var FirecrawlApp = class {
|
|
|
3101
3101
|
);
|
|
3102
3102
|
if (response.status === 200) {
|
|
3103
3103
|
const id = response.data.id;
|
|
3104
|
-
return this.monitorJobStatus(id, headers, pollInterval);
|
|
3104
|
+
return this.monitorJobStatus(id, headers, pollInterval, "batch");
|
|
3105
3105
|
} else {
|
|
3106
3106
|
this.handleError(response, "start batch scrape job");
|
|
3107
3107
|
}
|
|
@@ -3116,7 +3116,7 @@ var FirecrawlApp = class {
|
|
|
3116
3116
|
}
|
|
3117
3117
|
async asyncBatchScrapeUrls(urls, params, idempotencyKey, webhook, ignoreInvalidURLs) {
|
|
3118
3118
|
const headers = this.prepareHeaders(idempotencyKey);
|
|
3119
|
-
let jsonData = { urls, webhook, ignoreInvalidURLs, ...params, origin: typeof params
|
|
3119
|
+
let jsonData = { urls, webhook, ignoreInvalidURLs, ...params, origin: typeof params?.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` };
|
|
3120
3120
|
try {
|
|
3121
3121
|
const response = await this.postRequest(
|
|
3122
3122
|
this.apiUrl + `/v1/batch/scrape`,
|
|
@@ -3260,7 +3260,7 @@ var FirecrawlApp = class {
|
|
|
3260
3260
|
try {
|
|
3261
3261
|
const response = await this.postRequest(
|
|
3262
3262
|
this.apiUrl + `/v1/extract`,
|
|
3263
|
-
{ ...jsonData, schema: jsonSchema, origin: typeof params
|
|
3263
|
+
{ ...jsonData, schema: jsonSchema, origin: typeof params?.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` },
|
|
3264
3264
|
headers
|
|
3265
3265
|
);
|
|
3266
3266
|
if (response.status === 200) {
|
|
@@ -3313,7 +3313,7 @@ var FirecrawlApp = class {
|
|
|
3313
3313
|
try {
|
|
3314
3314
|
const response = await this.postRequest(
|
|
3315
3315
|
this.apiUrl + `/v1/extract`,
|
|
3316
|
-
{ ...jsonData, schema: jsonSchema, origin: typeof params
|
|
3316
|
+
{ ...jsonData, schema: jsonSchema, origin: typeof params?.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` },
|
|
3317
3317
|
headers
|
|
3318
3318
|
);
|
|
3319
3319
|
if (response.status === 200) {
|
|
@@ -3405,21 +3405,23 @@ var FirecrawlApp = class {
|
|
|
3405
3405
|
}
|
|
3406
3406
|
}
|
|
3407
3407
|
/**
|
|
3408
|
-
* Monitors the status of a crawl job until completion or failure.
|
|
3409
|
-
* @param id - The ID of the crawl operation.
|
|
3408
|
+
* Monitors the status of a crawl or batch scrape job until completion or failure.
|
|
3409
|
+
* @param id - The ID of the crawl or batch scrape operation.
|
|
3410
3410
|
* @param headers - The headers for the request.
|
|
3411
3411
|
* @param checkInterval - Interval in seconds for job status checks.
|
|
3412
|
-
* @param
|
|
3412
|
+
* @param jobType - Which status endpoint to poll. Defaults to `"crawl"`.
|
|
3413
3413
|
* @returns The final job status or data.
|
|
3414
3414
|
*/
|
|
3415
|
-
async monitorJobStatus(id, headers, checkInterval) {
|
|
3415
|
+
async monitorJobStatus(id, headers, checkInterval, jobType = "crawl") {
|
|
3416
|
+
const statusPath = jobType === "batch" ? "batch/scrape" : "crawl";
|
|
3417
|
+
const jobLabel = jobType === "batch" ? "Batch scrape" : "Crawl";
|
|
3416
3418
|
let failedTries = 0;
|
|
3417
3419
|
let networkRetries = 0;
|
|
3418
3420
|
const maxNetworkRetries = 3;
|
|
3419
3421
|
while (true) {
|
|
3420
3422
|
try {
|
|
3421
3423
|
let statusResponse = await this.getRequest(
|
|
3422
|
-
`${this.apiUrl}/v1
|
|
3424
|
+
`${this.apiUrl}/v1/${statusPath}/${id}`,
|
|
3423
3425
|
headers
|
|
3424
3426
|
);
|
|
3425
3427
|
if (statusResponse.status === 200) {
|
|
@@ -3440,7 +3442,7 @@ var FirecrawlApp = class {
|
|
|
3440
3442
|
statusData.data = data;
|
|
3441
3443
|
return statusData;
|
|
3442
3444
|
} else {
|
|
3443
|
-
throw new FirecrawlError(
|
|
3445
|
+
throw new FirecrawlError(`${jobLabel} job completed but no data was returned`, 500);
|
|
3444
3446
|
}
|
|
3445
3447
|
} else if (["active", "paused", "pending", "queued", "waiting", "scraping"].includes(statusData.status)) {
|
|
3446
3448
|
checkInterval = Math.max(checkInterval, 2);
|
|
@@ -3449,14 +3451,14 @@ var FirecrawlApp = class {
|
|
|
3449
3451
|
);
|
|
3450
3452
|
} else {
|
|
3451
3453
|
throw new FirecrawlError(
|
|
3452
|
-
|
|
3454
|
+
`${jobLabel} job failed or was stopped. Status: ${statusData.status}`,
|
|
3453
3455
|
500
|
|
3454
3456
|
);
|
|
3455
3457
|
}
|
|
3456
3458
|
} else {
|
|
3457
3459
|
failedTries++;
|
|
3458
3460
|
if (failedTries >= 3) {
|
|
3459
|
-
this.handleError(statusResponse,
|
|
3461
|
+
this.handleError(statusResponse, `check ${jobLabel.toLowerCase()} status`);
|
|
3460
3462
|
}
|
|
3461
3463
|
}
|
|
3462
3464
|
} catch (error) {
|
|
@@ -3593,7 +3595,7 @@ var FirecrawlApp = class {
|
|
|
3593
3595
|
*/
|
|
3594
3596
|
async asyncDeepResearch(query, params) {
|
|
3595
3597
|
const headers = this.prepareHeaders();
|
|
3596
|
-
let jsonData = { query, ...params, origin: typeof params
|
|
3598
|
+
let jsonData = { query, ...params, origin: typeof params?.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` };
|
|
3597
3599
|
if (jsonData?.jsonOptions?.schema) {
|
|
3598
3600
|
jsonData = {
|
|
3599
3601
|
...jsonData,
|
|
@@ -3712,7 +3714,7 @@ var FirecrawlApp = class {
|
|
|
3712
3714
|
async __asyncDeepResearch(topic, params) {
|
|
3713
3715
|
const headers = this.prepareHeaders();
|
|
3714
3716
|
try {
|
|
3715
|
-
let jsonData = { topic, ...params, origin: typeof params
|
|
3717
|
+
let jsonData = { topic, ...params, origin: typeof params?.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` };
|
|
3716
3718
|
const response = await this.postRequest(
|
|
3717
3719
|
`${this.apiUrl}/v1/deep-research`,
|
|
3718
3720
|
jsonData,
|
|
@@ -3812,7 +3814,7 @@ var FirecrawlApp = class {
|
|
|
3812
3814
|
*/
|
|
3813
3815
|
async asyncGenerateLLMsText(url, params) {
|
|
3814
3816
|
const headers = this.prepareHeaders();
|
|
3815
|
-
let jsonData = { url, ...params, origin: typeof params
|
|
3817
|
+
let jsonData = { url, ...params, origin: typeof params?.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` };
|
|
3816
3818
|
try {
|
|
3817
3819
|
const response = await this.postRequest(
|
|
3818
3820
|
`${this.apiUrl}/v1/llmstxt`,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mendable/firecrawl-js",
|
|
3
|
-
"version": "4.42.
|
|
3
|
+
"version": "4.42.4",
|
|
4
4
|
"description": "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -74,6 +74,6 @@
|
|
|
74
74
|
"build-and-publish": "pnpm run build && pnpm publish --access public",
|
|
75
75
|
"publish-beta": "pnpm run build && pnpm publish --access public --tag beta",
|
|
76
76
|
"test": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/e2e/v2/*.test.ts --detectOpenHandles",
|
|
77
|
-
"test:unit": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/unit
|
|
77
|
+
"test:unit": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/unit"
|
|
78
78
|
}
|
|
79
79
|
}
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
import { describe, test, expect, jest, afterEach, beforeEach } from "@jest/globals";
|
|
2
|
+
import axios from "axios";
|
|
3
|
+
import FirecrawlApp from "../../../v1";
|
|
4
|
+
|
|
5
|
+
const API_URL = "https://api.firecrawl.dev";
|
|
6
|
+
const API_KEY = "fc-test";
|
|
7
|
+
|
|
8
|
+
const scrapingPage = () => ({ success: true, status: "scraping", total: 2, completed: 0, data: [] });
|
|
9
|
+
const completedPage = (next: string) => ({
|
|
10
|
+
success: true,
|
|
11
|
+
status: "completed",
|
|
12
|
+
total: 2,
|
|
13
|
+
completed: 2,
|
|
14
|
+
expiresAt: "2030-01-01T00:00:00Z",
|
|
15
|
+
data: [{ markdown: "a" }],
|
|
16
|
+
next,
|
|
17
|
+
});
|
|
18
|
+
const lastPage = () => ({ success: true, status: "completed", data: [{ markdown: "b" }] });
|
|
19
|
+
|
|
20
|
+
function mockPoll(path: string, nextOrigin = API_URL) {
|
|
21
|
+
const sent: string[] = [];
|
|
22
|
+
let polls = 0;
|
|
23
|
+
jest.spyOn(axios, "get").mockImplementation(async (url: string) => {
|
|
24
|
+
sent.push(url);
|
|
25
|
+
if (new URL(url).searchParams.has("skip")) return { status: 200, data: lastPage() };
|
|
26
|
+
polls++;
|
|
27
|
+
return { status: 200, data: polls === 1 ? scrapingPage() : completedPage(`${nextOrigin}${path}?skip=1`) };
|
|
28
|
+
});
|
|
29
|
+
return sent;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
describe("v1 batch scrape waits on the batch scrape status endpoint", () => {
|
|
33
|
+
const app = new FirecrawlApp({ apiKey: API_KEY, apiUrl: API_URL });
|
|
34
|
+
|
|
35
|
+
beforeEach(() => {
|
|
36
|
+
jest.spyOn(global, "setTimeout").mockImplementation(((fn: () => void) => {
|
|
37
|
+
fn();
|
|
38
|
+
return 0;
|
|
39
|
+
}) as any);
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
afterEach(() => {
|
|
43
|
+
jest.restoreAllMocks();
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
test("batchScrapeUrls polls /v1/batch/scrape/{id} and follows pinned next pages", async () => {
|
|
47
|
+
const post = jest.spyOn(axios, "post").mockResolvedValue({ status: 200, data: { success: true, id: "abc" } });
|
|
48
|
+
const path = "/v1/batch/scrape/abc";
|
|
49
|
+
const sent = mockPoll(path, "https://evil.example");
|
|
50
|
+
|
|
51
|
+
const res: any = await app.batchScrapeUrls(["https://example.com", "https://example.org"], {});
|
|
52
|
+
|
|
53
|
+
expect(post.mock.calls[0][0]).toBe(`${API_URL}/v1/batch/scrape`);
|
|
54
|
+
expect(sent).toEqual([`${API_URL}${path}`, `${API_URL}${path}`, `${API_URL}${path}?skip=1`]);
|
|
55
|
+
expect(sent.some(url => url.includes("/v1/crawl/"))).toBe(false);
|
|
56
|
+
expect(res.status).toBe("completed");
|
|
57
|
+
expect(res.data).toHaveLength(2);
|
|
58
|
+
});
|
|
59
|
+
|
|
60
|
+
test("batchScrapeUrls reports batch scrape failures as batch scrape errors", async () => {
|
|
61
|
+
jest.spyOn(axios, "post").mockResolvedValue({ status: 200, data: { success: true, id: "abc" } });
|
|
62
|
+
const sent: string[] = [];
|
|
63
|
+
jest.spyOn(axios, "get").mockImplementation(async (url: string) => {
|
|
64
|
+
sent.push(url);
|
|
65
|
+
return { status: 200, data: { success: true, status: "failed", data: [] } };
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
await expect(app.batchScrapeUrls(["https://example.com"], {})).rejects.toThrow("Batch scrape job failed or was stopped");
|
|
69
|
+
expect(sent.every(url => url === `${API_URL}/v1/batch/scrape/abc`)).toBe(true);
|
|
70
|
+
});
|
|
71
|
+
|
|
72
|
+
test("monitorJobStatus still polls /v1/crawl/{id} by default", async () => {
|
|
73
|
+
const path = "/v1/crawl/abc";
|
|
74
|
+
const sent = mockPoll(path);
|
|
75
|
+
|
|
76
|
+
const res: any = await app.monitorJobStatus("abc", app.prepareHeaders(), 0);
|
|
77
|
+
|
|
78
|
+
expect(sent).toEqual([`${API_URL}${path}`, `${API_URL}${path}`, `${API_URL}${path}?skip=1`]);
|
|
79
|
+
expect(res.data).toHaveLength(2);
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
test("monitorJobStatus polls /v1/batch/scrape/{id} for batch jobs", async () => {
|
|
83
|
+
const path = "/v1/batch/scrape/abc";
|
|
84
|
+
const sent = mockPoll(path);
|
|
85
|
+
|
|
86
|
+
const res: any = await app.monitorJobStatus("abc", app.prepareHeaders(), 0, "batch");
|
|
87
|
+
|
|
88
|
+
expect(sent).toEqual([`${API_URL}${path}`, `${API_URL}${path}`, `${API_URL}${path}?skip=1`]);
|
|
89
|
+
expect(res.data).toHaveLength(2);
|
|
90
|
+
});
|
|
91
|
+
});
|
|
@@ -1,154 +1,107 @@
|
|
|
1
|
-
import FirecrawlApp from '../../../
|
|
1
|
+
import FirecrawlApp from '../../../v1';
|
|
2
2
|
import { describe, test, expect, jest, beforeEach, afterEach } from '@jest/globals';
|
|
3
3
|
|
|
4
|
+
const successResponse = {
|
|
5
|
+
status: 200,
|
|
6
|
+
data: { status: 'completed', data: [{ url: 'test.com', markdown: 'test' }] },
|
|
7
|
+
};
|
|
8
|
+
|
|
9
|
+
function networkError(message: string, code: string) {
|
|
10
|
+
return Object.assign(new Error(message), { code });
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
function httpError(message: string, status: number) {
|
|
14
|
+
return Object.assign(new Error(message), { response: { status, data: { error: message } } });
|
|
15
|
+
}
|
|
16
|
+
|
|
4
17
|
describe('monitorJobStatus retry logic', () => {
|
|
5
18
|
let app: FirecrawlApp;
|
|
6
|
-
let
|
|
7
|
-
|
|
19
|
+
let delays: number[];
|
|
20
|
+
|
|
8
21
|
beforeEach(() => {
|
|
9
22
|
app = new FirecrawlApp({ apiKey: 'test-key', apiUrl: 'https://test.com' });
|
|
10
|
-
|
|
11
|
-
|
|
23
|
+
delays = [];
|
|
24
|
+
jest.useFakeTimers();
|
|
25
|
+
const fakeSetTimeout = globalThis.setTimeout;
|
|
26
|
+
jest.spyOn(globalThis, 'setTimeout').mockImplementation(((fn: () => void, ms?: number) => {
|
|
27
|
+
delays.push(ms ?? 0);
|
|
28
|
+
return fakeSetTimeout(fn, ms);
|
|
29
|
+
}) as any);
|
|
12
30
|
});
|
|
13
31
|
|
|
14
32
|
afterEach(() => {
|
|
15
|
-
|
|
16
|
-
jest.
|
|
33
|
+
jest.restoreAllMocks();
|
|
34
|
+
jest.useRealTimers();
|
|
17
35
|
});
|
|
18
36
|
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
status
|
|
25
|
-
|
|
26
|
-
};
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
37
|
+
function failFirst(n: number, error: Error, statuses: string[] = []) {
|
|
38
|
+
let calls = 0;
|
|
39
|
+
app.getRequest = (async () => {
|
|
40
|
+
calls++;
|
|
41
|
+
if (calls <= n) throw error;
|
|
42
|
+
const status = statuses.shift();
|
|
43
|
+
return status ? { status: 200, data: { status } } : successResponse;
|
|
44
|
+
}) as any;
|
|
45
|
+
return () => calls;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
async function monitor() {
|
|
49
|
+
const result = app.monitorJobStatus('test-id', {} as any, 1);
|
|
50
|
+
result.catch(() => {});
|
|
51
|
+
await jest.runAllTimersAsync();
|
|
52
|
+
return result;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
test.each([
|
|
56
|
+
['socket hang up', networkError('socket hang up', 'ECONNRESET')],
|
|
57
|
+
['ETIMEDOUT', networkError('timeout', 'ETIMEDOUT')],
|
|
58
|
+
['HTTP 408', httpError('Request timeout', 408)],
|
|
59
|
+
['HTTP 504', httpError('Gateway timeout', 504)],
|
|
60
|
+
])('retries once after a %s error and returns the job data', async (_label, error) => {
|
|
61
|
+
const calls = failFirst(1, error);
|
|
62
|
+
|
|
63
|
+
const result = await monitor();
|
|
64
|
+
|
|
65
|
+
expect(calls()).toBe(2);
|
|
42
66
|
expect(result).toEqual(successResponse.data);
|
|
43
|
-
expect(
|
|
44
|
-
expect.stringContaining('Network error during job status check (attempt 1/3): socket hang up')
|
|
45
|
-
);
|
|
67
|
+
expect(delays).toEqual([1000]);
|
|
46
68
|
});
|
|
47
69
|
|
|
48
|
-
test('
|
|
49
|
-
const
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
data: { status: 'completed', data: [{ url: 'test.com', markdown: 'test' }] }
|
|
55
|
-
};
|
|
56
|
-
|
|
57
|
-
const originalGetRequest = app.getRequest;
|
|
58
|
-
let callCount = 0;
|
|
59
|
-
|
|
60
|
-
app.getRequest = async function(url: string, headers: any) {
|
|
61
|
-
callCount++;
|
|
62
|
-
if (callCount === 1) {
|
|
63
|
-
throw timeoutError;
|
|
64
|
-
}
|
|
65
|
-
return successResponse;
|
|
66
|
-
};
|
|
67
|
-
|
|
68
|
-
const result = await app.monitorJobStatus('test-id', {}, 1);
|
|
69
|
-
|
|
70
|
-
expect(callCount).toBe(2);
|
|
70
|
+
test('uses exponential backoff between retries', async () => {
|
|
71
|
+
const calls = failFirst(2, networkError('socket hang up', 'ECONNRESET'));
|
|
72
|
+
|
|
73
|
+
const result = await monitor();
|
|
74
|
+
|
|
75
|
+
expect(calls()).toBe(3);
|
|
71
76
|
expect(result).toEqual(successResponse.data);
|
|
77
|
+
expect(delays).toEqual([1000, 2000]);
|
|
72
78
|
});
|
|
73
79
|
|
|
74
|
-
test('
|
|
75
|
-
const
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
app.getRequest = async function(url: string, headers: any) {
|
|
79
|
-
throw socketHangUpError;
|
|
80
|
-
};
|
|
81
|
-
|
|
82
|
-
await expect(app.monitorJobStatus('test-id', {}, 1)).rejects.toThrow('socket hang up');
|
|
83
|
-
|
|
84
|
-
expect(console.warn).toHaveBeenCalledTimes(3);
|
|
85
|
-
}, 15000);
|
|
86
|
-
|
|
87
|
-
test('should not retry on non-retryable errors', async () => {
|
|
88
|
-
const authError = new Error('Unauthorized') as any;
|
|
89
|
-
authError.response = { status: 401, data: { error: 'Unauthorized' } };
|
|
90
|
-
|
|
91
|
-
app.getRequest = async function(url: string, headers: any) {
|
|
92
|
-
throw authError;
|
|
93
|
-
};
|
|
94
|
-
|
|
95
|
-
await expect(app.monitorJobStatus('test-id', {}, 1)).rejects.toThrow('Unauthorized');
|
|
96
|
-
|
|
97
|
-
expect(console.warn).not.toHaveBeenCalled();
|
|
98
|
-
});
|
|
80
|
+
test('records both the retry backoff and the 2s poll wait for an active job', async () => {
|
|
81
|
+
const calls = failFirst(1, networkError('socket hang up', 'ECONNRESET'), ['active']);
|
|
82
|
+
|
|
83
|
+
const result = await monitor();
|
|
99
84
|
|
|
100
|
-
|
|
101
|
-
const timeoutError = new Error('Request timeout') as any;
|
|
102
|
-
timeoutError.response = { status: 408, data: { error: 'Request timeout' } };
|
|
103
|
-
|
|
104
|
-
const successResponse = {
|
|
105
|
-
status: 200,
|
|
106
|
-
data: { status: 'completed', data: [{ url: 'test.com', markdown: 'test' }] }
|
|
107
|
-
};
|
|
108
|
-
|
|
109
|
-
const originalGetRequest = app.getRequest;
|
|
110
|
-
let callCount = 0;
|
|
111
|
-
|
|
112
|
-
app.getRequest = async function(url: string, headers: any) {
|
|
113
|
-
callCount++;
|
|
114
|
-
if (callCount === 1) {
|
|
115
|
-
throw timeoutError;
|
|
116
|
-
}
|
|
117
|
-
return successResponse;
|
|
118
|
-
};
|
|
119
|
-
|
|
120
|
-
const result = await app.monitorJobStatus('test-id', {}, 1);
|
|
121
|
-
|
|
122
|
-
expect(callCount).toBe(2);
|
|
85
|
+
expect(calls()).toBe(3);
|
|
123
86
|
expect(result).toEqual(successResponse.data);
|
|
87
|
+
expect(delays).toEqual([1000, 2000]);
|
|
124
88
|
});
|
|
125
89
|
|
|
126
|
-
test('
|
|
127
|
-
const
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
}
|
|
143
|
-
return successResponse;
|
|
144
|
-
};
|
|
145
|
-
|
|
146
|
-
const startTime = Date.now();
|
|
147
|
-
const result = await app.monitorJobStatus('test-id', {}, 1);
|
|
148
|
-
const endTime = Date.now();
|
|
149
|
-
|
|
150
|
-
expect(callCount).toBe(3);
|
|
151
|
-
expect(result).toEqual(successResponse.data);
|
|
152
|
-
expect(endTime - startTime).toBeGreaterThan(3000);
|
|
90
|
+
test('fails after max retries are exceeded', async () => {
|
|
91
|
+
const calls = failFirst(Infinity, networkError('socket hang up', 'ECONNRESET'));
|
|
92
|
+
|
|
93
|
+
await expect(monitor()).rejects.toThrow('socket hang up');
|
|
94
|
+
|
|
95
|
+
expect(calls()).toBe(4);
|
|
96
|
+
expect(delays).toEqual([1000, 2000, 4000]);
|
|
97
|
+
});
|
|
98
|
+
|
|
99
|
+
test('does not retry non-retryable errors', async () => {
|
|
100
|
+
const calls = failFirst(Infinity, httpError('Unauthorized', 401));
|
|
101
|
+
|
|
102
|
+
await expect(monitor()).rejects.toThrow('Unauthorized');
|
|
103
|
+
|
|
104
|
+
expect(calls()).toBe(1);
|
|
105
|
+
expect(delays).toEqual([]);
|
|
153
106
|
});
|
|
154
107
|
});
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import { describe, test, expect, jest, beforeEach, afterEach } from "@jest/globals";
|
|
2
|
+
import axios from "axios";
|
|
3
|
+
import FirecrawlApp from "../../../v1";
|
|
4
|
+
|
|
5
|
+
const API_URL = "https://api.firecrawl.dev";
|
|
6
|
+
|
|
7
|
+
class FakeWebSocket {
|
|
8
|
+
onopen: unknown = null;
|
|
9
|
+
onmessage: unknown = null;
|
|
10
|
+
onerror: unknown = null;
|
|
11
|
+
onclose: unknown = null;
|
|
12
|
+
close() {}
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
describe("v1 methods work without params", () => {
|
|
16
|
+
let app: FirecrawlApp;
|
|
17
|
+
let posts: Array<{ url: string; data: any }>;
|
|
18
|
+
const originalWebSocket = (globalThis as any).WebSocket;
|
|
19
|
+
|
|
20
|
+
beforeEach(() => {
|
|
21
|
+
app = new FirecrawlApp({ apiKey: "fc-test", apiUrl: API_URL });
|
|
22
|
+
posts = [];
|
|
23
|
+
jest.spyOn(axios, "post").mockImplementation(async (url: string, data?: any) => {
|
|
24
|
+
posts.push({ url, data });
|
|
25
|
+
return { status: 200, data: { success: true, id: "job-1", data: [], links: [] } };
|
|
26
|
+
});
|
|
27
|
+
jest.spyOn(axios, "get").mockImplementation(async () => ({
|
|
28
|
+
status: 200,
|
|
29
|
+
data: { success: true, status: "completed", data: [{ markdown: "ok" }] },
|
|
30
|
+
}));
|
|
31
|
+
(globalThis as any).WebSocket = FakeWebSocket;
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
afterEach(() => {
|
|
35
|
+
jest.restoreAllMocks();
|
|
36
|
+
(globalThis as any).WebSocket = originalWebSocket;
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
const cases: Array<[string, string, (app: any) => Promise<unknown>]> = [
|
|
40
|
+
["scrapeUrl", "/v1/scrape", app => app.scrapeUrl("https://example.com")],
|
|
41
|
+
["search", "/v1/search", app => app.search("firecrawl")],
|
|
42
|
+
["crawlUrl", "/v1/crawl", app => app.crawlUrl("https://example.com")],
|
|
43
|
+
["asyncCrawlUrl", "/v1/crawl", app => app.asyncCrawlUrl("https://example.com")],
|
|
44
|
+
["crawlUrlAndWatch", "/v1/crawl", app => app.crawlUrlAndWatch("https://example.com")],
|
|
45
|
+
["mapUrl", "/v1/map", app => app.mapUrl("https://example.com")],
|
|
46
|
+
["batchScrapeUrls", "/v1/batch/scrape", app => app.batchScrapeUrls(["https://example.com"])],
|
|
47
|
+
["asyncBatchScrapeUrls", "/v1/batch/scrape", app => app.asyncBatchScrapeUrls(["https://example.com"])],
|
|
48
|
+
["batchScrapeUrlsAndWatch", "/v1/batch/scrape", app => app.batchScrapeUrlsAndWatch(["https://example.com"])],
|
|
49
|
+
["extract", "/v1/extract", app => app.extract(["https://example.com"])],
|
|
50
|
+
["asyncExtract", "/v1/extract", app => app.asyncExtract(["https://example.com"])],
|
|
51
|
+
["deepResearch", "/v1/deep-research", app => app.deepResearch("firecrawl")],
|
|
52
|
+
["asyncDeepResearch", "/v1/deep-research", app => app.asyncDeepResearch("firecrawl")],
|
|
53
|
+
["__deepResearch", "/v1/deep-research", app => app.__deepResearch("firecrawl")],
|
|
54
|
+
["__asyncDeepResearch", "/v1/deep-research", app => app.__asyncDeepResearch("firecrawl")],
|
|
55
|
+
["generateLLMsText", "/v1/llmstxt", app => app.generateLLMsText("https://example.com")],
|
|
56
|
+
["asyncGenerateLLMsText", "/v1/llmstxt", app => app.asyncGenerateLLMsText("https://example.com")],
|
|
57
|
+
];
|
|
58
|
+
|
|
59
|
+
test.each(cases)("%s sends the default origin", async (_method, path, call) => {
|
|
60
|
+
await expect(call(app)).resolves.toBeDefined();
|
|
61
|
+
expect(posts).toHaveLength(1);
|
|
62
|
+
expect(posts[0].url).toBe(`${API_URL}${path}`);
|
|
63
|
+
expect(posts[0].data.origin).toBe(`js-sdk@${app.version}`);
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
test("an mcp origin passed in params is still forwarded", async () => {
|
|
67
|
+
await app.batchScrapeUrls(["https://example.com"], { origin: "mcp-server" } as any);
|
|
68
|
+
expect(posts[0].data.origin).toBe("mcp-server");
|
|
69
|
+
});
|
|
70
|
+
});
|
|
@@ -1,11 +1,60 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
*/
|
|
1
|
+
import { describe, test, expect } from "@jest/globals";
|
|
2
|
+
import axios, { type AxiosAdapter } from "axios";
|
|
4
3
|
import { FirecrawlClient } from "../../../v2/client";
|
|
5
4
|
|
|
5
|
+
const API_URL = "https://api.firecrawl.dev";
|
|
6
|
+
|
|
7
|
+
function makeClient(apiKey: string) {
|
|
8
|
+
const client = new FirecrawlClient({ apiKey, apiUrl: API_URL });
|
|
9
|
+
const sent: Array<{ method?: string; url: string; authorization: unknown; body: unknown }> = [];
|
|
10
|
+
const adapter: AxiosAdapter = async config => {
|
|
11
|
+
sent.push({
|
|
12
|
+
method: config.method,
|
|
13
|
+
url: axios.getUri(config),
|
|
14
|
+
authorization: config.headers.Authorization,
|
|
15
|
+
body: JSON.parse(config.data),
|
|
16
|
+
});
|
|
17
|
+
return {
|
|
18
|
+
data: { success: true, data: { markdown: "# hello" } },
|
|
19
|
+
status: 200,
|
|
20
|
+
statusText: "OK",
|
|
21
|
+
headers: {},
|
|
22
|
+
config,
|
|
23
|
+
};
|
|
24
|
+
};
|
|
25
|
+
(client as any).http.instance.defaults.adapter = adapter;
|
|
26
|
+
return { client, sent };
|
|
27
|
+
}
|
|
28
|
+
|
|
6
29
|
describe("v2.scrape unit", () => {
|
|
7
|
-
test("
|
|
8
|
-
|
|
30
|
+
test("constructs without an API key and scrapes keyless, without an Authorization header", async () => {
|
|
31
|
+
const { client, sent } = makeClient("");
|
|
32
|
+
const doc = await client.scrape(" https://example.com ", { formats: ["markdown"] });
|
|
33
|
+
|
|
34
|
+
expect(doc.markdown).toBe("# hello");
|
|
35
|
+
expect(sent).toHaveLength(1);
|
|
36
|
+
expect(sent[0]!.method).toBe("post");
|
|
37
|
+
expect(sent[0]!.url).toBe(`${API_URL}/v2/scrape`);
|
|
38
|
+
expect(sent[0]!.authorization).toBeUndefined();
|
|
39
|
+
expect(sent[0]!.body).toEqual({
|
|
40
|
+
url: "https://example.com",
|
|
41
|
+
formats: ["markdown"],
|
|
42
|
+
origin: expect.stringMatching(/^js-sdk@/),
|
|
43
|
+
});
|
|
9
44
|
});
|
|
10
|
-
});
|
|
11
45
|
|
|
46
|
+
test("sends the API key as a bearer token when one is set", async () => {
|
|
47
|
+
const { client, sent } = makeClient("fc-test");
|
|
48
|
+
await client.scrape("https://example.com");
|
|
49
|
+
|
|
50
|
+
expect(sent).toHaveLength(1);
|
|
51
|
+
expect(sent[0]!.authorization).toBe("Bearer fc-test");
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
test("rejects an empty URL before sending a request", async () => {
|
|
55
|
+
const { client, sent } = makeClient("fc-test");
|
|
56
|
+
|
|
57
|
+
await expect(client.scrape(" ")).rejects.toThrow("URL cannot be empty");
|
|
58
|
+
expect(sent).toHaveLength(0);
|
|
59
|
+
});
|
|
60
|
+
});
|
package/src/v1/index.ts
CHANGED
|
@@ -726,7 +726,7 @@ export default class FirecrawlApp {
|
|
|
726
726
|
"Content-Type": "application/json",
|
|
727
727
|
Authorization: `Bearer ${this.apiKey}`,
|
|
728
728
|
} as AxiosRequestHeaders;
|
|
729
|
-
let jsonData: any = { url, ...params, origin: typeof (params as any)
|
|
729
|
+
let jsonData: any = { url, ...params, origin: typeof (params as any)?.origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` };
|
|
730
730
|
if (jsonData?.extract?.schema) {
|
|
731
731
|
jsonData = {
|
|
732
732
|
...jsonData,
|
|
@@ -793,7 +793,7 @@ export default class FirecrawlApp {
|
|
|
793
793
|
lang: params?.lang ?? "en",
|
|
794
794
|
country: params?.country ?? "us",
|
|
795
795
|
location: params?.location,
|
|
796
|
-
origin: typeof (params as any)
|
|
796
|
+
origin: typeof (params as any)?.origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}`,
|
|
797
797
|
timeout: params?.timeout ?? 60000,
|
|
798
798
|
scrapeOptions: params?.scrapeOptions ?? { formats: [] },
|
|
799
799
|
};
|
|
@@ -857,7 +857,7 @@ export default class FirecrawlApp {
|
|
|
857
857
|
idempotencyKey?: string
|
|
858
858
|
): Promise<CrawlStatusResponse | ErrorResponse> {
|
|
859
859
|
const headers = this.prepareHeaders(idempotencyKey);
|
|
860
|
-
let jsonData: any = { url, ...params, origin: typeof (params as any)
|
|
860
|
+
let jsonData: any = { url, ...params, origin: typeof (params as any)?.origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` };
|
|
861
861
|
try {
|
|
862
862
|
const response: AxiosResponse = await this.postRequest(
|
|
863
863
|
this.apiUrl + `/v1/crawl`,
|
|
@@ -886,7 +886,7 @@ export default class FirecrawlApp {
|
|
|
886
886
|
idempotencyKey?: string
|
|
887
887
|
): Promise<CrawlResponse | ErrorResponse> {
|
|
888
888
|
const headers = this.prepareHeaders(idempotencyKey);
|
|
889
|
-
let jsonData: any = { url, ...params, origin: typeof (params as any)
|
|
889
|
+
let jsonData: any = { url, ...params, origin: typeof (params as any)?.origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` };
|
|
890
890
|
try {
|
|
891
891
|
const response: AxiosResponse = await this.postRequest(
|
|
892
892
|
this.apiUrl + `/v1/crawl`,
|
|
@@ -1062,7 +1062,7 @@ export default class FirecrawlApp {
|
|
|
1062
1062
|
*/
|
|
1063
1063
|
async mapUrl(url: string, params?: MapParams): Promise<MapResponse | ErrorResponse> {
|
|
1064
1064
|
const headers = this.prepareHeaders();
|
|
1065
|
-
let jsonData: any = { url, ...params, origin: typeof (params as any)
|
|
1065
|
+
let jsonData: any = { url, ...params, origin: typeof (params as any)?.origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` };
|
|
1066
1066
|
|
|
1067
1067
|
try {
|
|
1068
1068
|
const response: AxiosResponse = await this.postRequest(
|
|
@@ -1101,7 +1101,7 @@ export default class FirecrawlApp {
|
|
|
1101
1101
|
maxConcurrency?: number,
|
|
1102
1102
|
): Promise<BatchScrapeStatusResponse | ErrorResponse> {
|
|
1103
1103
|
const headers = this.prepareHeaders(idempotencyKey);
|
|
1104
|
-
let jsonData: any = { urls, webhook, ignoreInvalidURLs, maxConcurrency, ...params, origin: typeof (params as any)
|
|
1104
|
+
let jsonData: any = { urls, webhook, ignoreInvalidURLs, maxConcurrency, ...params, origin: typeof (params as any)?.origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` };
|
|
1105
1105
|
if (jsonData?.extract?.schema) {
|
|
1106
1106
|
jsonData = {
|
|
1107
1107
|
...jsonData,
|
|
@@ -1128,7 +1128,7 @@ export default class FirecrawlApp {
|
|
|
1128
1128
|
);
|
|
1129
1129
|
if (response.status === 200) {
|
|
1130
1130
|
const id: string = response.data.id;
|
|
1131
|
-
return this.monitorJobStatus(id, headers, pollInterval);
|
|
1131
|
+
return this.monitorJobStatus(id, headers, pollInterval, "batch");
|
|
1132
1132
|
} else {
|
|
1133
1133
|
this.handleError(response, "start batch scrape job");
|
|
1134
1134
|
}
|
|
@@ -1150,7 +1150,7 @@ export default class FirecrawlApp {
|
|
|
1150
1150
|
ignoreInvalidURLs?: boolean,
|
|
1151
1151
|
): Promise<BatchScrapeResponse | ErrorResponse> {
|
|
1152
1152
|
const headers = this.prepareHeaders(idempotencyKey);
|
|
1153
|
-
let jsonData: any = { urls, webhook, ignoreInvalidURLs, ...params, origin: typeof (params as any)
|
|
1153
|
+
let jsonData: any = { urls, webhook, ignoreInvalidURLs, ...params, origin: typeof (params as any)?.origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` };
|
|
1154
1154
|
try {
|
|
1155
1155
|
const response: AxiosResponse = await this.postRequest(
|
|
1156
1156
|
this.apiUrl + `/v1/batch/scrape`,
|
|
@@ -1314,7 +1314,7 @@ export default class FirecrawlApp {
|
|
|
1314
1314
|
try {
|
|
1315
1315
|
const response: AxiosResponse = await this.postRequest(
|
|
1316
1316
|
this.apiUrl + `/v1/extract`,
|
|
1317
|
-
{ ...jsonData, schema: jsonSchema, origin: typeof (params as any)
|
|
1317
|
+
{ ...jsonData, schema: jsonSchema, origin: typeof (params as any)?.origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` },
|
|
1318
1318
|
headers
|
|
1319
1319
|
);
|
|
1320
1320
|
|
|
@@ -1374,7 +1374,7 @@ export default class FirecrawlApp {
|
|
|
1374
1374
|
try {
|
|
1375
1375
|
const response: AxiosResponse = await this.postRequest(
|
|
1376
1376
|
this.apiUrl + `/v1/extract`,
|
|
1377
|
-
{ ...jsonData, schema: jsonSchema, origin: typeof (params as any)
|
|
1377
|
+
{ ...jsonData, schema: jsonSchema, origin: typeof (params as any)?.origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` },
|
|
1378
1378
|
headers
|
|
1379
1379
|
);
|
|
1380
1380
|
|
|
@@ -1484,18 +1484,21 @@ export default class FirecrawlApp {
|
|
|
1484
1484
|
}
|
|
1485
1485
|
|
|
1486
1486
|
/**
|
|
1487
|
-
* Monitors the status of a crawl job until completion or failure.
|
|
1488
|
-
* @param id - The ID of the crawl operation.
|
|
1487
|
+
* Monitors the status of a crawl or batch scrape job until completion or failure.
|
|
1488
|
+
* @param id - The ID of the crawl or batch scrape operation.
|
|
1489
1489
|
* @param headers - The headers for the request.
|
|
1490
1490
|
* @param checkInterval - Interval in seconds for job status checks.
|
|
1491
|
-
* @param
|
|
1491
|
+
* @param jobType - Which status endpoint to poll. Defaults to `"crawl"`.
|
|
1492
1492
|
* @returns The final job status or data.
|
|
1493
1493
|
*/
|
|
1494
1494
|
async monitorJobStatus(
|
|
1495
1495
|
id: string,
|
|
1496
1496
|
headers: AxiosRequestHeaders,
|
|
1497
|
-
checkInterval: number
|
|
1497
|
+
checkInterval: number,
|
|
1498
|
+
jobType: "crawl" | "batch" = "crawl"
|
|
1498
1499
|
): Promise<CrawlStatusResponse | ErrorResponse> {
|
|
1500
|
+
const statusPath = jobType === "batch" ? "batch/scrape" : "crawl";
|
|
1501
|
+
const jobLabel = jobType === "batch" ? "Batch scrape" : "Crawl";
|
|
1499
1502
|
let failedTries = 0;
|
|
1500
1503
|
let networkRetries = 0;
|
|
1501
1504
|
const maxNetworkRetries = 3;
|
|
@@ -1503,7 +1506,7 @@ export default class FirecrawlApp {
|
|
|
1503
1506
|
while (true) {
|
|
1504
1507
|
try {
|
|
1505
1508
|
let statusResponse: AxiosResponse = await this.getRequest(
|
|
1506
|
-
`${this.apiUrl}/v1
|
|
1509
|
+
`${this.apiUrl}/v1/${statusPath}/${id}`,
|
|
1507
1510
|
headers
|
|
1508
1511
|
);
|
|
1509
1512
|
|
|
@@ -1526,7 +1529,7 @@ export default class FirecrawlApp {
|
|
|
1526
1529
|
statusData.data = data;
|
|
1527
1530
|
return statusData;
|
|
1528
1531
|
} else {
|
|
1529
|
-
throw new FirecrawlError(
|
|
1532
|
+
throw new FirecrawlError(`${jobLabel} job completed but no data was returned`, 500);
|
|
1530
1533
|
}
|
|
1531
1534
|
} else if (
|
|
1532
1535
|
["active", "paused", "pending", "queued", "waiting", "scraping"].includes(statusData.status)
|
|
@@ -1537,14 +1540,14 @@ export default class FirecrawlApp {
|
|
|
1537
1540
|
);
|
|
1538
1541
|
} else {
|
|
1539
1542
|
throw new FirecrawlError(
|
|
1540
|
-
|
|
1543
|
+
`${jobLabel} job failed or was stopped. Status: ${statusData.status}`,
|
|
1541
1544
|
500
|
|
1542
1545
|
);
|
|
1543
1546
|
}
|
|
1544
1547
|
} else {
|
|
1545
1548
|
failedTries++;
|
|
1546
1549
|
if (failedTries >= 3) {
|
|
1547
|
-
this.handleError(statusResponse,
|
|
1550
|
+
this.handleError(statusResponse, `check ${jobLabel.toLowerCase()} status`);
|
|
1548
1551
|
}
|
|
1549
1552
|
}
|
|
1550
1553
|
} catch (error: any) {
|
|
@@ -1739,7 +1742,7 @@ export default class FirecrawlApp {
|
|
|
1739
1742
|
*/
|
|
1740
1743
|
async asyncDeepResearch(query: string, params: DeepResearchParams<zt.ZodSchema>): Promise<DeepResearchResponse | ErrorResponse> {
|
|
1741
1744
|
const headers = this.prepareHeaders();
|
|
1742
|
-
let jsonData: any = { query, ...params, origin: typeof (params as any)
|
|
1745
|
+
let jsonData: any = { query, ...params, origin: typeof (params as any)?.origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` };
|
|
1743
1746
|
|
|
1744
1747
|
if (jsonData?.jsonOptions?.schema) {
|
|
1745
1748
|
jsonData = {
|
|
@@ -1887,7 +1890,7 @@ export default class FirecrawlApp {
|
|
|
1887
1890
|
async __asyncDeepResearch(topic: string, params: DeepResearchParams): Promise<DeepResearchResponse | ErrorResponse> {
|
|
1888
1891
|
const headers = this.prepareHeaders();
|
|
1889
1892
|
try {
|
|
1890
|
-
let jsonData: any = { topic, ...params, origin: typeof (params as any)
|
|
1893
|
+
let jsonData: any = { topic, ...params, origin: typeof (params as any)?.origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` };
|
|
1891
1894
|
const response: AxiosResponse = await this.postRequest(
|
|
1892
1895
|
`${this.apiUrl}/v1/deep-research`,
|
|
1893
1896
|
jsonData,
|
|
@@ -2002,7 +2005,7 @@ export default class FirecrawlApp {
|
|
|
2002
2005
|
*/
|
|
2003
2006
|
async asyncGenerateLLMsText(url: string, params?: GenerateLLMsTextParams): Promise<GenerateLLMsTextResponse | ErrorResponse> {
|
|
2004
2007
|
const headers = this.prepareHeaders();
|
|
2005
|
-
let jsonData: any = { url, ...params, origin: typeof (params as any)
|
|
2008
|
+
let jsonData: any = { url, ...params, origin: typeof (params as any)?.origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` };
|
|
2006
2009
|
try {
|
|
2007
2010
|
const response: AxiosResponse = await this.postRequest(
|
|
2008
2011
|
`${this.apiUrl}/v1/llmstxt`,
|