@mendable/firecrawl-js 4.44.1 → 4.46.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/audit-ci.jsonc +1 -15
- package/dist/{chunk-JU6NENYM.js → chunk-R4XT7PME.js} +5 -9
- package/dist/index.cjs +77 -21
- package/dist/index.d.cts +14 -2
- package/dist/index.d.ts +14 -2
- package/dist/index.js +74 -14
- package/dist/{package-FEZU4ANT.js → package-EI4Q33HD.js} +1 -1
- package/package.json +5 -9
- package/pnpm-workspace.yaml +0 -6
- package/src/__tests__/e2e/v1/index.test.ts +1 -1
- package/src/__tests__/e2e/v2/batch.test.ts +1 -1
- package/src/__tests__/e2e/v2/crawl.test.ts +1 -1
- package/src/__tests__/e2e/v2/extract.test.ts +1 -1
- package/src/__tests__/e2e/v2/map.test.ts +1 -1
- package/src/__tests__/e2e/v2/parse.test.ts +1 -1
- package/src/__tests__/e2e/v2/scrape.test.ts +1 -1
- package/src/__tests__/e2e/v2/search.test.ts +1 -1
- package/src/__tests__/e2e/v2/usage.test.ts +1 -1
- package/src/__tests__/e2e/v2/utils/rateLimit.ts +2 -2
- package/src/__tests__/e2e/v2/watcher.test.ts +1 -1
- package/src/__tests__/unit/v1/batch-scrape-poll.test.ts +7 -7
- package/src/__tests__/unit/v1/monitor-job-status-retry.test.ts +6 -6
- package/src/__tests__/unit/v1/next-url-pinning.test.ts +3 -3
- package/src/__tests__/unit/v1/optional-params.test.ts +4 -4
- package/src/__tests__/unit/v2/agent-effort-trace.unit.test.ts +6 -6
- package/src/__tests__/unit/v2/agent-hints.test.ts +3 -3
- package/src/__tests__/unit/v2/agent-list.unit.test.ts +4 -4
- package/src/__tests__/unit/v2/agent-thread.unit.test.ts +7 -7
- package/src/__tests__/unit/v2/agent.test.ts +1 -1
- package/src/__tests__/unit/v2/audit-metadata.unit.test.ts +3 -3
- package/src/__tests__/unit/v2/branding.test.ts +2 -2
- package/src/__tests__/unit/v2/clientOptions.test.ts +1 -0
- package/src/__tests__/unit/v2/crawl-cancel-abort.unit.test.ts +189 -0
- package/src/__tests__/unit/v2/developer.test.ts +4 -4
- package/src/__tests__/unit/v2/e2eRateLimitRetry.test.ts +16 -16
- package/src/__tests__/unit/v2/errorHandler.test.ts +1 -1
- package/src/__tests__/unit/v2/extract.unit.test.ts +4 -4
- package/src/__tests__/unit/v2/gov.test.ts +4 -4
- package/src/__tests__/unit/v2/menu.test.ts +2 -2
- package/src/__tests__/unit/v2/monitor-search-target.test.ts +1 -0
- package/src/__tests__/unit/v2/next-url-pinning.test.ts +1 -1
- package/src/__tests__/unit/v2/pagination.test.ts +6 -6
- package/src/__tests__/unit/v2/parse-formats.unit.test.ts +1 -1
- package/src/__tests__/unit/v2/parse.unit.test.ts +1 -1
- package/src/__tests__/unit/v2/product.test.ts +2 -2
- package/src/__tests__/unit/v2/prototype-discoverability.test.ts +1 -0
- package/src/__tests__/unit/v2/research.test.ts +1 -1
- package/src/__tests__/unit/v2/scrape-browser.unit.test.ts +8 -8
- package/src/__tests__/unit/v2/scrape.unit.test.ts +1 -1
- package/src/__tests__/unit/v2/scrapeAutoResume.test.ts +13 -13
- package/src/__tests__/unit/v2/search-country.unit.test.ts +2 -2
- package/src/__tests__/unit/v2/search-data-property.test.ts +3 -3
- package/src/__tests__/unit/v2/search-developer.unit.test.ts +2 -2
- package/src/__tests__/unit/v2/search-highlights.unit.test.ts +2 -2
- package/src/__tests__/unit/v2/threat-protection.unit.test.ts +3 -3
- package/src/__tests__/unit/v2/tools.unit.test.ts +3 -3
- package/src/__tests__/unit/v2/v1-aliases.test.ts +2 -2
- package/src/__tests__/unit/v2/validation.test.ts +1 -1
- package/src/__tests__/unit/v2/zodSchemaToJson.test.ts +1 -1
- package/src/v1/index.ts +1 -3
- package/src/v2/client.ts +8 -5
- package/src/v2/methods/crawl.ts +90 -6
- package/src/v2/types.ts +8 -0
- package/vitest.config.ts +7 -0
- package/jest.config.js +0 -16
package/audit-ci.jsonc
CHANGED
|
@@ -1,19 +1,5 @@
|
|
|
1
1
|
{
|
|
2
2
|
"$schema": "https://github.com/IBM/audit-ci/raw/main/docs/schema.json",
|
|
3
3
|
"low": true,
|
|
4
|
-
"allowlist": [
|
|
5
|
-
// GHSA-vfj7-8cjw-p6xm (braces):
|
|
6
|
-
// Latest published braces is 3.0.3; advisory range is <=3.0.3 and
|
|
7
|
-
// no 3.0.4+ exists on npm. Same-major jest 30.x still resolves
|
|
8
|
-
// 3.0.3 via micromatch. Cannot override to a non-existent version.
|
|
9
|
-
// Owner: team-engineering-reviewers. Tracking: create internal ticket.
|
|
10
|
-
// Expires: 2026-10-17.
|
|
11
|
-
"GHSA-vfj7-8cjw-p6xm",
|
|
12
|
-
// GHSA-hp3w-g68c-fv3c (sprintf-js):
|
|
13
|
-
// Advisory asks for >=1.1.4; latest published sprintf-js is
|
|
14
|
-
// 1.1.3. Cannot override to a non-existent version.
|
|
15
|
-
// Owner: team-engineering-reviewers. Tracking: create internal ticket.
|
|
16
|
-
// Expires: 2026-10-20.
|
|
17
|
-
"GHSA-hp3w-g68c-fv3c"
|
|
18
|
-
]
|
|
4
|
+
"allowlist": []
|
|
19
5
|
}
|
|
@@ -12,7 +12,7 @@ var require_package = __commonJS({
|
|
|
12
12
|
"package.json"(exports, module) {
|
|
13
13
|
module.exports = {
|
|
14
14
|
name: "@mendable/firecrawl-js",
|
|
15
|
-
version: "4.
|
|
15
|
+
version: "4.46.0",
|
|
16
16
|
description: "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
|
|
17
17
|
main: "dist/index.js",
|
|
18
18
|
types: "dist/index.d.ts",
|
|
@@ -28,8 +28,8 @@ var require_package = __commonJS({
|
|
|
28
28
|
build: "tsup",
|
|
29
29
|
"build-and-publish": "pnpm run build && pnpm publish --access public",
|
|
30
30
|
"publish-beta": "pnpm run build && pnpm publish --access public --tag beta",
|
|
31
|
-
test: "
|
|
32
|
-
"test:unit": "
|
|
31
|
+
test: "vitest run --reporter=verbose src/__tests__/e2e/v2",
|
|
32
|
+
"test:unit": "vitest run --reporter=verbose src/__tests__/unit"
|
|
33
33
|
},
|
|
34
34
|
repository: {
|
|
35
35
|
type: "git",
|
|
@@ -49,18 +49,14 @@ var require_package = __commonJS({
|
|
|
49
49
|
},
|
|
50
50
|
homepage: "https://github.com/firecrawl/firecrawl#readme",
|
|
51
51
|
devDependencies: {
|
|
52
|
-
"@jest/globals": "^30.2.0",
|
|
53
52
|
"@types/dotenv": "^8.2.0",
|
|
54
|
-
"@types/jest": "^30.0.0",
|
|
55
|
-
"@types/mocha": "^10.0.6",
|
|
56
53
|
"@types/node": "^20.12.12",
|
|
57
54
|
"@types/uuid": "^9.0.8",
|
|
58
55
|
dotenv: "^16.4.5",
|
|
59
|
-
jest: "^30.2.0",
|
|
60
|
-
"ts-jest": "^29.4.5",
|
|
61
56
|
tsup: "^8.5.0",
|
|
62
57
|
typescript: "^5.4.5",
|
|
63
|
-
uuid: "^14.0.0"
|
|
58
|
+
uuid: "^14.0.0",
|
|
59
|
+
vitest: "^4.1.11"
|
|
64
60
|
},
|
|
65
61
|
keywords: [
|
|
66
62
|
"firecrawl",
|
package/dist/index.cjs
CHANGED
|
@@ -39,7 +39,7 @@ var require_package = __commonJS({
|
|
|
39
39
|
"package.json"(exports2, module2) {
|
|
40
40
|
module2.exports = {
|
|
41
41
|
name: "@mendable/firecrawl-js",
|
|
42
|
-
version: "4.
|
|
42
|
+
version: "4.46.0",
|
|
43
43
|
description: "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
|
|
44
44
|
main: "dist/index.js",
|
|
45
45
|
types: "dist/index.d.ts",
|
|
@@ -55,8 +55,8 @@ var require_package = __commonJS({
|
|
|
55
55
|
build: "tsup",
|
|
56
56
|
"build-and-publish": "pnpm run build && pnpm publish --access public",
|
|
57
57
|
"publish-beta": "pnpm run build && pnpm publish --access public --tag beta",
|
|
58
|
-
test: "
|
|
59
|
-
"test:unit": "
|
|
58
|
+
test: "vitest run --reporter=verbose src/__tests__/e2e/v2",
|
|
59
|
+
"test:unit": "vitest run --reporter=verbose src/__tests__/unit"
|
|
60
60
|
},
|
|
61
61
|
repository: {
|
|
62
62
|
type: "git",
|
|
@@ -76,18 +76,14 @@ var require_package = __commonJS({
|
|
|
76
76
|
},
|
|
77
77
|
homepage: "https://github.com/firecrawl/firecrawl#readme",
|
|
78
78
|
devDependencies: {
|
|
79
|
-
"@jest/globals": "^30.2.0",
|
|
80
79
|
"@types/dotenv": "^8.2.0",
|
|
81
|
-
"@types/jest": "^30.0.0",
|
|
82
|
-
"@types/mocha": "^10.0.6",
|
|
83
80
|
"@types/node": "^20.12.12",
|
|
84
81
|
"@types/uuid": "^9.0.8",
|
|
85
82
|
dotenv: "^16.4.5",
|
|
86
|
-
jest: "^30.2.0",
|
|
87
|
-
"ts-jest": "^29.4.5",
|
|
88
83
|
tsup: "^8.5.0",
|
|
89
84
|
typescript: "^5.4.5",
|
|
90
|
-
uuid: "^14.0.0"
|
|
85
|
+
uuid: "^14.0.0",
|
|
86
|
+
vitest: "^4.1.11"
|
|
91
87
|
},
|
|
92
88
|
keywords: [
|
|
93
89
|
"firecrawl",
|
|
@@ -636,7 +632,7 @@ function processingContinuesDelayMs(err) {
|
|
|
636
632
|
return Math.min(RESUME_MAX_DELAY_MS, Math.max(RESUME_MIN_DELAY_MS, seconds * 1e3));
|
|
637
633
|
}
|
|
638
634
|
async function scrape(http, url, options, deps = {}) {
|
|
639
|
-
const
|
|
635
|
+
const sleep2 = deps.sleepImpl ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
|
|
640
636
|
if (!url || !url.trim()) {
|
|
641
637
|
throw new Error("URL cannot be empty");
|
|
642
638
|
}
|
|
@@ -669,7 +665,7 @@ async function scrape(http, url, options, deps = {}) {
|
|
|
669
665
|
if (delayMs !== void 0 && resumes < RESUME_MAX_ATTEMPTS && resumeWaitedMs + delayMs <= RESUME_MAX_TOTAL_WAIT_MS) {
|
|
670
666
|
resumes += 1;
|
|
671
667
|
resumeWaitedMs += delayMs;
|
|
672
|
-
await
|
|
668
|
+
await sleep2(delayMs);
|
|
673
669
|
continue;
|
|
674
670
|
}
|
|
675
671
|
if (err?.isAxiosError) return normalizeAxiosError(err, "scrape");
|
|
@@ -1255,22 +1251,72 @@ async function getCrawlStatus(http, jobId, pagination) {
|
|
|
1255
1251
|
async function cancelCrawl(http, jobId) {
|
|
1256
1252
|
try {
|
|
1257
1253
|
const res = await http.delete(`/v2/crawl/${jobId}`);
|
|
1254
|
+
if (res.status === 409) return false;
|
|
1258
1255
|
if (res.status !== 200) throwForBadResponse(res, "cancel crawl");
|
|
1259
1256
|
return res.data?.status === "cancelled";
|
|
1260
1257
|
} catch (err) {
|
|
1261
|
-
if (err?.isAxiosError)
|
|
1258
|
+
if (err?.isAxiosError) {
|
|
1259
|
+
if (err.response?.status === 409) return false;
|
|
1260
|
+
return normalizeAxiosError(err, "cancel crawl");
|
|
1261
|
+
}
|
|
1262
1262
|
throw err;
|
|
1263
1263
|
}
|
|
1264
1264
|
}
|
|
1265
|
-
|
|
1265
|
+
function abortReason(signal) {
|
|
1266
|
+
if (signal.reason !== void 0) return signal.reason;
|
|
1267
|
+
return new DOMException("This operation was aborted", "AbortError");
|
|
1268
|
+
}
|
|
1269
|
+
function raceAbort(promise, signal) {
|
|
1270
|
+
if (!signal) return promise;
|
|
1271
|
+
if (signal.aborted) {
|
|
1272
|
+
promise.catch(() => {
|
|
1273
|
+
});
|
|
1274
|
+
return Promise.reject(abortReason(signal));
|
|
1275
|
+
}
|
|
1276
|
+
return new Promise((resolve, reject) => {
|
|
1277
|
+
const onAbort = () => {
|
|
1278
|
+
promise.catch(() => {
|
|
1279
|
+
});
|
|
1280
|
+
reject(abortReason(signal));
|
|
1281
|
+
};
|
|
1282
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
1283
|
+
promise.then(
|
|
1284
|
+
(value) => {
|
|
1285
|
+
signal.removeEventListener("abort", onAbort);
|
|
1286
|
+
resolve(value);
|
|
1287
|
+
},
|
|
1288
|
+
(err) => {
|
|
1289
|
+
signal.removeEventListener("abort", onAbort);
|
|
1290
|
+
reject(err);
|
|
1291
|
+
}
|
|
1292
|
+
);
|
|
1293
|
+
});
|
|
1294
|
+
}
|
|
1295
|
+
function sleep(ms, signal) {
|
|
1296
|
+
return new Promise((resolve, reject) => {
|
|
1297
|
+
if (signal?.aborted) return reject(abortReason(signal));
|
|
1298
|
+
const onAbort = () => {
|
|
1299
|
+
clearTimeout(timer);
|
|
1300
|
+
reject(abortReason(signal));
|
|
1301
|
+
};
|
|
1302
|
+
const timer = setTimeout(() => {
|
|
1303
|
+
signal?.removeEventListener("abort", onAbort);
|
|
1304
|
+
resolve();
|
|
1305
|
+
}, ms);
|
|
1306
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
1307
|
+
});
|
|
1308
|
+
}
|
|
1309
|
+
async function waitForCrawlCompletion(http, jobId, pollInterval = 2, timeout, signal) {
|
|
1266
1310
|
const start = Date.now();
|
|
1267
1311
|
while (true) {
|
|
1312
|
+
if (signal?.aborted) throw abortReason(signal);
|
|
1268
1313
|
try {
|
|
1269
|
-
const status = await getCrawlStatus(http, jobId);
|
|
1314
|
+
const status = await raceAbort(getCrawlStatus(http, jobId), signal);
|
|
1270
1315
|
if (["completed", "failed", "cancelled"].includes(status.status)) {
|
|
1271
1316
|
return status;
|
|
1272
1317
|
}
|
|
1273
1318
|
} catch (err) {
|
|
1319
|
+
if (signal?.aborted) throw abortReason(signal);
|
|
1274
1320
|
if (!isRetryableError(err)) {
|
|
1275
1321
|
if (err instanceof SdkError) {
|
|
1276
1322
|
const errorWithJobId = new SdkError(
|
|
@@ -1288,12 +1334,19 @@ async function waitForCrawlCompletion(http, jobId, pollInterval = 2, timeout) {
|
|
|
1288
1334
|
if (timeout != null && Date.now() - start > timeout * 1e3) {
|
|
1289
1335
|
throw new JobTimeoutError(jobId, timeout, "crawl");
|
|
1290
1336
|
}
|
|
1291
|
-
await
|
|
1337
|
+
await sleep(Math.max(1e3, pollInterval * 1e3), signal);
|
|
1292
1338
|
}
|
|
1293
1339
|
}
|
|
1294
|
-
async function crawl(http, request, pollInterval = 2, timeout) {
|
|
1340
|
+
async function crawl(http, request, pollInterval = 2, timeout, signal) {
|
|
1341
|
+
if (signal?.aborted) throw abortReason(signal);
|
|
1295
1342
|
const started = await startCrawl(http, request);
|
|
1296
|
-
|
|
1343
|
+
try {
|
|
1344
|
+
return await waitForCrawlCompletion(http, started.id, pollInterval, timeout, signal);
|
|
1345
|
+
} catch (err) {
|
|
1346
|
+
if (signal?.aborted) cancelCrawl(http, started.id).catch(() => {
|
|
1347
|
+
});
|
|
1348
|
+
throw err;
|
|
1349
|
+
}
|
|
1297
1350
|
}
|
|
1298
1351
|
async function getCrawlErrors(http, crawlId) {
|
|
1299
1352
|
try {
|
|
@@ -2508,7 +2561,7 @@ var FirecrawlClient = class {
|
|
|
2508
2561
|
/**
|
|
2509
2562
|
* Cancel a crawl job.
|
|
2510
2563
|
* @param jobId Crawl job id.
|
|
2511
|
-
* @returns True if cancelled.
|
|
2564
|
+
* @returns True if cancelled. False if the crawl was already completed (the API answers 409).
|
|
2512
2565
|
*/
|
|
2513
2566
|
async cancelCrawl(jobId) {
|
|
2514
2567
|
return cancelCrawl(this.http, jobId);
|
|
@@ -2516,11 +2569,14 @@ var FirecrawlClient = class {
|
|
|
2516
2569
|
/**
|
|
2517
2570
|
* Convenience waiter: start a crawl and poll until it finishes.
|
|
2518
2571
|
* @param url Root URL to crawl.
|
|
2519
|
-
* @param req Crawl configuration plus waiter controls (pollInterval, timeout seconds).
|
|
2572
|
+
* @param req Crawl configuration plus waiter controls (pollInterval, timeout seconds, signal).
|
|
2573
|
+
* When `signal` aborts, polling stops, the job gets a best-effort cancel request,
|
|
2574
|
+
* and the promise rejects with `signal.reason`.
|
|
2520
2575
|
* @returns Final job snapshot.
|
|
2521
2576
|
*/
|
|
2522
2577
|
async crawl(url, req = {}) {
|
|
2523
|
-
|
|
2578
|
+
const { pollInterval, timeout, signal, ...options } = req;
|
|
2579
|
+
return crawl(this.http, { url, ...options }, pollInterval, timeout, signal);
|
|
2524
2580
|
}
|
|
2525
2581
|
/**
|
|
2526
2582
|
* Retrieve crawl errors and robots.txt blocks.
|
|
@@ -2875,7 +2931,7 @@ var FirecrawlApp = class {
|
|
|
2875
2931
|
const packageJson = await Promise.resolve().then(() => __toESM(require_package(), 1));
|
|
2876
2932
|
return packageJson.default.version;
|
|
2877
2933
|
} catch (error) {
|
|
2878
|
-
const isTest = typeof process !== "undefined" &&
|
|
2934
|
+
const isTest = typeof process !== "undefined" && false;
|
|
2879
2935
|
if (!isTest) {
|
|
2880
2936
|
console.error("Error getting version:", error);
|
|
2881
2937
|
}
|
package/dist/index.d.cts
CHANGED
|
@@ -16,6 +16,7 @@ interface JsonFormat extends Format {
|
|
|
16
16
|
type: "json";
|
|
17
17
|
prompt?: string;
|
|
18
18
|
schema?: Record<string, unknown> | ZodTypeAny;
|
|
19
|
+
/** @deprecated Use the top-level `checkPromptInjection` scrape option. */
|
|
19
20
|
checkPromptInjection?: boolean;
|
|
20
21
|
}
|
|
21
22
|
interface ScreenshotFormat {
|
|
@@ -200,6 +201,13 @@ interface ScrapeOptions {
|
|
|
200
201
|
minAge?: number;
|
|
201
202
|
storeInCache?: boolean;
|
|
202
203
|
lockdown?: boolean;
|
|
204
|
+
/**
|
|
205
|
+
* Scans the page content for prompt injection with any format except
|
|
206
|
+
* rawBase64, before LLM-backed formats run. A detection fails the scrape
|
|
207
|
+
* with SCRAPE_PROMPT_INJECTION_DETECTED.
|
|
208
|
+
* Adds 4 credits when the check scans the whole page.
|
|
209
|
+
*/
|
|
210
|
+
checkPromptInjection?: boolean;
|
|
203
211
|
redactPII?: boolean | RedactPIIOptions;
|
|
204
212
|
threatProtection?: ThreatProtectionOptions;
|
|
205
213
|
auditMetadata?: AuditMetadata$1;
|
|
@@ -2396,18 +2404,21 @@ declare class FirecrawlClient {
|
|
|
2396
2404
|
/**
|
|
2397
2405
|
* Cancel a crawl job.
|
|
2398
2406
|
* @param jobId Crawl job id.
|
|
2399
|
-
* @returns True if cancelled.
|
|
2407
|
+
* @returns True if cancelled. False if the crawl was already completed (the API answers 409).
|
|
2400
2408
|
*/
|
|
2401
2409
|
cancelCrawl(jobId: string): Promise<boolean>;
|
|
2402
2410
|
/**
|
|
2403
2411
|
* Convenience waiter: start a crawl and poll until it finishes.
|
|
2404
2412
|
* @param url Root URL to crawl.
|
|
2405
|
-
* @param req Crawl configuration plus waiter controls (pollInterval, timeout seconds).
|
|
2413
|
+
* @param req Crawl configuration plus waiter controls (pollInterval, timeout seconds, signal).
|
|
2414
|
+
* When `signal` aborts, polling stops, the job gets a best-effort cancel request,
|
|
2415
|
+
* and the promise rejects with `signal.reason`.
|
|
2406
2416
|
* @returns Final job snapshot.
|
|
2407
2417
|
*/
|
|
2408
2418
|
crawl(url: string, req?: CrawlOptions & {
|
|
2409
2419
|
pollInterval?: number;
|
|
2410
2420
|
timeout?: number;
|
|
2421
|
+
signal?: AbortSignal;
|
|
2411
2422
|
}): Promise<CrawlJob>;
|
|
2412
2423
|
/**
|
|
2413
2424
|
* Retrieve crawl errors and robots.txt blocks.
|
|
@@ -2620,6 +2631,7 @@ declare class FirecrawlClient {
|
|
|
2620
2631
|
crawlUrl(url: string, req?: CrawlOptions & {
|
|
2621
2632
|
pollInterval?: number;
|
|
2622
2633
|
timeout?: number;
|
|
2634
|
+
signal?: AbortSignal;
|
|
2623
2635
|
}): Promise<CrawlJob>;
|
|
2624
2636
|
/** @deprecated V1 compatibility alias for agent recovery. Prefer startCrawl(). */
|
|
2625
2637
|
asyncCrawlUrl(url: string, req?: CrawlOptions): Promise<CrawlResponse$1>;
|
package/dist/index.d.ts
CHANGED
|
@@ -16,6 +16,7 @@ interface JsonFormat extends Format {
|
|
|
16
16
|
type: "json";
|
|
17
17
|
prompt?: string;
|
|
18
18
|
schema?: Record<string, unknown> | ZodTypeAny;
|
|
19
|
+
/** @deprecated Use the top-level `checkPromptInjection` scrape option. */
|
|
19
20
|
checkPromptInjection?: boolean;
|
|
20
21
|
}
|
|
21
22
|
interface ScreenshotFormat {
|
|
@@ -200,6 +201,13 @@ interface ScrapeOptions {
|
|
|
200
201
|
minAge?: number;
|
|
201
202
|
storeInCache?: boolean;
|
|
202
203
|
lockdown?: boolean;
|
|
204
|
+
/**
|
|
205
|
+
* Scans the page content for prompt injection with any format except
|
|
206
|
+
* rawBase64, before LLM-backed formats run. A detection fails the scrape
|
|
207
|
+
* with SCRAPE_PROMPT_INJECTION_DETECTED.
|
|
208
|
+
* Adds 4 credits when the check scans the whole page.
|
|
209
|
+
*/
|
|
210
|
+
checkPromptInjection?: boolean;
|
|
203
211
|
redactPII?: boolean | RedactPIIOptions;
|
|
204
212
|
threatProtection?: ThreatProtectionOptions;
|
|
205
213
|
auditMetadata?: AuditMetadata$1;
|
|
@@ -2396,18 +2404,21 @@ declare class FirecrawlClient {
|
|
|
2396
2404
|
/**
|
|
2397
2405
|
* Cancel a crawl job.
|
|
2398
2406
|
* @param jobId Crawl job id.
|
|
2399
|
-
* @returns True if cancelled.
|
|
2407
|
+
* @returns True if cancelled. False if the crawl was already completed (the API answers 409).
|
|
2400
2408
|
*/
|
|
2401
2409
|
cancelCrawl(jobId: string): Promise<boolean>;
|
|
2402
2410
|
/**
|
|
2403
2411
|
* Convenience waiter: start a crawl and poll until it finishes.
|
|
2404
2412
|
* @param url Root URL to crawl.
|
|
2405
|
-
* @param req Crawl configuration plus waiter controls (pollInterval, timeout seconds).
|
|
2413
|
+
* @param req Crawl configuration plus waiter controls (pollInterval, timeout seconds, signal).
|
|
2414
|
+
* When `signal` aborts, polling stops, the job gets a best-effort cancel request,
|
|
2415
|
+
* and the promise rejects with `signal.reason`.
|
|
2406
2416
|
* @returns Final job snapshot.
|
|
2407
2417
|
*/
|
|
2408
2418
|
crawl(url: string, req?: CrawlOptions & {
|
|
2409
2419
|
pollInterval?: number;
|
|
2410
2420
|
timeout?: number;
|
|
2421
|
+
signal?: AbortSignal;
|
|
2411
2422
|
}): Promise<CrawlJob>;
|
|
2412
2423
|
/**
|
|
2413
2424
|
* Retrieve crawl errors and robots.txt blocks.
|
|
@@ -2620,6 +2631,7 @@ declare class FirecrawlClient {
|
|
|
2620
2631
|
crawlUrl(url: string, req?: CrawlOptions & {
|
|
2621
2632
|
pollInterval?: number;
|
|
2622
2633
|
timeout?: number;
|
|
2634
|
+
signal?: AbortSignal;
|
|
2623
2635
|
}): Promise<CrawlJob>;
|
|
2624
2636
|
/** @deprecated V1 compatibility alias for agent recovery. Prefer startCrawl(). */
|
|
2625
2637
|
asyncCrawlUrl(url: string, req?: CrawlOptions): Promise<CrawlResponse$1>;
|
package/dist/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
require_package
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-R4XT7PME.js";
|
|
4
4
|
|
|
5
5
|
// src/v2/utils/httpClient.ts
|
|
6
6
|
import axios from "axios";
|
|
@@ -504,7 +504,7 @@ function processingContinuesDelayMs(err) {
|
|
|
504
504
|
return Math.min(RESUME_MAX_DELAY_MS, Math.max(RESUME_MIN_DELAY_MS, seconds * 1e3));
|
|
505
505
|
}
|
|
506
506
|
async function scrape(http, url, options, deps = {}) {
|
|
507
|
-
const
|
|
507
|
+
const sleep2 = deps.sleepImpl ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
|
|
508
508
|
if (!url || !url.trim()) {
|
|
509
509
|
throw new Error("URL cannot be empty");
|
|
510
510
|
}
|
|
@@ -537,7 +537,7 @@ async function scrape(http, url, options, deps = {}) {
|
|
|
537
537
|
if (delayMs !== void 0 && resumes < RESUME_MAX_ATTEMPTS && resumeWaitedMs + delayMs <= RESUME_MAX_TOTAL_WAIT_MS) {
|
|
538
538
|
resumes += 1;
|
|
539
539
|
resumeWaitedMs += delayMs;
|
|
540
|
-
await
|
|
540
|
+
await sleep2(delayMs);
|
|
541
541
|
continue;
|
|
542
542
|
}
|
|
543
543
|
if (err?.isAxiosError) return normalizeAxiosError(err, "scrape");
|
|
@@ -1123,22 +1123,72 @@ async function getCrawlStatus(http, jobId, pagination) {
|
|
|
1123
1123
|
async function cancelCrawl(http, jobId) {
|
|
1124
1124
|
try {
|
|
1125
1125
|
const res = await http.delete(`/v2/crawl/${jobId}`);
|
|
1126
|
+
if (res.status === 409) return false;
|
|
1126
1127
|
if (res.status !== 200) throwForBadResponse(res, "cancel crawl");
|
|
1127
1128
|
return res.data?.status === "cancelled";
|
|
1128
1129
|
} catch (err) {
|
|
1129
|
-
if (err?.isAxiosError)
|
|
1130
|
+
if (err?.isAxiosError) {
|
|
1131
|
+
if (err.response?.status === 409) return false;
|
|
1132
|
+
return normalizeAxiosError(err, "cancel crawl");
|
|
1133
|
+
}
|
|
1130
1134
|
throw err;
|
|
1131
1135
|
}
|
|
1132
1136
|
}
|
|
1133
|
-
|
|
1137
|
+
function abortReason(signal) {
|
|
1138
|
+
if (signal.reason !== void 0) return signal.reason;
|
|
1139
|
+
return new DOMException("This operation was aborted", "AbortError");
|
|
1140
|
+
}
|
|
1141
|
+
function raceAbort(promise, signal) {
|
|
1142
|
+
if (!signal) return promise;
|
|
1143
|
+
if (signal.aborted) {
|
|
1144
|
+
promise.catch(() => {
|
|
1145
|
+
});
|
|
1146
|
+
return Promise.reject(abortReason(signal));
|
|
1147
|
+
}
|
|
1148
|
+
return new Promise((resolve, reject) => {
|
|
1149
|
+
const onAbort = () => {
|
|
1150
|
+
promise.catch(() => {
|
|
1151
|
+
});
|
|
1152
|
+
reject(abortReason(signal));
|
|
1153
|
+
};
|
|
1154
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
1155
|
+
promise.then(
|
|
1156
|
+
(value) => {
|
|
1157
|
+
signal.removeEventListener("abort", onAbort);
|
|
1158
|
+
resolve(value);
|
|
1159
|
+
},
|
|
1160
|
+
(err) => {
|
|
1161
|
+
signal.removeEventListener("abort", onAbort);
|
|
1162
|
+
reject(err);
|
|
1163
|
+
}
|
|
1164
|
+
);
|
|
1165
|
+
});
|
|
1166
|
+
}
|
|
1167
|
+
function sleep(ms, signal) {
|
|
1168
|
+
return new Promise((resolve, reject) => {
|
|
1169
|
+
if (signal?.aborted) return reject(abortReason(signal));
|
|
1170
|
+
const onAbort = () => {
|
|
1171
|
+
clearTimeout(timer);
|
|
1172
|
+
reject(abortReason(signal));
|
|
1173
|
+
};
|
|
1174
|
+
const timer = setTimeout(() => {
|
|
1175
|
+
signal?.removeEventListener("abort", onAbort);
|
|
1176
|
+
resolve();
|
|
1177
|
+
}, ms);
|
|
1178
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
1179
|
+
});
|
|
1180
|
+
}
|
|
1181
|
+
async function waitForCrawlCompletion(http, jobId, pollInterval = 2, timeout, signal) {
|
|
1134
1182
|
const start = Date.now();
|
|
1135
1183
|
while (true) {
|
|
1184
|
+
if (signal?.aborted) throw abortReason(signal);
|
|
1136
1185
|
try {
|
|
1137
|
-
const status = await getCrawlStatus(http, jobId);
|
|
1186
|
+
const status = await raceAbort(getCrawlStatus(http, jobId), signal);
|
|
1138
1187
|
if (["completed", "failed", "cancelled"].includes(status.status)) {
|
|
1139
1188
|
return status;
|
|
1140
1189
|
}
|
|
1141
1190
|
} catch (err) {
|
|
1191
|
+
if (signal?.aborted) throw abortReason(signal);
|
|
1142
1192
|
if (!isRetryableError(err)) {
|
|
1143
1193
|
if (err instanceof SdkError) {
|
|
1144
1194
|
const errorWithJobId = new SdkError(
|
|
@@ -1156,12 +1206,19 @@ async function waitForCrawlCompletion(http, jobId, pollInterval = 2, timeout) {
|
|
|
1156
1206
|
if (timeout != null && Date.now() - start > timeout * 1e3) {
|
|
1157
1207
|
throw new JobTimeoutError(jobId, timeout, "crawl");
|
|
1158
1208
|
}
|
|
1159
|
-
await
|
|
1209
|
+
await sleep(Math.max(1e3, pollInterval * 1e3), signal);
|
|
1160
1210
|
}
|
|
1161
1211
|
}
|
|
1162
|
-
async function crawl(http, request, pollInterval = 2, timeout) {
|
|
1212
|
+
async function crawl(http, request, pollInterval = 2, timeout, signal) {
|
|
1213
|
+
if (signal?.aborted) throw abortReason(signal);
|
|
1163
1214
|
const started = await startCrawl(http, request);
|
|
1164
|
-
|
|
1215
|
+
try {
|
|
1216
|
+
return await waitForCrawlCompletion(http, started.id, pollInterval, timeout, signal);
|
|
1217
|
+
} catch (err) {
|
|
1218
|
+
if (signal?.aborted) cancelCrawl(http, started.id).catch(() => {
|
|
1219
|
+
});
|
|
1220
|
+
throw err;
|
|
1221
|
+
}
|
|
1165
1222
|
}
|
|
1166
1223
|
async function getCrawlErrors(http, crawlId) {
|
|
1167
1224
|
try {
|
|
@@ -2376,7 +2433,7 @@ var FirecrawlClient = class {
|
|
|
2376
2433
|
/**
|
|
2377
2434
|
* Cancel a crawl job.
|
|
2378
2435
|
* @param jobId Crawl job id.
|
|
2379
|
-
* @returns True if cancelled.
|
|
2436
|
+
* @returns True if cancelled. False if the crawl was already completed (the API answers 409).
|
|
2380
2437
|
*/
|
|
2381
2438
|
async cancelCrawl(jobId) {
|
|
2382
2439
|
return cancelCrawl(this.http, jobId);
|
|
@@ -2384,11 +2441,14 @@ var FirecrawlClient = class {
|
|
|
2384
2441
|
/**
|
|
2385
2442
|
* Convenience waiter: start a crawl and poll until it finishes.
|
|
2386
2443
|
* @param url Root URL to crawl.
|
|
2387
|
-
* @param req Crawl configuration plus waiter controls (pollInterval, timeout seconds).
|
|
2444
|
+
* @param req Crawl configuration plus waiter controls (pollInterval, timeout seconds, signal).
|
|
2445
|
+
* When `signal` aborts, polling stops, the job gets a best-effort cancel request,
|
|
2446
|
+
* and the promise rejects with `signal.reason`.
|
|
2388
2447
|
* @returns Final job snapshot.
|
|
2389
2448
|
*/
|
|
2390
2449
|
async crawl(url, req = {}) {
|
|
2391
|
-
|
|
2450
|
+
const { pollInterval, timeout, signal, ...options } = req;
|
|
2451
|
+
return crawl(this.http, { url, ...options }, pollInterval, timeout, signal);
|
|
2392
2452
|
}
|
|
2393
2453
|
/**
|
|
2394
2454
|
* Retrieve crawl errors and robots.txt blocks.
|
|
@@ -2740,10 +2800,10 @@ var FirecrawlApp = class {
|
|
|
2740
2800
|
if (typeof process !== "undefined" && process.env && process.env.npm_package_version) {
|
|
2741
2801
|
return process.env.npm_package_version;
|
|
2742
2802
|
}
|
|
2743
|
-
const packageJson = await import("./package-
|
|
2803
|
+
const packageJson = await import("./package-EI4Q33HD.js");
|
|
2744
2804
|
return packageJson.default.version;
|
|
2745
2805
|
} catch (error) {
|
|
2746
|
-
const isTest = typeof process !== "undefined" &&
|
|
2806
|
+
const isTest = typeof process !== "undefined" && false;
|
|
2747
2807
|
if (!isTest) {
|
|
2748
2808
|
console.error("Error getting version:", error);
|
|
2749
2809
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mendable/firecrawl-js",
|
|
3
|
-
"version": "4.
|
|
3
|
+
"version": "4.46.0",
|
|
4
4
|
"description": "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -30,18 +30,14 @@
|
|
|
30
30
|
},
|
|
31
31
|
"homepage": "https://github.com/firecrawl/firecrawl#readme",
|
|
32
32
|
"devDependencies": {
|
|
33
|
-
"@jest/globals": "^30.2.0",
|
|
34
33
|
"@types/dotenv": "^8.2.0",
|
|
35
|
-
"@types/jest": "^30.0.0",
|
|
36
|
-
"@types/mocha": "^10.0.6",
|
|
37
34
|
"@types/node": "^20.12.12",
|
|
38
35
|
"@types/uuid": "^9.0.8",
|
|
39
36
|
"dotenv": "^16.4.5",
|
|
40
|
-
"jest": "^30.2.0",
|
|
41
|
-
"ts-jest": "^29.4.5",
|
|
42
37
|
"tsup": "^8.5.0",
|
|
43
38
|
"typescript": "^5.4.5",
|
|
44
|
-
"uuid": "^14.0.0"
|
|
39
|
+
"uuid": "^14.0.0",
|
|
40
|
+
"vitest": "^4.1.11"
|
|
45
41
|
},
|
|
46
42
|
"keywords": [
|
|
47
43
|
"firecrawl",
|
|
@@ -73,7 +69,7 @@
|
|
|
73
69
|
"build": "tsup",
|
|
74
70
|
"build-and-publish": "pnpm run build && pnpm publish --access public",
|
|
75
71
|
"publish-beta": "pnpm run build && pnpm publish --access public --tag beta",
|
|
76
|
-
"test": "
|
|
77
|
-
"test:unit": "
|
|
72
|
+
"test": "vitest run --reporter=verbose src/__tests__/e2e/v2",
|
|
73
|
+
"test:unit": "vitest run --reporter=verbose src/__tests__/unit"
|
|
78
74
|
}
|
|
79
75
|
}
|
package/pnpm-workspace.yaml
CHANGED
|
@@ -1,17 +1,13 @@
|
|
|
1
1
|
allowBuilds:
|
|
2
2
|
esbuild: true
|
|
3
|
-
unrs-resolver: true
|
|
4
3
|
overrides:
|
|
5
4
|
"@isaacs/brace-expansion@<=5.0.0": ">=5.0.1"
|
|
6
5
|
"minimatch@<10.2.3": ">=10.2.3"
|
|
7
6
|
"rollup@<4.59.0": ">=4.59.0"
|
|
8
7
|
"picomatch@<4.0.4": ">=4.0.4"
|
|
9
|
-
handlebars: ">=4.7.9"
|
|
10
8
|
# GHSA-q2hr-2g5m-vwhr / GHSA-qhr7-859c-m2p7 / GHSA-6j4f-fj2g-mc7p:
|
|
11
9
|
# pin the 5.x line only. Exact 5.0.12 cannot resolve to 6.x.
|
|
12
10
|
"brace-expansion@>=5.0.0 <5.0.12": "5.0.12"
|
|
13
|
-
"js-yaml@>=3.0.0 <3.15.2": "3.15.2"
|
|
14
|
-
"js-yaml@>=4.0.0 <4.3.2": "4.3.2"
|
|
15
11
|
# GHSA-vh66-26gq-q6x8 / GHSA-9fr6-4gfg-395g / GHSA-c29m-xwm3-cm6r /
|
|
16
12
|
# GHSA-mghh-pgcx-3jjj / GHSA-x97p-jq2g-jp4f / GHSA-3pq3-5fj3-cg6v /
|
|
17
13
|
# GHSA-542g-h47m-68v8 / GHSA-j8rh-479h-cp32 / GHSA-4hqw-qxg8-jxx2 /
|
|
@@ -20,6 +16,4 @@ overrides:
|
|
|
20
16
|
"axios@>=1.0.0 <1.20.0": "1.20.0"
|
|
21
17
|
"follow-redirects@<1.16.0": ">=1.16.0 <2.0.0"
|
|
22
18
|
"form-data@>=4.0.0 <4.0.6": "4.0.6"
|
|
23
|
-
"@babel/core@>=7.0.0 <7.29.6": "7.29.7"
|
|
24
19
|
"esbuild@>=0.17.0 <0.28.1": "0.28.1"
|
|
25
|
-
"browserslist@<=4.28.6": ">=4.28.7 <5.0.0"
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import FirecrawlApp, { type CrawlParams, type CrawlResponse, type CrawlStatusResponse, type MapResponse, type ScrapeResponse } from '../../../index';
|
|
2
2
|
import { v4 as uuidv4 } from 'uuid';
|
|
3
3
|
import dotenv from 'dotenv';
|
|
4
|
-
import { describe, test, expect } from '
|
|
4
|
+
import { describe, test, expect } from 'vitest';
|
|
5
5
|
|
|
6
6
|
dotenv.config();
|
|
7
7
|
|
|
@@ -5,7 +5,7 @@ import Firecrawl from "../../../index";
|
|
|
5
5
|
import { config } from "dotenv";
|
|
6
6
|
import { getIdentity, getApiUrl } from "./utils/idmux";
|
|
7
7
|
import { testTimeoutMs, waitForJob, withRateLimitRetry } from "./utils/rateLimit";
|
|
8
|
-
import { describe, test, expect, beforeAll } from "
|
|
8
|
+
import { describe, test, expect, beforeAll } from "vitest";
|
|
9
9
|
|
|
10
10
|
config();
|
|
11
11
|
|
|
@@ -5,7 +5,7 @@ import Firecrawl from "../../../index";
|
|
|
5
5
|
import { config } from "dotenv";
|
|
6
6
|
import { getIdentity, getApiUrl } from "./utils/idmux";
|
|
7
7
|
import { testTimeoutMs, waitForJob, withRateLimitRetry } from "./utils/rateLimit";
|
|
8
|
-
import { describe, test, expect, beforeAll } from "
|
|
8
|
+
import { describe, test, expect, beforeAll } from "vitest";
|
|
9
9
|
|
|
10
10
|
config();
|
|
11
11
|
|
|
@@ -5,7 +5,7 @@ import Firecrawl from "../../../index";
|
|
|
5
5
|
import { config } from "dotenv";
|
|
6
6
|
import { getIdentity, getApiUrl } from "./utils/idmux";
|
|
7
7
|
import { testTimeoutMs, waitForJob, withRateLimitRetry } from "./utils/rateLimit";
|
|
8
|
-
import { describe, test, expect, beforeAll } from "
|
|
8
|
+
import { describe, test, expect, beforeAll } from "vitest";
|
|
9
9
|
import { z } from "zod";
|
|
10
10
|
|
|
11
11
|
config();
|
|
@@ -5,7 +5,7 @@ import Firecrawl from "../../../index";
|
|
|
5
5
|
import { config } from "dotenv";
|
|
6
6
|
import { getIdentity, getApiUrl } from "./utils/idmux";
|
|
7
7
|
import { testTimeoutMs, withRateLimitRetry } from "./utils/rateLimit";
|
|
8
|
-
import { describe, test, expect, beforeAll } from "
|
|
8
|
+
import { describe, test, expect, beforeAll } from "vitest";
|
|
9
9
|
|
|
10
10
|
config();
|
|
11
11
|
|
|
@@ -2,7 +2,7 @@ import Firecrawl from "../../../index";
|
|
|
2
2
|
import { config } from "dotenv";
|
|
3
3
|
import { getIdentity, getApiUrl } from "./utils/idmux";
|
|
4
4
|
import { testTimeoutMs, withRateLimitRetry } from "./utils/rateLimit";
|
|
5
|
-
import { describe, test, expect, beforeAll } from "
|
|
5
|
+
import { describe, test, expect, beforeAll } from "vitest";
|
|
6
6
|
import { SdkError } from "../../../v2/types";
|
|
7
7
|
|
|
8
8
|
config();
|
|
@@ -6,7 +6,7 @@ import { z } from "zod";
|
|
|
6
6
|
import { config } from "dotenv";
|
|
7
7
|
import { getIdentity, getApiUrl } from "./utils/idmux";
|
|
8
8
|
import { testTimeoutMs, withRateLimitRetry } from "./utils/rateLimit";
|
|
9
|
-
import { describe, test, expect, beforeAll } from "
|
|
9
|
+
import { describe, test, expect, beforeAll } from "vitest";
|
|
10
10
|
|
|
11
11
|
config();
|
|
12
12
|
|