firecrawl 4.44.1 → 4.45.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-JU6NENYM.js → chunk-LKCOCWYW.js} +1 -1
- package/dist/index.cjs +72 -12
- package/dist/index.d.cts +6 -2
- package/dist/index.d.ts +6 -2
- package/dist/index.js +73 -13
- package/dist/{package-FEZU4ANT.js → package-C6PK3CK7.js} +1 -1
- package/package.json +1 -1
- package/src/__tests__/unit/v2/crawl-cancel-abort.unit.test.ts +189 -0
- package/src/v2/client.ts +8 -5
- package/src/v2/methods/crawl.ts +90 -6
|
@@ -12,7 +12,7 @@ var require_package = __commonJS({
|
|
|
12
12
|
"package.json"(exports, module) {
|
|
13
13
|
module.exports = {
|
|
14
14
|
name: "@mendable/firecrawl-js",
|
|
15
|
-
version: "4.
|
|
15
|
+
version: "4.45.0",
|
|
16
16
|
description: "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
|
|
17
17
|
main: "dist/index.js",
|
|
18
18
|
types: "dist/index.d.ts",
|
package/dist/index.cjs
CHANGED
|
@@ -39,7 +39,7 @@ var require_package = __commonJS({
|
|
|
39
39
|
"package.json"(exports2, module2) {
|
|
40
40
|
module2.exports = {
|
|
41
41
|
name: "@mendable/firecrawl-js",
|
|
42
|
-
version: "4.
|
|
42
|
+
version: "4.45.0",
|
|
43
43
|
description: "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
|
|
44
44
|
main: "dist/index.js",
|
|
45
45
|
types: "dist/index.d.ts",
|
|
@@ -636,7 +636,7 @@ function processingContinuesDelayMs(err) {
|
|
|
636
636
|
return Math.min(RESUME_MAX_DELAY_MS, Math.max(RESUME_MIN_DELAY_MS, seconds * 1e3));
|
|
637
637
|
}
|
|
638
638
|
async function scrape(http, url, options, deps = {}) {
|
|
639
|
-
const
|
|
639
|
+
const sleep2 = deps.sleepImpl ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
|
|
640
640
|
if (!url || !url.trim()) {
|
|
641
641
|
throw new Error("URL cannot be empty");
|
|
642
642
|
}
|
|
@@ -669,7 +669,7 @@ async function scrape(http, url, options, deps = {}) {
|
|
|
669
669
|
if (delayMs !== void 0 && resumes < RESUME_MAX_ATTEMPTS && resumeWaitedMs + delayMs <= RESUME_MAX_TOTAL_WAIT_MS) {
|
|
670
670
|
resumes += 1;
|
|
671
671
|
resumeWaitedMs += delayMs;
|
|
672
|
-
await
|
|
672
|
+
await sleep2(delayMs);
|
|
673
673
|
continue;
|
|
674
674
|
}
|
|
675
675
|
if (err?.isAxiosError) return normalizeAxiosError(err, "scrape");
|
|
@@ -1255,22 +1255,72 @@ async function getCrawlStatus(http, jobId, pagination) {
|
|
|
1255
1255
|
async function cancelCrawl(http, jobId) {
|
|
1256
1256
|
try {
|
|
1257
1257
|
const res = await http.delete(`/v2/crawl/${jobId}`);
|
|
1258
|
+
if (res.status === 409) return false;
|
|
1258
1259
|
if (res.status !== 200) throwForBadResponse(res, "cancel crawl");
|
|
1259
1260
|
return res.data?.status === "cancelled";
|
|
1260
1261
|
} catch (err) {
|
|
1261
|
-
if (err?.isAxiosError)
|
|
1262
|
+
if (err?.isAxiosError) {
|
|
1263
|
+
if (err.response?.status === 409) return false;
|
|
1264
|
+
return normalizeAxiosError(err, "cancel crawl");
|
|
1265
|
+
}
|
|
1262
1266
|
throw err;
|
|
1263
1267
|
}
|
|
1264
1268
|
}
|
|
1265
|
-
|
|
1269
|
+
function abortReason(signal) {
|
|
1270
|
+
if (signal.reason !== void 0) return signal.reason;
|
|
1271
|
+
return new DOMException("This operation was aborted", "AbortError");
|
|
1272
|
+
}
|
|
1273
|
+
function raceAbort(promise, signal) {
|
|
1274
|
+
if (!signal) return promise;
|
|
1275
|
+
if (signal.aborted) {
|
|
1276
|
+
promise.catch(() => {
|
|
1277
|
+
});
|
|
1278
|
+
return Promise.reject(abortReason(signal));
|
|
1279
|
+
}
|
|
1280
|
+
return new Promise((resolve, reject) => {
|
|
1281
|
+
const onAbort = () => {
|
|
1282
|
+
promise.catch(() => {
|
|
1283
|
+
});
|
|
1284
|
+
reject(abortReason(signal));
|
|
1285
|
+
};
|
|
1286
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
1287
|
+
promise.then(
|
|
1288
|
+
(value) => {
|
|
1289
|
+
signal.removeEventListener("abort", onAbort);
|
|
1290
|
+
resolve(value);
|
|
1291
|
+
},
|
|
1292
|
+
(err) => {
|
|
1293
|
+
signal.removeEventListener("abort", onAbort);
|
|
1294
|
+
reject(err);
|
|
1295
|
+
}
|
|
1296
|
+
);
|
|
1297
|
+
});
|
|
1298
|
+
}
|
|
1299
|
+
function sleep(ms, signal) {
|
|
1300
|
+
return new Promise((resolve, reject) => {
|
|
1301
|
+
if (signal?.aborted) return reject(abortReason(signal));
|
|
1302
|
+
const onAbort = () => {
|
|
1303
|
+
clearTimeout(timer);
|
|
1304
|
+
reject(abortReason(signal));
|
|
1305
|
+
};
|
|
1306
|
+
const timer = setTimeout(() => {
|
|
1307
|
+
signal?.removeEventListener("abort", onAbort);
|
|
1308
|
+
resolve();
|
|
1309
|
+
}, ms);
|
|
1310
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
1311
|
+
});
|
|
1312
|
+
}
|
|
1313
|
+
async function waitForCrawlCompletion(http, jobId, pollInterval = 2, timeout, signal) {
|
|
1266
1314
|
const start = Date.now();
|
|
1267
1315
|
while (true) {
|
|
1316
|
+
if (signal?.aborted) throw abortReason(signal);
|
|
1268
1317
|
try {
|
|
1269
|
-
const status = await getCrawlStatus(http, jobId);
|
|
1318
|
+
const status = await raceAbort(getCrawlStatus(http, jobId), signal);
|
|
1270
1319
|
if (["completed", "failed", "cancelled"].includes(status.status)) {
|
|
1271
1320
|
return status;
|
|
1272
1321
|
}
|
|
1273
1322
|
} catch (err) {
|
|
1323
|
+
if (signal?.aborted) throw abortReason(signal);
|
|
1274
1324
|
if (!isRetryableError(err)) {
|
|
1275
1325
|
if (err instanceof SdkError) {
|
|
1276
1326
|
const errorWithJobId = new SdkError(
|
|
@@ -1288,12 +1338,19 @@ async function waitForCrawlCompletion(http, jobId, pollInterval = 2, timeout) {
|
|
|
1288
1338
|
if (timeout != null && Date.now() - start > timeout * 1e3) {
|
|
1289
1339
|
throw new JobTimeoutError(jobId, timeout, "crawl");
|
|
1290
1340
|
}
|
|
1291
|
-
await
|
|
1341
|
+
await sleep(Math.max(1e3, pollInterval * 1e3), signal);
|
|
1292
1342
|
}
|
|
1293
1343
|
}
|
|
1294
|
-
async function crawl(http, request, pollInterval = 2, timeout) {
|
|
1344
|
+
async function crawl(http, request, pollInterval = 2, timeout, signal) {
|
|
1345
|
+
if (signal?.aborted) throw abortReason(signal);
|
|
1295
1346
|
const started = await startCrawl(http, request);
|
|
1296
|
-
|
|
1347
|
+
try {
|
|
1348
|
+
return await waitForCrawlCompletion(http, started.id, pollInterval, timeout, signal);
|
|
1349
|
+
} catch (err) {
|
|
1350
|
+
if (signal?.aborted) cancelCrawl(http, started.id).catch(() => {
|
|
1351
|
+
});
|
|
1352
|
+
throw err;
|
|
1353
|
+
}
|
|
1297
1354
|
}
|
|
1298
1355
|
async function getCrawlErrors(http, crawlId) {
|
|
1299
1356
|
try {
|
|
@@ -2508,7 +2565,7 @@ var FirecrawlClient = class {
|
|
|
2508
2565
|
/**
|
|
2509
2566
|
* Cancel a crawl job.
|
|
2510
2567
|
* @param jobId Crawl job id.
|
|
2511
|
-
* @returns True if cancelled.
|
|
2568
|
+
* @returns True if cancelled. False if the crawl was already completed (the API answers 409).
|
|
2512
2569
|
*/
|
|
2513
2570
|
async cancelCrawl(jobId) {
|
|
2514
2571
|
return cancelCrawl(this.http, jobId);
|
|
@@ -2516,11 +2573,14 @@ var FirecrawlClient = class {
|
|
|
2516
2573
|
/**
|
|
2517
2574
|
* Convenience waiter: start a crawl and poll until it finishes.
|
|
2518
2575
|
* @param url Root URL to crawl.
|
|
2519
|
-
* @param req Crawl configuration plus waiter controls (pollInterval, timeout seconds).
|
|
2576
|
+
* @param req Crawl configuration plus waiter controls (pollInterval, timeout seconds, signal).
|
|
2577
|
+
* When `signal` aborts, polling stops, the job gets a best-effort cancel request,
|
|
2578
|
+
* and the promise rejects with `signal.reason`.
|
|
2520
2579
|
* @returns Final job snapshot.
|
|
2521
2580
|
*/
|
|
2522
2581
|
async crawl(url, req = {}) {
|
|
2523
|
-
|
|
2582
|
+
const { pollInterval, timeout, signal, ...options } = req;
|
|
2583
|
+
return crawl(this.http, { url, ...options }, pollInterval, timeout, signal);
|
|
2524
2584
|
}
|
|
2525
2585
|
/**
|
|
2526
2586
|
* Retrieve crawl errors and robots.txt blocks.
|
package/dist/index.d.cts
CHANGED
|
@@ -2396,18 +2396,21 @@ declare class FirecrawlClient {
|
|
|
2396
2396
|
/**
|
|
2397
2397
|
* Cancel a crawl job.
|
|
2398
2398
|
* @param jobId Crawl job id.
|
|
2399
|
-
* @returns True if cancelled.
|
|
2399
|
+
* @returns True if cancelled. False if the crawl was already completed (the API answers 409).
|
|
2400
2400
|
*/
|
|
2401
2401
|
cancelCrawl(jobId: string): Promise<boolean>;
|
|
2402
2402
|
/**
|
|
2403
2403
|
* Convenience waiter: start a crawl and poll until it finishes.
|
|
2404
2404
|
* @param url Root URL to crawl.
|
|
2405
|
-
* @param req Crawl configuration plus waiter controls (pollInterval, timeout seconds).
|
|
2405
|
+
* @param req Crawl configuration plus waiter controls (pollInterval, timeout seconds, signal).
|
|
2406
|
+
* When `signal` aborts, polling stops, the job gets a best-effort cancel request,
|
|
2407
|
+
* and the promise rejects with `signal.reason`.
|
|
2406
2408
|
* @returns Final job snapshot.
|
|
2407
2409
|
*/
|
|
2408
2410
|
crawl(url: string, req?: CrawlOptions & {
|
|
2409
2411
|
pollInterval?: number;
|
|
2410
2412
|
timeout?: number;
|
|
2413
|
+
signal?: AbortSignal;
|
|
2411
2414
|
}): Promise<CrawlJob>;
|
|
2412
2415
|
/**
|
|
2413
2416
|
* Retrieve crawl errors and robots.txt blocks.
|
|
@@ -2620,6 +2623,7 @@ declare class FirecrawlClient {
|
|
|
2620
2623
|
crawlUrl(url: string, req?: CrawlOptions & {
|
|
2621
2624
|
pollInterval?: number;
|
|
2622
2625
|
timeout?: number;
|
|
2626
|
+
signal?: AbortSignal;
|
|
2623
2627
|
}): Promise<CrawlJob>;
|
|
2624
2628
|
/** @deprecated V1 compatibility alias for agent recovery. Prefer startCrawl(). */
|
|
2625
2629
|
asyncCrawlUrl(url: string, req?: CrawlOptions): Promise<CrawlResponse$1>;
|
package/dist/index.d.ts
CHANGED
|
@@ -2396,18 +2396,21 @@ declare class FirecrawlClient {
|
|
|
2396
2396
|
/**
|
|
2397
2397
|
* Cancel a crawl job.
|
|
2398
2398
|
* @param jobId Crawl job id.
|
|
2399
|
-
* @returns True if cancelled.
|
|
2399
|
+
* @returns True if cancelled. False if the crawl was already completed (the API answers 409).
|
|
2400
2400
|
*/
|
|
2401
2401
|
cancelCrawl(jobId: string): Promise<boolean>;
|
|
2402
2402
|
/**
|
|
2403
2403
|
* Convenience waiter: start a crawl and poll until it finishes.
|
|
2404
2404
|
* @param url Root URL to crawl.
|
|
2405
|
-
* @param req Crawl configuration plus waiter controls (pollInterval, timeout seconds).
|
|
2405
|
+
* @param req Crawl configuration plus waiter controls (pollInterval, timeout seconds, signal).
|
|
2406
|
+
* When `signal` aborts, polling stops, the job gets a best-effort cancel request,
|
|
2407
|
+
* and the promise rejects with `signal.reason`.
|
|
2406
2408
|
* @returns Final job snapshot.
|
|
2407
2409
|
*/
|
|
2408
2410
|
crawl(url: string, req?: CrawlOptions & {
|
|
2409
2411
|
pollInterval?: number;
|
|
2410
2412
|
timeout?: number;
|
|
2413
|
+
signal?: AbortSignal;
|
|
2411
2414
|
}): Promise<CrawlJob>;
|
|
2412
2415
|
/**
|
|
2413
2416
|
* Retrieve crawl errors and robots.txt blocks.
|
|
@@ -2620,6 +2623,7 @@ declare class FirecrawlClient {
|
|
|
2620
2623
|
crawlUrl(url: string, req?: CrawlOptions & {
|
|
2621
2624
|
pollInterval?: number;
|
|
2622
2625
|
timeout?: number;
|
|
2626
|
+
signal?: AbortSignal;
|
|
2623
2627
|
}): Promise<CrawlJob>;
|
|
2624
2628
|
/** @deprecated V1 compatibility alias for agent recovery. Prefer startCrawl(). */
|
|
2625
2629
|
asyncCrawlUrl(url: string, req?: CrawlOptions): Promise<CrawlResponse$1>;
|
package/dist/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
require_package
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-LKCOCWYW.js";
|
|
4
4
|
|
|
5
5
|
// src/v2/utils/httpClient.ts
|
|
6
6
|
import axios from "axios";
|
|
@@ -504,7 +504,7 @@ function processingContinuesDelayMs(err) {
|
|
|
504
504
|
return Math.min(RESUME_MAX_DELAY_MS, Math.max(RESUME_MIN_DELAY_MS, seconds * 1e3));
|
|
505
505
|
}
|
|
506
506
|
async function scrape(http, url, options, deps = {}) {
|
|
507
|
-
const
|
|
507
|
+
const sleep2 = deps.sleepImpl ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
|
|
508
508
|
if (!url || !url.trim()) {
|
|
509
509
|
throw new Error("URL cannot be empty");
|
|
510
510
|
}
|
|
@@ -537,7 +537,7 @@ async function scrape(http, url, options, deps = {}) {
|
|
|
537
537
|
if (delayMs !== void 0 && resumes < RESUME_MAX_ATTEMPTS && resumeWaitedMs + delayMs <= RESUME_MAX_TOTAL_WAIT_MS) {
|
|
538
538
|
resumes += 1;
|
|
539
539
|
resumeWaitedMs += delayMs;
|
|
540
|
-
await
|
|
540
|
+
await sleep2(delayMs);
|
|
541
541
|
continue;
|
|
542
542
|
}
|
|
543
543
|
if (err?.isAxiosError) return normalizeAxiosError(err, "scrape");
|
|
@@ -1123,22 +1123,72 @@ async function getCrawlStatus(http, jobId, pagination) {
|
|
|
1123
1123
|
async function cancelCrawl(http, jobId) {
|
|
1124
1124
|
try {
|
|
1125
1125
|
const res = await http.delete(`/v2/crawl/${jobId}`);
|
|
1126
|
+
if (res.status === 409) return false;
|
|
1126
1127
|
if (res.status !== 200) throwForBadResponse(res, "cancel crawl");
|
|
1127
1128
|
return res.data?.status === "cancelled";
|
|
1128
1129
|
} catch (err) {
|
|
1129
|
-
if (err?.isAxiosError)
|
|
1130
|
+
if (err?.isAxiosError) {
|
|
1131
|
+
if (err.response?.status === 409) return false;
|
|
1132
|
+
return normalizeAxiosError(err, "cancel crawl");
|
|
1133
|
+
}
|
|
1130
1134
|
throw err;
|
|
1131
1135
|
}
|
|
1132
1136
|
}
|
|
1133
|
-
|
|
1137
|
+
function abortReason(signal) {
|
|
1138
|
+
if (signal.reason !== void 0) return signal.reason;
|
|
1139
|
+
return new DOMException("This operation was aborted", "AbortError");
|
|
1140
|
+
}
|
|
1141
|
+
function raceAbort(promise, signal) {
|
|
1142
|
+
if (!signal) return promise;
|
|
1143
|
+
if (signal.aborted) {
|
|
1144
|
+
promise.catch(() => {
|
|
1145
|
+
});
|
|
1146
|
+
return Promise.reject(abortReason(signal));
|
|
1147
|
+
}
|
|
1148
|
+
return new Promise((resolve, reject) => {
|
|
1149
|
+
const onAbort = () => {
|
|
1150
|
+
promise.catch(() => {
|
|
1151
|
+
});
|
|
1152
|
+
reject(abortReason(signal));
|
|
1153
|
+
};
|
|
1154
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
1155
|
+
promise.then(
|
|
1156
|
+
(value) => {
|
|
1157
|
+
signal.removeEventListener("abort", onAbort);
|
|
1158
|
+
resolve(value);
|
|
1159
|
+
},
|
|
1160
|
+
(err) => {
|
|
1161
|
+
signal.removeEventListener("abort", onAbort);
|
|
1162
|
+
reject(err);
|
|
1163
|
+
}
|
|
1164
|
+
);
|
|
1165
|
+
});
|
|
1166
|
+
}
|
|
1167
|
+
function sleep(ms, signal) {
|
|
1168
|
+
return new Promise((resolve, reject) => {
|
|
1169
|
+
if (signal?.aborted) return reject(abortReason(signal));
|
|
1170
|
+
const onAbort = () => {
|
|
1171
|
+
clearTimeout(timer);
|
|
1172
|
+
reject(abortReason(signal));
|
|
1173
|
+
};
|
|
1174
|
+
const timer = setTimeout(() => {
|
|
1175
|
+
signal?.removeEventListener("abort", onAbort);
|
|
1176
|
+
resolve();
|
|
1177
|
+
}, ms);
|
|
1178
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
1179
|
+
});
|
|
1180
|
+
}
|
|
1181
|
+
async function waitForCrawlCompletion(http, jobId, pollInterval = 2, timeout, signal) {
|
|
1134
1182
|
const start = Date.now();
|
|
1135
1183
|
while (true) {
|
|
1184
|
+
if (signal?.aborted) throw abortReason(signal);
|
|
1136
1185
|
try {
|
|
1137
|
-
const status = await getCrawlStatus(http, jobId);
|
|
1186
|
+
const status = await raceAbort(getCrawlStatus(http, jobId), signal);
|
|
1138
1187
|
if (["completed", "failed", "cancelled"].includes(status.status)) {
|
|
1139
1188
|
return status;
|
|
1140
1189
|
}
|
|
1141
1190
|
} catch (err) {
|
|
1191
|
+
if (signal?.aborted) throw abortReason(signal);
|
|
1142
1192
|
if (!isRetryableError(err)) {
|
|
1143
1193
|
if (err instanceof SdkError) {
|
|
1144
1194
|
const errorWithJobId = new SdkError(
|
|
@@ -1156,12 +1206,19 @@ async function waitForCrawlCompletion(http, jobId, pollInterval = 2, timeout) {
|
|
|
1156
1206
|
if (timeout != null && Date.now() - start > timeout * 1e3) {
|
|
1157
1207
|
throw new JobTimeoutError(jobId, timeout, "crawl");
|
|
1158
1208
|
}
|
|
1159
|
-
await
|
|
1209
|
+
await sleep(Math.max(1e3, pollInterval * 1e3), signal);
|
|
1160
1210
|
}
|
|
1161
1211
|
}
|
|
1162
|
-
async function crawl(http, request, pollInterval = 2, timeout) {
|
|
1212
|
+
async function crawl(http, request, pollInterval = 2, timeout, signal) {
|
|
1213
|
+
if (signal?.aborted) throw abortReason(signal);
|
|
1163
1214
|
const started = await startCrawl(http, request);
|
|
1164
|
-
|
|
1215
|
+
try {
|
|
1216
|
+
return await waitForCrawlCompletion(http, started.id, pollInterval, timeout, signal);
|
|
1217
|
+
} catch (err) {
|
|
1218
|
+
if (signal?.aborted) cancelCrawl(http, started.id).catch(() => {
|
|
1219
|
+
});
|
|
1220
|
+
throw err;
|
|
1221
|
+
}
|
|
1165
1222
|
}
|
|
1166
1223
|
async function getCrawlErrors(http, crawlId) {
|
|
1167
1224
|
try {
|
|
@@ -2376,7 +2433,7 @@ var FirecrawlClient = class {
|
|
|
2376
2433
|
/**
|
|
2377
2434
|
* Cancel a crawl job.
|
|
2378
2435
|
* @param jobId Crawl job id.
|
|
2379
|
-
* @returns True if cancelled.
|
|
2436
|
+
* @returns True if cancelled. False if the crawl was already completed (the API answers 409).
|
|
2380
2437
|
*/
|
|
2381
2438
|
async cancelCrawl(jobId) {
|
|
2382
2439
|
return cancelCrawl(this.http, jobId);
|
|
@@ -2384,11 +2441,14 @@ var FirecrawlClient = class {
|
|
|
2384
2441
|
/**
|
|
2385
2442
|
* Convenience waiter: start a crawl and poll until it finishes.
|
|
2386
2443
|
* @param url Root URL to crawl.
|
|
2387
|
-
* @param req Crawl configuration plus waiter controls (pollInterval, timeout seconds).
|
|
2444
|
+
* @param req Crawl configuration plus waiter controls (pollInterval, timeout seconds, signal).
|
|
2445
|
+
* When `signal` aborts, polling stops, the job gets a best-effort cancel request,
|
|
2446
|
+
* and the promise rejects with `signal.reason`.
|
|
2388
2447
|
* @returns Final job snapshot.
|
|
2389
2448
|
*/
|
|
2390
2449
|
async crawl(url, req = {}) {
|
|
2391
|
-
|
|
2450
|
+
const { pollInterval, timeout, signal, ...options } = req;
|
|
2451
|
+
return crawl(this.http, { url, ...options }, pollInterval, timeout, signal);
|
|
2392
2452
|
}
|
|
2393
2453
|
/**
|
|
2394
2454
|
* Retrieve crawl errors and robots.txt blocks.
|
|
@@ -2740,7 +2800,7 @@ var FirecrawlApp = class {
|
|
|
2740
2800
|
if (typeof process !== "undefined" && process.env && process.env.npm_package_version) {
|
|
2741
2801
|
return process.env.npm_package_version;
|
|
2742
2802
|
}
|
|
2743
|
-
const packageJson = await import("./package-
|
|
2803
|
+
const packageJson = await import("./package-C6PK3CK7.js");
|
|
2744
2804
|
return packageJson.default.version;
|
|
2745
2805
|
} catch (error) {
|
|
2746
2806
|
const isTest = typeof process !== "undefined" && (process.env.JEST_WORKER_ID != null || false);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "firecrawl",
|
|
3
|
-
"version": "4.
|
|
3
|
+
"version": "4.45.0",
|
|
4
4
|
"description": "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
import { describe, test, expect } from "@jest/globals";
|
|
2
|
+
import axios, { type AxiosAdapter } from "axios";
|
|
3
|
+
import { FirecrawlClient } from "../../../v2/client";
|
|
4
|
+
import { HttpClient } from "../../../v2/utils/httpClient";
|
|
5
|
+
import { waitForCrawlCompletion } from "../../../v2/methods/crawl";
|
|
6
|
+
import { SdkError } from "../../../v2/types";
|
|
7
|
+
|
|
8
|
+
const API_URL = "https://api.firecrawl.dev";
|
|
9
|
+
const JOB_ID = "job-123";
|
|
10
|
+
|
|
11
|
+
type Sent = { method: string; path: string };
|
|
12
|
+
type Reply = { status: number; data: unknown };
|
|
13
|
+
|
|
14
|
+
function withAdapter<T extends { defaults: any }>(instance: T, route: (req: Sent) => Reply): Sent[] {
|
|
15
|
+
const sent: Sent[] = [];
|
|
16
|
+
const adapter: AxiosAdapter = async (config) => {
|
|
17
|
+
const req = {
|
|
18
|
+
method: (config.method || "get").toLowerCase(),
|
|
19
|
+
path: new URL(axios.getUri(config)).pathname,
|
|
20
|
+
};
|
|
21
|
+
sent.push(req);
|
|
22
|
+
const { status, data } = route(req);
|
|
23
|
+
const response = { data, status, statusText: String(status), headers: {}, config };
|
|
24
|
+
const validate = config.validateStatus;
|
|
25
|
+
if (validate && !validate(status)) {
|
|
26
|
+
throw new axios.AxiosError(`Request failed with status code ${status}`, undefined, config, null, response);
|
|
27
|
+
}
|
|
28
|
+
return response;
|
|
29
|
+
};
|
|
30
|
+
instance.defaults.adapter = adapter;
|
|
31
|
+
return sent;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function makeClient(route: (req: Sent) => Reply) {
|
|
35
|
+
const client = new FirecrawlClient({ apiKey: "fc-test", apiUrl: API_URL });
|
|
36
|
+
const sent = withAdapter((client as any).http.instance, route);
|
|
37
|
+
return { client, sent };
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
// The cancel runs in the background, so let its request reach the adapter.
|
|
41
|
+
const flush = () => new Promise((r) => setTimeout(r, 0));
|
|
42
|
+
|
|
43
|
+
const running: Reply = { status: 200, data: { success: true, status: "scraping", completed: 0, total: 1, data: [] } };
|
|
44
|
+
const started: Reply = { status: 200, data: { success: true, id: JOB_ID, url: "https://example.com" } };
|
|
45
|
+
|
|
46
|
+
describe("v2 cancelCrawl", () => {
|
|
47
|
+
test("returns false when the crawl is already completed (409)", async () => {
|
|
48
|
+
const { client, sent } = makeClient(() => ({ status: 409, data: { error: "Crawl is already completed" } }));
|
|
49
|
+
await expect(client.cancelCrawl(JOB_ID)).resolves.toBe(false);
|
|
50
|
+
expect(sent).toEqual([{ method: "delete", path: `/v2/crawl/${JOB_ID}` }]);
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
test("returns false on 409 when the transport does not throw for 4xx", async () => {
|
|
54
|
+
const { client } = makeClient(() => ({ status: 409, data: { error: "Crawl is already completed" } }));
|
|
55
|
+
(client as any).http.instance.defaults.validateStatus = () => true;
|
|
56
|
+
await expect(client.cancelCrawl(JOB_ID)).resolves.toBe(false);
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
test("returns true when the crawl is cancelled", async () => {
|
|
60
|
+
const { client } = makeClient(() => ({ status: 200, data: { status: "cancelled" } }));
|
|
61
|
+
await expect(client.cancelCrawl(JOB_ID)).resolves.toBe(true);
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
test("still throws for other errors", async () => {
|
|
65
|
+
const { client } = makeClient(() => ({ status: 404, data: { error: "Job not found" } }));
|
|
66
|
+
const err = await client.cancelCrawl(JOB_ID).catch((e) => e);
|
|
67
|
+
expect(err).toBeInstanceOf(SdkError);
|
|
68
|
+
expect(err.status).toBe(404);
|
|
69
|
+
});
|
|
70
|
+
});
|
|
71
|
+
|
|
72
|
+
describe("v2 crawl with an AbortSignal", () => {
|
|
73
|
+
test("abort during polling stops polling, cancels the job, and rejects with the abort reason", async () => {
|
|
74
|
+
const controller = new AbortController();
|
|
75
|
+
let polls = 0;
|
|
76
|
+
const { client, sent } = makeClient((req) => {
|
|
77
|
+
if (req.method === "post") return started;
|
|
78
|
+
if (req.method === "delete") return { status: 200, data: { status: "cancelled" } };
|
|
79
|
+
polls += 1;
|
|
80
|
+
if (polls === 1) setTimeout(() => controller.abort(), 10);
|
|
81
|
+
return running;
|
|
82
|
+
});
|
|
83
|
+
|
|
84
|
+
const t0 = Date.now();
|
|
85
|
+
const err = await client.crawl("https://example.com", { pollInterval: 30, signal: controller.signal }).catch((e) => e);
|
|
86
|
+
|
|
87
|
+
expect(Date.now() - t0).toBeLessThan(5000);
|
|
88
|
+
expect(err).toBe(controller.signal.reason);
|
|
89
|
+
expect((err as Error).name).toBe("AbortError");
|
|
90
|
+
await flush();
|
|
91
|
+
expect(sent.map((r) => r.method)).toEqual(["post", "get", "delete"]);
|
|
92
|
+
expect(sent[2]!.path).toBe(`/v2/crawl/${JOB_ID}`);
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
test("rejects with a custom abort reason and ignores a failing cancel", async () => {
|
|
96
|
+
const controller = new AbortController();
|
|
97
|
+
const reason = new Error("caller gave up");
|
|
98
|
+
const { client, sent } = makeClient((req) => {
|
|
99
|
+
if (req.method === "post") return started;
|
|
100
|
+
if (req.method === "delete") return { status: 500, data: { error: "boom" } };
|
|
101
|
+
setTimeout(() => controller.abort(reason), 10);
|
|
102
|
+
return running;
|
|
103
|
+
});
|
|
104
|
+
|
|
105
|
+
await expect(client.crawl("https://example.com", { pollInterval: 30, signal: controller.signal })).rejects.toBe(reason);
|
|
106
|
+
await flush();
|
|
107
|
+
expect(sent.some((r) => r.method === "delete")).toBe(true);
|
|
108
|
+
});
|
|
109
|
+
|
|
110
|
+
test("a hanging cancel does not delay the rejection", async () => {
|
|
111
|
+
const controller = new AbortController();
|
|
112
|
+
const client = new FirecrawlClient({ apiKey: "fc-test", apiUrl: API_URL });
|
|
113
|
+
const methods: string[] = [];
|
|
114
|
+
(client as any).http.instance.defaults.adapter = (async (config) => {
|
|
115
|
+
methods.push(config.method!);
|
|
116
|
+
if (config.method === "delete") return new Promise(() => {});
|
|
117
|
+
if (config.method === "get") setTimeout(() => controller.abort(), 10);
|
|
118
|
+
const data = config.method === "post" ? started.data : running.data;
|
|
119
|
+
return { data, status: 200, statusText: "OK", headers: {}, config };
|
|
120
|
+
}) as AxiosAdapter;
|
|
121
|
+
|
|
122
|
+
// Read the reason after the abort: before it, signal.reason is undefined.
|
|
123
|
+
const err = await client.crawl("https://example.com", { pollInterval: 30, signal: controller.signal }).catch((e) => e);
|
|
124
|
+
expect(err).toBe(controller.signal.reason);
|
|
125
|
+
expect((err as Error).name).toBe("AbortError");
|
|
126
|
+
await flush();
|
|
127
|
+
expect(methods).toEqual(["post", "get", "delete"]);
|
|
128
|
+
});
|
|
129
|
+
|
|
130
|
+
test("abort while the start request is in flight cancels the job once the id arrives", async () => {
|
|
131
|
+
const controller = new AbortController();
|
|
132
|
+
const { client, sent } = makeClient((req) => {
|
|
133
|
+
if (req.method === "post") {
|
|
134
|
+
controller.abort();
|
|
135
|
+
return started;
|
|
136
|
+
}
|
|
137
|
+
if (req.method === "delete") return { status: 200, data: { status: "cancelled" } };
|
|
138
|
+
return running;
|
|
139
|
+
});
|
|
140
|
+
|
|
141
|
+
const err = await client.crawl("https://example.com", { signal: controller.signal }).catch((e) => e);
|
|
142
|
+
expect(err).toBe(controller.signal.reason);
|
|
143
|
+
expect((err as Error).name).toBe("AbortError");
|
|
144
|
+
await flush();
|
|
145
|
+
expect(sent.map((r) => r.method)).toEqual(["post", "delete"]);
|
|
146
|
+
});
|
|
147
|
+
|
|
148
|
+
test("an already aborted signal sends no request", async () => {
|
|
149
|
+
const controller = new AbortController();
|
|
150
|
+
controller.abort();
|
|
151
|
+
const { client, sent } = makeClient(() => started);
|
|
152
|
+
|
|
153
|
+
await expect(client.crawl("https://example.com", { signal: controller.signal })).rejects.toBe(controller.signal.reason);
|
|
154
|
+
expect(sent).toHaveLength(0);
|
|
155
|
+
});
|
|
156
|
+
|
|
157
|
+
test("the signal is not sent in the crawl request body", async () => {
|
|
158
|
+
const controller = new AbortController();
|
|
159
|
+
const client = new FirecrawlClient({ apiKey: "fc-test", apiUrl: API_URL });
|
|
160
|
+
const bodies: unknown[] = [];
|
|
161
|
+
(client as any).http.instance.defaults.adapter = (async (config) => {
|
|
162
|
+
if (config.method === "post") bodies.push(JSON.parse(config.data as string));
|
|
163
|
+
const data = config.method === "post" ? started.data : { success: true, status: "completed", completed: 1, total: 1, data: [] };
|
|
164
|
+
return { data, status: 200, statusText: "OK", headers: {}, config };
|
|
165
|
+
}) as AxiosAdapter;
|
|
166
|
+
|
|
167
|
+
const job = await client.crawl("https://example.com", { limit: 1, signal: controller.signal });
|
|
168
|
+
expect(job.status).toBe("completed");
|
|
169
|
+
expect(bodies[0]).toMatchObject({ url: "https://example.com", limit: 1 });
|
|
170
|
+
expect(Object.keys(bodies[0] as object)).not.toContain("signal");
|
|
171
|
+
});
|
|
172
|
+
});
|
|
173
|
+
|
|
174
|
+
describe("v2 waitForCrawlCompletion with an AbortSignal", () => {
|
|
175
|
+
test("stops polling and rejects without cancelling the job", async () => {
|
|
176
|
+
const http = new HttpClient({ apiKey: "fc-test", apiUrl: API_URL });
|
|
177
|
+
const controller = new AbortController();
|
|
178
|
+
const sent = withAdapter((http as any).instance, () => {
|
|
179
|
+
setTimeout(() => controller.abort(), 10);
|
|
180
|
+
return running;
|
|
181
|
+
});
|
|
182
|
+
|
|
183
|
+
const err = await waitForCrawlCompletion(http, JOB_ID, 30, undefined, controller.signal).catch((e) => e);
|
|
184
|
+
expect(err).toBe(controller.signal.reason);
|
|
185
|
+
expect((err as Error).name).toBe("AbortError");
|
|
186
|
+
await new Promise((r) => setTimeout(r, 50));
|
|
187
|
+
expect(sent.map((r) => r.method)).toEqual(["get"]);
|
|
188
|
+
});
|
|
189
|
+
});
|
package/src/v2/client.ts
CHANGED
|
@@ -394,7 +394,7 @@ export class FirecrawlClient {
|
|
|
394
394
|
/**
|
|
395
395
|
* Cancel a crawl job.
|
|
396
396
|
* @param jobId Crawl job id.
|
|
397
|
-
* @returns True if cancelled.
|
|
397
|
+
* @returns True if cancelled. False if the crawl was already completed (the API answers 409).
|
|
398
398
|
*/
|
|
399
399
|
async cancelCrawl(jobId: string): Promise<boolean> {
|
|
400
400
|
return cancelCrawl(this.http, jobId);
|
|
@@ -402,11 +402,14 @@ export class FirecrawlClient {
|
|
|
402
402
|
/**
|
|
403
403
|
* Convenience waiter: start a crawl and poll until it finishes.
|
|
404
404
|
* @param url Root URL to crawl.
|
|
405
|
-
* @param req Crawl configuration plus waiter controls (pollInterval, timeout seconds).
|
|
405
|
+
* @param req Crawl configuration plus waiter controls (pollInterval, timeout seconds, signal).
|
|
406
|
+
* When `signal` aborts, polling stops, the job gets a best-effort cancel request,
|
|
407
|
+
* and the promise rejects with `signal.reason`.
|
|
406
408
|
* @returns Final job snapshot.
|
|
407
409
|
*/
|
|
408
|
-
async crawl(url: string, req: CrawlOptions & { pollInterval?: number; timeout?: number } = {}): Promise<CrawlJob> {
|
|
409
|
-
|
|
410
|
+
async crawl(url: string, req: CrawlOptions & { pollInterval?: number; timeout?: number; signal?: AbortSignal } = {}): Promise<CrawlJob> {
|
|
411
|
+
const { pollInterval, timeout, signal, ...options } = req;
|
|
412
|
+
return crawlWaiter(this.http, { url, ...options }, pollInterval, timeout, signal);
|
|
410
413
|
}
|
|
411
414
|
/**
|
|
412
415
|
* Retrieve crawl errors and robots.txt blocks.
|
|
@@ -724,7 +727,7 @@ export class FirecrawlClient {
|
|
|
724
727
|
}
|
|
725
728
|
|
|
726
729
|
/** @deprecated V1 compatibility alias for agent recovery. Prefer crawl(). */
|
|
727
|
-
async crawlUrl(url: string, req: CrawlOptions & { pollInterval?: number; timeout?: number } = {}): Promise<CrawlJob> {
|
|
730
|
+
async crawlUrl(url: string, req: CrawlOptions & { pollInterval?: number; timeout?: number; signal?: AbortSignal } = {}): Promise<CrawlJob> {
|
|
728
731
|
return this.crawl(url, req);
|
|
729
732
|
}
|
|
730
733
|
|
package/src/v2/methods/crawl.ts
CHANGED
|
@@ -107,28 +107,94 @@ export async function getCrawlStatus(
|
|
|
107
107
|
}
|
|
108
108
|
}
|
|
109
109
|
|
|
110
|
+
/**
|
|
111
|
+
* Cancel a crawl job. Resolves `false` when the API answers 409 because the
|
|
112
|
+
* crawl already finished, so there is nothing left to cancel.
|
|
113
|
+
*/
|
|
110
114
|
export async function cancelCrawl(http: HttpClient, jobId: string): Promise<boolean> {
|
|
111
115
|
try {
|
|
112
116
|
const res = await http.delete<{ status: string }>(`/v2/crawl/${jobId}`);
|
|
117
|
+
if (res.status === 409) return false;
|
|
113
118
|
if (res.status !== 200) throwForBadResponse(res, "cancel crawl");
|
|
114
119
|
return res.data?.status === "cancelled";
|
|
115
120
|
} catch (err: any) {
|
|
116
|
-
if (err?.isAxiosError)
|
|
121
|
+
if (err?.isAxiosError) {
|
|
122
|
+
if (err.response?.status === 409) return false;
|
|
123
|
+
return normalizeAxiosError(err, "cancel crawl");
|
|
124
|
+
}
|
|
117
125
|
throw err;
|
|
118
126
|
}
|
|
119
127
|
}
|
|
120
128
|
|
|
121
|
-
|
|
129
|
+
function abortReason(signal: AbortSignal): unknown {
|
|
130
|
+
if (signal.reason !== undefined) return signal.reason;
|
|
131
|
+
return new DOMException("This operation was aborted", "AbortError");
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/** Reject with the abort reason as soon as `signal` aborts. */
|
|
135
|
+
function raceAbort<T>(promise: Promise<T>, signal?: AbortSignal): Promise<T> {
|
|
136
|
+
if (!signal) return promise;
|
|
137
|
+
if (signal.aborted) {
|
|
138
|
+
promise.catch(() => {});
|
|
139
|
+
return Promise.reject(abortReason(signal));
|
|
140
|
+
}
|
|
141
|
+
return new Promise<T>((resolve, reject) => {
|
|
142
|
+
const onAbort = () => {
|
|
143
|
+
promise.catch(() => {});
|
|
144
|
+
reject(abortReason(signal));
|
|
145
|
+
};
|
|
146
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
147
|
+
promise.then(
|
|
148
|
+
(value) => {
|
|
149
|
+
signal.removeEventListener("abort", onAbort);
|
|
150
|
+
resolve(value);
|
|
151
|
+
},
|
|
152
|
+
(err) => {
|
|
153
|
+
signal.removeEventListener("abort", onAbort);
|
|
154
|
+
reject(err);
|
|
155
|
+
},
|
|
156
|
+
);
|
|
157
|
+
});
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
function sleep(ms: number, signal?: AbortSignal): Promise<void> {
|
|
161
|
+
return new Promise<void>((resolve, reject) => {
|
|
162
|
+
if (signal?.aborted) return reject(abortReason(signal));
|
|
163
|
+
const onAbort = () => {
|
|
164
|
+
clearTimeout(timer);
|
|
165
|
+
reject(abortReason(signal!));
|
|
166
|
+
};
|
|
167
|
+
const timer = setTimeout(() => {
|
|
168
|
+
signal?.removeEventListener("abort", onAbort);
|
|
169
|
+
resolve();
|
|
170
|
+
}, ms);
|
|
171
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
172
|
+
});
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
/**
|
|
176
|
+
* Poll a crawl until it reaches a terminal status. When `signal` aborts, polling
|
|
177
|
+
* stops and the promise rejects with the abort reason. The job is not cancelled.
|
|
178
|
+
*/
|
|
179
|
+
export async function waitForCrawlCompletion(
|
|
180
|
+
http: HttpClient,
|
|
181
|
+
jobId: string,
|
|
182
|
+
pollInterval = 2,
|
|
183
|
+
timeout?: number,
|
|
184
|
+
signal?: AbortSignal,
|
|
185
|
+
): Promise<CrawlJob> {
|
|
122
186
|
const start = Date.now();
|
|
123
187
|
|
|
124
188
|
while (true) {
|
|
189
|
+
if (signal?.aborted) throw abortReason(signal);
|
|
125
190
|
try {
|
|
126
|
-
const status = await getCrawlStatus(http, jobId);
|
|
191
|
+
const status = await raceAbort(getCrawlStatus(http, jobId), signal);
|
|
127
192
|
|
|
128
193
|
if (["completed", "failed", "cancelled"].includes(status.status)) {
|
|
129
194
|
return status;
|
|
130
195
|
}
|
|
131
196
|
} catch (err: any) {
|
|
197
|
+
if (signal?.aborted) throw abortReason(signal);
|
|
132
198
|
// Don't retry on permanent errors (4xx) - re-throw immediately with jobId context
|
|
133
199
|
if (!isRetryableError(err)) {
|
|
134
200
|
// Create new error with jobId for better debugging (non-retryable errors like 404)
|
|
@@ -151,13 +217,31 @@ export async function waitForCrawlCompletion(http: HttpClient, jobId: string, po
|
|
|
151
217
|
throw new JobTimeoutError(jobId, timeout, 'crawl');
|
|
152
218
|
}
|
|
153
219
|
|
|
154
|
-
await
|
|
220
|
+
await sleep(Math.max(1000, pollInterval * 1000), signal);
|
|
155
221
|
}
|
|
156
222
|
}
|
|
157
223
|
|
|
158
|
-
|
|
224
|
+
/**
|
|
225
|
+
* Start a crawl and wait for it to finish. When `signal` aborts, polling stops,
|
|
226
|
+
* the job gets a best-effort cancel request, and the promise rejects with the
|
|
227
|
+
* abort reason at once, without waiting for the cancel. A signal that is already
|
|
228
|
+
* aborted rejects before any request.
|
|
229
|
+
*/
|
|
230
|
+
export async function crawl(
|
|
231
|
+
http: HttpClient,
|
|
232
|
+
request: CrawlRequest,
|
|
233
|
+
pollInterval = 2,
|
|
234
|
+
timeout?: number,
|
|
235
|
+
signal?: AbortSignal,
|
|
236
|
+
): Promise<CrawlJob> {
|
|
237
|
+
if (signal?.aborted) throw abortReason(signal);
|
|
159
238
|
const started = await startCrawl(http, request);
|
|
160
|
-
|
|
239
|
+
try {
|
|
240
|
+
return await waitForCrawlCompletion(http, started.id, pollInterval, timeout, signal);
|
|
241
|
+
} catch (err) {
|
|
242
|
+
if (signal?.aborted) cancelCrawl(http, started.id).catch(() => {});
|
|
243
|
+
throw err;
|
|
244
|
+
}
|
|
161
245
|
}
|
|
162
246
|
|
|
163
247
|
export async function getCrawlErrors(http: HttpClient, crawlId: string): Promise<CrawlErrorsResponse> {
|