firecrawl 4.44.0 → 4.45.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-BD2TNHAD.js → chunk-LKCOCWYW.js} +1 -1
- package/dist/index.cjs +78 -13
- package/dist/index.d.cts +6 -2
- package/dist/index.d.ts +6 -2
- package/dist/index.js +79 -14
- package/dist/{package-WSFVH42J.js → package-C6PK3CK7.js} +1 -1
- package/package.json +1 -1
- package/src/__tests__/unit/v2/crawl-cancel-abort.unit.test.ts +189 -0
- package/src/__tests__/unit/v2/tools.unit.test.ts +4 -2
- package/src/v2/client.ts +8 -5
- package/src/v2/methods/crawl.ts +90 -6
- package/src/v2/methods/tools.ts +9 -1
|
@@ -12,7 +12,7 @@ var require_package = __commonJS({
|
|
|
12
12
|
"package.json"(exports, module) {
|
|
13
13
|
module.exports = {
|
|
14
14
|
name: "@mendable/firecrawl-js",
|
|
15
|
-
version: "4.
|
|
15
|
+
version: "4.45.0",
|
|
16
16
|
description: "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
|
|
17
17
|
main: "dist/index.js",
|
|
18
18
|
types: "dist/index.d.ts",
|
package/dist/index.cjs
CHANGED
|
@@ -39,7 +39,7 @@ var require_package = __commonJS({
|
|
|
39
39
|
"package.json"(exports2, module2) {
|
|
40
40
|
module2.exports = {
|
|
41
41
|
name: "@mendable/firecrawl-js",
|
|
42
|
-
version: "4.
|
|
42
|
+
version: "4.45.0",
|
|
43
43
|
description: "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
|
|
44
44
|
main: "dist/index.js",
|
|
45
45
|
types: "dist/index.d.ts",
|
|
@@ -636,7 +636,7 @@ function processingContinuesDelayMs(err) {
|
|
|
636
636
|
return Math.min(RESUME_MAX_DELAY_MS, Math.max(RESUME_MIN_DELAY_MS, seconds * 1e3));
|
|
637
637
|
}
|
|
638
638
|
async function scrape(http, url, options, deps = {}) {
|
|
639
|
-
const
|
|
639
|
+
const sleep2 = deps.sleepImpl ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
|
|
640
640
|
if (!url || !url.trim()) {
|
|
641
641
|
throw new Error("URL cannot be empty");
|
|
642
642
|
}
|
|
@@ -669,7 +669,7 @@ async function scrape(http, url, options, deps = {}) {
|
|
|
669
669
|
if (delayMs !== void 0 && resumes < RESUME_MAX_ATTEMPTS && resumeWaitedMs + delayMs <= RESUME_MAX_TOTAL_WAIT_MS) {
|
|
670
670
|
resumes += 1;
|
|
671
671
|
resumeWaitedMs += delayMs;
|
|
672
|
-
await
|
|
672
|
+
await sleep2(delayMs);
|
|
673
673
|
continue;
|
|
674
674
|
}
|
|
675
675
|
if (err?.isAxiosError) return normalizeAxiosError(err, "scrape");
|
|
@@ -897,6 +897,8 @@ async function search(http, request) {
|
|
|
897
897
|
|
|
898
898
|
// src/v2/methods/tools.ts
|
|
899
899
|
var ALEXANDRIA_MAX_CALLS = 10;
|
|
900
|
+
var ALEXANDRIA_MAX_TIMEOUT_MS = 12e4;
|
|
901
|
+
var ALEXANDRIA_RESPONSE_MARGIN_MS = 3e4;
|
|
900
902
|
function prepareAlexandriaPayload(calls, opts) {
|
|
901
903
|
if (!Array.isArray(calls) || calls.length === 0) {
|
|
902
904
|
throw new Error("alexandria requires at least one call");
|
|
@@ -946,7 +948,10 @@ async function scrapeAlexandria(http, calls, opts = {}) {
|
|
|
946
948
|
const res = await http.post("/v2/scrape", payload, {
|
|
947
949
|
headers: { "x-request-id": requestId },
|
|
948
950
|
// Allow response delivery after the API's capped execution deadline.
|
|
949
|
-
timeoutMs: Math.min(
|
|
951
|
+
timeoutMs: Math.min(
|
|
952
|
+
opts.timeout ?? ALEXANDRIA_MAX_TIMEOUT_MS,
|
|
953
|
+
ALEXANDRIA_MAX_TIMEOUT_MS
|
|
954
|
+
) + ALEXANDRIA_RESPONSE_MARGIN_MS
|
|
950
955
|
});
|
|
951
956
|
if (res.status !== 200 || !res.data?.success) {
|
|
952
957
|
throwForBadResponse(res, "alexandria");
|
|
@@ -1250,22 +1255,72 @@ async function getCrawlStatus(http, jobId, pagination) {
|
|
|
1250
1255
|
async function cancelCrawl(http, jobId) {
|
|
1251
1256
|
try {
|
|
1252
1257
|
const res = await http.delete(`/v2/crawl/${jobId}`);
|
|
1258
|
+
if (res.status === 409) return false;
|
|
1253
1259
|
if (res.status !== 200) throwForBadResponse(res, "cancel crawl");
|
|
1254
1260
|
return res.data?.status === "cancelled";
|
|
1255
1261
|
} catch (err) {
|
|
1256
|
-
if (err?.isAxiosError)
|
|
1262
|
+
if (err?.isAxiosError) {
|
|
1263
|
+
if (err.response?.status === 409) return false;
|
|
1264
|
+
return normalizeAxiosError(err, "cancel crawl");
|
|
1265
|
+
}
|
|
1257
1266
|
throw err;
|
|
1258
1267
|
}
|
|
1259
1268
|
}
|
|
1260
|
-
|
|
1269
|
+
function abortReason(signal) {
|
|
1270
|
+
if (signal.reason !== void 0) return signal.reason;
|
|
1271
|
+
return new DOMException("This operation was aborted", "AbortError");
|
|
1272
|
+
}
|
|
1273
|
+
function raceAbort(promise, signal) {
|
|
1274
|
+
if (!signal) return promise;
|
|
1275
|
+
if (signal.aborted) {
|
|
1276
|
+
promise.catch(() => {
|
|
1277
|
+
});
|
|
1278
|
+
return Promise.reject(abortReason(signal));
|
|
1279
|
+
}
|
|
1280
|
+
return new Promise((resolve, reject) => {
|
|
1281
|
+
const onAbort = () => {
|
|
1282
|
+
promise.catch(() => {
|
|
1283
|
+
});
|
|
1284
|
+
reject(abortReason(signal));
|
|
1285
|
+
};
|
|
1286
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
1287
|
+
promise.then(
|
|
1288
|
+
(value) => {
|
|
1289
|
+
signal.removeEventListener("abort", onAbort);
|
|
1290
|
+
resolve(value);
|
|
1291
|
+
},
|
|
1292
|
+
(err) => {
|
|
1293
|
+
signal.removeEventListener("abort", onAbort);
|
|
1294
|
+
reject(err);
|
|
1295
|
+
}
|
|
1296
|
+
);
|
|
1297
|
+
});
|
|
1298
|
+
}
|
|
1299
|
+
function sleep(ms, signal) {
|
|
1300
|
+
return new Promise((resolve, reject) => {
|
|
1301
|
+
if (signal?.aborted) return reject(abortReason(signal));
|
|
1302
|
+
const onAbort = () => {
|
|
1303
|
+
clearTimeout(timer);
|
|
1304
|
+
reject(abortReason(signal));
|
|
1305
|
+
};
|
|
1306
|
+
const timer = setTimeout(() => {
|
|
1307
|
+
signal?.removeEventListener("abort", onAbort);
|
|
1308
|
+
resolve();
|
|
1309
|
+
}, ms);
|
|
1310
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
1311
|
+
});
|
|
1312
|
+
}
|
|
1313
|
+
async function waitForCrawlCompletion(http, jobId, pollInterval = 2, timeout, signal) {
|
|
1261
1314
|
const start = Date.now();
|
|
1262
1315
|
while (true) {
|
|
1316
|
+
if (signal?.aborted) throw abortReason(signal);
|
|
1263
1317
|
try {
|
|
1264
|
-
const status = await getCrawlStatus(http, jobId);
|
|
1318
|
+
const status = await raceAbort(getCrawlStatus(http, jobId), signal);
|
|
1265
1319
|
if (["completed", "failed", "cancelled"].includes(status.status)) {
|
|
1266
1320
|
return status;
|
|
1267
1321
|
}
|
|
1268
1322
|
} catch (err) {
|
|
1323
|
+
if (signal?.aborted) throw abortReason(signal);
|
|
1269
1324
|
if (!isRetryableError(err)) {
|
|
1270
1325
|
if (err instanceof SdkError) {
|
|
1271
1326
|
const errorWithJobId = new SdkError(
|
|
@@ -1283,12 +1338,19 @@ async function waitForCrawlCompletion(http, jobId, pollInterval = 2, timeout) {
|
|
|
1283
1338
|
if (timeout != null && Date.now() - start > timeout * 1e3) {
|
|
1284
1339
|
throw new JobTimeoutError(jobId, timeout, "crawl");
|
|
1285
1340
|
}
|
|
1286
|
-
await
|
|
1341
|
+
await sleep(Math.max(1e3, pollInterval * 1e3), signal);
|
|
1287
1342
|
}
|
|
1288
1343
|
}
|
|
1289
|
-
async function crawl(http, request, pollInterval = 2, timeout) {
|
|
1344
|
+
async function crawl(http, request, pollInterval = 2, timeout, signal) {
|
|
1345
|
+
if (signal?.aborted) throw abortReason(signal);
|
|
1290
1346
|
const started = await startCrawl(http, request);
|
|
1291
|
-
|
|
1347
|
+
try {
|
|
1348
|
+
return await waitForCrawlCompletion(http, started.id, pollInterval, timeout, signal);
|
|
1349
|
+
} catch (err) {
|
|
1350
|
+
if (signal?.aborted) cancelCrawl(http, started.id).catch(() => {
|
|
1351
|
+
});
|
|
1352
|
+
throw err;
|
|
1353
|
+
}
|
|
1292
1354
|
}
|
|
1293
1355
|
async function getCrawlErrors(http, crawlId) {
|
|
1294
1356
|
try {
|
|
@@ -2503,7 +2565,7 @@ var FirecrawlClient = class {
|
|
|
2503
2565
|
/**
|
|
2504
2566
|
* Cancel a crawl job.
|
|
2505
2567
|
* @param jobId Crawl job id.
|
|
2506
|
-
* @returns True if cancelled.
|
|
2568
|
+
* @returns True if cancelled. False if the crawl was already completed (the API answers 409).
|
|
2507
2569
|
*/
|
|
2508
2570
|
async cancelCrawl(jobId) {
|
|
2509
2571
|
return cancelCrawl(this.http, jobId);
|
|
@@ -2511,11 +2573,14 @@ var FirecrawlClient = class {
|
|
|
2511
2573
|
/**
|
|
2512
2574
|
* Convenience waiter: start a crawl and poll until it finishes.
|
|
2513
2575
|
* @param url Root URL to crawl.
|
|
2514
|
-
* @param req Crawl configuration plus waiter controls (pollInterval, timeout seconds).
|
|
2576
|
+
* @param req Crawl configuration plus waiter controls (pollInterval, timeout seconds, signal).
|
|
2577
|
+
* When `signal` aborts, polling stops, the job gets a best-effort cancel request,
|
|
2578
|
+
* and the promise rejects with `signal.reason`.
|
|
2515
2579
|
* @returns Final job snapshot.
|
|
2516
2580
|
*/
|
|
2517
2581
|
async crawl(url, req = {}) {
|
|
2518
|
-
|
|
2582
|
+
const { pollInterval, timeout, signal, ...options } = req;
|
|
2583
|
+
return crawl(this.http, { url, ...options }, pollInterval, timeout, signal);
|
|
2519
2584
|
}
|
|
2520
2585
|
/**
|
|
2521
2586
|
* Retrieve crawl errors and robots.txt blocks.
|
package/dist/index.d.cts
CHANGED
|
@@ -2396,18 +2396,21 @@ declare class FirecrawlClient {
|
|
|
2396
2396
|
/**
|
|
2397
2397
|
* Cancel a crawl job.
|
|
2398
2398
|
* @param jobId Crawl job id.
|
|
2399
|
-
* @returns True if cancelled.
|
|
2399
|
+
* @returns True if cancelled. False if the crawl was already completed (the API answers 409).
|
|
2400
2400
|
*/
|
|
2401
2401
|
cancelCrawl(jobId: string): Promise<boolean>;
|
|
2402
2402
|
/**
|
|
2403
2403
|
* Convenience waiter: start a crawl and poll until it finishes.
|
|
2404
2404
|
* @param url Root URL to crawl.
|
|
2405
|
-
* @param req Crawl configuration plus waiter controls (pollInterval, timeout seconds).
|
|
2405
|
+
* @param req Crawl configuration plus waiter controls (pollInterval, timeout seconds, signal).
|
|
2406
|
+
* When `signal` aborts, polling stops, the job gets a best-effort cancel request,
|
|
2407
|
+
* and the promise rejects with `signal.reason`.
|
|
2406
2408
|
* @returns Final job snapshot.
|
|
2407
2409
|
*/
|
|
2408
2410
|
crawl(url: string, req?: CrawlOptions & {
|
|
2409
2411
|
pollInterval?: number;
|
|
2410
2412
|
timeout?: number;
|
|
2413
|
+
signal?: AbortSignal;
|
|
2411
2414
|
}): Promise<CrawlJob>;
|
|
2412
2415
|
/**
|
|
2413
2416
|
* Retrieve crawl errors and robots.txt blocks.
|
|
@@ -2620,6 +2623,7 @@ declare class FirecrawlClient {
|
|
|
2620
2623
|
crawlUrl(url: string, req?: CrawlOptions & {
|
|
2621
2624
|
pollInterval?: number;
|
|
2622
2625
|
timeout?: number;
|
|
2626
|
+
signal?: AbortSignal;
|
|
2623
2627
|
}): Promise<CrawlJob>;
|
|
2624
2628
|
/** @deprecated V1 compatibility alias for agent recovery. Prefer startCrawl(). */
|
|
2625
2629
|
asyncCrawlUrl(url: string, req?: CrawlOptions): Promise<CrawlResponse$1>;
|
package/dist/index.d.ts
CHANGED
|
@@ -2396,18 +2396,21 @@ declare class FirecrawlClient {
|
|
|
2396
2396
|
/**
|
|
2397
2397
|
* Cancel a crawl job.
|
|
2398
2398
|
* @param jobId Crawl job id.
|
|
2399
|
-
* @returns True if cancelled.
|
|
2399
|
+
* @returns True if cancelled. False if the crawl was already completed (the API answers 409).
|
|
2400
2400
|
*/
|
|
2401
2401
|
cancelCrawl(jobId: string): Promise<boolean>;
|
|
2402
2402
|
/**
|
|
2403
2403
|
* Convenience waiter: start a crawl and poll until it finishes.
|
|
2404
2404
|
* @param url Root URL to crawl.
|
|
2405
|
-
* @param req Crawl configuration plus waiter controls (pollInterval, timeout seconds).
|
|
2405
|
+
* @param req Crawl configuration plus waiter controls (pollInterval, timeout seconds, signal).
|
|
2406
|
+
* When `signal` aborts, polling stops, the job gets a best-effort cancel request,
|
|
2407
|
+
* and the promise rejects with `signal.reason`.
|
|
2406
2408
|
* @returns Final job snapshot.
|
|
2407
2409
|
*/
|
|
2408
2410
|
crawl(url: string, req?: CrawlOptions & {
|
|
2409
2411
|
pollInterval?: number;
|
|
2410
2412
|
timeout?: number;
|
|
2413
|
+
signal?: AbortSignal;
|
|
2411
2414
|
}): Promise<CrawlJob>;
|
|
2412
2415
|
/**
|
|
2413
2416
|
* Retrieve crawl errors and robots.txt blocks.
|
|
@@ -2620,6 +2623,7 @@ declare class FirecrawlClient {
|
|
|
2620
2623
|
crawlUrl(url: string, req?: CrawlOptions & {
|
|
2621
2624
|
pollInterval?: number;
|
|
2622
2625
|
timeout?: number;
|
|
2626
|
+
signal?: AbortSignal;
|
|
2623
2627
|
}): Promise<CrawlJob>;
|
|
2624
2628
|
/** @deprecated V1 compatibility alias for agent recovery. Prefer startCrawl(). */
|
|
2625
2629
|
asyncCrawlUrl(url: string, req?: CrawlOptions): Promise<CrawlResponse$1>;
|
package/dist/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
require_package
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-LKCOCWYW.js";
|
|
4
4
|
|
|
5
5
|
// src/v2/utils/httpClient.ts
|
|
6
6
|
import axios from "axios";
|
|
@@ -504,7 +504,7 @@ function processingContinuesDelayMs(err) {
|
|
|
504
504
|
return Math.min(RESUME_MAX_DELAY_MS, Math.max(RESUME_MIN_DELAY_MS, seconds * 1e3));
|
|
505
505
|
}
|
|
506
506
|
async function scrape(http, url, options, deps = {}) {
|
|
507
|
-
const
|
|
507
|
+
const sleep2 = deps.sleepImpl ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
|
|
508
508
|
if (!url || !url.trim()) {
|
|
509
509
|
throw new Error("URL cannot be empty");
|
|
510
510
|
}
|
|
@@ -537,7 +537,7 @@ async function scrape(http, url, options, deps = {}) {
|
|
|
537
537
|
if (delayMs !== void 0 && resumes < RESUME_MAX_ATTEMPTS && resumeWaitedMs + delayMs <= RESUME_MAX_TOTAL_WAIT_MS) {
|
|
538
538
|
resumes += 1;
|
|
539
539
|
resumeWaitedMs += delayMs;
|
|
540
|
-
await
|
|
540
|
+
await sleep2(delayMs);
|
|
541
541
|
continue;
|
|
542
542
|
}
|
|
543
543
|
if (err?.isAxiosError) return normalizeAxiosError(err, "scrape");
|
|
@@ -765,6 +765,8 @@ async function search(http, request) {
|
|
|
765
765
|
|
|
766
766
|
// src/v2/methods/tools.ts
|
|
767
767
|
var ALEXANDRIA_MAX_CALLS = 10;
|
|
768
|
+
var ALEXANDRIA_MAX_TIMEOUT_MS = 12e4;
|
|
769
|
+
var ALEXANDRIA_RESPONSE_MARGIN_MS = 3e4;
|
|
768
770
|
function prepareAlexandriaPayload(calls, opts) {
|
|
769
771
|
if (!Array.isArray(calls) || calls.length === 0) {
|
|
770
772
|
throw new Error("alexandria requires at least one call");
|
|
@@ -814,7 +816,10 @@ async function scrapeAlexandria(http, calls, opts = {}) {
|
|
|
814
816
|
const res = await http.post("/v2/scrape", payload, {
|
|
815
817
|
headers: { "x-request-id": requestId },
|
|
816
818
|
// Allow response delivery after the API's capped execution deadline.
|
|
817
|
-
timeoutMs: Math.min(
|
|
819
|
+
timeoutMs: Math.min(
|
|
820
|
+
opts.timeout ?? ALEXANDRIA_MAX_TIMEOUT_MS,
|
|
821
|
+
ALEXANDRIA_MAX_TIMEOUT_MS
|
|
822
|
+
) + ALEXANDRIA_RESPONSE_MARGIN_MS
|
|
818
823
|
});
|
|
819
824
|
if (res.status !== 200 || !res.data?.success) {
|
|
820
825
|
throwForBadResponse(res, "alexandria");
|
|
@@ -1118,22 +1123,72 @@ async function getCrawlStatus(http, jobId, pagination) {
|
|
|
1118
1123
|
async function cancelCrawl(http, jobId) {
|
|
1119
1124
|
try {
|
|
1120
1125
|
const res = await http.delete(`/v2/crawl/${jobId}`);
|
|
1126
|
+
if (res.status === 409) return false;
|
|
1121
1127
|
if (res.status !== 200) throwForBadResponse(res, "cancel crawl");
|
|
1122
1128
|
return res.data?.status === "cancelled";
|
|
1123
1129
|
} catch (err) {
|
|
1124
|
-
if (err?.isAxiosError)
|
|
1130
|
+
if (err?.isAxiosError) {
|
|
1131
|
+
if (err.response?.status === 409) return false;
|
|
1132
|
+
return normalizeAxiosError(err, "cancel crawl");
|
|
1133
|
+
}
|
|
1125
1134
|
throw err;
|
|
1126
1135
|
}
|
|
1127
1136
|
}
|
|
1128
|
-
|
|
1137
|
+
function abortReason(signal) {
|
|
1138
|
+
if (signal.reason !== void 0) return signal.reason;
|
|
1139
|
+
return new DOMException("This operation was aborted", "AbortError");
|
|
1140
|
+
}
|
|
1141
|
+
function raceAbort(promise, signal) {
|
|
1142
|
+
if (!signal) return promise;
|
|
1143
|
+
if (signal.aborted) {
|
|
1144
|
+
promise.catch(() => {
|
|
1145
|
+
});
|
|
1146
|
+
return Promise.reject(abortReason(signal));
|
|
1147
|
+
}
|
|
1148
|
+
return new Promise((resolve, reject) => {
|
|
1149
|
+
const onAbort = () => {
|
|
1150
|
+
promise.catch(() => {
|
|
1151
|
+
});
|
|
1152
|
+
reject(abortReason(signal));
|
|
1153
|
+
};
|
|
1154
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
1155
|
+
promise.then(
|
|
1156
|
+
(value) => {
|
|
1157
|
+
signal.removeEventListener("abort", onAbort);
|
|
1158
|
+
resolve(value);
|
|
1159
|
+
},
|
|
1160
|
+
(err) => {
|
|
1161
|
+
signal.removeEventListener("abort", onAbort);
|
|
1162
|
+
reject(err);
|
|
1163
|
+
}
|
|
1164
|
+
);
|
|
1165
|
+
});
|
|
1166
|
+
}
|
|
1167
|
+
function sleep(ms, signal) {
|
|
1168
|
+
return new Promise((resolve, reject) => {
|
|
1169
|
+
if (signal?.aborted) return reject(abortReason(signal));
|
|
1170
|
+
const onAbort = () => {
|
|
1171
|
+
clearTimeout(timer);
|
|
1172
|
+
reject(abortReason(signal));
|
|
1173
|
+
};
|
|
1174
|
+
const timer = setTimeout(() => {
|
|
1175
|
+
signal?.removeEventListener("abort", onAbort);
|
|
1176
|
+
resolve();
|
|
1177
|
+
}, ms);
|
|
1178
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
1179
|
+
});
|
|
1180
|
+
}
|
|
1181
|
+
async function waitForCrawlCompletion(http, jobId, pollInterval = 2, timeout, signal) {
|
|
1129
1182
|
const start = Date.now();
|
|
1130
1183
|
while (true) {
|
|
1184
|
+
if (signal?.aborted) throw abortReason(signal);
|
|
1131
1185
|
try {
|
|
1132
|
-
const status = await getCrawlStatus(http, jobId);
|
|
1186
|
+
const status = await raceAbort(getCrawlStatus(http, jobId), signal);
|
|
1133
1187
|
if (["completed", "failed", "cancelled"].includes(status.status)) {
|
|
1134
1188
|
return status;
|
|
1135
1189
|
}
|
|
1136
1190
|
} catch (err) {
|
|
1191
|
+
if (signal?.aborted) throw abortReason(signal);
|
|
1137
1192
|
if (!isRetryableError(err)) {
|
|
1138
1193
|
if (err instanceof SdkError) {
|
|
1139
1194
|
const errorWithJobId = new SdkError(
|
|
@@ -1151,12 +1206,19 @@ async function waitForCrawlCompletion(http, jobId, pollInterval = 2, timeout) {
|
|
|
1151
1206
|
if (timeout != null && Date.now() - start > timeout * 1e3) {
|
|
1152
1207
|
throw new JobTimeoutError(jobId, timeout, "crawl");
|
|
1153
1208
|
}
|
|
1154
|
-
await
|
|
1209
|
+
await sleep(Math.max(1e3, pollInterval * 1e3), signal);
|
|
1155
1210
|
}
|
|
1156
1211
|
}
|
|
1157
|
-
async function crawl(http, request, pollInterval = 2, timeout) {
|
|
1212
|
+
async function crawl(http, request, pollInterval = 2, timeout, signal) {
|
|
1213
|
+
if (signal?.aborted) throw abortReason(signal);
|
|
1158
1214
|
const started = await startCrawl(http, request);
|
|
1159
|
-
|
|
1215
|
+
try {
|
|
1216
|
+
return await waitForCrawlCompletion(http, started.id, pollInterval, timeout, signal);
|
|
1217
|
+
} catch (err) {
|
|
1218
|
+
if (signal?.aborted) cancelCrawl(http, started.id).catch(() => {
|
|
1219
|
+
});
|
|
1220
|
+
throw err;
|
|
1221
|
+
}
|
|
1160
1222
|
}
|
|
1161
1223
|
async function getCrawlErrors(http, crawlId) {
|
|
1162
1224
|
try {
|
|
@@ -2371,7 +2433,7 @@ var FirecrawlClient = class {
|
|
|
2371
2433
|
/**
|
|
2372
2434
|
* Cancel a crawl job.
|
|
2373
2435
|
* @param jobId Crawl job id.
|
|
2374
|
-
* @returns True if cancelled.
|
|
2436
|
+
* @returns True if cancelled. False if the crawl was already completed (the API answers 409).
|
|
2375
2437
|
*/
|
|
2376
2438
|
async cancelCrawl(jobId) {
|
|
2377
2439
|
return cancelCrawl(this.http, jobId);
|
|
@@ -2379,11 +2441,14 @@ var FirecrawlClient = class {
|
|
|
2379
2441
|
/**
|
|
2380
2442
|
* Convenience waiter: start a crawl and poll until it finishes.
|
|
2381
2443
|
* @param url Root URL to crawl.
|
|
2382
|
-
* @param req Crawl configuration plus waiter controls (pollInterval, timeout seconds).
|
|
2444
|
+
* @param req Crawl configuration plus waiter controls (pollInterval, timeout seconds, signal).
|
|
2445
|
+
* When `signal` aborts, polling stops, the job gets a best-effort cancel request,
|
|
2446
|
+
* and the promise rejects with `signal.reason`.
|
|
2383
2447
|
* @returns Final job snapshot.
|
|
2384
2448
|
*/
|
|
2385
2449
|
async crawl(url, req = {}) {
|
|
2386
|
-
|
|
2450
|
+
const { pollInterval, timeout, signal, ...options } = req;
|
|
2451
|
+
return crawl(this.http, { url, ...options }, pollInterval, timeout, signal);
|
|
2387
2452
|
}
|
|
2388
2453
|
/**
|
|
2389
2454
|
* Retrieve crawl errors and robots.txt blocks.
|
|
@@ -2735,7 +2800,7 @@ var FirecrawlApp = class {
|
|
|
2735
2800
|
if (typeof process !== "undefined" && process.env && process.env.npm_package_version) {
|
|
2736
2801
|
return process.env.npm_package_version;
|
|
2737
2802
|
}
|
|
2738
|
-
const packageJson = await import("./package-
|
|
2803
|
+
const packageJson = await import("./package-C6PK3CK7.js");
|
|
2739
2804
|
return packageJson.default.version;
|
|
2740
2805
|
} catch (error) {
|
|
2741
2806
|
const isTest = typeof process !== "undefined" && (process.env.JEST_WORKER_ID != null || false);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "firecrawl",
|
|
3
|
-
"version": "4.
|
|
3
|
+
"version": "4.45.0",
|
|
4
4
|
"description": "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
import { describe, test, expect } from "@jest/globals";
|
|
2
|
+
import axios, { type AxiosAdapter } from "axios";
|
|
3
|
+
import { FirecrawlClient } from "../../../v2/client";
|
|
4
|
+
import { HttpClient } from "../../../v2/utils/httpClient";
|
|
5
|
+
import { waitForCrawlCompletion } from "../../../v2/methods/crawl";
|
|
6
|
+
import { SdkError } from "../../../v2/types";
|
|
7
|
+
|
|
8
|
+
const API_URL = "https://api.firecrawl.dev";
|
|
9
|
+
const JOB_ID = "job-123";
|
|
10
|
+
|
|
11
|
+
type Sent = { method: string; path: string };
|
|
12
|
+
type Reply = { status: number; data: unknown };
|
|
13
|
+
|
|
14
|
+
function withAdapter<T extends { defaults: any }>(instance: T, route: (req: Sent) => Reply): Sent[] {
|
|
15
|
+
const sent: Sent[] = [];
|
|
16
|
+
const adapter: AxiosAdapter = async (config) => {
|
|
17
|
+
const req = {
|
|
18
|
+
method: (config.method || "get").toLowerCase(),
|
|
19
|
+
path: new URL(axios.getUri(config)).pathname,
|
|
20
|
+
};
|
|
21
|
+
sent.push(req);
|
|
22
|
+
const { status, data } = route(req);
|
|
23
|
+
const response = { data, status, statusText: String(status), headers: {}, config };
|
|
24
|
+
const validate = config.validateStatus;
|
|
25
|
+
if (validate && !validate(status)) {
|
|
26
|
+
throw new axios.AxiosError(`Request failed with status code ${status}`, undefined, config, null, response);
|
|
27
|
+
}
|
|
28
|
+
return response;
|
|
29
|
+
};
|
|
30
|
+
instance.defaults.adapter = adapter;
|
|
31
|
+
return sent;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function makeClient(route: (req: Sent) => Reply) {
|
|
35
|
+
const client = new FirecrawlClient({ apiKey: "fc-test", apiUrl: API_URL });
|
|
36
|
+
const sent = withAdapter((client as any).http.instance, route);
|
|
37
|
+
return { client, sent };
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
// The cancel runs in the background, so let its request reach the adapter.
|
|
41
|
+
const flush = () => new Promise((r) => setTimeout(r, 0));
|
|
42
|
+
|
|
43
|
+
const running: Reply = { status: 200, data: { success: true, status: "scraping", completed: 0, total: 1, data: [] } };
|
|
44
|
+
const started: Reply = { status: 200, data: { success: true, id: JOB_ID, url: "https://example.com" } };
|
|
45
|
+
|
|
46
|
+
describe("v2 cancelCrawl", () => {
|
|
47
|
+
test("returns false when the crawl is already completed (409)", async () => {
|
|
48
|
+
const { client, sent } = makeClient(() => ({ status: 409, data: { error: "Crawl is already completed" } }));
|
|
49
|
+
await expect(client.cancelCrawl(JOB_ID)).resolves.toBe(false);
|
|
50
|
+
expect(sent).toEqual([{ method: "delete", path: `/v2/crawl/${JOB_ID}` }]);
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
test("returns false on 409 when the transport does not throw for 4xx", async () => {
|
|
54
|
+
const { client } = makeClient(() => ({ status: 409, data: { error: "Crawl is already completed" } }));
|
|
55
|
+
(client as any).http.instance.defaults.validateStatus = () => true;
|
|
56
|
+
await expect(client.cancelCrawl(JOB_ID)).resolves.toBe(false);
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
test("returns true when the crawl is cancelled", async () => {
|
|
60
|
+
const { client } = makeClient(() => ({ status: 200, data: { status: "cancelled" } }));
|
|
61
|
+
await expect(client.cancelCrawl(JOB_ID)).resolves.toBe(true);
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
test("still throws for other errors", async () => {
|
|
65
|
+
const { client } = makeClient(() => ({ status: 404, data: { error: "Job not found" } }));
|
|
66
|
+
const err = await client.cancelCrawl(JOB_ID).catch((e) => e);
|
|
67
|
+
expect(err).toBeInstanceOf(SdkError);
|
|
68
|
+
expect(err.status).toBe(404);
|
|
69
|
+
});
|
|
70
|
+
});
|
|
71
|
+
|
|
72
|
+
describe("v2 crawl with an AbortSignal", () => {
|
|
73
|
+
test("abort during polling stops polling, cancels the job, and rejects with the abort reason", async () => {
|
|
74
|
+
const controller = new AbortController();
|
|
75
|
+
let polls = 0;
|
|
76
|
+
const { client, sent } = makeClient((req) => {
|
|
77
|
+
if (req.method === "post") return started;
|
|
78
|
+
if (req.method === "delete") return { status: 200, data: { status: "cancelled" } };
|
|
79
|
+
polls += 1;
|
|
80
|
+
if (polls === 1) setTimeout(() => controller.abort(), 10);
|
|
81
|
+
return running;
|
|
82
|
+
});
|
|
83
|
+
|
|
84
|
+
const t0 = Date.now();
|
|
85
|
+
const err = await client.crawl("https://example.com", { pollInterval: 30, signal: controller.signal }).catch((e) => e);
|
|
86
|
+
|
|
87
|
+
expect(Date.now() - t0).toBeLessThan(5000);
|
|
88
|
+
expect(err).toBe(controller.signal.reason);
|
|
89
|
+
expect((err as Error).name).toBe("AbortError");
|
|
90
|
+
await flush();
|
|
91
|
+
expect(sent.map((r) => r.method)).toEqual(["post", "get", "delete"]);
|
|
92
|
+
expect(sent[2]!.path).toBe(`/v2/crawl/${JOB_ID}`);
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
test("rejects with a custom abort reason and ignores a failing cancel", async () => {
|
|
96
|
+
const controller = new AbortController();
|
|
97
|
+
const reason = new Error("caller gave up");
|
|
98
|
+
const { client, sent } = makeClient((req) => {
|
|
99
|
+
if (req.method === "post") return started;
|
|
100
|
+
if (req.method === "delete") return { status: 500, data: { error: "boom" } };
|
|
101
|
+
setTimeout(() => controller.abort(reason), 10);
|
|
102
|
+
return running;
|
|
103
|
+
});
|
|
104
|
+
|
|
105
|
+
await expect(client.crawl("https://example.com", { pollInterval: 30, signal: controller.signal })).rejects.toBe(reason);
|
|
106
|
+
await flush();
|
|
107
|
+
expect(sent.some((r) => r.method === "delete")).toBe(true);
|
|
108
|
+
});
|
|
109
|
+
|
|
110
|
+
test("a hanging cancel does not delay the rejection", async () => {
|
|
111
|
+
const controller = new AbortController();
|
|
112
|
+
const client = new FirecrawlClient({ apiKey: "fc-test", apiUrl: API_URL });
|
|
113
|
+
const methods: string[] = [];
|
|
114
|
+
(client as any).http.instance.defaults.adapter = (async (config) => {
|
|
115
|
+
methods.push(config.method!);
|
|
116
|
+
if (config.method === "delete") return new Promise(() => {});
|
|
117
|
+
if (config.method === "get") setTimeout(() => controller.abort(), 10);
|
|
118
|
+
const data = config.method === "post" ? started.data : running.data;
|
|
119
|
+
return { data, status: 200, statusText: "OK", headers: {}, config };
|
|
120
|
+
}) as AxiosAdapter;
|
|
121
|
+
|
|
122
|
+
// Read the reason after the abort: before it, signal.reason is undefined.
|
|
123
|
+
const err = await client.crawl("https://example.com", { pollInterval: 30, signal: controller.signal }).catch((e) => e);
|
|
124
|
+
expect(err).toBe(controller.signal.reason);
|
|
125
|
+
expect((err as Error).name).toBe("AbortError");
|
|
126
|
+
await flush();
|
|
127
|
+
expect(methods).toEqual(["post", "get", "delete"]);
|
|
128
|
+
});
|
|
129
|
+
|
|
130
|
+
test("abort while the start request is in flight cancels the job once the id arrives", async () => {
|
|
131
|
+
const controller = new AbortController();
|
|
132
|
+
const { client, sent } = makeClient((req) => {
|
|
133
|
+
if (req.method === "post") {
|
|
134
|
+
controller.abort();
|
|
135
|
+
return started;
|
|
136
|
+
}
|
|
137
|
+
if (req.method === "delete") return { status: 200, data: { status: "cancelled" } };
|
|
138
|
+
return running;
|
|
139
|
+
});
|
|
140
|
+
|
|
141
|
+
const err = await client.crawl("https://example.com", { signal: controller.signal }).catch((e) => e);
|
|
142
|
+
expect(err).toBe(controller.signal.reason);
|
|
143
|
+
expect((err as Error).name).toBe("AbortError");
|
|
144
|
+
await flush();
|
|
145
|
+
expect(sent.map((r) => r.method)).toEqual(["post", "delete"]);
|
|
146
|
+
});
|
|
147
|
+
|
|
148
|
+
test("an already aborted signal sends no request", async () => {
|
|
149
|
+
const controller = new AbortController();
|
|
150
|
+
controller.abort();
|
|
151
|
+
const { client, sent } = makeClient(() => started);
|
|
152
|
+
|
|
153
|
+
await expect(client.crawl("https://example.com", { signal: controller.signal })).rejects.toBe(controller.signal.reason);
|
|
154
|
+
expect(sent).toHaveLength(0);
|
|
155
|
+
});
|
|
156
|
+
|
|
157
|
+
test("the signal is not sent in the crawl request body", async () => {
|
|
158
|
+
const controller = new AbortController();
|
|
159
|
+
const client = new FirecrawlClient({ apiKey: "fc-test", apiUrl: API_URL });
|
|
160
|
+
const bodies: unknown[] = [];
|
|
161
|
+
(client as any).http.instance.defaults.adapter = (async (config) => {
|
|
162
|
+
if (config.method === "post") bodies.push(JSON.parse(config.data as string));
|
|
163
|
+
const data = config.method === "post" ? started.data : { success: true, status: "completed", completed: 1, total: 1, data: [] };
|
|
164
|
+
return { data, status: 200, statusText: "OK", headers: {}, config };
|
|
165
|
+
}) as AxiosAdapter;
|
|
166
|
+
|
|
167
|
+
const job = await client.crawl("https://example.com", { limit: 1, signal: controller.signal });
|
|
168
|
+
expect(job.status).toBe("completed");
|
|
169
|
+
expect(bodies[0]).toMatchObject({ url: "https://example.com", limit: 1 });
|
|
170
|
+
expect(Object.keys(bodies[0] as object)).not.toContain("signal");
|
|
171
|
+
});
|
|
172
|
+
});
|
|
173
|
+
|
|
174
|
+
describe("v2 waitForCrawlCompletion with an AbortSignal", () => {
|
|
175
|
+
test("stops polling and rejects without cancelling the job", async () => {
|
|
176
|
+
const http = new HttpClient({ apiKey: "fc-test", apiUrl: API_URL });
|
|
177
|
+
const controller = new AbortController();
|
|
178
|
+
const sent = withAdapter((http as any).instance, () => {
|
|
179
|
+
setTimeout(() => controller.abort(), 10);
|
|
180
|
+
return running;
|
|
181
|
+
});
|
|
182
|
+
|
|
183
|
+
const err = await waitForCrawlCompletion(http, JOB_ID, 30, undefined, controller.signal).catch((e) => e);
|
|
184
|
+
expect(err).toBe(controller.signal.reason);
|
|
185
|
+
expect((err as Error).name).toBe("AbortError");
|
|
186
|
+
await new Promise((r) => setTimeout(r, 50));
|
|
187
|
+
expect(sent.map((r) => r.method)).toEqual(["get"]);
|
|
188
|
+
});
|
|
189
|
+
});
|
|
@@ -272,9 +272,11 @@ describe("Alexandria contracts and execution", () => {
|
|
|
272
272
|
});
|
|
273
273
|
|
|
274
274
|
test.each([
|
|
275
|
-
[undefined,
|
|
275
|
+
[undefined, 150000],
|
|
276
276
|
[1000, 31000],
|
|
277
|
-
[100000,
|
|
277
|
+
[100000, 130000],
|
|
278
|
+
[120000, 150000],
|
|
279
|
+
[300000, 150000],
|
|
278
280
|
])(
|
|
279
281
|
"Alexandria timeout %s allows response delivery (%s ms)",
|
|
280
282
|
async (timeout, timeoutMs) => {
|
package/src/v2/client.ts
CHANGED
|
@@ -394,7 +394,7 @@ export class FirecrawlClient {
|
|
|
394
394
|
/**
|
|
395
395
|
* Cancel a crawl job.
|
|
396
396
|
* @param jobId Crawl job id.
|
|
397
|
-
* @returns True if cancelled.
|
|
397
|
+
* @returns True if cancelled. False if the crawl was already completed (the API answers 409).
|
|
398
398
|
*/
|
|
399
399
|
async cancelCrawl(jobId: string): Promise<boolean> {
|
|
400
400
|
return cancelCrawl(this.http, jobId);
|
|
@@ -402,11 +402,14 @@ export class FirecrawlClient {
|
|
|
402
402
|
/**
|
|
403
403
|
* Convenience waiter: start a crawl and poll until it finishes.
|
|
404
404
|
* @param url Root URL to crawl.
|
|
405
|
-
* @param req Crawl configuration plus waiter controls (pollInterval, timeout seconds).
|
|
405
|
+
* @param req Crawl configuration plus waiter controls (pollInterval, timeout seconds, signal).
|
|
406
|
+
* When `signal` aborts, polling stops, the job gets a best-effort cancel request,
|
|
407
|
+
* and the promise rejects with `signal.reason`.
|
|
406
408
|
* @returns Final job snapshot.
|
|
407
409
|
*/
|
|
408
|
-
async crawl(url: string, req: CrawlOptions & { pollInterval?: number; timeout?: number } = {}): Promise<CrawlJob> {
|
|
409
|
-
|
|
410
|
+
async crawl(url: string, req: CrawlOptions & { pollInterval?: number; timeout?: number; signal?: AbortSignal } = {}): Promise<CrawlJob> {
|
|
411
|
+
const { pollInterval, timeout, signal, ...options } = req;
|
|
412
|
+
return crawlWaiter(this.http, { url, ...options }, pollInterval, timeout, signal);
|
|
410
413
|
}
|
|
411
414
|
/**
|
|
412
415
|
* Retrieve crawl errors and robots.txt blocks.
|
|
@@ -724,7 +727,7 @@ export class FirecrawlClient {
|
|
|
724
727
|
}
|
|
725
728
|
|
|
726
729
|
/** @deprecated V1 compatibility alias for agent recovery. Prefer crawl(). */
|
|
727
|
-
async crawlUrl(url: string, req: CrawlOptions & { pollInterval?: number; timeout?: number } = {}): Promise<CrawlJob> {
|
|
730
|
+
async crawlUrl(url: string, req: CrawlOptions & { pollInterval?: number; timeout?: number; signal?: AbortSignal } = {}): Promise<CrawlJob> {
|
|
728
731
|
return this.crawl(url, req);
|
|
729
732
|
}
|
|
730
733
|
|
package/src/v2/methods/crawl.ts
CHANGED
|
@@ -107,28 +107,94 @@ export async function getCrawlStatus(
|
|
|
107
107
|
}
|
|
108
108
|
}
|
|
109
109
|
|
|
110
|
+
/**
|
|
111
|
+
* Cancel a crawl job. Resolves `false` when the API answers 409 because the
|
|
112
|
+
* crawl already finished, so there is nothing left to cancel.
|
|
113
|
+
*/
|
|
110
114
|
export async function cancelCrawl(http: HttpClient, jobId: string): Promise<boolean> {
|
|
111
115
|
try {
|
|
112
116
|
const res = await http.delete<{ status: string }>(`/v2/crawl/${jobId}`);
|
|
117
|
+
if (res.status === 409) return false;
|
|
113
118
|
if (res.status !== 200) throwForBadResponse(res, "cancel crawl");
|
|
114
119
|
return res.data?.status === "cancelled";
|
|
115
120
|
} catch (err: any) {
|
|
116
|
-
if (err?.isAxiosError)
|
|
121
|
+
if (err?.isAxiosError) {
|
|
122
|
+
if (err.response?.status === 409) return false;
|
|
123
|
+
return normalizeAxiosError(err, "cancel crawl");
|
|
124
|
+
}
|
|
117
125
|
throw err;
|
|
118
126
|
}
|
|
119
127
|
}
|
|
120
128
|
|
|
121
|
-
|
|
129
|
+
function abortReason(signal: AbortSignal): unknown {
|
|
130
|
+
if (signal.reason !== undefined) return signal.reason;
|
|
131
|
+
return new DOMException("This operation was aborted", "AbortError");
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/** Reject with the abort reason as soon as `signal` aborts. */
|
|
135
|
+
function raceAbort<T>(promise: Promise<T>, signal?: AbortSignal): Promise<T> {
|
|
136
|
+
if (!signal) return promise;
|
|
137
|
+
if (signal.aborted) {
|
|
138
|
+
promise.catch(() => {});
|
|
139
|
+
return Promise.reject(abortReason(signal));
|
|
140
|
+
}
|
|
141
|
+
return new Promise<T>((resolve, reject) => {
|
|
142
|
+
const onAbort = () => {
|
|
143
|
+
promise.catch(() => {});
|
|
144
|
+
reject(abortReason(signal));
|
|
145
|
+
};
|
|
146
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
147
|
+
promise.then(
|
|
148
|
+
(value) => {
|
|
149
|
+
signal.removeEventListener("abort", onAbort);
|
|
150
|
+
resolve(value);
|
|
151
|
+
},
|
|
152
|
+
(err) => {
|
|
153
|
+
signal.removeEventListener("abort", onAbort);
|
|
154
|
+
reject(err);
|
|
155
|
+
},
|
|
156
|
+
);
|
|
157
|
+
});
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
function sleep(ms: number, signal?: AbortSignal): Promise<void> {
|
|
161
|
+
return new Promise<void>((resolve, reject) => {
|
|
162
|
+
if (signal?.aborted) return reject(abortReason(signal));
|
|
163
|
+
const onAbort = () => {
|
|
164
|
+
clearTimeout(timer);
|
|
165
|
+
reject(abortReason(signal!));
|
|
166
|
+
};
|
|
167
|
+
const timer = setTimeout(() => {
|
|
168
|
+
signal?.removeEventListener("abort", onAbort);
|
|
169
|
+
resolve();
|
|
170
|
+
}, ms);
|
|
171
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
172
|
+
});
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
/**
|
|
176
|
+
* Poll a crawl until it reaches a terminal status. When `signal` aborts, polling
|
|
177
|
+
* stops and the promise rejects with the abort reason. The job is not cancelled.
|
|
178
|
+
*/
|
|
179
|
+
export async function waitForCrawlCompletion(
|
|
180
|
+
http: HttpClient,
|
|
181
|
+
jobId: string,
|
|
182
|
+
pollInterval = 2,
|
|
183
|
+
timeout?: number,
|
|
184
|
+
signal?: AbortSignal,
|
|
185
|
+
): Promise<CrawlJob> {
|
|
122
186
|
const start = Date.now();
|
|
123
187
|
|
|
124
188
|
while (true) {
|
|
189
|
+
if (signal?.aborted) throw abortReason(signal);
|
|
125
190
|
try {
|
|
126
|
-
const status = await getCrawlStatus(http, jobId);
|
|
191
|
+
const status = await raceAbort(getCrawlStatus(http, jobId), signal);
|
|
127
192
|
|
|
128
193
|
if (["completed", "failed", "cancelled"].includes(status.status)) {
|
|
129
194
|
return status;
|
|
130
195
|
}
|
|
131
196
|
} catch (err: any) {
|
|
197
|
+
if (signal?.aborted) throw abortReason(signal);
|
|
132
198
|
// Don't retry on permanent errors (4xx) - re-throw immediately with jobId context
|
|
133
199
|
if (!isRetryableError(err)) {
|
|
134
200
|
// Create new error with jobId for better debugging (non-retryable errors like 404)
|
|
@@ -151,13 +217,31 @@ export async function waitForCrawlCompletion(http: HttpClient, jobId: string, po
|
|
|
151
217
|
throw new JobTimeoutError(jobId, timeout, 'crawl');
|
|
152
218
|
}
|
|
153
219
|
|
|
154
|
-
await
|
|
220
|
+
await sleep(Math.max(1000, pollInterval * 1000), signal);
|
|
155
221
|
}
|
|
156
222
|
}
|
|
157
223
|
|
|
158
|
-
|
|
224
|
+
/**
|
|
225
|
+
* Start a crawl and wait for it to finish. When `signal` aborts, polling stops,
|
|
226
|
+
* the job gets a best-effort cancel request, and the promise rejects with the
|
|
227
|
+
* abort reason at once, without waiting for the cancel. A signal that is already
|
|
228
|
+
* aborted rejects before any request.
|
|
229
|
+
*/
|
|
230
|
+
export async function crawl(
|
|
231
|
+
http: HttpClient,
|
|
232
|
+
request: CrawlRequest,
|
|
233
|
+
pollInterval = 2,
|
|
234
|
+
timeout?: number,
|
|
235
|
+
signal?: AbortSignal,
|
|
236
|
+
): Promise<CrawlJob> {
|
|
237
|
+
if (signal?.aborted) throw abortReason(signal);
|
|
159
238
|
const started = await startCrawl(http, request);
|
|
160
|
-
|
|
239
|
+
try {
|
|
240
|
+
return await waitForCrawlCompletion(http, started.id, pollInterval, timeout, signal);
|
|
241
|
+
} catch (err) {
|
|
242
|
+
if (signal?.aborted) cancelCrawl(http, started.id).catch(() => {});
|
|
243
|
+
throw err;
|
|
244
|
+
}
|
|
161
245
|
}
|
|
162
246
|
|
|
163
247
|
export async function getCrawlErrors(http: HttpClient, crawlId: string): Promise<CrawlErrorsResponse> {
|
package/src/v2/methods/tools.ts
CHANGED
|
@@ -14,6 +14,10 @@ import {
|
|
|
14
14
|
} from "../utils/errorHandler";
|
|
15
15
|
|
|
16
16
|
const ALEXANDRIA_MAX_CALLS = 10;
|
|
17
|
+
// Matches the API's default and maximum execution timeout for Alexandria calls.
|
|
18
|
+
const ALEXANDRIA_MAX_TIMEOUT_MS = 120_000;
|
|
19
|
+
// Extra time for the API to deliver a response after its execution deadline.
|
|
20
|
+
const ALEXANDRIA_RESPONSE_MARGIN_MS = 30_000;
|
|
17
21
|
|
|
18
22
|
function prepareAlexandriaPayload(
|
|
19
23
|
calls: AlexandriaCall[],
|
|
@@ -82,7 +86,11 @@ export async function scrapeAlexandria(
|
|
|
82
86
|
}>("/v2/scrape", payload, {
|
|
83
87
|
headers: { "x-request-id": requestId },
|
|
84
88
|
// Allow response delivery after the API's capped execution deadline.
|
|
85
|
-
timeoutMs:
|
|
89
|
+
timeoutMs:
|
|
90
|
+
Math.min(
|
|
91
|
+
opts.timeout ?? ALEXANDRIA_MAX_TIMEOUT_MS,
|
|
92
|
+
ALEXANDRIA_MAX_TIMEOUT_MS,
|
|
93
|
+
) + ALEXANDRIA_RESPONSE_MARGIN_MS,
|
|
86
94
|
});
|
|
87
95
|
if (res.status !== 200 || !res.data?.success) {
|
|
88
96
|
throwForBadResponse(res, "alexandria");
|