firecrawl 4.44.0 → 4.45.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -12,7 +12,7 @@ var require_package = __commonJS({
12
12
  "package.json"(exports, module) {
13
13
  module.exports = {
14
14
  name: "@mendable/firecrawl-js",
15
- version: "4.44.0",
15
+ version: "4.45.0",
16
16
  description: "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
17
17
  main: "dist/index.js",
18
18
  types: "dist/index.d.ts",
package/dist/index.cjs CHANGED
@@ -39,7 +39,7 @@ var require_package = __commonJS({
39
39
  "package.json"(exports2, module2) {
40
40
  module2.exports = {
41
41
  name: "@mendable/firecrawl-js",
42
- version: "4.44.0",
42
+ version: "4.45.0",
43
43
  description: "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
44
44
  main: "dist/index.js",
45
45
  types: "dist/index.d.ts",
@@ -636,7 +636,7 @@ function processingContinuesDelayMs(err) {
636
636
  return Math.min(RESUME_MAX_DELAY_MS, Math.max(RESUME_MIN_DELAY_MS, seconds * 1e3));
637
637
  }
638
638
  async function scrape(http, url, options, deps = {}) {
639
- const sleep = deps.sleepImpl ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
639
+ const sleep2 = deps.sleepImpl ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
640
640
  if (!url || !url.trim()) {
641
641
  throw new Error("URL cannot be empty");
642
642
  }
@@ -669,7 +669,7 @@ async function scrape(http, url, options, deps = {}) {
669
669
  if (delayMs !== void 0 && resumes < RESUME_MAX_ATTEMPTS && resumeWaitedMs + delayMs <= RESUME_MAX_TOTAL_WAIT_MS) {
670
670
  resumes += 1;
671
671
  resumeWaitedMs += delayMs;
672
- await sleep(delayMs);
672
+ await sleep2(delayMs);
673
673
  continue;
674
674
  }
675
675
  if (err?.isAxiosError) return normalizeAxiosError(err, "scrape");
@@ -897,6 +897,8 @@ async function search(http, request) {
897
897
 
898
898
  // src/v2/methods/tools.ts
899
899
  var ALEXANDRIA_MAX_CALLS = 10;
900
+ var ALEXANDRIA_MAX_TIMEOUT_MS = 12e4;
901
+ var ALEXANDRIA_RESPONSE_MARGIN_MS = 3e4;
900
902
  function prepareAlexandriaPayload(calls, opts) {
901
903
  if (!Array.isArray(calls) || calls.length === 0) {
902
904
  throw new Error("alexandria requires at least one call");
@@ -946,7 +948,10 @@ async function scrapeAlexandria(http, calls, opts = {}) {
946
948
  const res = await http.post("/v2/scrape", payload, {
947
949
  headers: { "x-request-id": requestId },
948
950
  // Allow response delivery after the API's capped execution deadline.
949
- timeoutMs: Math.min(opts.timeout ?? 5e4, 5e4) + 3e4
951
+ timeoutMs: Math.min(
952
+ opts.timeout ?? ALEXANDRIA_MAX_TIMEOUT_MS,
953
+ ALEXANDRIA_MAX_TIMEOUT_MS
954
+ ) + ALEXANDRIA_RESPONSE_MARGIN_MS
950
955
  });
951
956
  if (res.status !== 200 || !res.data?.success) {
952
957
  throwForBadResponse(res, "alexandria");
@@ -1250,22 +1255,72 @@ async function getCrawlStatus(http, jobId, pagination) {
1250
1255
  async function cancelCrawl(http, jobId) {
1251
1256
  try {
1252
1257
  const res = await http.delete(`/v2/crawl/${jobId}`);
1258
+ if (res.status === 409) return false;
1253
1259
  if (res.status !== 200) throwForBadResponse(res, "cancel crawl");
1254
1260
  return res.data?.status === "cancelled";
1255
1261
  } catch (err) {
1256
- if (err?.isAxiosError) return normalizeAxiosError(err, "cancel crawl");
1262
+ if (err?.isAxiosError) {
1263
+ if (err.response?.status === 409) return false;
1264
+ return normalizeAxiosError(err, "cancel crawl");
1265
+ }
1257
1266
  throw err;
1258
1267
  }
1259
1268
  }
1260
- async function waitForCrawlCompletion(http, jobId, pollInterval = 2, timeout) {
1269
+ function abortReason(signal) {
1270
+ if (signal.reason !== void 0) return signal.reason;
1271
+ return new DOMException("This operation was aborted", "AbortError");
1272
+ }
1273
+ function raceAbort(promise, signal) {
1274
+ if (!signal) return promise;
1275
+ if (signal.aborted) {
1276
+ promise.catch(() => {
1277
+ });
1278
+ return Promise.reject(abortReason(signal));
1279
+ }
1280
+ return new Promise((resolve, reject) => {
1281
+ const onAbort = () => {
1282
+ promise.catch(() => {
1283
+ });
1284
+ reject(abortReason(signal));
1285
+ };
1286
+ signal.addEventListener("abort", onAbort, { once: true });
1287
+ promise.then(
1288
+ (value) => {
1289
+ signal.removeEventListener("abort", onAbort);
1290
+ resolve(value);
1291
+ },
1292
+ (err) => {
1293
+ signal.removeEventListener("abort", onAbort);
1294
+ reject(err);
1295
+ }
1296
+ );
1297
+ });
1298
+ }
1299
+ function sleep(ms, signal) {
1300
+ return new Promise((resolve, reject) => {
1301
+ if (signal?.aborted) return reject(abortReason(signal));
1302
+ const onAbort = () => {
1303
+ clearTimeout(timer);
1304
+ reject(abortReason(signal));
1305
+ };
1306
+ const timer = setTimeout(() => {
1307
+ signal?.removeEventListener("abort", onAbort);
1308
+ resolve();
1309
+ }, ms);
1310
+ signal?.addEventListener("abort", onAbort, { once: true });
1311
+ });
1312
+ }
1313
+ async function waitForCrawlCompletion(http, jobId, pollInterval = 2, timeout, signal) {
1261
1314
  const start = Date.now();
1262
1315
  while (true) {
1316
+ if (signal?.aborted) throw abortReason(signal);
1263
1317
  try {
1264
- const status = await getCrawlStatus(http, jobId);
1318
+ const status = await raceAbort(getCrawlStatus(http, jobId), signal);
1265
1319
  if (["completed", "failed", "cancelled"].includes(status.status)) {
1266
1320
  return status;
1267
1321
  }
1268
1322
  } catch (err) {
1323
+ if (signal?.aborted) throw abortReason(signal);
1269
1324
  if (!isRetryableError(err)) {
1270
1325
  if (err instanceof SdkError) {
1271
1326
  const errorWithJobId = new SdkError(
@@ -1283,12 +1338,19 @@ async function waitForCrawlCompletion(http, jobId, pollInterval = 2, timeout) {
1283
1338
  if (timeout != null && Date.now() - start > timeout * 1e3) {
1284
1339
  throw new JobTimeoutError(jobId, timeout, "crawl");
1285
1340
  }
1286
- await new Promise((r) => setTimeout(r, Math.max(1e3, pollInterval * 1e3)));
1341
+ await sleep(Math.max(1e3, pollInterval * 1e3), signal);
1287
1342
  }
1288
1343
  }
1289
- async function crawl(http, request, pollInterval = 2, timeout) {
1344
+ async function crawl(http, request, pollInterval = 2, timeout, signal) {
1345
+ if (signal?.aborted) throw abortReason(signal);
1290
1346
  const started = await startCrawl(http, request);
1291
- return waitForCrawlCompletion(http, started.id, pollInterval, timeout);
1347
+ try {
1348
+ return await waitForCrawlCompletion(http, started.id, pollInterval, timeout, signal);
1349
+ } catch (err) {
1350
+ if (signal?.aborted) cancelCrawl(http, started.id).catch(() => {
1351
+ });
1352
+ throw err;
1353
+ }
1292
1354
  }
1293
1355
  async function getCrawlErrors(http, crawlId) {
1294
1356
  try {
@@ -2503,7 +2565,7 @@ var FirecrawlClient = class {
2503
2565
  /**
2504
2566
  * Cancel a crawl job.
2505
2567
  * @param jobId Crawl job id.
2506
- * @returns True if cancelled.
2568
+ * @returns True if cancelled. False if the crawl was already completed (the API answers 409).
2507
2569
  */
2508
2570
  async cancelCrawl(jobId) {
2509
2571
  return cancelCrawl(this.http, jobId);
@@ -2511,11 +2573,14 @@ var FirecrawlClient = class {
2511
2573
  /**
2512
2574
  * Convenience waiter: start a crawl and poll until it finishes.
2513
2575
  * @param url Root URL to crawl.
2514
- * @param req Crawl configuration plus waiter controls (pollInterval, timeout seconds).
2576
+ * @param req Crawl configuration plus waiter controls (pollInterval, timeout seconds, signal).
2577
+ * When `signal` aborts, polling stops, the job gets a best-effort cancel request,
2578
+ * and the promise rejects with `signal.reason`.
2515
2579
  * @returns Final job snapshot.
2516
2580
  */
2517
2581
  async crawl(url, req = {}) {
2518
- return crawl(this.http, { url, ...req }, req.pollInterval, req.timeout);
2582
+ const { pollInterval, timeout, signal, ...options } = req;
2583
+ return crawl(this.http, { url, ...options }, pollInterval, timeout, signal);
2519
2584
  }
2520
2585
  /**
2521
2586
  * Retrieve crawl errors and robots.txt blocks.
package/dist/index.d.cts CHANGED
@@ -2396,18 +2396,21 @@ declare class FirecrawlClient {
2396
2396
  /**
2397
2397
  * Cancel a crawl job.
2398
2398
  * @param jobId Crawl job id.
2399
- * @returns True if cancelled.
2399
+ * @returns True if cancelled. False if the crawl was already completed (the API answers 409).
2400
2400
  */
2401
2401
  cancelCrawl(jobId: string): Promise<boolean>;
2402
2402
  /**
2403
2403
  * Convenience waiter: start a crawl and poll until it finishes.
2404
2404
  * @param url Root URL to crawl.
2405
- * @param req Crawl configuration plus waiter controls (pollInterval, timeout seconds).
2405
+ * @param req Crawl configuration plus waiter controls (pollInterval, timeout seconds, signal).
2406
+ * When `signal` aborts, polling stops, the job gets a best-effort cancel request,
2407
+ * and the promise rejects with `signal.reason`.
2406
2408
  * @returns Final job snapshot.
2407
2409
  */
2408
2410
  crawl(url: string, req?: CrawlOptions & {
2409
2411
  pollInterval?: number;
2410
2412
  timeout?: number;
2413
+ signal?: AbortSignal;
2411
2414
  }): Promise<CrawlJob>;
2412
2415
  /**
2413
2416
  * Retrieve crawl errors and robots.txt blocks.
@@ -2620,6 +2623,7 @@ declare class FirecrawlClient {
2620
2623
  crawlUrl(url: string, req?: CrawlOptions & {
2621
2624
  pollInterval?: number;
2622
2625
  timeout?: number;
2626
+ signal?: AbortSignal;
2623
2627
  }): Promise<CrawlJob>;
2624
2628
  /** @deprecated V1 compatibility alias for agent recovery. Prefer startCrawl(). */
2625
2629
  asyncCrawlUrl(url: string, req?: CrawlOptions): Promise<CrawlResponse$1>;
package/dist/index.d.ts CHANGED
@@ -2396,18 +2396,21 @@ declare class FirecrawlClient {
2396
2396
  /**
2397
2397
  * Cancel a crawl job.
2398
2398
  * @param jobId Crawl job id.
2399
- * @returns True if cancelled.
2399
+ * @returns True if cancelled. False if the crawl was already completed (the API answers 409).
2400
2400
  */
2401
2401
  cancelCrawl(jobId: string): Promise<boolean>;
2402
2402
  /**
2403
2403
  * Convenience waiter: start a crawl and poll until it finishes.
2404
2404
  * @param url Root URL to crawl.
2405
- * @param req Crawl configuration plus waiter controls (pollInterval, timeout seconds).
2405
+ * @param req Crawl configuration plus waiter controls (pollInterval, timeout seconds, signal).
2406
+ * When `signal` aborts, polling stops, the job gets a best-effort cancel request,
2407
+ * and the promise rejects with `signal.reason`.
2406
2408
  * @returns Final job snapshot.
2407
2409
  */
2408
2410
  crawl(url: string, req?: CrawlOptions & {
2409
2411
  pollInterval?: number;
2410
2412
  timeout?: number;
2413
+ signal?: AbortSignal;
2411
2414
  }): Promise<CrawlJob>;
2412
2415
  /**
2413
2416
  * Retrieve crawl errors and robots.txt blocks.
@@ -2620,6 +2623,7 @@ declare class FirecrawlClient {
2620
2623
  crawlUrl(url: string, req?: CrawlOptions & {
2621
2624
  pollInterval?: number;
2622
2625
  timeout?: number;
2626
+ signal?: AbortSignal;
2623
2627
  }): Promise<CrawlJob>;
2624
2628
  /** @deprecated V1 compatibility alias for agent recovery. Prefer startCrawl(). */
2625
2629
  asyncCrawlUrl(url: string, req?: CrawlOptions): Promise<CrawlResponse$1>;
package/dist/index.js CHANGED
@@ -1,6 +1,6 @@
1
1
  import {
2
2
  require_package
3
- } from "./chunk-BD2TNHAD.js";
3
+ } from "./chunk-LKCOCWYW.js";
4
4
 
5
5
  // src/v2/utils/httpClient.ts
6
6
  import axios from "axios";
@@ -504,7 +504,7 @@ function processingContinuesDelayMs(err) {
504
504
  return Math.min(RESUME_MAX_DELAY_MS, Math.max(RESUME_MIN_DELAY_MS, seconds * 1e3));
505
505
  }
506
506
  async function scrape(http, url, options, deps = {}) {
507
- const sleep = deps.sleepImpl ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
507
+ const sleep2 = deps.sleepImpl ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
508
508
  if (!url || !url.trim()) {
509
509
  throw new Error("URL cannot be empty");
510
510
  }
@@ -537,7 +537,7 @@ async function scrape(http, url, options, deps = {}) {
537
537
  if (delayMs !== void 0 && resumes < RESUME_MAX_ATTEMPTS && resumeWaitedMs + delayMs <= RESUME_MAX_TOTAL_WAIT_MS) {
538
538
  resumes += 1;
539
539
  resumeWaitedMs += delayMs;
540
- await sleep(delayMs);
540
+ await sleep2(delayMs);
541
541
  continue;
542
542
  }
543
543
  if (err?.isAxiosError) return normalizeAxiosError(err, "scrape");
@@ -765,6 +765,8 @@ async function search(http, request) {
765
765
 
766
766
  // src/v2/methods/tools.ts
767
767
  var ALEXANDRIA_MAX_CALLS = 10;
768
+ var ALEXANDRIA_MAX_TIMEOUT_MS = 12e4;
769
+ var ALEXANDRIA_RESPONSE_MARGIN_MS = 3e4;
768
770
  function prepareAlexandriaPayload(calls, opts) {
769
771
  if (!Array.isArray(calls) || calls.length === 0) {
770
772
  throw new Error("alexandria requires at least one call");
@@ -814,7 +816,10 @@ async function scrapeAlexandria(http, calls, opts = {}) {
814
816
  const res = await http.post("/v2/scrape", payload, {
815
817
  headers: { "x-request-id": requestId },
816
818
  // Allow response delivery after the API's capped execution deadline.
817
- timeoutMs: Math.min(opts.timeout ?? 5e4, 5e4) + 3e4
819
+ timeoutMs: Math.min(
820
+ opts.timeout ?? ALEXANDRIA_MAX_TIMEOUT_MS,
821
+ ALEXANDRIA_MAX_TIMEOUT_MS
822
+ ) + ALEXANDRIA_RESPONSE_MARGIN_MS
818
823
  });
819
824
  if (res.status !== 200 || !res.data?.success) {
820
825
  throwForBadResponse(res, "alexandria");
@@ -1118,22 +1123,72 @@ async function getCrawlStatus(http, jobId, pagination) {
1118
1123
  async function cancelCrawl(http, jobId) {
1119
1124
  try {
1120
1125
  const res = await http.delete(`/v2/crawl/${jobId}`);
1126
+ if (res.status === 409) return false;
1121
1127
  if (res.status !== 200) throwForBadResponse(res, "cancel crawl");
1122
1128
  return res.data?.status === "cancelled";
1123
1129
  } catch (err) {
1124
- if (err?.isAxiosError) return normalizeAxiosError(err, "cancel crawl");
1130
+ if (err?.isAxiosError) {
1131
+ if (err.response?.status === 409) return false;
1132
+ return normalizeAxiosError(err, "cancel crawl");
1133
+ }
1125
1134
  throw err;
1126
1135
  }
1127
1136
  }
1128
- async function waitForCrawlCompletion(http, jobId, pollInterval = 2, timeout) {
1137
+ function abortReason(signal) {
1138
+ if (signal.reason !== void 0) return signal.reason;
1139
+ return new DOMException("This operation was aborted", "AbortError");
1140
+ }
1141
+ function raceAbort(promise, signal) {
1142
+ if (!signal) return promise;
1143
+ if (signal.aborted) {
1144
+ promise.catch(() => {
1145
+ });
1146
+ return Promise.reject(abortReason(signal));
1147
+ }
1148
+ return new Promise((resolve, reject) => {
1149
+ const onAbort = () => {
1150
+ promise.catch(() => {
1151
+ });
1152
+ reject(abortReason(signal));
1153
+ };
1154
+ signal.addEventListener("abort", onAbort, { once: true });
1155
+ promise.then(
1156
+ (value) => {
1157
+ signal.removeEventListener("abort", onAbort);
1158
+ resolve(value);
1159
+ },
1160
+ (err) => {
1161
+ signal.removeEventListener("abort", onAbort);
1162
+ reject(err);
1163
+ }
1164
+ );
1165
+ });
1166
+ }
1167
+ function sleep(ms, signal) {
1168
+ return new Promise((resolve, reject) => {
1169
+ if (signal?.aborted) return reject(abortReason(signal));
1170
+ const onAbort = () => {
1171
+ clearTimeout(timer);
1172
+ reject(abortReason(signal));
1173
+ };
1174
+ const timer = setTimeout(() => {
1175
+ signal?.removeEventListener("abort", onAbort);
1176
+ resolve();
1177
+ }, ms);
1178
+ signal?.addEventListener("abort", onAbort, { once: true });
1179
+ });
1180
+ }
1181
+ async function waitForCrawlCompletion(http, jobId, pollInterval = 2, timeout, signal) {
1129
1182
  const start = Date.now();
1130
1183
  while (true) {
1184
+ if (signal?.aborted) throw abortReason(signal);
1131
1185
  try {
1132
- const status = await getCrawlStatus(http, jobId);
1186
+ const status = await raceAbort(getCrawlStatus(http, jobId), signal);
1133
1187
  if (["completed", "failed", "cancelled"].includes(status.status)) {
1134
1188
  return status;
1135
1189
  }
1136
1190
  } catch (err) {
1191
+ if (signal?.aborted) throw abortReason(signal);
1137
1192
  if (!isRetryableError(err)) {
1138
1193
  if (err instanceof SdkError) {
1139
1194
  const errorWithJobId = new SdkError(
@@ -1151,12 +1206,19 @@ async function waitForCrawlCompletion(http, jobId, pollInterval = 2, timeout) {
1151
1206
  if (timeout != null && Date.now() - start > timeout * 1e3) {
1152
1207
  throw new JobTimeoutError(jobId, timeout, "crawl");
1153
1208
  }
1154
- await new Promise((r) => setTimeout(r, Math.max(1e3, pollInterval * 1e3)));
1209
+ await sleep(Math.max(1e3, pollInterval * 1e3), signal);
1155
1210
  }
1156
1211
  }
1157
- async function crawl(http, request, pollInterval = 2, timeout) {
1212
+ async function crawl(http, request, pollInterval = 2, timeout, signal) {
1213
+ if (signal?.aborted) throw abortReason(signal);
1158
1214
  const started = await startCrawl(http, request);
1159
- return waitForCrawlCompletion(http, started.id, pollInterval, timeout);
1215
+ try {
1216
+ return await waitForCrawlCompletion(http, started.id, pollInterval, timeout, signal);
1217
+ } catch (err) {
1218
+ if (signal?.aborted) cancelCrawl(http, started.id).catch(() => {
1219
+ });
1220
+ throw err;
1221
+ }
1160
1222
  }
1161
1223
  async function getCrawlErrors(http, crawlId) {
1162
1224
  try {
@@ -2371,7 +2433,7 @@ var FirecrawlClient = class {
2371
2433
  /**
2372
2434
  * Cancel a crawl job.
2373
2435
  * @param jobId Crawl job id.
2374
- * @returns True if cancelled.
2436
+ * @returns True if cancelled. False if the crawl was already completed (the API answers 409).
2375
2437
  */
2376
2438
  async cancelCrawl(jobId) {
2377
2439
  return cancelCrawl(this.http, jobId);
@@ -2379,11 +2441,14 @@ var FirecrawlClient = class {
2379
2441
  /**
2380
2442
  * Convenience waiter: start a crawl and poll until it finishes.
2381
2443
  * @param url Root URL to crawl.
2382
- * @param req Crawl configuration plus waiter controls (pollInterval, timeout seconds).
2444
+ * @param req Crawl configuration plus waiter controls (pollInterval, timeout seconds, signal).
2445
+ * When `signal` aborts, polling stops, the job gets a best-effort cancel request,
2446
+ * and the promise rejects with `signal.reason`.
2383
2447
  * @returns Final job snapshot.
2384
2448
  */
2385
2449
  async crawl(url, req = {}) {
2386
- return crawl(this.http, { url, ...req }, req.pollInterval, req.timeout);
2450
+ const { pollInterval, timeout, signal, ...options } = req;
2451
+ return crawl(this.http, { url, ...options }, pollInterval, timeout, signal);
2387
2452
  }
2388
2453
  /**
2389
2454
  * Retrieve crawl errors and robots.txt blocks.
@@ -2735,7 +2800,7 @@ var FirecrawlApp = class {
2735
2800
  if (typeof process !== "undefined" && process.env && process.env.npm_package_version) {
2736
2801
  return process.env.npm_package_version;
2737
2802
  }
2738
- const packageJson = await import("./package-WSFVH42J.js");
2803
+ const packageJson = await import("./package-C6PK3CK7.js");
2739
2804
  return packageJson.default.version;
2740
2805
  } catch (error) {
2741
2806
  const isTest = typeof process !== "undefined" && (process.env.JEST_WORKER_ID != null || false);
@@ -1,4 +1,4 @@
1
1
  import {
2
2
  require_package
3
- } from "./chunk-BD2TNHAD.js";
3
+ } from "./chunk-LKCOCWYW.js";
4
4
  export default require_package();
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "firecrawl",
3
- "version": "4.44.0",
3
+ "version": "4.45.0",
4
4
  "description": "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
5
5
  "main": "dist/index.js",
6
6
  "types": "dist/index.d.ts",
@@ -0,0 +1,189 @@
1
+ import { describe, test, expect } from "@jest/globals";
2
+ import axios, { type AxiosAdapter } from "axios";
3
+ import { FirecrawlClient } from "../../../v2/client";
4
+ import { HttpClient } from "../../../v2/utils/httpClient";
5
+ import { waitForCrawlCompletion } from "../../../v2/methods/crawl";
6
+ import { SdkError } from "../../../v2/types";
7
+
8
+ const API_URL = "https://api.firecrawl.dev";
9
+ const JOB_ID = "job-123";
10
+
11
+ type Sent = { method: string; path: string };
12
+ type Reply = { status: number; data: unknown };
13
+
14
+ function withAdapter<T extends { defaults: any }>(instance: T, route: (req: Sent) => Reply): Sent[] {
15
+ const sent: Sent[] = [];
16
+ const adapter: AxiosAdapter = async (config) => {
17
+ const req = {
18
+ method: (config.method || "get").toLowerCase(),
19
+ path: new URL(axios.getUri(config)).pathname,
20
+ };
21
+ sent.push(req);
22
+ const { status, data } = route(req);
23
+ const response = { data, status, statusText: String(status), headers: {}, config };
24
+ const validate = config.validateStatus;
25
+ if (validate && !validate(status)) {
26
+ throw new axios.AxiosError(`Request failed with status code ${status}`, undefined, config, null, response);
27
+ }
28
+ return response;
29
+ };
30
+ instance.defaults.adapter = adapter;
31
+ return sent;
32
+ }
33
+
34
+ function makeClient(route: (req: Sent) => Reply) {
35
+ const client = new FirecrawlClient({ apiKey: "fc-test", apiUrl: API_URL });
36
+ const sent = withAdapter((client as any).http.instance, route);
37
+ return { client, sent };
38
+ }
39
+
40
+ // The cancel runs in the background, so let its request reach the adapter.
41
+ const flush = () => new Promise((r) => setTimeout(r, 0));
42
+
43
+ const running: Reply = { status: 200, data: { success: true, status: "scraping", completed: 0, total: 1, data: [] } };
44
+ const started: Reply = { status: 200, data: { success: true, id: JOB_ID, url: "https://example.com" } };
45
+
46
+ describe("v2 cancelCrawl", () => {
47
+ test("returns false when the crawl is already completed (409)", async () => {
48
+ const { client, sent } = makeClient(() => ({ status: 409, data: { error: "Crawl is already completed" } }));
49
+ await expect(client.cancelCrawl(JOB_ID)).resolves.toBe(false);
50
+ expect(sent).toEqual([{ method: "delete", path: `/v2/crawl/${JOB_ID}` }]);
51
+ });
52
+
53
+ test("returns false on 409 when the transport does not throw for 4xx", async () => {
54
+ const { client } = makeClient(() => ({ status: 409, data: { error: "Crawl is already completed" } }));
55
+ (client as any).http.instance.defaults.validateStatus = () => true;
56
+ await expect(client.cancelCrawl(JOB_ID)).resolves.toBe(false);
57
+ });
58
+
59
+ test("returns true when the crawl is cancelled", async () => {
60
+ const { client } = makeClient(() => ({ status: 200, data: { status: "cancelled" } }));
61
+ await expect(client.cancelCrawl(JOB_ID)).resolves.toBe(true);
62
+ });
63
+
64
+ test("still throws for other errors", async () => {
65
+ const { client } = makeClient(() => ({ status: 404, data: { error: "Job not found" } }));
66
+ const err = await client.cancelCrawl(JOB_ID).catch((e) => e);
67
+ expect(err).toBeInstanceOf(SdkError);
68
+ expect(err.status).toBe(404);
69
+ });
70
+ });
71
+
72
+ describe("v2 crawl with an AbortSignal", () => {
73
+ test("abort during polling stops polling, cancels the job, and rejects with the abort reason", async () => {
74
+ const controller = new AbortController();
75
+ let polls = 0;
76
+ const { client, sent } = makeClient((req) => {
77
+ if (req.method === "post") return started;
78
+ if (req.method === "delete") return { status: 200, data: { status: "cancelled" } };
79
+ polls += 1;
80
+ if (polls === 1) setTimeout(() => controller.abort(), 10);
81
+ return running;
82
+ });
83
+
84
+ const t0 = Date.now();
85
+ const err = await client.crawl("https://example.com", { pollInterval: 30, signal: controller.signal }).catch((e) => e);
86
+
87
+ expect(Date.now() - t0).toBeLessThan(5000);
88
+ expect(err).toBe(controller.signal.reason);
89
+ expect((err as Error).name).toBe("AbortError");
90
+ await flush();
91
+ expect(sent.map((r) => r.method)).toEqual(["post", "get", "delete"]);
92
+ expect(sent[2]!.path).toBe(`/v2/crawl/${JOB_ID}`);
93
+ });
94
+
95
+ test("rejects with a custom abort reason and ignores a failing cancel", async () => {
96
+ const controller = new AbortController();
97
+ const reason = new Error("caller gave up");
98
+ const { client, sent } = makeClient((req) => {
99
+ if (req.method === "post") return started;
100
+ if (req.method === "delete") return { status: 500, data: { error: "boom" } };
101
+ setTimeout(() => controller.abort(reason), 10);
102
+ return running;
103
+ });
104
+
105
+ await expect(client.crawl("https://example.com", { pollInterval: 30, signal: controller.signal })).rejects.toBe(reason);
106
+ await flush();
107
+ expect(sent.some((r) => r.method === "delete")).toBe(true);
108
+ });
109
+
110
+ test("a hanging cancel does not delay the rejection", async () => {
111
+ const controller = new AbortController();
112
+ const client = new FirecrawlClient({ apiKey: "fc-test", apiUrl: API_URL });
113
+ const methods: string[] = [];
114
+ (client as any).http.instance.defaults.adapter = (async (config) => {
115
+ methods.push(config.method!);
116
+ if (config.method === "delete") return new Promise(() => {});
117
+ if (config.method === "get") setTimeout(() => controller.abort(), 10);
118
+ const data = config.method === "post" ? started.data : running.data;
119
+ return { data, status: 200, statusText: "OK", headers: {}, config };
120
+ }) as AxiosAdapter;
121
+
122
+ // Read the reason after the abort: before it, signal.reason is undefined.
123
+ const err = await client.crawl("https://example.com", { pollInterval: 30, signal: controller.signal }).catch((e) => e);
124
+ expect(err).toBe(controller.signal.reason);
125
+ expect((err as Error).name).toBe("AbortError");
126
+ await flush();
127
+ expect(methods).toEqual(["post", "get", "delete"]);
128
+ });
129
+
130
+ test("abort while the start request is in flight cancels the job once the id arrives", async () => {
131
+ const controller = new AbortController();
132
+ const { client, sent } = makeClient((req) => {
133
+ if (req.method === "post") {
134
+ controller.abort();
135
+ return started;
136
+ }
137
+ if (req.method === "delete") return { status: 200, data: { status: "cancelled" } };
138
+ return running;
139
+ });
140
+
141
+ const err = await client.crawl("https://example.com", { signal: controller.signal }).catch((e) => e);
142
+ expect(err).toBe(controller.signal.reason);
143
+ expect((err as Error).name).toBe("AbortError");
144
+ await flush();
145
+ expect(sent.map((r) => r.method)).toEqual(["post", "delete"]);
146
+ });
147
+
148
+ test("an already aborted signal sends no request", async () => {
149
+ const controller = new AbortController();
150
+ controller.abort();
151
+ const { client, sent } = makeClient(() => started);
152
+
153
+ await expect(client.crawl("https://example.com", { signal: controller.signal })).rejects.toBe(controller.signal.reason);
154
+ expect(sent).toHaveLength(0);
155
+ });
156
+
157
+ test("the signal is not sent in the crawl request body", async () => {
158
+ const controller = new AbortController();
159
+ const client = new FirecrawlClient({ apiKey: "fc-test", apiUrl: API_URL });
160
+ const bodies: unknown[] = [];
161
+ (client as any).http.instance.defaults.adapter = (async (config) => {
162
+ if (config.method === "post") bodies.push(JSON.parse(config.data as string));
163
+ const data = config.method === "post" ? started.data : { success: true, status: "completed", completed: 1, total: 1, data: [] };
164
+ return { data, status: 200, statusText: "OK", headers: {}, config };
165
+ }) as AxiosAdapter;
166
+
167
+ const job = await client.crawl("https://example.com", { limit: 1, signal: controller.signal });
168
+ expect(job.status).toBe("completed");
169
+ expect(bodies[0]).toMatchObject({ url: "https://example.com", limit: 1 });
170
+ expect(Object.keys(bodies[0] as object)).not.toContain("signal");
171
+ });
172
+ });
173
+
174
+ describe("v2 waitForCrawlCompletion with an AbortSignal", () => {
175
+ test("stops polling and rejects without cancelling the job", async () => {
176
+ const http = new HttpClient({ apiKey: "fc-test", apiUrl: API_URL });
177
+ const controller = new AbortController();
178
+ const sent = withAdapter((http as any).instance, () => {
179
+ setTimeout(() => controller.abort(), 10);
180
+ return running;
181
+ });
182
+
183
+ const err = await waitForCrawlCompletion(http, JOB_ID, 30, undefined, controller.signal).catch((e) => e);
184
+ expect(err).toBe(controller.signal.reason);
185
+ expect((err as Error).name).toBe("AbortError");
186
+ await new Promise((r) => setTimeout(r, 50));
187
+ expect(sent.map((r) => r.method)).toEqual(["get"]);
188
+ });
189
+ });
@@ -272,9 +272,11 @@ describe("Alexandria contracts and execution", () => {
272
272
  });
273
273
 
274
274
  test.each([
275
- [undefined, 80000],
275
+ [undefined, 150000],
276
276
  [1000, 31000],
277
- [100000, 80000],
277
+ [100000, 130000],
278
+ [120000, 150000],
279
+ [300000, 150000],
278
280
  ])(
279
281
  "Alexandria timeout %s allows response delivery (%s ms)",
280
282
  async (timeout, timeoutMs) => {
package/src/v2/client.ts CHANGED
@@ -394,7 +394,7 @@ export class FirecrawlClient {
394
394
  /**
395
395
  * Cancel a crawl job.
396
396
  * @param jobId Crawl job id.
397
- * @returns True if cancelled.
397
+ * @returns True if cancelled. False if the crawl was already completed (the API answers 409).
398
398
  */
399
399
  async cancelCrawl(jobId: string): Promise<boolean> {
400
400
  return cancelCrawl(this.http, jobId);
@@ -402,11 +402,14 @@ export class FirecrawlClient {
402
402
  /**
403
403
  * Convenience waiter: start a crawl and poll until it finishes.
404
404
  * @param url Root URL to crawl.
405
- * @param req Crawl configuration plus waiter controls (pollInterval, timeout seconds).
405
+ * @param req Crawl configuration plus waiter controls (pollInterval, timeout seconds, signal).
406
+ * When `signal` aborts, polling stops, the job gets a best-effort cancel request,
407
+ * and the promise rejects with `signal.reason`.
406
408
  * @returns Final job snapshot.
407
409
  */
408
- async crawl(url: string, req: CrawlOptions & { pollInterval?: number; timeout?: number } = {}): Promise<CrawlJob> {
409
- return crawlWaiter(this.http, { url, ...req }, req.pollInterval, req.timeout);
410
+ async crawl(url: string, req: CrawlOptions & { pollInterval?: number; timeout?: number; signal?: AbortSignal } = {}): Promise<CrawlJob> {
411
+ const { pollInterval, timeout, signal, ...options } = req;
412
+ return crawlWaiter(this.http, { url, ...options }, pollInterval, timeout, signal);
410
413
  }
411
414
  /**
412
415
  * Retrieve crawl errors and robots.txt blocks.
@@ -724,7 +727,7 @@ export class FirecrawlClient {
724
727
  }
725
728
 
726
729
  /** @deprecated V1 compatibility alias for agent recovery. Prefer crawl(). */
727
- async crawlUrl(url: string, req: CrawlOptions & { pollInterval?: number; timeout?: number } = {}): Promise<CrawlJob> {
730
+ async crawlUrl(url: string, req: CrawlOptions & { pollInterval?: number; timeout?: number; signal?: AbortSignal } = {}): Promise<CrawlJob> {
728
731
  return this.crawl(url, req);
729
732
  }
730
733
 
@@ -107,28 +107,94 @@ export async function getCrawlStatus(
107
107
  }
108
108
  }
109
109
 
110
+ /**
111
+ * Cancel a crawl job. Resolves `false` when the API answers 409 because the
112
+ * crawl already finished, so there is nothing left to cancel.
113
+ */
110
114
  export async function cancelCrawl(http: HttpClient, jobId: string): Promise<boolean> {
111
115
  try {
112
116
  const res = await http.delete<{ status: string }>(`/v2/crawl/${jobId}`);
117
+ if (res.status === 409) return false;
113
118
  if (res.status !== 200) throwForBadResponse(res, "cancel crawl");
114
119
  return res.data?.status === "cancelled";
115
120
  } catch (err: any) {
116
- if (err?.isAxiosError) return normalizeAxiosError(err, "cancel crawl");
121
+ if (err?.isAxiosError) {
122
+ if (err.response?.status === 409) return false;
123
+ return normalizeAxiosError(err, "cancel crawl");
124
+ }
117
125
  throw err;
118
126
  }
119
127
  }
120
128
 
121
- export async function waitForCrawlCompletion(http: HttpClient, jobId: string, pollInterval = 2, timeout?: number): Promise<CrawlJob> {
129
+ function abortReason(signal: AbortSignal): unknown {
130
+ if (signal.reason !== undefined) return signal.reason;
131
+ return new DOMException("This operation was aborted", "AbortError");
132
+ }
133
+
134
+ /** Reject with the abort reason as soon as `signal` aborts. */
135
+ function raceAbort<T>(promise: Promise<T>, signal?: AbortSignal): Promise<T> {
136
+ if (!signal) return promise;
137
+ if (signal.aborted) {
138
+ promise.catch(() => {});
139
+ return Promise.reject(abortReason(signal));
140
+ }
141
+ return new Promise<T>((resolve, reject) => {
142
+ const onAbort = () => {
143
+ promise.catch(() => {});
144
+ reject(abortReason(signal));
145
+ };
146
+ signal.addEventListener("abort", onAbort, { once: true });
147
+ promise.then(
148
+ (value) => {
149
+ signal.removeEventListener("abort", onAbort);
150
+ resolve(value);
151
+ },
152
+ (err) => {
153
+ signal.removeEventListener("abort", onAbort);
154
+ reject(err);
155
+ },
156
+ );
157
+ });
158
+ }
159
+
160
+ function sleep(ms: number, signal?: AbortSignal): Promise<void> {
161
+ return new Promise<void>((resolve, reject) => {
162
+ if (signal?.aborted) return reject(abortReason(signal));
163
+ const onAbort = () => {
164
+ clearTimeout(timer);
165
+ reject(abortReason(signal!));
166
+ };
167
+ const timer = setTimeout(() => {
168
+ signal?.removeEventListener("abort", onAbort);
169
+ resolve();
170
+ }, ms);
171
+ signal?.addEventListener("abort", onAbort, { once: true });
172
+ });
173
+ }
174
+
175
+ /**
176
+ * Poll a crawl until it reaches a terminal status. When `signal` aborts, polling
177
+ * stops and the promise rejects with the abort reason. The job is not cancelled.
178
+ */
179
+ export async function waitForCrawlCompletion(
180
+ http: HttpClient,
181
+ jobId: string,
182
+ pollInterval = 2,
183
+ timeout?: number,
184
+ signal?: AbortSignal,
185
+ ): Promise<CrawlJob> {
122
186
  const start = Date.now();
123
187
 
124
188
  while (true) {
189
+ if (signal?.aborted) throw abortReason(signal);
125
190
  try {
126
- const status = await getCrawlStatus(http, jobId);
191
+ const status = await raceAbort(getCrawlStatus(http, jobId), signal);
127
192
 
128
193
  if (["completed", "failed", "cancelled"].includes(status.status)) {
129
194
  return status;
130
195
  }
131
196
  } catch (err: any) {
197
+ if (signal?.aborted) throw abortReason(signal);
132
198
  // Don't retry on permanent errors (4xx) - re-throw immediately with jobId context
133
199
  if (!isRetryableError(err)) {
134
200
  // Create new error with jobId for better debugging (non-retryable errors like 404)
@@ -151,13 +217,31 @@ export async function waitForCrawlCompletion(http: HttpClient, jobId: string, po
151
217
  throw new JobTimeoutError(jobId, timeout, 'crawl');
152
218
  }
153
219
 
154
- await new Promise((r) => setTimeout(r, Math.max(1000, pollInterval * 1000)));
220
+ await sleep(Math.max(1000, pollInterval * 1000), signal);
155
221
  }
156
222
  }
157
223
 
158
- export async function crawl(http: HttpClient, request: CrawlRequest, pollInterval = 2, timeout?: number): Promise<CrawlJob> {
224
+ /**
225
+ * Start a crawl and wait for it to finish. When `signal` aborts, polling stops,
226
+ * the job gets a best-effort cancel request, and the promise rejects with the
227
+ * abort reason at once, without waiting for the cancel. A signal that is already
228
+ * aborted rejects before any request.
229
+ */
230
+ export async function crawl(
231
+ http: HttpClient,
232
+ request: CrawlRequest,
233
+ pollInterval = 2,
234
+ timeout?: number,
235
+ signal?: AbortSignal,
236
+ ): Promise<CrawlJob> {
237
+ if (signal?.aborted) throw abortReason(signal);
159
238
  const started = await startCrawl(http, request);
160
- return waitForCrawlCompletion(http, started.id, pollInterval, timeout);
239
+ try {
240
+ return await waitForCrawlCompletion(http, started.id, pollInterval, timeout, signal);
241
+ } catch (err) {
242
+ if (signal?.aborted) cancelCrawl(http, started.id).catch(() => {});
243
+ throw err;
244
+ }
161
245
  }
162
246
 
163
247
  export async function getCrawlErrors(http: HttpClient, crawlId: string): Promise<CrawlErrorsResponse> {
@@ -14,6 +14,10 @@ import {
14
14
  } from "../utils/errorHandler";
15
15
 
16
16
  const ALEXANDRIA_MAX_CALLS = 10;
17
+ // Matches the API's default and maximum execution timeout for Alexandria calls.
18
+ const ALEXANDRIA_MAX_TIMEOUT_MS = 120_000;
19
+ // Extra time for the API to deliver a response after its execution deadline.
20
+ const ALEXANDRIA_RESPONSE_MARGIN_MS = 30_000;
17
21
 
18
22
  function prepareAlexandriaPayload(
19
23
  calls: AlexandriaCall[],
@@ -82,7 +86,11 @@ export async function scrapeAlexandria(
82
86
  }>("/v2/scrape", payload, {
83
87
  headers: { "x-request-id": requestId },
84
88
  // Allow response delivery after the API's capped execution deadline.
85
- timeoutMs: Math.min(opts.timeout ?? 50000, 50000) + 30000,
89
+ timeoutMs:
90
+ Math.min(
91
+ opts.timeout ?? ALEXANDRIA_MAX_TIMEOUT_MS,
92
+ ALEXANDRIA_MAX_TIMEOUT_MS,
93
+ ) + ALEXANDRIA_RESPONSE_MARGIN_MS,
86
94
  });
87
95
  if (res.status !== 200 || !res.data?.success) {
88
96
  throwForBadResponse(res, "alexandria");