firecrawl 4.44.1 → 4.45.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -12,7 +12,7 @@ var require_package = __commonJS({
12
12
  "package.json"(exports, module) {
13
13
  module.exports = {
14
14
  name: "@mendable/firecrawl-js",
15
- version: "4.44.1",
15
+ version: "4.45.0",
16
16
  description: "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
17
17
  main: "dist/index.js",
18
18
  types: "dist/index.d.ts",
package/dist/index.cjs CHANGED
@@ -39,7 +39,7 @@ var require_package = __commonJS({
39
39
  "package.json"(exports2, module2) {
40
40
  module2.exports = {
41
41
  name: "@mendable/firecrawl-js",
42
- version: "4.44.1",
42
+ version: "4.45.0",
43
43
  description: "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
44
44
  main: "dist/index.js",
45
45
  types: "dist/index.d.ts",
@@ -636,7 +636,7 @@ function processingContinuesDelayMs(err) {
636
636
  return Math.min(RESUME_MAX_DELAY_MS, Math.max(RESUME_MIN_DELAY_MS, seconds * 1e3));
637
637
  }
638
638
  async function scrape(http, url, options, deps = {}) {
639
- const sleep = deps.sleepImpl ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
639
+ const sleep2 = deps.sleepImpl ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
640
640
  if (!url || !url.trim()) {
641
641
  throw new Error("URL cannot be empty");
642
642
  }
@@ -669,7 +669,7 @@ async function scrape(http, url, options, deps = {}) {
669
669
  if (delayMs !== void 0 && resumes < RESUME_MAX_ATTEMPTS && resumeWaitedMs + delayMs <= RESUME_MAX_TOTAL_WAIT_MS) {
670
670
  resumes += 1;
671
671
  resumeWaitedMs += delayMs;
672
- await sleep(delayMs);
672
+ await sleep2(delayMs);
673
673
  continue;
674
674
  }
675
675
  if (err?.isAxiosError) return normalizeAxiosError(err, "scrape");
@@ -1255,22 +1255,72 @@ async function getCrawlStatus(http, jobId, pagination) {
1255
1255
  async function cancelCrawl(http, jobId) {
1256
1256
  try {
1257
1257
  const res = await http.delete(`/v2/crawl/${jobId}`);
1258
+ if (res.status === 409) return false;
1258
1259
  if (res.status !== 200) throwForBadResponse(res, "cancel crawl");
1259
1260
  return res.data?.status === "cancelled";
1260
1261
  } catch (err) {
1261
- if (err?.isAxiosError) return normalizeAxiosError(err, "cancel crawl");
1262
+ if (err?.isAxiosError) {
1263
+ if (err.response?.status === 409) return false;
1264
+ return normalizeAxiosError(err, "cancel crawl");
1265
+ }
1262
1266
  throw err;
1263
1267
  }
1264
1268
  }
1265
- async function waitForCrawlCompletion(http, jobId, pollInterval = 2, timeout) {
1269
+ function abortReason(signal) {
1270
+ if (signal.reason !== void 0) return signal.reason;
1271
+ return new DOMException("This operation was aborted", "AbortError");
1272
+ }
1273
+ function raceAbort(promise, signal) {
1274
+ if (!signal) return promise;
1275
+ if (signal.aborted) {
1276
+ promise.catch(() => {
1277
+ });
1278
+ return Promise.reject(abortReason(signal));
1279
+ }
1280
+ return new Promise((resolve, reject) => {
1281
+ const onAbort = () => {
1282
+ promise.catch(() => {
1283
+ });
1284
+ reject(abortReason(signal));
1285
+ };
1286
+ signal.addEventListener("abort", onAbort, { once: true });
1287
+ promise.then(
1288
+ (value) => {
1289
+ signal.removeEventListener("abort", onAbort);
1290
+ resolve(value);
1291
+ },
1292
+ (err) => {
1293
+ signal.removeEventListener("abort", onAbort);
1294
+ reject(err);
1295
+ }
1296
+ );
1297
+ });
1298
+ }
1299
+ function sleep(ms, signal) {
1300
+ return new Promise((resolve, reject) => {
1301
+ if (signal?.aborted) return reject(abortReason(signal));
1302
+ const onAbort = () => {
1303
+ clearTimeout(timer);
1304
+ reject(abortReason(signal));
1305
+ };
1306
+ const timer = setTimeout(() => {
1307
+ signal?.removeEventListener("abort", onAbort);
1308
+ resolve();
1309
+ }, ms);
1310
+ signal?.addEventListener("abort", onAbort, { once: true });
1311
+ });
1312
+ }
1313
+ async function waitForCrawlCompletion(http, jobId, pollInterval = 2, timeout, signal) {
1266
1314
  const start = Date.now();
1267
1315
  while (true) {
1316
+ if (signal?.aborted) throw abortReason(signal);
1268
1317
  try {
1269
- const status = await getCrawlStatus(http, jobId);
1318
+ const status = await raceAbort(getCrawlStatus(http, jobId), signal);
1270
1319
  if (["completed", "failed", "cancelled"].includes(status.status)) {
1271
1320
  return status;
1272
1321
  }
1273
1322
  } catch (err) {
1323
+ if (signal?.aborted) throw abortReason(signal);
1274
1324
  if (!isRetryableError(err)) {
1275
1325
  if (err instanceof SdkError) {
1276
1326
  const errorWithJobId = new SdkError(
@@ -1288,12 +1338,19 @@ async function waitForCrawlCompletion(http, jobId, pollInterval = 2, timeout) {
1288
1338
  if (timeout != null && Date.now() - start > timeout * 1e3) {
1289
1339
  throw new JobTimeoutError(jobId, timeout, "crawl");
1290
1340
  }
1291
- await new Promise((r) => setTimeout(r, Math.max(1e3, pollInterval * 1e3)));
1341
+ await sleep(Math.max(1e3, pollInterval * 1e3), signal);
1292
1342
  }
1293
1343
  }
1294
- async function crawl(http, request, pollInterval = 2, timeout) {
1344
+ async function crawl(http, request, pollInterval = 2, timeout, signal) {
1345
+ if (signal?.aborted) throw abortReason(signal);
1295
1346
  const started = await startCrawl(http, request);
1296
- return waitForCrawlCompletion(http, started.id, pollInterval, timeout);
1347
+ try {
1348
+ return await waitForCrawlCompletion(http, started.id, pollInterval, timeout, signal);
1349
+ } catch (err) {
1350
+ if (signal?.aborted) cancelCrawl(http, started.id).catch(() => {
1351
+ });
1352
+ throw err;
1353
+ }
1297
1354
  }
1298
1355
  async function getCrawlErrors(http, crawlId) {
1299
1356
  try {
@@ -2508,7 +2565,7 @@ var FirecrawlClient = class {
2508
2565
  /**
2509
2566
  * Cancel a crawl job.
2510
2567
  * @param jobId Crawl job id.
2511
- * @returns True if cancelled.
2568
+ * @returns True if cancelled. False if the crawl was already completed (the API answers 409).
2512
2569
  */
2513
2570
  async cancelCrawl(jobId) {
2514
2571
  return cancelCrawl(this.http, jobId);
@@ -2516,11 +2573,14 @@ var FirecrawlClient = class {
2516
2573
  /**
2517
2574
  * Convenience waiter: start a crawl and poll until it finishes.
2518
2575
  * @param url Root URL to crawl.
2519
- * @param req Crawl configuration plus waiter controls (pollInterval, timeout seconds).
2576
+ * @param req Crawl configuration plus waiter controls (pollInterval, timeout seconds, signal).
2577
+ * When `signal` aborts, polling stops, the job gets a best-effort cancel request,
2578
+ * and the promise rejects with `signal.reason`.
2520
2579
  * @returns Final job snapshot.
2521
2580
  */
2522
2581
  async crawl(url, req = {}) {
2523
- return crawl(this.http, { url, ...req }, req.pollInterval, req.timeout);
2582
+ const { pollInterval, timeout, signal, ...options } = req;
2583
+ return crawl(this.http, { url, ...options }, pollInterval, timeout, signal);
2524
2584
  }
2525
2585
  /**
2526
2586
  * Retrieve crawl errors and robots.txt blocks.
package/dist/index.d.cts CHANGED
@@ -2396,18 +2396,21 @@ declare class FirecrawlClient {
2396
2396
  /**
2397
2397
  * Cancel a crawl job.
2398
2398
  * @param jobId Crawl job id.
2399
- * @returns True if cancelled.
2399
+ * @returns True if cancelled. False if the crawl was already completed (the API answers 409).
2400
2400
  */
2401
2401
  cancelCrawl(jobId: string): Promise<boolean>;
2402
2402
  /**
2403
2403
  * Convenience waiter: start a crawl and poll until it finishes.
2404
2404
  * @param url Root URL to crawl.
2405
- * @param req Crawl configuration plus waiter controls (pollInterval, timeout seconds).
2405
+ * @param req Crawl configuration plus waiter controls (pollInterval, timeout seconds, signal).
2406
+ * When `signal` aborts, polling stops, the job gets a best-effort cancel request,
2407
+ * and the promise rejects with `signal.reason`.
2406
2408
  * @returns Final job snapshot.
2407
2409
  */
2408
2410
  crawl(url: string, req?: CrawlOptions & {
2409
2411
  pollInterval?: number;
2410
2412
  timeout?: number;
2413
+ signal?: AbortSignal;
2411
2414
  }): Promise<CrawlJob>;
2412
2415
  /**
2413
2416
  * Retrieve crawl errors and robots.txt blocks.
@@ -2620,6 +2623,7 @@ declare class FirecrawlClient {
2620
2623
  crawlUrl(url: string, req?: CrawlOptions & {
2621
2624
  pollInterval?: number;
2622
2625
  timeout?: number;
2626
+ signal?: AbortSignal;
2623
2627
  }): Promise<CrawlJob>;
2624
2628
  /** @deprecated V1 compatibility alias for agent recovery. Prefer startCrawl(). */
2625
2629
  asyncCrawlUrl(url: string, req?: CrawlOptions): Promise<CrawlResponse$1>;
package/dist/index.d.ts CHANGED
@@ -2396,18 +2396,21 @@ declare class FirecrawlClient {
2396
2396
  /**
2397
2397
  * Cancel a crawl job.
2398
2398
  * @param jobId Crawl job id.
2399
- * @returns True if cancelled.
2399
+ * @returns True if cancelled. False if the crawl was already completed (the API answers 409).
2400
2400
  */
2401
2401
  cancelCrawl(jobId: string): Promise<boolean>;
2402
2402
  /**
2403
2403
  * Convenience waiter: start a crawl and poll until it finishes.
2404
2404
  * @param url Root URL to crawl.
2405
- * @param req Crawl configuration plus waiter controls (pollInterval, timeout seconds).
2405
+ * @param req Crawl configuration plus waiter controls (pollInterval, timeout seconds, signal).
2406
+ * When `signal` aborts, polling stops, the job gets a best-effort cancel request,
2407
+ * and the promise rejects with `signal.reason`.
2406
2408
  * @returns Final job snapshot.
2407
2409
  */
2408
2410
  crawl(url: string, req?: CrawlOptions & {
2409
2411
  pollInterval?: number;
2410
2412
  timeout?: number;
2413
+ signal?: AbortSignal;
2411
2414
  }): Promise<CrawlJob>;
2412
2415
  /**
2413
2416
  * Retrieve crawl errors and robots.txt blocks.
@@ -2620,6 +2623,7 @@ declare class FirecrawlClient {
2620
2623
  crawlUrl(url: string, req?: CrawlOptions & {
2621
2624
  pollInterval?: number;
2622
2625
  timeout?: number;
2626
+ signal?: AbortSignal;
2623
2627
  }): Promise<CrawlJob>;
2624
2628
  /** @deprecated V1 compatibility alias for agent recovery. Prefer startCrawl(). */
2625
2629
  asyncCrawlUrl(url: string, req?: CrawlOptions): Promise<CrawlResponse$1>;
package/dist/index.js CHANGED
@@ -1,6 +1,6 @@
1
1
  import {
2
2
  require_package
3
- } from "./chunk-JU6NENYM.js";
3
+ } from "./chunk-LKCOCWYW.js";
4
4
 
5
5
  // src/v2/utils/httpClient.ts
6
6
  import axios from "axios";
@@ -504,7 +504,7 @@ function processingContinuesDelayMs(err) {
504
504
  return Math.min(RESUME_MAX_DELAY_MS, Math.max(RESUME_MIN_DELAY_MS, seconds * 1e3));
505
505
  }
506
506
  async function scrape(http, url, options, deps = {}) {
507
- const sleep = deps.sleepImpl ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
507
+ const sleep2 = deps.sleepImpl ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
508
508
  if (!url || !url.trim()) {
509
509
  throw new Error("URL cannot be empty");
510
510
  }
@@ -537,7 +537,7 @@ async function scrape(http, url, options, deps = {}) {
537
537
  if (delayMs !== void 0 && resumes < RESUME_MAX_ATTEMPTS && resumeWaitedMs + delayMs <= RESUME_MAX_TOTAL_WAIT_MS) {
538
538
  resumes += 1;
539
539
  resumeWaitedMs += delayMs;
540
- await sleep(delayMs);
540
+ await sleep2(delayMs);
541
541
  continue;
542
542
  }
543
543
  if (err?.isAxiosError) return normalizeAxiosError(err, "scrape");
@@ -1123,22 +1123,72 @@ async function getCrawlStatus(http, jobId, pagination) {
1123
1123
  async function cancelCrawl(http, jobId) {
1124
1124
  try {
1125
1125
  const res = await http.delete(`/v2/crawl/${jobId}`);
1126
+ if (res.status === 409) return false;
1126
1127
  if (res.status !== 200) throwForBadResponse(res, "cancel crawl");
1127
1128
  return res.data?.status === "cancelled";
1128
1129
  } catch (err) {
1129
- if (err?.isAxiosError) return normalizeAxiosError(err, "cancel crawl");
1130
+ if (err?.isAxiosError) {
1131
+ if (err.response?.status === 409) return false;
1132
+ return normalizeAxiosError(err, "cancel crawl");
1133
+ }
1130
1134
  throw err;
1131
1135
  }
1132
1136
  }
1133
- async function waitForCrawlCompletion(http, jobId, pollInterval = 2, timeout) {
1137
+ function abortReason(signal) {
1138
+ if (signal.reason !== void 0) return signal.reason;
1139
+ return new DOMException("This operation was aborted", "AbortError");
1140
+ }
1141
+ function raceAbort(promise, signal) {
1142
+ if (!signal) return promise;
1143
+ if (signal.aborted) {
1144
+ promise.catch(() => {
1145
+ });
1146
+ return Promise.reject(abortReason(signal));
1147
+ }
1148
+ return new Promise((resolve, reject) => {
1149
+ const onAbort = () => {
1150
+ promise.catch(() => {
1151
+ });
1152
+ reject(abortReason(signal));
1153
+ };
1154
+ signal.addEventListener("abort", onAbort, { once: true });
1155
+ promise.then(
1156
+ (value) => {
1157
+ signal.removeEventListener("abort", onAbort);
1158
+ resolve(value);
1159
+ },
1160
+ (err) => {
1161
+ signal.removeEventListener("abort", onAbort);
1162
+ reject(err);
1163
+ }
1164
+ );
1165
+ });
1166
+ }
1167
+ function sleep(ms, signal) {
1168
+ return new Promise((resolve, reject) => {
1169
+ if (signal?.aborted) return reject(abortReason(signal));
1170
+ const onAbort = () => {
1171
+ clearTimeout(timer);
1172
+ reject(abortReason(signal));
1173
+ };
1174
+ const timer = setTimeout(() => {
1175
+ signal?.removeEventListener("abort", onAbort);
1176
+ resolve();
1177
+ }, ms);
1178
+ signal?.addEventListener("abort", onAbort, { once: true });
1179
+ });
1180
+ }
1181
+ async function waitForCrawlCompletion(http, jobId, pollInterval = 2, timeout, signal) {
1134
1182
  const start = Date.now();
1135
1183
  while (true) {
1184
+ if (signal?.aborted) throw abortReason(signal);
1136
1185
  try {
1137
- const status = await getCrawlStatus(http, jobId);
1186
+ const status = await raceAbort(getCrawlStatus(http, jobId), signal);
1138
1187
  if (["completed", "failed", "cancelled"].includes(status.status)) {
1139
1188
  return status;
1140
1189
  }
1141
1190
  } catch (err) {
1191
+ if (signal?.aborted) throw abortReason(signal);
1142
1192
  if (!isRetryableError(err)) {
1143
1193
  if (err instanceof SdkError) {
1144
1194
  const errorWithJobId = new SdkError(
@@ -1156,12 +1206,19 @@ async function waitForCrawlCompletion(http, jobId, pollInterval = 2, timeout) {
1156
1206
  if (timeout != null && Date.now() - start > timeout * 1e3) {
1157
1207
  throw new JobTimeoutError(jobId, timeout, "crawl");
1158
1208
  }
1159
- await new Promise((r) => setTimeout(r, Math.max(1e3, pollInterval * 1e3)));
1209
+ await sleep(Math.max(1e3, pollInterval * 1e3), signal);
1160
1210
  }
1161
1211
  }
1162
- async function crawl(http, request, pollInterval = 2, timeout) {
1212
+ async function crawl(http, request, pollInterval = 2, timeout, signal) {
1213
+ if (signal?.aborted) throw abortReason(signal);
1163
1214
  const started = await startCrawl(http, request);
1164
- return waitForCrawlCompletion(http, started.id, pollInterval, timeout);
1215
+ try {
1216
+ return await waitForCrawlCompletion(http, started.id, pollInterval, timeout, signal);
1217
+ } catch (err) {
1218
+ if (signal?.aborted) cancelCrawl(http, started.id).catch(() => {
1219
+ });
1220
+ throw err;
1221
+ }
1165
1222
  }
1166
1223
  async function getCrawlErrors(http, crawlId) {
1167
1224
  try {
@@ -2376,7 +2433,7 @@ var FirecrawlClient = class {
2376
2433
  /**
2377
2434
  * Cancel a crawl job.
2378
2435
  * @param jobId Crawl job id.
2379
- * @returns True if cancelled.
2436
+ * @returns True if cancelled. False if the crawl was already completed (the API answers 409).
2380
2437
  */
2381
2438
  async cancelCrawl(jobId) {
2382
2439
  return cancelCrawl(this.http, jobId);
@@ -2384,11 +2441,14 @@ var FirecrawlClient = class {
2384
2441
  /**
2385
2442
  * Convenience waiter: start a crawl and poll until it finishes.
2386
2443
  * @param url Root URL to crawl.
2387
- * @param req Crawl configuration plus waiter controls (pollInterval, timeout seconds).
2444
+ * @param req Crawl configuration plus waiter controls (pollInterval, timeout seconds, signal).
2445
+ * When `signal` aborts, polling stops, the job gets a best-effort cancel request,
2446
+ * and the promise rejects with `signal.reason`.
2388
2447
  * @returns Final job snapshot.
2389
2448
  */
2390
2449
  async crawl(url, req = {}) {
2391
- return crawl(this.http, { url, ...req }, req.pollInterval, req.timeout);
2450
+ const { pollInterval, timeout, signal, ...options } = req;
2451
+ return crawl(this.http, { url, ...options }, pollInterval, timeout, signal);
2392
2452
  }
2393
2453
  /**
2394
2454
  * Retrieve crawl errors and robots.txt blocks.
@@ -2740,7 +2800,7 @@ var FirecrawlApp = class {
2740
2800
  if (typeof process !== "undefined" && process.env && process.env.npm_package_version) {
2741
2801
  return process.env.npm_package_version;
2742
2802
  }
2743
- const packageJson = await import("./package-FEZU4ANT.js");
2803
+ const packageJson = await import("./package-C6PK3CK7.js");
2744
2804
  return packageJson.default.version;
2745
2805
  } catch (error) {
2746
2806
  const isTest = typeof process !== "undefined" && (process.env.JEST_WORKER_ID != null || false);
@@ -1,4 +1,4 @@
1
1
  import {
2
2
  require_package
3
- } from "./chunk-JU6NENYM.js";
3
+ } from "./chunk-LKCOCWYW.js";
4
4
  export default require_package();
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "firecrawl",
3
- "version": "4.44.1",
3
+ "version": "4.45.0",
4
4
  "description": "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
5
5
  "main": "dist/index.js",
6
6
  "types": "dist/index.d.ts",
@@ -0,0 +1,189 @@
1
+ import { describe, test, expect } from "@jest/globals";
2
+ import axios, { type AxiosAdapter } from "axios";
3
+ import { FirecrawlClient } from "../../../v2/client";
4
+ import { HttpClient } from "../../../v2/utils/httpClient";
5
+ import { waitForCrawlCompletion } from "../../../v2/methods/crawl";
6
+ import { SdkError } from "../../../v2/types";
7
+
8
+ const API_URL = "https://api.firecrawl.dev";
9
+ const JOB_ID = "job-123";
10
+
11
+ type Sent = { method: string; path: string };
12
+ type Reply = { status: number; data: unknown };
13
+
14
+ function withAdapter<T extends { defaults: any }>(instance: T, route: (req: Sent) => Reply): Sent[] {
15
+ const sent: Sent[] = [];
16
+ const adapter: AxiosAdapter = async (config) => {
17
+ const req = {
18
+ method: (config.method || "get").toLowerCase(),
19
+ path: new URL(axios.getUri(config)).pathname,
20
+ };
21
+ sent.push(req);
22
+ const { status, data } = route(req);
23
+ const response = { data, status, statusText: String(status), headers: {}, config };
24
+ const validate = config.validateStatus;
25
+ if (validate && !validate(status)) {
26
+ throw new axios.AxiosError(`Request failed with status code ${status}`, undefined, config, null, response);
27
+ }
28
+ return response;
29
+ };
30
+ instance.defaults.adapter = adapter;
31
+ return sent;
32
+ }
33
+
34
+ function makeClient(route: (req: Sent) => Reply) {
35
+ const client = new FirecrawlClient({ apiKey: "fc-test", apiUrl: API_URL });
36
+ const sent = withAdapter((client as any).http.instance, route);
37
+ return { client, sent };
38
+ }
39
+
40
+ // The cancel runs in the background, so let its request reach the adapter.
41
+ const flush = () => new Promise((r) => setTimeout(r, 0));
42
+
43
+ const running: Reply = { status: 200, data: { success: true, status: "scraping", completed: 0, total: 1, data: [] } };
44
+ const started: Reply = { status: 200, data: { success: true, id: JOB_ID, url: "https://example.com" } };
45
+
46
+ describe("v2 cancelCrawl", () => {
47
+ test("returns false when the crawl is already completed (409)", async () => {
48
+ const { client, sent } = makeClient(() => ({ status: 409, data: { error: "Crawl is already completed" } }));
49
+ await expect(client.cancelCrawl(JOB_ID)).resolves.toBe(false);
50
+ expect(sent).toEqual([{ method: "delete", path: `/v2/crawl/${JOB_ID}` }]);
51
+ });
52
+
53
+ test("returns false on 409 when the transport does not throw for 4xx", async () => {
54
+ const { client } = makeClient(() => ({ status: 409, data: { error: "Crawl is already completed" } }));
55
+ (client as any).http.instance.defaults.validateStatus = () => true;
56
+ await expect(client.cancelCrawl(JOB_ID)).resolves.toBe(false);
57
+ });
58
+
59
+ test("returns true when the crawl is cancelled", async () => {
60
+ const { client } = makeClient(() => ({ status: 200, data: { status: "cancelled" } }));
61
+ await expect(client.cancelCrawl(JOB_ID)).resolves.toBe(true);
62
+ });
63
+
64
+ test("still throws for other errors", async () => {
65
+ const { client } = makeClient(() => ({ status: 404, data: { error: "Job not found" } }));
66
+ const err = await client.cancelCrawl(JOB_ID).catch((e) => e);
67
+ expect(err).toBeInstanceOf(SdkError);
68
+ expect(err.status).toBe(404);
69
+ });
70
+ });
71
+
72
+ describe("v2 crawl with an AbortSignal", () => {
73
+ test("abort during polling stops polling, cancels the job, and rejects with the abort reason", async () => {
74
+ const controller = new AbortController();
75
+ let polls = 0;
76
+ const { client, sent } = makeClient((req) => {
77
+ if (req.method === "post") return started;
78
+ if (req.method === "delete") return { status: 200, data: { status: "cancelled" } };
79
+ polls += 1;
80
+ if (polls === 1) setTimeout(() => controller.abort(), 10);
81
+ return running;
82
+ });
83
+
84
+ const t0 = Date.now();
85
+ const err = await client.crawl("https://example.com", { pollInterval: 30, signal: controller.signal }).catch((e) => e);
86
+
87
+ expect(Date.now() - t0).toBeLessThan(5000);
88
+ expect(err).toBe(controller.signal.reason);
89
+ expect((err as Error).name).toBe("AbortError");
90
+ await flush();
91
+ expect(sent.map((r) => r.method)).toEqual(["post", "get", "delete"]);
92
+ expect(sent[2]!.path).toBe(`/v2/crawl/${JOB_ID}`);
93
+ });
94
+
95
+ test("rejects with a custom abort reason and ignores a failing cancel", async () => {
96
+ const controller = new AbortController();
97
+ const reason = new Error("caller gave up");
98
+ const { client, sent } = makeClient((req) => {
99
+ if (req.method === "post") return started;
100
+ if (req.method === "delete") return { status: 500, data: { error: "boom" } };
101
+ setTimeout(() => controller.abort(reason), 10);
102
+ return running;
103
+ });
104
+
105
+ await expect(client.crawl("https://example.com", { pollInterval: 30, signal: controller.signal })).rejects.toBe(reason);
106
+ await flush();
107
+ expect(sent.some((r) => r.method === "delete")).toBe(true);
108
+ });
109
+
110
+ test("a hanging cancel does not delay the rejection", async () => {
111
+ const controller = new AbortController();
112
+ const client = new FirecrawlClient({ apiKey: "fc-test", apiUrl: API_URL });
113
+ const methods: string[] = [];
114
+ (client as any).http.instance.defaults.adapter = (async (config) => {
115
+ methods.push(config.method!);
116
+ if (config.method === "delete") return new Promise(() => {});
117
+ if (config.method === "get") setTimeout(() => controller.abort(), 10);
118
+ const data = config.method === "post" ? started.data : running.data;
119
+ return { data, status: 200, statusText: "OK", headers: {}, config };
120
+ }) as AxiosAdapter;
121
+
122
+ // Read the reason after the abort: before it, signal.reason is undefined.
123
+ const err = await client.crawl("https://example.com", { pollInterval: 30, signal: controller.signal }).catch((e) => e);
124
+ expect(err).toBe(controller.signal.reason);
125
+ expect((err as Error).name).toBe("AbortError");
126
+ await flush();
127
+ expect(methods).toEqual(["post", "get", "delete"]);
128
+ });
129
+
130
+ test("abort while the start request is in flight cancels the job once the id arrives", async () => {
131
+ const controller = new AbortController();
132
+ const { client, sent } = makeClient((req) => {
133
+ if (req.method === "post") {
134
+ controller.abort();
135
+ return started;
136
+ }
137
+ if (req.method === "delete") return { status: 200, data: { status: "cancelled" } };
138
+ return running;
139
+ });
140
+
141
+ const err = await client.crawl("https://example.com", { signal: controller.signal }).catch((e) => e);
142
+ expect(err).toBe(controller.signal.reason);
143
+ expect((err as Error).name).toBe("AbortError");
144
+ await flush();
145
+ expect(sent.map((r) => r.method)).toEqual(["post", "delete"]);
146
+ });
147
+
148
+ test("an already aborted signal sends no request", async () => {
149
+ const controller = new AbortController();
150
+ controller.abort();
151
+ const { client, sent } = makeClient(() => started);
152
+
153
+ await expect(client.crawl("https://example.com", { signal: controller.signal })).rejects.toBe(controller.signal.reason);
154
+ expect(sent).toHaveLength(0);
155
+ });
156
+
157
+ test("the signal is not sent in the crawl request body", async () => {
158
+ const controller = new AbortController();
159
+ const client = new FirecrawlClient({ apiKey: "fc-test", apiUrl: API_URL });
160
+ const bodies: unknown[] = [];
161
+ (client as any).http.instance.defaults.adapter = (async (config) => {
162
+ if (config.method === "post") bodies.push(JSON.parse(config.data as string));
163
+ const data = config.method === "post" ? started.data : { success: true, status: "completed", completed: 1, total: 1, data: [] };
164
+ return { data, status: 200, statusText: "OK", headers: {}, config };
165
+ }) as AxiosAdapter;
166
+
167
+ const job = await client.crawl("https://example.com", { limit: 1, signal: controller.signal });
168
+ expect(job.status).toBe("completed");
169
+ expect(bodies[0]).toMatchObject({ url: "https://example.com", limit: 1 });
170
+ expect(Object.keys(bodies[0] as object)).not.toContain("signal");
171
+ });
172
+ });
173
+
174
+ describe("v2 waitForCrawlCompletion with an AbortSignal", () => {
175
+ test("stops polling and rejects without cancelling the job", async () => {
176
+ const http = new HttpClient({ apiKey: "fc-test", apiUrl: API_URL });
177
+ const controller = new AbortController();
178
+ const sent = withAdapter((http as any).instance, () => {
179
+ setTimeout(() => controller.abort(), 10);
180
+ return running;
181
+ });
182
+
183
+ const err = await waitForCrawlCompletion(http, JOB_ID, 30, undefined, controller.signal).catch((e) => e);
184
+ expect(err).toBe(controller.signal.reason);
185
+ expect((err as Error).name).toBe("AbortError");
186
+ await new Promise((r) => setTimeout(r, 50));
187
+ expect(sent.map((r) => r.method)).toEqual(["get"]);
188
+ });
189
+ });
package/src/v2/client.ts CHANGED
@@ -394,7 +394,7 @@ export class FirecrawlClient {
394
394
  /**
395
395
  * Cancel a crawl job.
396
396
  * @param jobId Crawl job id.
397
- * @returns True if cancelled.
397
+ * @returns True if cancelled. False if the crawl was already completed (the API answers 409).
398
398
  */
399
399
  async cancelCrawl(jobId: string): Promise<boolean> {
400
400
  return cancelCrawl(this.http, jobId);
@@ -402,11 +402,14 @@ export class FirecrawlClient {
402
402
  /**
403
403
  * Convenience waiter: start a crawl and poll until it finishes.
404
404
  * @param url Root URL to crawl.
405
- * @param req Crawl configuration plus waiter controls (pollInterval, timeout seconds).
405
+ * @param req Crawl configuration plus waiter controls (pollInterval, timeout seconds, signal).
406
+ * When `signal` aborts, polling stops, the job gets a best-effort cancel request,
407
+ * and the promise rejects with `signal.reason`.
406
408
  * @returns Final job snapshot.
407
409
  */
408
- async crawl(url: string, req: CrawlOptions & { pollInterval?: number; timeout?: number } = {}): Promise<CrawlJob> {
409
- return crawlWaiter(this.http, { url, ...req }, req.pollInterval, req.timeout);
410
+ async crawl(url: string, req: CrawlOptions & { pollInterval?: number; timeout?: number; signal?: AbortSignal } = {}): Promise<CrawlJob> {
411
+ const { pollInterval, timeout, signal, ...options } = req;
412
+ return crawlWaiter(this.http, { url, ...options }, pollInterval, timeout, signal);
410
413
  }
411
414
  /**
412
415
  * Retrieve crawl errors and robots.txt blocks.
@@ -724,7 +727,7 @@ export class FirecrawlClient {
724
727
  }
725
728
 
726
729
  /** @deprecated V1 compatibility alias for agent recovery. Prefer crawl(). */
727
- async crawlUrl(url: string, req: CrawlOptions & { pollInterval?: number; timeout?: number } = {}): Promise<CrawlJob> {
730
+ async crawlUrl(url: string, req: CrawlOptions & { pollInterval?: number; timeout?: number; signal?: AbortSignal } = {}): Promise<CrawlJob> {
728
731
  return this.crawl(url, req);
729
732
  }
730
733
 
@@ -107,28 +107,94 @@ export async function getCrawlStatus(
107
107
  }
108
108
  }
109
109
 
110
+ /**
111
+ * Cancel a crawl job. Resolves `false` when the API answers 409 because the
112
+ * crawl already finished, so there is nothing left to cancel.
113
+ */
110
114
  export async function cancelCrawl(http: HttpClient, jobId: string): Promise<boolean> {
111
115
  try {
112
116
  const res = await http.delete<{ status: string }>(`/v2/crawl/${jobId}`);
117
+ if (res.status === 409) return false;
113
118
  if (res.status !== 200) throwForBadResponse(res, "cancel crawl");
114
119
  return res.data?.status === "cancelled";
115
120
  } catch (err: any) {
116
- if (err?.isAxiosError) return normalizeAxiosError(err, "cancel crawl");
121
+ if (err?.isAxiosError) {
122
+ if (err.response?.status === 409) return false;
123
+ return normalizeAxiosError(err, "cancel crawl");
124
+ }
117
125
  throw err;
118
126
  }
119
127
  }
120
128
 
121
- export async function waitForCrawlCompletion(http: HttpClient, jobId: string, pollInterval = 2, timeout?: number): Promise<CrawlJob> {
129
+ function abortReason(signal: AbortSignal): unknown {
130
+ if (signal.reason !== undefined) return signal.reason;
131
+ return new DOMException("This operation was aborted", "AbortError");
132
+ }
133
+
134
+ /** Reject with the abort reason as soon as `signal` aborts. */
135
+ function raceAbort<T>(promise: Promise<T>, signal?: AbortSignal): Promise<T> {
136
+ if (!signal) return promise;
137
+ if (signal.aborted) {
138
+ promise.catch(() => {});
139
+ return Promise.reject(abortReason(signal));
140
+ }
141
+ return new Promise<T>((resolve, reject) => {
142
+ const onAbort = () => {
143
+ promise.catch(() => {});
144
+ reject(abortReason(signal));
145
+ };
146
+ signal.addEventListener("abort", onAbort, { once: true });
147
+ promise.then(
148
+ (value) => {
149
+ signal.removeEventListener("abort", onAbort);
150
+ resolve(value);
151
+ },
152
+ (err) => {
153
+ signal.removeEventListener("abort", onAbort);
154
+ reject(err);
155
+ },
156
+ );
157
+ });
158
+ }
159
+
160
+ function sleep(ms: number, signal?: AbortSignal): Promise<void> {
161
+ return new Promise<void>((resolve, reject) => {
162
+ if (signal?.aborted) return reject(abortReason(signal));
163
+ const onAbort = () => {
164
+ clearTimeout(timer);
165
+ reject(abortReason(signal!));
166
+ };
167
+ const timer = setTimeout(() => {
168
+ signal?.removeEventListener("abort", onAbort);
169
+ resolve();
170
+ }, ms);
171
+ signal?.addEventListener("abort", onAbort, { once: true });
172
+ });
173
+ }
174
+
175
+ /**
176
+ * Poll a crawl until it reaches a terminal status. When `signal` aborts, polling
177
+ * stops and the promise rejects with the abort reason. The job is not cancelled.
178
+ */
179
+ export async function waitForCrawlCompletion(
180
+ http: HttpClient,
181
+ jobId: string,
182
+ pollInterval = 2,
183
+ timeout?: number,
184
+ signal?: AbortSignal,
185
+ ): Promise<CrawlJob> {
122
186
  const start = Date.now();
123
187
 
124
188
  while (true) {
189
+ if (signal?.aborted) throw abortReason(signal);
125
190
  try {
126
- const status = await getCrawlStatus(http, jobId);
191
+ const status = await raceAbort(getCrawlStatus(http, jobId), signal);
127
192
 
128
193
  if (["completed", "failed", "cancelled"].includes(status.status)) {
129
194
  return status;
130
195
  }
131
196
  } catch (err: any) {
197
+ if (signal?.aborted) throw abortReason(signal);
132
198
  // Don't retry on permanent errors (4xx) - re-throw immediately with jobId context
133
199
  if (!isRetryableError(err)) {
134
200
  // Create new error with jobId for better debugging (non-retryable errors like 404)
@@ -151,13 +217,31 @@ export async function waitForCrawlCompletion(http: HttpClient, jobId: string, po
151
217
  throw new JobTimeoutError(jobId, timeout, 'crawl');
152
218
  }
153
219
 
154
- await new Promise((r) => setTimeout(r, Math.max(1000, pollInterval * 1000)));
220
+ await sleep(Math.max(1000, pollInterval * 1000), signal);
155
221
  }
156
222
  }
157
223
 
158
- export async function crawl(http: HttpClient, request: CrawlRequest, pollInterval = 2, timeout?: number): Promise<CrawlJob> {
224
+ /**
225
+ * Start a crawl and wait for it to finish. When `signal` aborts, polling stops,
226
+ * the job gets a best-effort cancel request, and the promise rejects with the
227
+ * abort reason at once, without waiting for the cancel. A signal that is already
228
+ * aborted rejects before any request.
229
+ */
230
+ export async function crawl(
231
+ http: HttpClient,
232
+ request: CrawlRequest,
233
+ pollInterval = 2,
234
+ timeout?: number,
235
+ signal?: AbortSignal,
236
+ ): Promise<CrawlJob> {
237
+ if (signal?.aborted) throw abortReason(signal);
159
238
  const started = await startCrawl(http, request);
160
- return waitForCrawlCompletion(http, started.id, pollInterval, timeout);
239
+ try {
240
+ return await waitForCrawlCompletion(http, started.id, pollInterval, timeout, signal);
241
+ } catch (err) {
242
+ if (signal?.aborted) cancelCrawl(http, started.id).catch(() => {});
243
+ throw err;
244
+ }
161
245
  }
162
246
 
163
247
  export async function getCrawlErrors(http: HttpClient, crawlId: string): Promise<CrawlErrorsResponse> {