@mendable/firecrawl-js 4.42.2 → 4.42.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -12,7 +12,7 @@ var require_package = __commonJS({
12
12
  "package.json"(exports, module) {
13
13
  module.exports = {
14
14
  name: "@mendable/firecrawl-js",
15
- version: "4.42.2",
15
+ version: "4.42.4",
16
16
  description: "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
17
17
  main: "dist/index.js",
18
18
  types: "dist/index.d.ts",
@@ -29,7 +29,7 @@ var require_package = __commonJS({
29
29
  "build-and-publish": "pnpm run build && pnpm publish --access public",
30
30
  "publish-beta": "pnpm run build && pnpm publish --access public --tag beta",
31
31
  test: "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/e2e/v2/*.test.ts --detectOpenHandles",
32
- "test:unit": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/unit/v2/*.test.ts"
32
+ "test:unit": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/unit"
33
33
  },
34
34
  repository: {
35
35
  type: "git",
package/dist/index.cjs CHANGED
@@ -39,7 +39,7 @@ var require_package = __commonJS({
39
39
  "package.json"(exports2, module2) {
40
40
  module2.exports = {
41
41
  name: "@mendable/firecrawl-js",
42
- version: "4.42.2",
42
+ version: "4.42.4",
43
43
  description: "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
44
44
  main: "dist/index.js",
45
45
  types: "dist/index.d.ts",
@@ -56,7 +56,7 @@ var require_package = __commonJS({
56
56
  "build-and-publish": "pnpm run build && pnpm publish --access public",
57
57
  "publish-beta": "pnpm run build && pnpm publish --access public --tag beta",
58
58
  test: "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/e2e/v2/*.test.ts --detectOpenHandles",
59
- "test:unit": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/unit/v2/*.test.ts"
59
+ "test:unit": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/unit"
60
60
  },
61
61
  repository: {
62
62
  type: "git",
@@ -2875,7 +2875,7 @@ var FirecrawlApp = class {
2875
2875
  "Content-Type": "application/json",
2876
2876
  Authorization: `Bearer ${this.apiKey}`
2877
2877
  };
2878
- let jsonData = { url, ...params, origin: typeof params.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` };
2878
+ let jsonData = { url, ...params, origin: typeof params?.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` };
2879
2879
  if (jsonData?.extract?.schema) {
2880
2880
  jsonData = {
2881
2881
  ...jsonData,
@@ -2939,7 +2939,7 @@ var FirecrawlApp = class {
2939
2939
  lang: params?.lang ?? "en",
2940
2940
  country: params?.country ?? "us",
2941
2941
  location: params?.location,
2942
- origin: typeof params.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}`,
2942
+ origin: typeof params?.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}`,
2943
2943
  timeout: params?.timeout ?? 6e4,
2944
2944
  scrapeOptions: params?.scrapeOptions ?? { formats: [] }
2945
2945
  };
@@ -2994,7 +2994,7 @@ var FirecrawlApp = class {
2994
2994
  */
2995
2995
  async crawlUrl(url, params, pollInterval = 2, idempotencyKey) {
2996
2996
  const headers = this.prepareHeaders(idempotencyKey);
2997
- let jsonData = { url, ...params, origin: typeof params.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` };
2997
+ let jsonData = { url, ...params, origin: typeof params?.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` };
2998
2998
  try {
2999
2999
  const response = await this.postRequest(
3000
3000
  this.apiUrl + `/v1/crawl`,
@@ -3018,7 +3018,7 @@ var FirecrawlApp = class {
3018
3018
  }
3019
3019
  async asyncCrawlUrl(url, params, idempotencyKey) {
3020
3020
  const headers = this.prepareHeaders(idempotencyKey);
3021
- let jsonData = { url, ...params, origin: typeof params.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` };
3021
+ let jsonData = { url, ...params, origin: typeof params?.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` };
3022
3022
  try {
3023
3023
  const response = await this.postRequest(
3024
3024
  this.apiUrl + `/v1/crawl`,
@@ -3177,7 +3177,7 @@ var FirecrawlApp = class {
3177
3177
  */
3178
3178
  async mapUrl(url, params) {
3179
3179
  const headers = this.prepareHeaders();
3180
- let jsonData = { url, ...params, origin: typeof params.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` };
3180
+ let jsonData = { url, ...params, origin: typeof params?.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` };
3181
3181
  try {
3182
3182
  const response = await this.postRequest(
3183
3183
  this.apiUrl + `/v1/map`,
@@ -3206,7 +3206,7 @@ var FirecrawlApp = class {
3206
3206
  */
3207
3207
  async batchScrapeUrls(urls, params, pollInterval = 2, idempotencyKey, webhook, ignoreInvalidURLs, maxConcurrency) {
3208
3208
  const headers = this.prepareHeaders(idempotencyKey);
3209
- let jsonData = { urls, webhook, ignoreInvalidURLs, maxConcurrency, ...params, origin: typeof params.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` };
3209
+ let jsonData = { urls, webhook, ignoreInvalidURLs, maxConcurrency, ...params, origin: typeof params?.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` };
3210
3210
  if (jsonData?.extract?.schema) {
3211
3211
  jsonData = {
3212
3212
  ...jsonData,
@@ -3233,7 +3233,7 @@ var FirecrawlApp = class {
3233
3233
  );
3234
3234
  if (response.status === 200) {
3235
3235
  const id = response.data.id;
3236
- return this.monitorJobStatus(id, headers, pollInterval);
3236
+ return this.monitorJobStatus(id, headers, pollInterval, "batch");
3237
3237
  } else {
3238
3238
  this.handleError(response, "start batch scrape job");
3239
3239
  }
@@ -3248,7 +3248,7 @@ var FirecrawlApp = class {
3248
3248
  }
3249
3249
  async asyncBatchScrapeUrls(urls, params, idempotencyKey, webhook, ignoreInvalidURLs) {
3250
3250
  const headers = this.prepareHeaders(idempotencyKey);
3251
- let jsonData = { urls, webhook, ignoreInvalidURLs, ...params, origin: typeof params.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` };
3251
+ let jsonData = { urls, webhook, ignoreInvalidURLs, ...params, origin: typeof params?.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` };
3252
3252
  try {
3253
3253
  const response = await this.postRequest(
3254
3254
  this.apiUrl + `/v1/batch/scrape`,
@@ -3392,7 +3392,7 @@ var FirecrawlApp = class {
3392
3392
  try {
3393
3393
  const response = await this.postRequest(
3394
3394
  this.apiUrl + `/v1/extract`,
3395
- { ...jsonData, schema: jsonSchema, origin: typeof params.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` },
3395
+ { ...jsonData, schema: jsonSchema, origin: typeof params?.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` },
3396
3396
  headers
3397
3397
  );
3398
3398
  if (response.status === 200) {
@@ -3445,7 +3445,7 @@ var FirecrawlApp = class {
3445
3445
  try {
3446
3446
  const response = await this.postRequest(
3447
3447
  this.apiUrl + `/v1/extract`,
3448
- { ...jsonData, schema: jsonSchema, origin: typeof params.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` },
3448
+ { ...jsonData, schema: jsonSchema, origin: typeof params?.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` },
3449
3449
  headers
3450
3450
  );
3451
3451
  if (response.status === 200) {
@@ -3537,21 +3537,23 @@ var FirecrawlApp = class {
3537
3537
  }
3538
3538
  }
3539
3539
  /**
3540
- * Monitors the status of a crawl job until completion or failure.
3541
- * @param id - The ID of the crawl operation.
3540
+ * Monitors the status of a crawl or batch scrape job until completion or failure.
3541
+ * @param id - The ID of the crawl or batch scrape operation.
3542
3542
  * @param headers - The headers for the request.
3543
3543
  * @param checkInterval - Interval in seconds for job status checks.
3544
- * @param checkUrl - Optional URL to check the status (used for v1 API)
3544
+ * @param jobType - Which status endpoint to poll. Defaults to `"crawl"`.
3545
3545
  * @returns The final job status or data.
3546
3546
  */
3547
- async monitorJobStatus(id, headers, checkInterval) {
3547
+ async monitorJobStatus(id, headers, checkInterval, jobType = "crawl") {
3548
+ const statusPath = jobType === "batch" ? "batch/scrape" : "crawl";
3549
+ const jobLabel = jobType === "batch" ? "Batch scrape" : "Crawl";
3548
3550
  let failedTries = 0;
3549
3551
  let networkRetries = 0;
3550
3552
  const maxNetworkRetries = 3;
3551
3553
  while (true) {
3552
3554
  try {
3553
3555
  let statusResponse = await this.getRequest(
3554
- `${this.apiUrl}/v1/crawl/${id}`,
3556
+ `${this.apiUrl}/v1/${statusPath}/${id}`,
3555
3557
  headers
3556
3558
  );
3557
3559
  if (statusResponse.status === 200) {
@@ -3572,7 +3574,7 @@ var FirecrawlApp = class {
3572
3574
  statusData.data = data;
3573
3575
  return statusData;
3574
3576
  } else {
3575
- throw new FirecrawlError("Crawl job completed but no data was returned", 500);
3577
+ throw new FirecrawlError(`${jobLabel} job completed but no data was returned`, 500);
3576
3578
  }
3577
3579
  } else if (["active", "paused", "pending", "queued", "waiting", "scraping"].includes(statusData.status)) {
3578
3580
  checkInterval = Math.max(checkInterval, 2);
@@ -3581,14 +3583,14 @@ var FirecrawlApp = class {
3581
3583
  );
3582
3584
  } else {
3583
3585
  throw new FirecrawlError(
3584
- `Crawl job failed or was stopped. Status: ${statusData.status}`,
3586
+ `${jobLabel} job failed or was stopped. Status: ${statusData.status}`,
3585
3587
  500
3586
3588
  );
3587
3589
  }
3588
3590
  } else {
3589
3591
  failedTries++;
3590
3592
  if (failedTries >= 3) {
3591
- this.handleError(statusResponse, "check crawl status");
3593
+ this.handleError(statusResponse, `check ${jobLabel.toLowerCase()} status`);
3592
3594
  }
3593
3595
  }
3594
3596
  } catch (error) {
@@ -3725,7 +3727,7 @@ var FirecrawlApp = class {
3725
3727
  */
3726
3728
  async asyncDeepResearch(query, params) {
3727
3729
  const headers = this.prepareHeaders();
3728
- let jsonData = { query, ...params, origin: typeof params.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` };
3730
+ let jsonData = { query, ...params, origin: typeof params?.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` };
3729
3731
  if (jsonData?.jsonOptions?.schema) {
3730
3732
  jsonData = {
3731
3733
  ...jsonData,
@@ -3844,7 +3846,7 @@ var FirecrawlApp = class {
3844
3846
  async __asyncDeepResearch(topic, params) {
3845
3847
  const headers = this.prepareHeaders();
3846
3848
  try {
3847
- let jsonData = { topic, ...params, origin: typeof params.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` };
3849
+ let jsonData = { topic, ...params, origin: typeof params?.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` };
3848
3850
  const response = await this.postRequest(
3849
3851
  `${this.apiUrl}/v1/deep-research`,
3850
3852
  jsonData,
@@ -3944,7 +3946,7 @@ var FirecrawlApp = class {
3944
3946
  */
3945
3947
  async asyncGenerateLLMsText(url, params) {
3946
3948
  const headers = this.prepareHeaders();
3947
- let jsonData = { url, ...params, origin: typeof params.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` };
3949
+ let jsonData = { url, ...params, origin: typeof params?.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` };
3948
3950
  try {
3949
3951
  const response = await this.postRequest(
3950
3952
  `${this.apiUrl}/v1/llmstxt`,
package/dist/index.d.cts CHANGED
@@ -3388,14 +3388,14 @@ declare class FirecrawlApp {
3388
3388
  */
3389
3389
  deleteRequest(url: string, headers: AxiosRequestHeaders): Promise<AxiosResponse>;
3390
3390
  /**
3391
- * Monitors the status of a crawl job until completion or failure.
3392
- * @param id - The ID of the crawl operation.
3391
+ * Monitors the status of a crawl or batch scrape job until completion or failure.
3392
+ * @param id - The ID of the crawl or batch scrape operation.
3393
3393
  * @param headers - The headers for the request.
3394
3394
  * @param checkInterval - Interval in seconds for job status checks.
3395
- * @param checkUrl - Optional URL to check the status (used for v1 API)
3395
+ * @param jobType - Which status endpoint to poll. Defaults to `"crawl"`.
3396
3396
  * @returns The final job status or data.
3397
3397
  */
3398
- monitorJobStatus(id: string, headers: AxiosRequestHeaders, checkInterval: number): Promise<CrawlStatusResponse | ErrorResponse>;
3398
+ monitorJobStatus(id: string, headers: AxiosRequestHeaders, checkInterval: number, jobType?: "crawl" | "batch"): Promise<CrawlStatusResponse | ErrorResponse>;
3399
3399
  /**
3400
3400
  * Determines if an error is retryable (transient network error)
3401
3401
  * @param error - The error to check
package/dist/index.d.ts CHANGED
@@ -3388,14 +3388,14 @@ declare class FirecrawlApp {
3388
3388
  */
3389
3389
  deleteRequest(url: string, headers: AxiosRequestHeaders): Promise<AxiosResponse>;
3390
3390
  /**
3391
- * Monitors the status of a crawl job until completion or failure.
3392
- * @param id - The ID of the crawl operation.
3391
+ * Monitors the status of a crawl or batch scrape job until completion or failure.
3392
+ * @param id - The ID of the crawl or batch scrape operation.
3393
3393
  * @param headers - The headers for the request.
3394
3394
  * @param checkInterval - Interval in seconds for job status checks.
3395
- * @param checkUrl - Optional URL to check the status (used for v1 API)
3395
+ * @param jobType - Which status endpoint to poll. Defaults to `"crawl"`.
3396
3396
  * @returns The final job status or data.
3397
3397
  */
3398
- monitorJobStatus(id: string, headers: AxiosRequestHeaders, checkInterval: number): Promise<CrawlStatusResponse | ErrorResponse>;
3398
+ monitorJobStatus(id: string, headers: AxiosRequestHeaders, checkInterval: number, jobType?: "crawl" | "batch"): Promise<CrawlStatusResponse | ErrorResponse>;
3399
3399
  /**
3400
3400
  * Determines if an error is retryable (transient network error)
3401
3401
  * @param error - The error to check
package/dist/index.js CHANGED
@@ -1,6 +1,6 @@
1
1
  import {
2
2
  require_package
3
- } from "./chunk-XZ5XI5DK.js";
3
+ } from "./chunk-TALYWLKJ.js";
4
4
 
5
5
  // src/v2/utils/httpClient.ts
6
6
  import axios from "axios";
@@ -2706,7 +2706,7 @@ var FirecrawlApp = class {
2706
2706
  if (typeof process !== "undefined" && process.env && process.env.npm_package_version) {
2707
2707
  return process.env.npm_package_version;
2708
2708
  }
2709
- const packageJson = await import("./package-JVVNUZPT.js");
2709
+ const packageJson = await import("./package-NTEZDZT4.js");
2710
2710
  return packageJson.default.version;
2711
2711
  } catch (error) {
2712
2712
  const isTest = typeof process !== "undefined" && (process.env.JEST_WORKER_ID != null || false);
@@ -2743,7 +2743,7 @@ var FirecrawlApp = class {
2743
2743
  "Content-Type": "application/json",
2744
2744
  Authorization: `Bearer ${this.apiKey}`
2745
2745
  };
2746
- let jsonData = { url, ...params, origin: typeof params.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` };
2746
+ let jsonData = { url, ...params, origin: typeof params?.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` };
2747
2747
  if (jsonData?.extract?.schema) {
2748
2748
  jsonData = {
2749
2749
  ...jsonData,
@@ -2807,7 +2807,7 @@ var FirecrawlApp = class {
2807
2807
  lang: params?.lang ?? "en",
2808
2808
  country: params?.country ?? "us",
2809
2809
  location: params?.location,
2810
- origin: typeof params.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}`,
2810
+ origin: typeof params?.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}`,
2811
2811
  timeout: params?.timeout ?? 6e4,
2812
2812
  scrapeOptions: params?.scrapeOptions ?? { formats: [] }
2813
2813
  };
@@ -2862,7 +2862,7 @@ var FirecrawlApp = class {
2862
2862
  */
2863
2863
  async crawlUrl(url, params, pollInterval = 2, idempotencyKey) {
2864
2864
  const headers = this.prepareHeaders(idempotencyKey);
2865
- let jsonData = { url, ...params, origin: typeof params.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` };
2865
+ let jsonData = { url, ...params, origin: typeof params?.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` };
2866
2866
  try {
2867
2867
  const response = await this.postRequest(
2868
2868
  this.apiUrl + `/v1/crawl`,
@@ -2886,7 +2886,7 @@ var FirecrawlApp = class {
2886
2886
  }
2887
2887
  async asyncCrawlUrl(url, params, idempotencyKey) {
2888
2888
  const headers = this.prepareHeaders(idempotencyKey);
2889
- let jsonData = { url, ...params, origin: typeof params.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` };
2889
+ let jsonData = { url, ...params, origin: typeof params?.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` };
2890
2890
  try {
2891
2891
  const response = await this.postRequest(
2892
2892
  this.apiUrl + `/v1/crawl`,
@@ -3045,7 +3045,7 @@ var FirecrawlApp = class {
3045
3045
  */
3046
3046
  async mapUrl(url, params) {
3047
3047
  const headers = this.prepareHeaders();
3048
- let jsonData = { url, ...params, origin: typeof params.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` };
3048
+ let jsonData = { url, ...params, origin: typeof params?.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` };
3049
3049
  try {
3050
3050
  const response = await this.postRequest(
3051
3051
  this.apiUrl + `/v1/map`,
@@ -3074,7 +3074,7 @@ var FirecrawlApp = class {
3074
3074
  */
3075
3075
  async batchScrapeUrls(urls, params, pollInterval = 2, idempotencyKey, webhook, ignoreInvalidURLs, maxConcurrency) {
3076
3076
  const headers = this.prepareHeaders(idempotencyKey);
3077
- let jsonData = { urls, webhook, ignoreInvalidURLs, maxConcurrency, ...params, origin: typeof params.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` };
3077
+ let jsonData = { urls, webhook, ignoreInvalidURLs, maxConcurrency, ...params, origin: typeof params?.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` };
3078
3078
  if (jsonData?.extract?.schema) {
3079
3079
  jsonData = {
3080
3080
  ...jsonData,
@@ -3101,7 +3101,7 @@ var FirecrawlApp = class {
3101
3101
  );
3102
3102
  if (response.status === 200) {
3103
3103
  const id = response.data.id;
3104
- return this.monitorJobStatus(id, headers, pollInterval);
3104
+ return this.monitorJobStatus(id, headers, pollInterval, "batch");
3105
3105
  } else {
3106
3106
  this.handleError(response, "start batch scrape job");
3107
3107
  }
@@ -3116,7 +3116,7 @@ var FirecrawlApp = class {
3116
3116
  }
3117
3117
  async asyncBatchScrapeUrls(urls, params, idempotencyKey, webhook, ignoreInvalidURLs) {
3118
3118
  const headers = this.prepareHeaders(idempotencyKey);
3119
- let jsonData = { urls, webhook, ignoreInvalidURLs, ...params, origin: typeof params.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` };
3119
+ let jsonData = { urls, webhook, ignoreInvalidURLs, ...params, origin: typeof params?.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` };
3120
3120
  try {
3121
3121
  const response = await this.postRequest(
3122
3122
  this.apiUrl + `/v1/batch/scrape`,
@@ -3260,7 +3260,7 @@ var FirecrawlApp = class {
3260
3260
  try {
3261
3261
  const response = await this.postRequest(
3262
3262
  this.apiUrl + `/v1/extract`,
3263
- { ...jsonData, schema: jsonSchema, origin: typeof params.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` },
3263
+ { ...jsonData, schema: jsonSchema, origin: typeof params?.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` },
3264
3264
  headers
3265
3265
  );
3266
3266
  if (response.status === 200) {
@@ -3313,7 +3313,7 @@ var FirecrawlApp = class {
3313
3313
  try {
3314
3314
  const response = await this.postRequest(
3315
3315
  this.apiUrl + `/v1/extract`,
3316
- { ...jsonData, schema: jsonSchema, origin: typeof params.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` },
3316
+ { ...jsonData, schema: jsonSchema, origin: typeof params?.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` },
3317
3317
  headers
3318
3318
  );
3319
3319
  if (response.status === 200) {
@@ -3405,21 +3405,23 @@ var FirecrawlApp = class {
3405
3405
  }
3406
3406
  }
3407
3407
  /**
3408
- * Monitors the status of a crawl job until completion or failure.
3409
- * @param id - The ID of the crawl operation.
3408
+ * Monitors the status of a crawl or batch scrape job until completion or failure.
3409
+ * @param id - The ID of the crawl or batch scrape operation.
3410
3410
  * @param headers - The headers for the request.
3411
3411
  * @param checkInterval - Interval in seconds for job status checks.
3412
- * @param checkUrl - Optional URL to check the status (used for v1 API)
3412
+ * @param jobType - Which status endpoint to poll. Defaults to `"crawl"`.
3413
3413
  * @returns The final job status or data.
3414
3414
  */
3415
- async monitorJobStatus(id, headers, checkInterval) {
3415
+ async monitorJobStatus(id, headers, checkInterval, jobType = "crawl") {
3416
+ const statusPath = jobType === "batch" ? "batch/scrape" : "crawl";
3417
+ const jobLabel = jobType === "batch" ? "Batch scrape" : "Crawl";
3416
3418
  let failedTries = 0;
3417
3419
  let networkRetries = 0;
3418
3420
  const maxNetworkRetries = 3;
3419
3421
  while (true) {
3420
3422
  try {
3421
3423
  let statusResponse = await this.getRequest(
3422
- `${this.apiUrl}/v1/crawl/${id}`,
3424
+ `${this.apiUrl}/v1/${statusPath}/${id}`,
3423
3425
  headers
3424
3426
  );
3425
3427
  if (statusResponse.status === 200) {
@@ -3440,7 +3442,7 @@ var FirecrawlApp = class {
3440
3442
  statusData.data = data;
3441
3443
  return statusData;
3442
3444
  } else {
3443
- throw new FirecrawlError("Crawl job completed but no data was returned", 500);
3445
+ throw new FirecrawlError(`${jobLabel} job completed but no data was returned`, 500);
3444
3446
  }
3445
3447
  } else if (["active", "paused", "pending", "queued", "waiting", "scraping"].includes(statusData.status)) {
3446
3448
  checkInterval = Math.max(checkInterval, 2);
@@ -3449,14 +3451,14 @@ var FirecrawlApp = class {
3449
3451
  );
3450
3452
  } else {
3451
3453
  throw new FirecrawlError(
3452
- `Crawl job failed or was stopped. Status: ${statusData.status}`,
3454
+ `${jobLabel} job failed or was stopped. Status: ${statusData.status}`,
3453
3455
  500
3454
3456
  );
3455
3457
  }
3456
3458
  } else {
3457
3459
  failedTries++;
3458
3460
  if (failedTries >= 3) {
3459
- this.handleError(statusResponse, "check crawl status");
3461
+ this.handleError(statusResponse, `check ${jobLabel.toLowerCase()} status`);
3460
3462
  }
3461
3463
  }
3462
3464
  } catch (error) {
@@ -3593,7 +3595,7 @@ var FirecrawlApp = class {
3593
3595
  */
3594
3596
  async asyncDeepResearch(query, params) {
3595
3597
  const headers = this.prepareHeaders();
3596
- let jsonData = { query, ...params, origin: typeof params.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` };
3598
+ let jsonData = { query, ...params, origin: typeof params?.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` };
3597
3599
  if (jsonData?.jsonOptions?.schema) {
3598
3600
  jsonData = {
3599
3601
  ...jsonData,
@@ -3712,7 +3714,7 @@ var FirecrawlApp = class {
3712
3714
  async __asyncDeepResearch(topic, params) {
3713
3715
  const headers = this.prepareHeaders();
3714
3716
  try {
3715
- let jsonData = { topic, ...params, origin: typeof params.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` };
3717
+ let jsonData = { topic, ...params, origin: typeof params?.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` };
3716
3718
  const response = await this.postRequest(
3717
3719
  `${this.apiUrl}/v1/deep-research`,
3718
3720
  jsonData,
@@ -3812,7 +3814,7 @@ var FirecrawlApp = class {
3812
3814
  */
3813
3815
  async asyncGenerateLLMsText(url, params) {
3814
3816
  const headers = this.prepareHeaders();
3815
- let jsonData = { url, ...params, origin: typeof params.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` };
3817
+ let jsonData = { url, ...params, origin: typeof params?.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` };
3816
3818
  try {
3817
3819
  const response = await this.postRequest(
3818
3820
  `${this.apiUrl}/v1/llmstxt`,
@@ -1,4 +1,4 @@
1
1
  import {
2
2
  require_package
3
- } from "./chunk-XZ5XI5DK.js";
3
+ } from "./chunk-TALYWLKJ.js";
4
4
  export default require_package();
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@mendable/firecrawl-js",
3
- "version": "4.42.2",
3
+ "version": "4.42.4",
4
4
  "description": "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
5
5
  "main": "dist/index.js",
6
6
  "types": "dist/index.d.ts",
@@ -74,6 +74,6 @@
74
74
  "build-and-publish": "pnpm run build && pnpm publish --access public",
75
75
  "publish-beta": "pnpm run build && pnpm publish --access public --tag beta",
76
76
  "test": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/e2e/v2/*.test.ts --detectOpenHandles",
77
- "test:unit": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/unit/v2/*.test.ts"
77
+ "test:unit": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/unit"
78
78
  }
79
79
  }
@@ -0,0 +1,91 @@
1
+ import { describe, test, expect, jest, afterEach, beforeEach } from "@jest/globals";
2
+ import axios from "axios";
3
+ import FirecrawlApp from "../../../v1";
4
+
5
+ const API_URL = "https://api.firecrawl.dev";
6
+ const API_KEY = "fc-test";
7
+
8
+ const scrapingPage = () => ({ success: true, status: "scraping", total: 2, completed: 0, data: [] });
9
+ const completedPage = (next: string) => ({
10
+ success: true,
11
+ status: "completed",
12
+ total: 2,
13
+ completed: 2,
14
+ expiresAt: "2030-01-01T00:00:00Z",
15
+ data: [{ markdown: "a" }],
16
+ next,
17
+ });
18
+ const lastPage = () => ({ success: true, status: "completed", data: [{ markdown: "b" }] });
19
+
20
+ function mockPoll(path: string, nextOrigin = API_URL) {
21
+ const sent: string[] = [];
22
+ let polls = 0;
23
+ jest.spyOn(axios, "get").mockImplementation(async (url: string) => {
24
+ sent.push(url);
25
+ if (new URL(url).searchParams.has("skip")) return { status: 200, data: lastPage() };
26
+ polls++;
27
+ return { status: 200, data: polls === 1 ? scrapingPage() : completedPage(`${nextOrigin}${path}?skip=1`) };
28
+ });
29
+ return sent;
30
+ }
31
+
32
+ describe("v1 batch scrape waits on the batch scrape status endpoint", () => {
33
+ const app = new FirecrawlApp({ apiKey: API_KEY, apiUrl: API_URL });
34
+
35
+ beforeEach(() => {
36
+ jest.spyOn(global, "setTimeout").mockImplementation(((fn: () => void) => {
37
+ fn();
38
+ return 0;
39
+ }) as any);
40
+ });
41
+
42
+ afterEach(() => {
43
+ jest.restoreAllMocks();
44
+ });
45
+
46
+ test("batchScrapeUrls polls /v1/batch/scrape/{id} and follows pinned next pages", async () => {
47
+ const post = jest.spyOn(axios, "post").mockResolvedValue({ status: 200, data: { success: true, id: "abc" } });
48
+ const path = "/v1/batch/scrape/abc";
49
+ const sent = mockPoll(path, "https://evil.example");
50
+
51
+ const res: any = await app.batchScrapeUrls(["https://example.com", "https://example.org"], {});
52
+
53
+ expect(post.mock.calls[0][0]).toBe(`${API_URL}/v1/batch/scrape`);
54
+ expect(sent).toEqual([`${API_URL}${path}`, `${API_URL}${path}`, `${API_URL}${path}?skip=1`]);
55
+ expect(sent.some(url => url.includes("/v1/crawl/"))).toBe(false);
56
+ expect(res.status).toBe("completed");
57
+ expect(res.data).toHaveLength(2);
58
+ });
59
+
60
+ test("batchScrapeUrls reports batch scrape failures as batch scrape errors", async () => {
61
+ jest.spyOn(axios, "post").mockResolvedValue({ status: 200, data: { success: true, id: "abc" } });
62
+ const sent: string[] = [];
63
+ jest.spyOn(axios, "get").mockImplementation(async (url: string) => {
64
+ sent.push(url);
65
+ return { status: 200, data: { success: true, status: "failed", data: [] } };
66
+ });
67
+
68
+ await expect(app.batchScrapeUrls(["https://example.com"], {})).rejects.toThrow("Batch scrape job failed or was stopped");
69
+ expect(sent.every(url => url === `${API_URL}/v1/batch/scrape/abc`)).toBe(true);
70
+ });
71
+
72
+ test("monitorJobStatus still polls /v1/crawl/{id} by default", async () => {
73
+ const path = "/v1/crawl/abc";
74
+ const sent = mockPoll(path);
75
+
76
+ const res: any = await app.monitorJobStatus("abc", app.prepareHeaders(), 0);
77
+
78
+ expect(sent).toEqual([`${API_URL}${path}`, `${API_URL}${path}`, `${API_URL}${path}?skip=1`]);
79
+ expect(res.data).toHaveLength(2);
80
+ });
81
+
82
+ test("monitorJobStatus polls /v1/batch/scrape/{id} for batch jobs", async () => {
83
+ const path = "/v1/batch/scrape/abc";
84
+ const sent = mockPoll(path);
85
+
86
+ const res: any = await app.monitorJobStatus("abc", app.prepareHeaders(), 0, "batch");
87
+
88
+ expect(sent).toEqual([`${API_URL}${path}`, `${API_URL}${path}`, `${API_URL}${path}?skip=1`]);
89
+ expect(res.data).toHaveLength(2);
90
+ });
91
+ });
@@ -1,154 +1,107 @@
1
- import FirecrawlApp from '../../../index';
1
+ import FirecrawlApp from '../../../v1';
2
2
  import { describe, test, expect, jest, beforeEach, afterEach } from '@jest/globals';
3
3
 
4
+ const successResponse = {
5
+ status: 200,
6
+ data: { status: 'completed', data: [{ url: 'test.com', markdown: 'test' }] },
7
+ };
8
+
9
+ function networkError(message: string, code: string) {
10
+ return Object.assign(new Error(message), { code });
11
+ }
12
+
13
+ function httpError(message: string, status: number) {
14
+ return Object.assign(new Error(message), { response: { status, data: { error: message } } });
15
+ }
16
+
4
17
  describe('monitorJobStatus retry logic', () => {
5
18
  let app: FirecrawlApp;
6
- let originalConsoleWarn: typeof console.warn;
7
-
19
+ let delays: number[];
20
+
8
21
  beforeEach(() => {
9
22
  app = new FirecrawlApp({ apiKey: 'test-key', apiUrl: 'https://test.com' });
10
- originalConsoleWarn = console.warn;
11
- console.warn = jest.fn();
23
+ delays = [];
24
+ jest.useFakeTimers();
25
+ const fakeSetTimeout = globalThis.setTimeout;
26
+ jest.spyOn(globalThis, 'setTimeout').mockImplementation(((fn: () => void, ms?: number) => {
27
+ delays.push(ms ?? 0);
28
+ return fakeSetTimeout(fn, ms);
29
+ }) as any);
12
30
  });
13
31
 
14
32
  afterEach(() => {
15
- console.warn = originalConsoleWarn;
16
- jest.clearAllMocks();
33
+ jest.restoreAllMocks();
34
+ jest.useRealTimers();
17
35
  });
18
36
 
19
- test('should retry on socket hang up error', async () => {
20
- const socketHangUpError = new Error('socket hang up') as any;
21
- socketHangUpError.code = 'ECONNRESET';
22
-
23
- const successResponse = {
24
- status: 200,
25
- data: { status: 'completed', data: [{ url: 'test.com', markdown: 'test' }] }
26
- };
27
-
28
- const originalGetRequest = app.getRequest;
29
- let callCount = 0;
30
-
31
- app.getRequest = async function(url: string, headers: any) {
32
- callCount++;
33
- if (callCount === 1) {
34
- throw socketHangUpError;
35
- }
36
- return successResponse;
37
- };
38
-
39
- const result = await app.monitorJobStatus('test-id', {}, 1);
40
-
41
- expect(callCount).toBe(2);
37
+ function failFirst(n: number, error: Error, statuses: string[] = []) {
38
+ let calls = 0;
39
+ app.getRequest = (async () => {
40
+ calls++;
41
+ if (calls <= n) throw error;
42
+ const status = statuses.shift();
43
+ return status ? { status: 200, data: { status } } : successResponse;
44
+ }) as any;
45
+ return () => calls;
46
+ }
47
+
48
+ async function monitor() {
49
+ const result = app.monitorJobStatus('test-id', {} as any, 1);
50
+ result.catch(() => {});
51
+ await jest.runAllTimersAsync();
52
+ return result;
53
+ }
54
+
55
+ test.each([
56
+ ['socket hang up', networkError('socket hang up', 'ECONNRESET')],
57
+ ['ETIMEDOUT', networkError('timeout', 'ETIMEDOUT')],
58
+ ['HTTP 408', httpError('Request timeout', 408)],
59
+ ['HTTP 504', httpError('Gateway timeout', 504)],
60
+ ])('retries once after a %s error and returns the job data', async (_label, error) => {
61
+ const calls = failFirst(1, error);
62
+
63
+ const result = await monitor();
64
+
65
+ expect(calls()).toBe(2);
42
66
  expect(result).toEqual(successResponse.data);
43
- expect(console.warn).toHaveBeenCalledWith(
44
- expect.stringContaining('Network error during job status check (attempt 1/3): socket hang up')
45
- );
67
+ expect(delays).toEqual([1000]);
46
68
  });
47
69
 
48
- test('should retry on ETIMEDOUT error', async () => {
49
- const timeoutError = new Error('timeout') as any;
50
- timeoutError.code = 'ETIMEDOUT';
51
-
52
- const successResponse = {
53
- status: 200,
54
- data: { status: 'completed', data: [{ url: 'test.com', markdown: 'test' }] }
55
- };
56
-
57
- const originalGetRequest = app.getRequest;
58
- let callCount = 0;
59
-
60
- app.getRequest = async function(url: string, headers: any) {
61
- callCount++;
62
- if (callCount === 1) {
63
- throw timeoutError;
64
- }
65
- return successResponse;
66
- };
67
-
68
- const result = await app.monitorJobStatus('test-id', {}, 1);
69
-
70
- expect(callCount).toBe(2);
70
+ test('uses exponential backoff between retries', async () => {
71
+ const calls = failFirst(2, networkError('socket hang up', 'ECONNRESET'));
72
+
73
+ const result = await monitor();
74
+
75
+ expect(calls()).toBe(3);
71
76
  expect(result).toEqual(successResponse.data);
77
+ expect(delays).toEqual([1000, 2000]);
72
78
  });
73
79
 
74
- test('should fail after max retries exceeded', async () => {
75
- const socketHangUpError = new Error('socket hang up') as any;
76
- socketHangUpError.code = 'ECONNRESET';
77
-
78
- app.getRequest = async function(url: string, headers: any) {
79
- throw socketHangUpError;
80
- };
81
-
82
- await expect(app.monitorJobStatus('test-id', {}, 1)).rejects.toThrow('socket hang up');
83
-
84
- expect(console.warn).toHaveBeenCalledTimes(3);
85
- }, 15000);
86
-
87
- test('should not retry on non-retryable errors', async () => {
88
- const authError = new Error('Unauthorized') as any;
89
- authError.response = { status: 401, data: { error: 'Unauthorized' } };
90
-
91
- app.getRequest = async function(url: string, headers: any) {
92
- throw authError;
93
- };
94
-
95
- await expect(app.monitorJobStatus('test-id', {}, 1)).rejects.toThrow('Unauthorized');
96
-
97
- expect(console.warn).not.toHaveBeenCalled();
98
- });
80
+ test('records both the retry backoff and the 2s poll wait for an active job', async () => {
81
+ const calls = failFirst(1, networkError('socket hang up', 'ECONNRESET'), ['active']);
82
+
83
+ const result = await monitor();
99
84
 
100
- test('should retry on HTTP timeout status codes', async () => {
101
- const timeoutError = new Error('Request timeout') as any;
102
- timeoutError.response = { status: 408, data: { error: 'Request timeout' } };
103
-
104
- const successResponse = {
105
- status: 200,
106
- data: { status: 'completed', data: [{ url: 'test.com', markdown: 'test' }] }
107
- };
108
-
109
- const originalGetRequest = app.getRequest;
110
- let callCount = 0;
111
-
112
- app.getRequest = async function(url: string, headers: any) {
113
- callCount++;
114
- if (callCount === 1) {
115
- throw timeoutError;
116
- }
117
- return successResponse;
118
- };
119
-
120
- const result = await app.monitorJobStatus('test-id', {}, 1);
121
-
122
- expect(callCount).toBe(2);
85
+ expect(calls()).toBe(3);
123
86
  expect(result).toEqual(successResponse.data);
87
+ expect(delays).toEqual([1000, 2000]);
124
88
  });
125
89
 
126
- test('should use exponential backoff for retries', async () => {
127
- const socketHangUpError = new Error('socket hang up') as any;
128
- socketHangUpError.code = 'ECONNRESET';
129
-
130
- const successResponse = {
131
- status: 200,
132
- data: { status: 'completed', data: [{ url: 'test.com', markdown: 'test' }] }
133
- };
134
-
135
- const originalGetRequest = app.getRequest;
136
- let callCount = 0;
137
-
138
- app.getRequest = async function(url: string, headers: any) {
139
- callCount++;
140
- if (callCount <= 2) {
141
- throw socketHangUpError;
142
- }
143
- return successResponse;
144
- };
145
-
146
- const startTime = Date.now();
147
- const result = await app.monitorJobStatus('test-id', {}, 1);
148
- const endTime = Date.now();
149
-
150
- expect(callCount).toBe(3);
151
- expect(result).toEqual(successResponse.data);
152
- expect(endTime - startTime).toBeGreaterThan(3000);
90
+ test('fails after max retries are exceeded', async () => {
91
+ const calls = failFirst(Infinity, networkError('socket hang up', 'ECONNRESET'));
92
+
93
+ await expect(monitor()).rejects.toThrow('socket hang up');
94
+
95
+ expect(calls()).toBe(4);
96
+ expect(delays).toEqual([1000, 2000, 4000]);
97
+ });
98
+
99
+ test('does not retry non-retryable errors', async () => {
100
+ const calls = failFirst(Infinity, httpError('Unauthorized', 401));
101
+
102
+ await expect(monitor()).rejects.toThrow('Unauthorized');
103
+
104
+ expect(calls()).toBe(1);
105
+ expect(delays).toEqual([]);
153
106
  });
154
107
  });
@@ -0,0 +1,70 @@
1
+ import { describe, test, expect, jest, beforeEach, afterEach } from "@jest/globals";
2
+ import axios from "axios";
3
+ import FirecrawlApp from "../../../v1";
4
+
5
+ const API_URL = "https://api.firecrawl.dev";
6
+
7
+ class FakeWebSocket {
8
+ onopen: unknown = null;
9
+ onmessage: unknown = null;
10
+ onerror: unknown = null;
11
+ onclose: unknown = null;
12
+ close() {}
13
+ }
14
+
15
+ describe("v1 methods work without params", () => {
16
+ let app: FirecrawlApp;
17
+ let posts: Array<{ url: string; data: any }>;
18
+ const originalWebSocket = (globalThis as any).WebSocket;
19
+
20
+ beforeEach(() => {
21
+ app = new FirecrawlApp({ apiKey: "fc-test", apiUrl: API_URL });
22
+ posts = [];
23
+ jest.spyOn(axios, "post").mockImplementation(async (url: string, data?: any) => {
24
+ posts.push({ url, data });
25
+ return { status: 200, data: { success: true, id: "job-1", data: [], links: [] } };
26
+ });
27
+ jest.spyOn(axios, "get").mockImplementation(async () => ({
28
+ status: 200,
29
+ data: { success: true, status: "completed", data: [{ markdown: "ok" }] },
30
+ }));
31
+ (globalThis as any).WebSocket = FakeWebSocket;
32
+ });
33
+
34
+ afterEach(() => {
35
+ jest.restoreAllMocks();
36
+ (globalThis as any).WebSocket = originalWebSocket;
37
+ });
38
+
39
+ const cases: Array<[string, string, (app: any) => Promise<unknown>]> = [
40
+ ["scrapeUrl", "/v1/scrape", app => app.scrapeUrl("https://example.com")],
41
+ ["search", "/v1/search", app => app.search("firecrawl")],
42
+ ["crawlUrl", "/v1/crawl", app => app.crawlUrl("https://example.com")],
43
+ ["asyncCrawlUrl", "/v1/crawl", app => app.asyncCrawlUrl("https://example.com")],
44
+ ["crawlUrlAndWatch", "/v1/crawl", app => app.crawlUrlAndWatch("https://example.com")],
45
+ ["mapUrl", "/v1/map", app => app.mapUrl("https://example.com")],
46
+ ["batchScrapeUrls", "/v1/batch/scrape", app => app.batchScrapeUrls(["https://example.com"])],
47
+ ["asyncBatchScrapeUrls", "/v1/batch/scrape", app => app.asyncBatchScrapeUrls(["https://example.com"])],
48
+ ["batchScrapeUrlsAndWatch", "/v1/batch/scrape", app => app.batchScrapeUrlsAndWatch(["https://example.com"])],
49
+ ["extract", "/v1/extract", app => app.extract(["https://example.com"])],
50
+ ["asyncExtract", "/v1/extract", app => app.asyncExtract(["https://example.com"])],
51
+ ["deepResearch", "/v1/deep-research", app => app.deepResearch("firecrawl")],
52
+ ["asyncDeepResearch", "/v1/deep-research", app => app.asyncDeepResearch("firecrawl")],
53
+ ["__deepResearch", "/v1/deep-research", app => app.__deepResearch("firecrawl")],
54
+ ["__asyncDeepResearch", "/v1/deep-research", app => app.__asyncDeepResearch("firecrawl")],
55
+ ["generateLLMsText", "/v1/llmstxt", app => app.generateLLMsText("https://example.com")],
56
+ ["asyncGenerateLLMsText", "/v1/llmstxt", app => app.asyncGenerateLLMsText("https://example.com")],
57
+ ];
58
+
59
+ test.each(cases)("%s sends the default origin", async (_method, path, call) => {
60
+ await expect(call(app)).resolves.toBeDefined();
61
+ expect(posts).toHaveLength(1);
62
+ expect(posts[0].url).toBe(`${API_URL}${path}`);
63
+ expect(posts[0].data.origin).toBe(`js-sdk@${app.version}`);
64
+ });
65
+
66
+ test("an mcp origin passed in params is still forwarded", async () => {
67
+ await app.batchScrapeUrls(["https://example.com"], { origin: "mcp-server" } as any);
68
+ expect(posts[0].data.origin).toBe("mcp-server");
69
+ });
70
+ });
@@ -1,11 +1,60 @@
1
- /**
2
- * Minimal unit test for v2 scrape (no mocking; sanity check payload path)
3
- */
1
+ import { describe, test, expect } from "@jest/globals";
2
+ import axios, { type AxiosAdapter } from "axios";
4
3
  import { FirecrawlClient } from "../../../v2/client";
5
4
 
5
+ const API_URL = "https://api.firecrawl.dev";
6
+
7
+ function makeClient(apiKey: string) {
8
+ const client = new FirecrawlClient({ apiKey, apiUrl: API_URL });
9
+ const sent: Array<{ method?: string; url: string; authorization: unknown; body: unknown }> = [];
10
+ const adapter: AxiosAdapter = async config => {
11
+ sent.push({
12
+ method: config.method,
13
+ url: axios.getUri(config),
14
+ authorization: config.headers.Authorization,
15
+ body: JSON.parse(config.data),
16
+ });
17
+ return {
18
+ data: { success: true, data: { markdown: "# hello" } },
19
+ status: 200,
20
+ statusText: "OK",
21
+ headers: {},
22
+ config,
23
+ };
24
+ };
25
+ (client as any).http.instance.defaults.adapter = adapter;
26
+ return { client, sent };
27
+ }
28
+
6
29
  describe("v2.scrape unit", () => {
7
- test("constructor requires apiKey", () => {
8
- expect(() => new FirecrawlClient({ apiKey: "", apiUrl: "https://api.firecrawl.dev" })).toThrow();
30
+ test("constructs without an API key and scrapes keyless, without an Authorization header", async () => {
31
+ const { client, sent } = makeClient("");
32
+ const doc = await client.scrape(" https://example.com ", { formats: ["markdown"] });
33
+
34
+ expect(doc.markdown).toBe("# hello");
35
+ expect(sent).toHaveLength(1);
36
+ expect(sent[0]!.method).toBe("post");
37
+ expect(sent[0]!.url).toBe(`${API_URL}/v2/scrape`);
38
+ expect(sent[0]!.authorization).toBeUndefined();
39
+ expect(sent[0]!.body).toEqual({
40
+ url: "https://example.com",
41
+ formats: ["markdown"],
42
+ origin: expect.stringMatching(/^js-sdk@/),
43
+ });
9
44
  });
10
- });
11
45
 
46
+ test("sends the API key as a bearer token when one is set", async () => {
47
+ const { client, sent } = makeClient("fc-test");
48
+ await client.scrape("https://example.com");
49
+
50
+ expect(sent).toHaveLength(1);
51
+ expect(sent[0]!.authorization).toBe("Bearer fc-test");
52
+ });
53
+
54
+ test("rejects an empty URL before sending a request", async () => {
55
+ const { client, sent } = makeClient("fc-test");
56
+
57
+ await expect(client.scrape(" ")).rejects.toThrow("URL cannot be empty");
58
+ expect(sent).toHaveLength(0);
59
+ });
60
+ });
package/src/v1/index.ts CHANGED
@@ -726,7 +726,7 @@ export default class FirecrawlApp {
726
726
  "Content-Type": "application/json",
727
727
  Authorization: `Bearer ${this.apiKey}`,
728
728
  } as AxiosRequestHeaders;
729
- let jsonData: any = { url, ...params, origin: typeof (params as any).origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` };
729
+ let jsonData: any = { url, ...params, origin: typeof (params as any)?.origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` };
730
730
  if (jsonData?.extract?.schema) {
731
731
  jsonData = {
732
732
  ...jsonData,
@@ -793,7 +793,7 @@ export default class FirecrawlApp {
793
793
  lang: params?.lang ?? "en",
794
794
  country: params?.country ?? "us",
795
795
  location: params?.location,
796
- origin: typeof (params as any).origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}`,
796
+ origin: typeof (params as any)?.origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}`,
797
797
  timeout: params?.timeout ?? 60000,
798
798
  scrapeOptions: params?.scrapeOptions ?? { formats: [] },
799
799
  };
@@ -857,7 +857,7 @@ export default class FirecrawlApp {
857
857
  idempotencyKey?: string
858
858
  ): Promise<CrawlStatusResponse | ErrorResponse> {
859
859
  const headers = this.prepareHeaders(idempotencyKey);
860
- let jsonData: any = { url, ...params, origin: typeof (params as any).origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` };
860
+ let jsonData: any = { url, ...params, origin: typeof (params as any)?.origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` };
861
861
  try {
862
862
  const response: AxiosResponse = await this.postRequest(
863
863
  this.apiUrl + `/v1/crawl`,
@@ -886,7 +886,7 @@ export default class FirecrawlApp {
886
886
  idempotencyKey?: string
887
887
  ): Promise<CrawlResponse | ErrorResponse> {
888
888
  const headers = this.prepareHeaders(idempotencyKey);
889
- let jsonData: any = { url, ...params, origin: typeof (params as any).origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` };
889
+ let jsonData: any = { url, ...params, origin: typeof (params as any)?.origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` };
890
890
  try {
891
891
  const response: AxiosResponse = await this.postRequest(
892
892
  this.apiUrl + `/v1/crawl`,
@@ -1062,7 +1062,7 @@ export default class FirecrawlApp {
1062
1062
  */
1063
1063
  async mapUrl(url: string, params?: MapParams): Promise<MapResponse | ErrorResponse> {
1064
1064
  const headers = this.prepareHeaders();
1065
- let jsonData: any = { url, ...params, origin: typeof (params as any).origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` };
1065
+ let jsonData: any = { url, ...params, origin: typeof (params as any)?.origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` };
1066
1066
 
1067
1067
  try {
1068
1068
  const response: AxiosResponse = await this.postRequest(
@@ -1101,7 +1101,7 @@ export default class FirecrawlApp {
1101
1101
  maxConcurrency?: number,
1102
1102
  ): Promise<BatchScrapeStatusResponse | ErrorResponse> {
1103
1103
  const headers = this.prepareHeaders(idempotencyKey);
1104
- let jsonData: any = { urls, webhook, ignoreInvalidURLs, maxConcurrency, ...params, origin: typeof (params as any).origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` };
1104
+ let jsonData: any = { urls, webhook, ignoreInvalidURLs, maxConcurrency, ...params, origin: typeof (params as any)?.origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` };
1105
1105
  if (jsonData?.extract?.schema) {
1106
1106
  jsonData = {
1107
1107
  ...jsonData,
@@ -1128,7 +1128,7 @@ export default class FirecrawlApp {
1128
1128
  );
1129
1129
  if (response.status === 200) {
1130
1130
  const id: string = response.data.id;
1131
- return this.monitorJobStatus(id, headers, pollInterval);
1131
+ return this.monitorJobStatus(id, headers, pollInterval, "batch");
1132
1132
  } else {
1133
1133
  this.handleError(response, "start batch scrape job");
1134
1134
  }
@@ -1150,7 +1150,7 @@ export default class FirecrawlApp {
1150
1150
  ignoreInvalidURLs?: boolean,
1151
1151
  ): Promise<BatchScrapeResponse | ErrorResponse> {
1152
1152
  const headers = this.prepareHeaders(idempotencyKey);
1153
- let jsonData: any = { urls, webhook, ignoreInvalidURLs, ...params, origin: typeof (params as any).origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` };
1153
+ let jsonData: any = { urls, webhook, ignoreInvalidURLs, ...params, origin: typeof (params as any)?.origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` };
1154
1154
  try {
1155
1155
  const response: AxiosResponse = await this.postRequest(
1156
1156
  this.apiUrl + `/v1/batch/scrape`,
@@ -1314,7 +1314,7 @@ export default class FirecrawlApp {
1314
1314
  try {
1315
1315
  const response: AxiosResponse = await this.postRequest(
1316
1316
  this.apiUrl + `/v1/extract`,
1317
- { ...jsonData, schema: jsonSchema, origin: typeof (params as any).origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` },
1317
+ { ...jsonData, schema: jsonSchema, origin: typeof (params as any)?.origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` },
1318
1318
  headers
1319
1319
  );
1320
1320
 
@@ -1374,7 +1374,7 @@ export default class FirecrawlApp {
1374
1374
  try {
1375
1375
  const response: AxiosResponse = await this.postRequest(
1376
1376
  this.apiUrl + `/v1/extract`,
1377
- { ...jsonData, schema: jsonSchema, origin: typeof (params as any).origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` },
1377
+ { ...jsonData, schema: jsonSchema, origin: typeof (params as any)?.origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` },
1378
1378
  headers
1379
1379
  );
1380
1380
 
@@ -1484,18 +1484,21 @@ export default class FirecrawlApp {
1484
1484
  }
1485
1485
 
1486
1486
  /**
1487
- * Monitors the status of a crawl job until completion or failure.
1488
- * @param id - The ID of the crawl operation.
1487
+ * Monitors the status of a crawl or batch scrape job until completion or failure.
1488
+ * @param id - The ID of the crawl or batch scrape operation.
1489
1489
  * @param headers - The headers for the request.
1490
1490
  * @param checkInterval - Interval in seconds for job status checks.
1491
- * @param checkUrl - Optional URL to check the status (used for v1 API)
1491
+ * @param jobType - Which status endpoint to poll. Defaults to `"crawl"`.
1492
1492
  * @returns The final job status or data.
1493
1493
  */
1494
1494
  async monitorJobStatus(
1495
1495
  id: string,
1496
1496
  headers: AxiosRequestHeaders,
1497
- checkInterval: number
1497
+ checkInterval: number,
1498
+ jobType: "crawl" | "batch" = "crawl"
1498
1499
  ): Promise<CrawlStatusResponse | ErrorResponse> {
1500
+ const statusPath = jobType === "batch" ? "batch/scrape" : "crawl";
1501
+ const jobLabel = jobType === "batch" ? "Batch scrape" : "Crawl";
1499
1502
  let failedTries = 0;
1500
1503
  let networkRetries = 0;
1501
1504
  const maxNetworkRetries = 3;
@@ -1503,7 +1506,7 @@ export default class FirecrawlApp {
1503
1506
  while (true) {
1504
1507
  try {
1505
1508
  let statusResponse: AxiosResponse = await this.getRequest(
1506
- `${this.apiUrl}/v1/crawl/${id}`,
1509
+ `${this.apiUrl}/v1/${statusPath}/${id}`,
1507
1510
  headers
1508
1511
  );
1509
1512
 
@@ -1526,7 +1529,7 @@ export default class FirecrawlApp {
1526
1529
  statusData.data = data;
1527
1530
  return statusData;
1528
1531
  } else {
1529
- throw new FirecrawlError("Crawl job completed but no data was returned", 500);
1532
+ throw new FirecrawlError(`${jobLabel} job completed but no data was returned`, 500);
1530
1533
  }
1531
1534
  } else if (
1532
1535
  ["active", "paused", "pending", "queued", "waiting", "scraping"].includes(statusData.status)
@@ -1537,14 +1540,14 @@ export default class FirecrawlApp {
1537
1540
  );
1538
1541
  } else {
1539
1542
  throw new FirecrawlError(
1540
- `Crawl job failed or was stopped. Status: ${statusData.status}`,
1543
+ `${jobLabel} job failed or was stopped. Status: ${statusData.status}`,
1541
1544
  500
1542
1545
  );
1543
1546
  }
1544
1547
  } else {
1545
1548
  failedTries++;
1546
1549
  if (failedTries >= 3) {
1547
- this.handleError(statusResponse, "check crawl status");
1550
+ this.handleError(statusResponse, `check ${jobLabel.toLowerCase()} status`);
1548
1551
  }
1549
1552
  }
1550
1553
  } catch (error: any) {
@@ -1739,7 +1742,7 @@ export default class FirecrawlApp {
1739
1742
  */
1740
1743
  async asyncDeepResearch(query: string, params: DeepResearchParams<zt.ZodSchema>): Promise<DeepResearchResponse | ErrorResponse> {
1741
1744
  const headers = this.prepareHeaders();
1742
- let jsonData: any = { query, ...params, origin: typeof (params as any).origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` };
1745
+ let jsonData: any = { query, ...params, origin: typeof (params as any)?.origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` };
1743
1746
 
1744
1747
  if (jsonData?.jsonOptions?.schema) {
1745
1748
  jsonData = {
@@ -1887,7 +1890,7 @@ export default class FirecrawlApp {
1887
1890
  async __asyncDeepResearch(topic: string, params: DeepResearchParams): Promise<DeepResearchResponse | ErrorResponse> {
1888
1891
  const headers = this.prepareHeaders();
1889
1892
  try {
1890
- let jsonData: any = { topic, ...params, origin: typeof (params as any).origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` };
1893
+ let jsonData: any = { topic, ...params, origin: typeof (params as any)?.origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` };
1891
1894
  const response: AxiosResponse = await this.postRequest(
1892
1895
  `${this.apiUrl}/v1/deep-research`,
1893
1896
  jsonData,
@@ -2002,7 +2005,7 @@ export default class FirecrawlApp {
2002
2005
  */
2003
2006
  async asyncGenerateLLMsText(url: string, params?: GenerateLLMsTextParams): Promise<GenerateLLMsTextResponse | ErrorResponse> {
2004
2007
  const headers = this.prepareHeaders();
2005
- let jsonData: any = { url, ...params, origin: typeof (params as any).origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` };
2008
+ let jsonData: any = { url, ...params, origin: typeof (params as any)?.origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` };
2006
2009
  try {
2007
2010
  const response: AxiosResponse = await this.postRequest(
2008
2011
  `${this.apiUrl}/v1/llmstxt`,