firecrawl 4.18.2 → 4.18.4

package/README.md

@@ -1,6 +1,6 @@
 # Firecrawl Node SDK
 
-The Firecrawl Node SDK is a library that allows you to easily scrape and crawl websites, and output the data in a format ready for use with language models (LLMs). It provides a simple and intuitive interface for interacting with the Firecrawl API.
+The Firecrawl Node SDK is a library that allows you to easily search, scrape, and interact with the web, and output the data in a format ready for use with language models (LLMs). It provides a simple and intuitive interface for the Firecrawl API.
 
 ## Installation
 

package/dist/{chunk-QU5TGKGP.js → chunk-ZUJQPZTD.js}

@@ -8,7 +8,7 @@ var require_package = __commonJS({
   "package.json"(exports, module) {
     module.exports = {
       name: "@mendable/firecrawl-js",
-      version: "4.18.2",
+      version: "4.18.4",
       description: "JavaScript SDK for Firecrawl API",
       main: "dist/index.js",
       types: "dist/index.d.ts",
@@ -78,7 +78,8 @@ var require_package = __commonJS({
           "picomatch@<4.0.4": ">=4.0.4",
           handlebars: ">=4.7.9",
           "brace-expansion": ">=5.0.5",
-          "axios@<1.15.0": "1.15.0"
+          "axios@<1.15.0": "1.15.0",
+          "follow-redirects@<1.16.0": ">=1.16.0 <2.0.0"
         }
       }
     };

package/dist/index.cjs

@@ -35,7 +35,7 @@ var require_package = __commonJS({
   "package.json"(exports2, module2) {
     module2.exports = {
       name: "@mendable/firecrawl-js",
-      version: "4.18.2",
+      version: "4.18.4",
       description: "JavaScript SDK for Firecrawl API",
       main: "dist/index.js",
       types: "dist/index.d.ts",
@@ -105,7 +105,8 @@ var require_package = __commonJS({
           "picomatch@<4.0.4": ">=4.0.4",
           handlebars: ">=4.7.9",
           "brace-expansion": ">=5.0.5",
-          "axios@<1.15.0": "1.15.0"
+          "axios@<1.15.0": "1.15.0",
+          "follow-redirects@<1.16.0": ">=1.16.0 <2.0.0"
         }
       }
     };
@@ -589,6 +590,7 @@ function prepareCrawlPayload(request) {
   if (request.includePaths) data.includePaths = request.includePaths;
   if (request.maxDiscoveryDepth != null) data.maxDiscoveryDepth = request.maxDiscoveryDepth;
   if (request.sitemap != null) data.sitemap = request.sitemap;
+  if (request.robotsUserAgent != null) data.robotsUserAgent = request.robotsUserAgent;
   if (request.ignoreQueryParameters != null) data.ignoreQueryParameters = request.ignoreQueryParameters;
   if (request.deduplicateSimilarURLs != null) data.deduplicateSimilarURLs = request.deduplicateSimilarURLs;
   if (request.limit != null) data.limit = request.limit;

package/dist/index.d.cts

@@ -400,6 +400,8 @@ interface CrawlOptions {
     crawlEntireDomain?: boolean;
     allowExternalLinks?: boolean;
     allowSubdomains?: boolean;
+    ignoreRobotsTxt?: boolean;
+    robotsUserAgent?: string | null;
     delay?: number | null;
     maxConcurrency?: number | null;
     webhook?: string | WebhookConfig | null;

package/dist/index.d.ts

@@ -400,6 +400,8 @@ interface CrawlOptions {
     crawlEntireDomain?: boolean;
     allowExternalLinks?: boolean;
     allowSubdomains?: boolean;
+    ignoreRobotsTxt?: boolean;
+    robotsUserAgent?: string | null;
     delay?: number | null;
     maxConcurrency?: number | null;
     webhook?: string | WebhookConfig | null;

package/dist/index.js

@@ -1,6 +1,6 @@
 import {
   require_package
-} from "./chunk-QU5TGKGP.js";
+} from "./chunk-ZUJQPZTD.js";
 
 // src/v2/utils/httpClient.ts
 import axios from "axios";
@@ -466,6 +466,7 @@ function prepareCrawlPayload(request) {
   if (request.includePaths) data.includePaths = request.includePaths;
   if (request.maxDiscoveryDepth != null) data.maxDiscoveryDepth = request.maxDiscoveryDepth;
   if (request.sitemap != null) data.sitemap = request.sitemap;
+  if (request.robotsUserAgent != null) data.robotsUserAgent = request.robotsUserAgent;
   if (request.ignoreQueryParameters != null) data.ignoreQueryParameters = request.ignoreQueryParameters;
   if (request.deduplicateSimilarURLs != null) data.deduplicateSimilarURLs = request.deduplicateSimilarURLs;
   if (request.limit != null) data.limit = request.limit;
@@ -1559,7 +1560,7 @@ var FirecrawlApp = class {
       if (typeof process !== "undefined" && process.env && process.env.npm_package_version) {
         return process.env.npm_package_version;
       }
-      const packageJson = await import("./package-ZEW57A2P.js");
+      const packageJson = await import("./package-43GY3VT3.js");
       return packageJson.default.version;
     } catch (error) {
       const isTest = typeof process !== "undefined" && (process.env.JEST_WORKER_ID != null || false);

package/dist/{package-ZEW57A2P.js → package-43GY3VT3.js}

@@ -1,4 +1,4 @@
 import {
   require_package
-} from "./chunk-QU5TGKGP.js";
+} from "./chunk-ZUJQPZTD.js";
 export default require_package();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "firecrawl",
-  "version": "4.18.2",
+  "version": "4.18.4",
   "description": "JavaScript SDK for Firecrawl API",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

package/src/v2/methods/crawl.ts

@@ -27,6 +27,7 @@ function prepareCrawlPayload(request: CrawlRequest): Record<string, unknown> {
   if (request.includePaths) data.includePaths = request.includePaths;
   if (request.maxDiscoveryDepth != null) data.maxDiscoveryDepth = request.maxDiscoveryDepth;
   if (request.sitemap != null) data.sitemap = request.sitemap;
+  if (request.robotsUserAgent != null) data.robotsUserAgent = request.robotsUserAgent;
   if (request.ignoreQueryParameters != null) data.ignoreQueryParameters = request.ignoreQueryParameters;
   if (request.deduplicateSimilarURLs != null) data.deduplicateSimilarURLs = request.deduplicateSimilarURLs;
   if (request.limit != null) data.limit = request.limit;

package/src/v2/types.ts

@@ -482,6 +482,8 @@ export interface CrawlOptions {
   crawlEntireDomain?: boolean;
   allowExternalLinks?: boolean;
   allowSubdomains?: boolean;
+  ignoreRobotsTxt?: boolean;
+  robotsUserAgent?: string | null;
   delay?: number | null;
   maxConcurrency?: number | null;
   webhook?: string | WebhookConfig | null;