@mendable/firecrawl 1.23.9 → 1.25.3

package/dist/index.cjs

@@ -5,6 +5,9 @@ var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
 var __getOwnPropNames = Object.getOwnPropertyNames;
 var __getProtoOf = Object.getPrototypeOf;
 var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __commonJS = (cb, mod) => function __require() {
+  return mod || (0, cb[__getOwnPropNames(cb)[0]])((mod = { exports: {} }).exports, mod), mod.exports;
+};
 var __export = (target, all) => {
   for (var name in all)
     __defProp(target, name, { get: all[name], enumerable: true });
@@ -27,6 +30,76 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
 ));
 var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
 
+// package.json
+var require_package = __commonJS({
+  "package.json"(exports2, module2) {
+    module2.exports = {
+      name: "@mendable/firecrawl-js",
+      version: "1.25.3",
+      description: "JavaScript SDK for Firecrawl API",
+      main: "dist/index.js",
+      types: "dist/index.d.ts",
+      exports: {
+        "./package.json": "./package.json",
+        ".": {
+          import: "./dist/index.js",
+          default: "./dist/index.cjs"
+        }
+      },
+      type: "module",
+      scripts: {
+        build: "tsup",
+        "build-and-publish": "npm run build && npm publish --access public",
+        "publish-beta": "npm run build && npm publish --access public --tag beta",
+        test: "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/v1/**/*.test.ts"
+      },
+      repository: {
+        type: "git",
+        url: "git+https://github.com/mendableai/firecrawl.git"
+      },
+      author: "Mendable.ai",
+      license: "MIT",
+      dependencies: {
+        "typescript-event-target": "^1.1.1",
+        zod: "^3.23.8",
+        "zod-to-json-schema": "^3.23.0",
+        axios: "^1.6.8"
+      },
+      bugs: {
+        url: "https://github.com/mendableai/firecrawl/issues"
+      },
+      homepage: "https://github.com/mendableai/firecrawl#readme",
+      devDependencies: {
+        "@jest/globals": "^29.7.0",
+        "@types/axios": "^0.14.0",
+        "@types/dotenv": "^8.2.0",
+        "@types/jest": "^29.5.14",
+        "@types/mocha": "^10.0.6",
+        "@types/node": "^20.12.12",
+        "@types/uuid": "^9.0.8",
+        dotenv: "^16.4.5",
+        jest: "^29.7.0",
+        "ts-jest": "^29.2.2",
+        tsup: "^8.2.4",
+        typescript: "^5.4.5",
+        uuid: "^9.0.1"
+      },
+      keywords: [
+        "firecrawl",
+        "mendable",
+        "crawler",
+        "web",
+        "scraper",
+        "api",
+        "sdk"
+      ],
+      engines: {
+        node: ">=22.0.0"
+      }
+    };
+  }
+});
+
 // src/index.ts
 var src_exports = {};
 __export(src_exports, {
@@ -59,9 +132,22 @@ var FirecrawlError = class extends Error {
 var FirecrawlApp = class {
   apiKey;
   apiUrl;
+  version = "1.25.1";
   isCloudService(url) {
     return url.includes("api.firecrawl.dev");
   }
+  async getVersion() {
+    try {
+      const packageJson = await Promise.resolve().then(() => __toESM(require_package(), 1));
+      return packageJson.default.version;
+    } catch (error) {
+      console.error("Error getting version:", error);
+      return "1.25.1";
+    }
+  }
+  async init() {
+    this.version = await this.getVersion();
+  }
   /**
    * Initializes a new instance of the FirecrawlApp class.
    * @param config - Configuration options for the FirecrawlApp instance.
@@ -73,6 +159,7 @@ var FirecrawlApp = class {
     }
     this.apiKey = apiKey || "";
     this.apiUrl = baseUrl;
+    this.init();
   }
   /**
    * Scrapes a URL using the Firecrawl API.
@@ -85,7 +172,7 @@ var FirecrawlApp = class {
       "Content-Type": "application/json",
       Authorization: `Bearer ${this.apiKey}`
     };
-    let jsonData = { url, ...params };
+    let jsonData = { url, ...params, origin: `js-sdk@${this.version}` };
     if (jsonData?.extract?.schema) {
       let schema = jsonData.extract.schema;
       try {
@@ -159,7 +246,7 @@ var FirecrawlApp = class {
       lang: params?.lang ?? "en",
       country: params?.country ?? "us",
       location: params?.location,
-      origin: params?.origin ?? "api",
+      origin: `js-sdk@${this.version}`,
       timeout: params?.timeout ?? 6e4,
       scrapeOptions: params?.scrapeOptions ?? { formats: [] }
     };
@@ -219,7 +306,7 @@ var FirecrawlApp = class {
    */
   async crawlUrl(url, params, pollInterval = 2, idempotencyKey) {
     const headers = this.prepareHeaders(idempotencyKey);
-    let jsonData = { url, ...params };
+    let jsonData = { url, ...params, origin: `js-sdk@${this.version}` };
     try {
       const response = await this.postRequest(
         this.apiUrl + `/v1/crawl`,
@@ -243,7 +330,7 @@ var FirecrawlApp = class {
   }
   async asyncCrawlUrl(url, params, idempotencyKey) {
     const headers = this.prepareHeaders(idempotencyKey);
-    let jsonData = { url, ...params };
+    let jsonData = { url, ...params, origin: `js-sdk@${this.version}` };
     try {
       const response = await this.postRequest(
         this.apiUrl + `/v1/crawl`,
@@ -402,7 +489,7 @@ var FirecrawlApp = class {
    */
   async mapUrl(url, params) {
     const headers = this.prepareHeaders();
-    let jsonData = { url, ...params };
+    let jsonData = { url, ...params, origin: `js-sdk@${this.version}` };
     try {
       const response = await this.postRequest(
         this.apiUrl + `/v1/map`,
@@ -431,7 +518,7 @@ var FirecrawlApp = class {
    */
   async batchScrapeUrls(urls, params, pollInterval = 2, idempotencyKey, webhook, ignoreInvalidURLs) {
     const headers = this.prepareHeaders(idempotencyKey);
-    let jsonData = { urls, webhook, ignoreInvalidURLs, ...params };
+    let jsonData = { urls, webhook, ignoreInvalidURLs, ...params, origin: `js-sdk@${this.version}` };
     if (jsonData?.extract?.schema) {
       let schema = jsonData.extract.schema;
       try {
@@ -483,7 +570,7 @@ var FirecrawlApp = class {
   }
   async asyncBatchScrapeUrls(urls, params, idempotencyKey, webhook, ignoreInvalidURLs) {
     const headers = this.prepareHeaders(idempotencyKey);
-    let jsonData = { urls, webhook, ignoreInvalidURLs, ...params ?? {} };
+    let jsonData = { urls, webhook, ignoreInvalidURLs, ...params, origin: `js-sdk@${this.version}` };
     try {
       const response = await this.postRequest(
         this.apiUrl + `/v1/batch/scrape`,
@@ -637,7 +724,7 @@ var FirecrawlApp = class {
     try {
       const response = await this.postRequest(
         this.apiUrl + `/v1/extract`,
-        { ...jsonData, schema: jsonSchema, origin: params?.origin || "api-sdk" },
+        { ...jsonData, schema: jsonSchema, origin: `js-sdk@${this.version}` },
         headers
       );
       if (response.status === 200) {
@@ -697,7 +784,7 @@ var FirecrawlApp = class {
     try {
       const response = await this.postRequest(
         this.apiUrl + `/v1/extract`,
-        { ...jsonData, schema: jsonSchema },
+        { ...jsonData, schema: jsonSchema, origin: `js-sdk@${this.version}` },
         headers
       );
       if (response.status === 200) {
@@ -849,6 +936,12 @@ var FirecrawlApp = class {
    * @param {string} action - The action being performed when the error occurred.
    */
   handleError(response, action) {
+    if (!response) {
+      throw new FirecrawlError(
+        `No response received while trying to ${action}. This may be a network error or the server is unreachable.`,
+        0
+      );
+    }
     if ([400, 402, 403, 408, 409, 500].includes(response.status)) {
       const errorMessage = response.data.error || "Unknown error occurred";
       const details = response.data.details ? ` - ${JSON.stringify(response.data.details)}` : "";
@@ -930,7 +1023,7 @@ var FirecrawlApp = class {
    */
   async asyncDeepResearch(query, params) {
     const headers = this.prepareHeaders();
-    let jsonData = { query, ...params };
+    let jsonData = { query, ...params, origin: `js-sdk@${this.version}` };
     if (jsonData?.jsonOptions?.schema) {
       let schema = jsonData.jsonOptions.schema;
       try {
@@ -1053,9 +1146,10 @@ var FirecrawlApp = class {
   async __asyncDeepResearch(topic, params) {
     const headers = this.prepareHeaders();
     try {
+      let jsonData = { topic, ...params, origin: `js-sdk@${this.version}` };
       const response = await this.postRequest(
         `${this.apiUrl}/v1/deep-research`,
-        { topic, ...params },
+        jsonData,
         headers
       );
       if (response.status === 200) {
@@ -1150,10 +1244,11 @@ var FirecrawlApp = class {
    */
   async asyncGenerateLLMsText(url, params) {
     const headers = this.prepareHeaders();
+    let jsonData = { url, ...params, origin: `js-sdk@${this.version}` };
     try {
       const response = await this.postRequest(
         `${this.apiUrl}/v1/llmstxt`,
-        { url, ...params },
+        jsonData,
         headers
       );
       if (response.status === 200) {

package/dist/index.d.cts

@@ -114,7 +114,8 @@ interface CrawlScrapeOptions {
     skipTlsVerification?: boolean;
     removeBase64Images?: boolean;
     blockAds?: boolean;
-    proxy?: "basic" | "stealth";
+    proxy?: "basic" | "stealth" | "auto";
+    storeInCache?: boolean;
 }
 type Action = {
     type: "wait";
@@ -160,6 +161,7 @@ interface ScrapeParams<LLMSchema extends zt.ZodSchema = any, ActionsSchema exten
         modes?: ("json" | "git-diff")[];
     };
     actions?: ActionsSchema;
+    agent?: AgentOptions;
 }
 interface ActionsResult {
     screenshots: string[];
@@ -204,6 +206,11 @@ interface CrawlParams {
     deduplicateSimilarURLs?: boolean;
     ignoreQueryParameters?: boolean;
     regexOnFullURL?: boolean;
+    /**
+     * Delay in seconds between scrapes. This helps respect website rate limits.
+     * If not provided, the crawler may use the robots.txt crawl delay if available.
+     */
+    delay?: number;
 }
 /**
  * Response interface for crawling operations.
@@ -279,6 +286,19 @@ interface MapResponse {
  * Parameters for extracting information from URLs.
  * Defines options for extracting information from URLs.
  */
+interface AgentOptions {
+    model?: string;
+    prompt?: string;
+    sessionId?: string;
+}
+/**
+ * Parameters for extracting information from URLs.
+ * Defines options for extracting information from URLs.
+ */
+interface AgentOptionsExtract {
+    model?: string;
+    sessionId?: string;
+}
 interface ExtractParams<LLMSchema extends zt.ZodSchema = any> {
     prompt?: string;
     schema?: LLMSchema | object;
@@ -289,6 +309,7 @@ interface ExtractParams<LLMSchema extends zt.ZodSchema = any> {
     origin?: string;
     showSources?: boolean;
     scrapeOptions?: CrawlScrapeOptions;
+    agent?: AgentOptionsExtract;
 }
 /**
  * Response interface for extracting information from URLs.
@@ -462,6 +483,11 @@ interface GenerateLLMsTextParams {
      * @default false
      */
     showFullText?: boolean;
+    /**
+     * Whether to use cached content if available
+     * @default true
+     */
+    cache?: boolean;
     /**
      * Experimental flag for streaming
      */
@@ -494,7 +520,10 @@ interface GenerateLLMsTextStatusResponse {
 declare class FirecrawlApp {
     apiKey: string;
     apiUrl: string;
+    version: string;
     private isCloudService;
+    private getVersion;
+    private init;
     /**
      * Initializes a new instance of the FirecrawlApp class.
      * @param config - Configuration options for the FirecrawlApp instance.
@@ -765,4 +794,4 @@ declare class CrawlWatcher extends TypedEventTarget<CrawlWatcherEvents> {
     close(): void;
 }
 
-export { type Action, type ActionsResult, type BatchScrapeResponse, type BatchScrapeStatusResponse, type CrawlErrorsResponse, type CrawlParams, type CrawlResponse, type CrawlScrapeOptions, type CrawlStatusResponse, CrawlWatcher, type DeepResearchParams, type DeepResearchResponse, type DeepResearchStatusResponse, type ErrorResponse, type ExtractParams, type ExtractResponse, type FirecrawlAppConfig, type FirecrawlDocument, type FirecrawlDocumentMetadata, FirecrawlError, type GenerateLLMsTextParams, type GenerateLLMsTextResponse, type GenerateLLMsTextStatusResponse, type MapParams, type MapResponse, type ScrapeParams, type ScrapeResponse, type SearchParams, type SearchResponse, FirecrawlApp as default };
+export { type Action, type ActionsResult, type AgentOptions, type AgentOptionsExtract, type BatchScrapeResponse, type BatchScrapeStatusResponse, type CrawlErrorsResponse, type CrawlParams, type CrawlResponse, type CrawlScrapeOptions, type CrawlStatusResponse, CrawlWatcher, type DeepResearchParams, type DeepResearchResponse, type DeepResearchStatusResponse, type ErrorResponse, type ExtractParams, type ExtractResponse, type FirecrawlAppConfig, type FirecrawlDocument, type FirecrawlDocumentMetadata, FirecrawlError, type GenerateLLMsTextParams, type GenerateLLMsTextResponse, type GenerateLLMsTextStatusResponse, type MapParams, type MapResponse, type ScrapeParams, type ScrapeResponse, type SearchParams, type SearchResponse, FirecrawlApp as default };

package/dist/index.d.ts

@@ -114,7 +114,8 @@ interface CrawlScrapeOptions {
     skipTlsVerification?: boolean;
     removeBase64Images?: boolean;
     blockAds?: boolean;
-    proxy?: "basic" | "stealth";
+    proxy?: "basic" | "stealth" | "auto";
+    storeInCache?: boolean;
 }
 type Action = {
     type: "wait";
@@ -160,6 +161,7 @@ interface ScrapeParams<LLMSchema extends zt.ZodSchema = any, ActionsSchema exten
         modes?: ("json" | "git-diff")[];
     };
     actions?: ActionsSchema;
+    agent?: AgentOptions;
 }
 interface ActionsResult {
     screenshots: string[];
@@ -204,6 +206,11 @@ interface CrawlParams {
     deduplicateSimilarURLs?: boolean;
     ignoreQueryParameters?: boolean;
     regexOnFullURL?: boolean;
+    /**
+     * Delay in seconds between scrapes. This helps respect website rate limits.
+     * If not provided, the crawler may use the robots.txt crawl delay if available.
+     */
+    delay?: number;
 }
 /**
  * Response interface for crawling operations.
@@ -279,6 +286,19 @@ interface MapResponse {
  * Parameters for extracting information from URLs.
  * Defines options for extracting information from URLs.
  */
+interface AgentOptions {
+    model?: string;
+    prompt?: string;
+    sessionId?: string;
+}
+/**
+ * Parameters for extracting information from URLs.
+ * Defines options for extracting information from URLs.
+ */
+interface AgentOptionsExtract {
+    model?: string;
+    sessionId?: string;
+}
 interface ExtractParams<LLMSchema extends zt.ZodSchema = any> {
     prompt?: string;
     schema?: LLMSchema | object;
@@ -289,6 +309,7 @@ interface ExtractParams<LLMSchema extends zt.ZodSchema = any> {
     origin?: string;
     showSources?: boolean;
     scrapeOptions?: CrawlScrapeOptions;
+    agent?: AgentOptionsExtract;
 }
 /**
  * Response interface for extracting information from URLs.
@@ -462,6 +483,11 @@ interface GenerateLLMsTextParams {
      * @default false
      */
     showFullText?: boolean;
+    /**
+     * Whether to use cached content if available
+     * @default true
+     */
+    cache?: boolean;
     /**
      * Experimental flag for streaming
      */
@@ -494,7 +520,10 @@ interface GenerateLLMsTextStatusResponse {
 declare class FirecrawlApp {
     apiKey: string;
     apiUrl: string;
+    version: string;
     private isCloudService;
+    private getVersion;
+    private init;
     /**
      * Initializes a new instance of the FirecrawlApp class.
      * @param config - Configuration options for the FirecrawlApp instance.
@@ -765,4 +794,4 @@ declare class CrawlWatcher extends TypedEventTarget<CrawlWatcherEvents> {
     close(): void;
 }
 
-export { type Action, type ActionsResult, type BatchScrapeResponse, type BatchScrapeStatusResponse, type CrawlErrorsResponse, type CrawlParams, type CrawlResponse, type CrawlScrapeOptions, type CrawlStatusResponse, CrawlWatcher, type DeepResearchParams, type DeepResearchResponse, type DeepResearchStatusResponse, type ErrorResponse, type ExtractParams, type ExtractResponse, type FirecrawlAppConfig, type FirecrawlDocument, type FirecrawlDocumentMetadata, FirecrawlError, type GenerateLLMsTextParams, type GenerateLLMsTextResponse, type GenerateLLMsTextStatusResponse, type MapParams, type MapResponse, type ScrapeParams, type ScrapeResponse, type SearchParams, type SearchResponse, FirecrawlApp as default };
+export { type Action, type ActionsResult, type AgentOptions, type AgentOptionsExtract, type BatchScrapeResponse, type BatchScrapeStatusResponse, type CrawlErrorsResponse, type CrawlParams, type CrawlResponse, type CrawlScrapeOptions, type CrawlStatusResponse, CrawlWatcher, type DeepResearchParams, type DeepResearchResponse, type DeepResearchStatusResponse, type ErrorResponse, type ExtractParams, type ExtractResponse, type FirecrawlAppConfig, type FirecrawlDocument, type FirecrawlDocumentMetadata, FirecrawlError, type GenerateLLMsTextParams, type GenerateLLMsTextResponse, type GenerateLLMsTextStatusResponse, type MapParams, type MapResponse, type ScrapeParams, type ScrapeResponse, type SearchParams, type SearchResponse, FirecrawlApp as default };

package/dist/index.js

@@ -23,9 +23,22 @@ var FirecrawlError = class extends Error {
 var FirecrawlApp = class {
   apiKey;
   apiUrl;
+  version = "1.25.1";
   isCloudService(url) {
     return url.includes("api.firecrawl.dev");
   }
+  async getVersion() {
+    try {
+      const packageJson = await import("./package-Y5V6L2WQ.js");
+      return packageJson.default.version;
+    } catch (error) {
+      console.error("Error getting version:", error);
+      return "1.25.1";
+    }
+  }
+  async init() {
+    this.version = await this.getVersion();
+  }
   /**
    * Initializes a new instance of the FirecrawlApp class.
    * @param config - Configuration options for the FirecrawlApp instance.
@@ -37,6 +50,7 @@ var FirecrawlApp = class {
     }
     this.apiKey = apiKey || "";
     this.apiUrl = baseUrl;
+    this.init();
   }
   /**
    * Scrapes a URL using the Firecrawl API.
@@ -49,7 +63,7 @@ var FirecrawlApp = class {
       "Content-Type": "application/json",
       Authorization: `Bearer ${this.apiKey}`
     };
-    let jsonData = { url, ...params };
+    let jsonData = { url, ...params, origin: `js-sdk@${this.version}` };
     if (jsonData?.extract?.schema) {
       let schema = jsonData.extract.schema;
       try {
@@ -123,7 +137,7 @@ var FirecrawlApp = class {
       lang: params?.lang ?? "en",
       country: params?.country ?? "us",
       location: params?.location,
-      origin: params?.origin ?? "api",
+      origin: `js-sdk@${this.version}`,
       timeout: params?.timeout ?? 6e4,
       scrapeOptions: params?.scrapeOptions ?? { formats: [] }
     };
@@ -183,7 +197,7 @@ var FirecrawlApp = class {
    */
   async crawlUrl(url, params, pollInterval = 2, idempotencyKey) {
     const headers = this.prepareHeaders(idempotencyKey);
-    let jsonData = { url, ...params };
+    let jsonData = { url, ...params, origin: `js-sdk@${this.version}` };
     try {
       const response = await this.postRequest(
         this.apiUrl + `/v1/crawl`,
@@ -207,7 +221,7 @@ var FirecrawlApp = class {
   }
   async asyncCrawlUrl(url, params, idempotencyKey) {
     const headers = this.prepareHeaders(idempotencyKey);
-    let jsonData = { url, ...params };
+    let jsonData = { url, ...params, origin: `js-sdk@${this.version}` };
     try {
       const response = await this.postRequest(
         this.apiUrl + `/v1/crawl`,
@@ -366,7 +380,7 @@ var FirecrawlApp = class {
    */
   async mapUrl(url, params) {
     const headers = this.prepareHeaders();
-    let jsonData = { url, ...params };
+    let jsonData = { url, ...params, origin: `js-sdk@${this.version}` };
     try {
       const response = await this.postRequest(
         this.apiUrl + `/v1/map`,
@@ -395,7 +409,7 @@ var FirecrawlApp = class {
    */
   async batchScrapeUrls(urls, params, pollInterval = 2, idempotencyKey, webhook, ignoreInvalidURLs) {
     const headers = this.prepareHeaders(idempotencyKey);
-    let jsonData = { urls, webhook, ignoreInvalidURLs, ...params };
+    let jsonData = { urls, webhook, ignoreInvalidURLs, ...params, origin: `js-sdk@${this.version}` };
     if (jsonData?.extract?.schema) {
       let schema = jsonData.extract.schema;
       try {
@@ -447,7 +461,7 @@ var FirecrawlApp = class {
   }
   async asyncBatchScrapeUrls(urls, params, idempotencyKey, webhook, ignoreInvalidURLs) {
     const headers = this.prepareHeaders(idempotencyKey);
-    let jsonData = { urls, webhook, ignoreInvalidURLs, ...params ?? {} };
+    let jsonData = { urls, webhook, ignoreInvalidURLs, ...params, origin: `js-sdk@${this.version}` };
     try {
       const response = await this.postRequest(
         this.apiUrl + `/v1/batch/scrape`,
@@ -601,7 +615,7 @@ var FirecrawlApp = class {
     try {
       const response = await this.postRequest(
         this.apiUrl + `/v1/extract`,
-        { ...jsonData, schema: jsonSchema, origin: params?.origin || "api-sdk" },
+        { ...jsonData, schema: jsonSchema, origin: `js-sdk@${this.version}` },
         headers
       );
       if (response.status === 200) {
@@ -661,7 +675,7 @@ var FirecrawlApp = class {
     try {
       const response = await this.postRequest(
         this.apiUrl + `/v1/extract`,
-        { ...jsonData, schema: jsonSchema },
+        { ...jsonData, schema: jsonSchema, origin: `js-sdk@${this.version}` },
         headers
       );
       if (response.status === 200) {
@@ -813,6 +827,12 @@ var FirecrawlApp = class {
    * @param {string} action - The action being performed when the error occurred.
    */
   handleError(response, action) {
+    if (!response) {
+      throw new FirecrawlError(
+        `No response received while trying to ${action}. This may be a network error or the server is unreachable.`,
+        0
+      );
+    }
     if ([400, 402, 403, 408, 409, 500].includes(response.status)) {
       const errorMessage = response.data.error || "Unknown error occurred";
       const details = response.data.details ? ` - ${JSON.stringify(response.data.details)}` : "";
@@ -894,7 +914,7 @@ var FirecrawlApp = class {
    */
   async asyncDeepResearch(query, params) {
     const headers = this.prepareHeaders();
-    let jsonData = { query, ...params };
+    let jsonData = { query, ...params, origin: `js-sdk@${this.version}` };
     if (jsonData?.jsonOptions?.schema) {
       let schema = jsonData.jsonOptions.schema;
       try {
@@ -1017,9 +1037,10 @@ var FirecrawlApp = class {
   async __asyncDeepResearch(topic, params) {
     const headers = this.prepareHeaders();
     try {
+      let jsonData = { topic, ...params, origin: `js-sdk@${this.version}` };
       const response = await this.postRequest(
         `${this.apiUrl}/v1/deep-research`,
-        { topic, ...params },
+        jsonData,
         headers
       );
       if (response.status === 200) {
@@ -1114,10 +1135,11 @@ var FirecrawlApp = class {
    */
   async asyncGenerateLLMsText(url, params) {
     const headers = this.prepareHeaders();
+    let jsonData = { url, ...params, origin: `js-sdk@${this.version}` };
     try {
       const response = await this.postRequest(
         `${this.apiUrl}/v1/llmstxt`,
-        { url, ...params },
+        jsonData,
         headers
       );
       if (response.status === 200) {

package/dist/package-Y5V6L2WQ.js

@@ -0,0 +1,102 @@
+// package.json
+var name = "@mendable/firecrawl-js";
+var version = "1.25.3";
+var description = "JavaScript SDK for Firecrawl API";
+var main = "dist/index.js";
+var types = "dist/index.d.ts";
+var exports = {
+  "./package.json": "./package.json",
+  ".": {
+    import: "./dist/index.js",
+    default: "./dist/index.cjs"
+  }
+};
+var type = "module";
+var scripts = {
+  build: "tsup",
+  "build-and-publish": "npm run build && npm publish --access public",
+  "publish-beta": "npm run build && npm publish --access public --tag beta",
+  test: "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/v1/**/*.test.ts"
+};
+var repository = {
+  type: "git",
+  url: "git+https://github.com/mendableai/firecrawl.git"
+};
+var author = "Mendable.ai";
+var license = "MIT";
+var dependencies = {
+  "typescript-event-target": "^1.1.1",
+  zod: "^3.23.8",
+  "zod-to-json-schema": "^3.23.0",
+  axios: "^1.6.8"
+};
+var bugs = {
+  url: "https://github.com/mendableai/firecrawl/issues"
+};
+var homepage = "https://github.com/mendableai/firecrawl#readme";
+var devDependencies = {
+  "@jest/globals": "^29.7.0",
+  "@types/axios": "^0.14.0",
+  "@types/dotenv": "^8.2.0",
+  "@types/jest": "^29.5.14",
+  "@types/mocha": "^10.0.6",
+  "@types/node": "^20.12.12",
+  "@types/uuid": "^9.0.8",
+  dotenv: "^16.4.5",
+  jest: "^29.7.0",
+  "ts-jest": "^29.2.2",
+  tsup: "^8.2.4",
+  typescript: "^5.4.5",
+  uuid: "^9.0.1"
+};
+var keywords = [
+  "firecrawl",
+  "mendable",
+  "crawler",
+  "web",
+  "scraper",
+  "api",
+  "sdk"
+];
+var engines = {
+  node: ">=22.0.0"
+};
+var package_default = {
+  name,
+  version,
+  description,
+  main,
+  types,
+  exports,
+  type,
+  scripts,
+  repository,
+  author,
+  license,
+  dependencies,
+  bugs,
+  homepage,
+  devDependencies,
+  keywords,
+  engines
+};
+export {
+  author,
+  bugs,
+  package_default as default,
+  dependencies,
+  description,
+  devDependencies,
+  engines,
+  exports,
+  homepage,
+  keywords,
+  license,
+  main,
+  name,
+  repository,
+  scripts,
+  type,
+  types,
+  version
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mendable/firecrawl",
-  "version": "1.23.9",
+  "version": "1.25.3",
   "description": "JavaScript SDK for Firecrawl API",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

package/src/index.ts

@@ -119,7 +119,8 @@ export interface CrawlScrapeOptions {
   skipTlsVerification?: boolean;
   removeBase64Images?: boolean;
   blockAds?: boolean;
-  proxy?: "basic" | "stealth";
+  proxy?: "basic" | "stealth" | "auto";
+  storeInCache?: boolean;
 }
 
 export type Action = {
@@ -167,6 +168,7 @@ export interface ScrapeParams<LLMSchema extends zt.ZodSchema = any, ActionsSchem
     modes?: ("json" | "git-diff")[];
   }
   actions?: ActionsSchema;
+  agent?: AgentOptions;
 }
 
 export interface ActionsResult {
@@ -214,6 +216,11 @@ export interface CrawlParams {
   deduplicateSimilarURLs?: boolean;
   ignoreQueryParameters?: boolean;
   regexOnFullURL?: boolean;
+  /**
+   * Delay in seconds between scrapes. This helps respect website rate limits.
+   * If not provided, the crawler may use the robots.txt crawl delay if available.
+   */
+  delay?: number;
 }
 
 /**
@@ -296,6 +303,21 @@ export interface MapResponse {
  * Parameters for extracting information from URLs.
  * Defines options for extracting information from URLs.
  */
+export interface AgentOptions {
+  model?: string;
+  prompt?: string;
+  sessionId?: string;
+}
+
+/**
+ * Parameters for extracting information from URLs.
+ * Defines options for extracting information from URLs.
+ */
+export interface AgentOptionsExtract {
+  model?: string;
+  sessionId?: string;
+}
+
 export interface ExtractParams<LLMSchema extends zt.ZodSchema = any> {
   prompt?: string;
   schema?: LLMSchema | object;
@@ -306,6 +328,7 @@ export interface ExtractParams<LLMSchema extends zt.ZodSchema = any> {
   origin?: string;
   showSources?: boolean;
   scrapeOptions?: CrawlScrapeOptions;
+  agent?: AgentOptionsExtract;
 }
 
 /**
@@ -498,6 +521,11 @@ export interface GenerateLLMsTextParams {
    * @default false
    */
   showFullText?: boolean;
+  /**
+   * Whether to use cached content if available
+   * @default true
+   */
+  cache?: boolean;
   /**
    * Experimental flag for streaming
    */
@@ -533,11 +561,26 @@ export interface GenerateLLMsTextStatusResponse {
 export default class FirecrawlApp {
   public apiKey: string;
   public apiUrl: string;
-
+  public version: string =  "1.25.1";
+  
   private isCloudService(url: string): boolean {
     return url.includes('api.firecrawl.dev');
   }
 
+  private async getVersion(): Promise<string> {
+    try {
+      const packageJson = await import('../package.json', { assert: { type: 'json' } });
+      return packageJson.default.version;
+    } catch (error) {
+      console.error("Error getting version:", error);
+      return  "1.25.1";
+    }
+  }
+
+  private async init() {
+    this.version = await this.getVersion();
+  }
+
   /**
    * Initializes a new instance of the FirecrawlApp class.
    * @param config - Configuration options for the FirecrawlApp instance.
@@ -551,6 +594,7 @@ export default class FirecrawlApp {
 
     this.apiKey = apiKey || '';
     this.apiUrl = baseUrl;
+    this.init();
   }
 
   /**
@@ -567,7 +611,7 @@ export default class FirecrawlApp {
       "Content-Type": "application/json",
       Authorization: `Bearer ${this.apiKey}`,
     } as AxiosRequestHeaders;
-    let jsonData: any = { url, ...params };
+    let jsonData: any = { url, ...params, origin: `js-sdk@${this.version}` };
     if (jsonData?.extract?.schema) {
       let schema = jsonData.extract.schema;
 
@@ -649,7 +693,7 @@ export default class FirecrawlApp {
       lang: params?.lang ?? "en",
       country: params?.country ?? "us",
       location: params?.location,
-      origin: params?.origin ?? "api",
+      origin: `js-sdk@${this.version}`,
       timeout: params?.timeout ?? 60000,
       scrapeOptions: params?.scrapeOptions ?? { formats: [] },
     };
@@ -721,7 +765,7 @@ export default class FirecrawlApp {
     idempotencyKey?: string
   ): Promise<CrawlStatusResponse | ErrorResponse> {
     const headers = this.prepareHeaders(idempotencyKey);
-    let jsonData: any = { url, ...params };
+    let jsonData: any = { url, ...params, origin: `js-sdk@${this.version}` };
     try {
       const response: AxiosResponse = await this.postRequest(
         this.apiUrl + `/v1/crawl`,
@@ -750,7 +794,7 @@ export default class FirecrawlApp {
     idempotencyKey?: string
   ): Promise<CrawlResponse | ErrorResponse> {
     const headers = this.prepareHeaders(idempotencyKey);
-    let jsonData: any = { url, ...params };
+    let jsonData: any = { url, ...params, origin: `js-sdk@${this.version}` };
     try {
       const response: AxiosResponse = await this.postRequest(
         this.apiUrl + `/v1/crawl`,
@@ -926,7 +970,7 @@ export default class FirecrawlApp {
    */
   async mapUrl(url: string, params?: MapParams): Promise<MapResponse | ErrorResponse> {
     const headers = this.prepareHeaders();
-    let jsonData: { url: string } & MapParams = { url, ...params };
+    let jsonData: any = { url, ...params, origin: `js-sdk@${this.version}` };
 
     try {
       const response: AxiosResponse = await this.postRequest(
@@ -964,7 +1008,7 @@ export default class FirecrawlApp {
     ignoreInvalidURLs?: boolean,
   ): Promise<BatchScrapeStatusResponse | ErrorResponse> {
     const headers = this.prepareHeaders(idempotencyKey);
-    let jsonData: any = { urls, webhook, ignoreInvalidURLs, ...params };
+    let jsonData: any = { urls, webhook, ignoreInvalidURLs, ...params, origin: `js-sdk@${this.version}` };
     if (jsonData?.extract?.schema) {
       let schema = jsonData.extract.schema;
 
@@ -1029,7 +1073,7 @@ export default class FirecrawlApp {
     ignoreInvalidURLs?: boolean,
   ): Promise<BatchScrapeResponse | ErrorResponse> {
     const headers = this.prepareHeaders(idempotencyKey);
-    let jsonData: any = { urls, webhook, ignoreInvalidURLs, ...(params ?? {}) };
+    let jsonData: any = { urls, webhook, ignoreInvalidURLs, ...params, origin: `js-sdk@${this.version}` };
     try {
       const response: AxiosResponse = await this.postRequest(
         this.apiUrl + `/v1/batch/scrape`,
@@ -1203,7 +1247,7 @@ export default class FirecrawlApp {
     try {
       const response: AxiosResponse = await this.postRequest(
         this.apiUrl + `/v1/extract`,
-        { ...jsonData, schema: jsonSchema, origin: params?.origin || "api-sdk" },
+        { ...jsonData, schema: jsonSchema, origin: `js-sdk@${this.version}` },
         headers
       );
 
@@ -1271,7 +1315,7 @@ export default class FirecrawlApp {
     try {
       const response: AxiosResponse = await this.postRequest(
         this.apiUrl + `/v1/extract`,
-        { ...jsonData, schema: jsonSchema },
+        { ...jsonData, schema: jsonSchema, origin: `js-sdk@${this.version}` },
         headers
       );
 
@@ -1448,6 +1492,13 @@ export default class FirecrawlApp {
    * @param {string} action - The action being performed when the error occurred.
    */
   handleError(response: AxiosResponse, action: string): void {
+    if (!response) {
+      throw new FirecrawlError(
+        `No response received while trying to ${action}. This may be a network error or the server is unreachable.`,
+        0
+      );
+    }
+
     if ([400, 402, 403, 408, 409, 500].includes(response.status)) {
       const errorMessage: string =
         response.data.error || "Unknown error occurred";
@@ -1562,7 +1613,7 @@ export default class FirecrawlApp {
    */
   async asyncDeepResearch(query: string, params: DeepResearchParams<zt.ZodSchema>): Promise<DeepResearchResponse | ErrorResponse> {
     const headers = this.prepareHeaders();
-    let jsonData: any = { query, ...params };
+    let jsonData: any = { query, ...params, origin: `js-sdk@${this.version}` };
 
     if (jsonData?.jsonOptions?.schema) {
       let schema = jsonData.jsonOptions.schema;
@@ -1570,7 +1621,7 @@ export default class FirecrawlApp {
       try {
         schema = zodToJsonSchema(schema);
       } catch (error) {
-        
+        // Ignore error if schema can't be parsed as Zod
       }
       jsonData = {
         ...jsonData,
@@ -1716,9 +1767,10 @@ export default class FirecrawlApp {
   async __asyncDeepResearch(topic: string, params: DeepResearchParams): Promise<DeepResearchResponse | ErrorResponse> {
     const headers = this.prepareHeaders();
     try {
+      let jsonData: any = { topic, ...params, origin: `js-sdk@${this.version}` };
       const response: AxiosResponse = await this.postRequest(
         `${this.apiUrl}/v1/deep-research`,
-        { topic, ...params },
+        jsonData,
         headers
       );
 
@@ -1828,10 +1880,11 @@ export default class FirecrawlApp {
    */
   async asyncGenerateLLMsText(url: string, params?: GenerateLLMsTextParams): Promise<GenerateLLMsTextResponse | ErrorResponse> {
     const headers = this.prepareHeaders();
+    let jsonData: any = { url, ...params, origin: `js-sdk@${this.version}` };
     try {
       const response: AxiosResponse = await this.postRequest(
         `${this.apiUrl}/v1/llmstxt`,
-        { url, ...params },
+        jsonData,
         headers
       );