@mendable/firecrawl 1.24.0 → 1.25.4

package/dist/index.cjs

@@ -5,6 +5,9 @@ var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
 var __getOwnPropNames = Object.getOwnPropertyNames;
 var __getProtoOf = Object.getPrototypeOf;
 var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __commonJS = (cb, mod) => function __require() {
+  return mod || (0, cb[__getOwnPropNames(cb)[0]])((mod = { exports: {} }).exports, mod), mod.exports;
+};
 var __export = (target, all) => {
   for (var name in all)
     __defProp(target, name, { get: all[name], enumerable: true });
@@ -27,6 +30,76 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
 ));
 var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
 
+// package.json
+var require_package = __commonJS({
+  "package.json"(exports2, module2) {
+    module2.exports = {
+      name: "@mendable/firecrawl-js",
+      version: "1.25.4",
+      description: "JavaScript SDK for Firecrawl API",
+      main: "dist/index.js",
+      types: "dist/index.d.ts",
+      exports: {
+        "./package.json": "./package.json",
+        ".": {
+          import: "./dist/index.js",
+          default: "./dist/index.cjs"
+        }
+      },
+      type: "module",
+      scripts: {
+        build: "tsup",
+        "build-and-publish": "npm run build && npm publish --access public",
+        "publish-beta": "npm run build && npm publish --access public --tag beta",
+        test: "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/v1/**/*.test.ts"
+      },
+      repository: {
+        type: "git",
+        url: "git+https://github.com/mendableai/firecrawl.git"
+      },
+      author: "Mendable.ai",
+      license: "MIT",
+      dependencies: {
+        "typescript-event-target": "^1.1.1",
+        zod: "^3.23.8",
+        "zod-to-json-schema": "^3.23.0",
+        axios: "^1.6.8"
+      },
+      bugs: {
+        url: "https://github.com/mendableai/firecrawl/issues"
+      },
+      homepage: "https://github.com/mendableai/firecrawl#readme",
+      devDependencies: {
+        "@jest/globals": "^29.7.0",
+        "@types/axios": "^0.14.0",
+        "@types/dotenv": "^8.2.0",
+        "@types/jest": "^29.5.14",
+        "@types/mocha": "^10.0.6",
+        "@types/node": "^20.12.12",
+        "@types/uuid": "^9.0.8",
+        dotenv: "^16.4.5",
+        jest: "^29.7.0",
+        "ts-jest": "^29.2.2",
+        tsup: "^8.2.4",
+        typescript: "^5.4.5",
+        uuid: "^9.0.1"
+      },
+      keywords: [
+        "firecrawl",
+        "mendable",
+        "crawler",
+        "web",
+        "scraper",
+        "api",
+        "sdk"
+      ],
+      engines: {
+        node: ">=22.0.0"
+      }
+    };
+  }
+});
+
 // src/index.ts
 var src_exports = {};
 __export(src_exports, {
@@ -59,9 +132,22 @@ var FirecrawlError = class extends Error {
 var FirecrawlApp = class {
   apiKey;
   apiUrl;
+  version = "1.25.1";
   isCloudService(url) {
     return url.includes("api.firecrawl.dev");
   }
+  async getVersion() {
+    try {
+      const packageJson = await Promise.resolve().then(() => __toESM(require_package(), 1));
+      return packageJson.default.version;
+    } catch (error) {
+      console.error("Error getting version:", error);
+      return "1.25.1";
+    }
+  }
+  async init() {
+    this.version = await this.getVersion();
+  }
   /**
    * Initializes a new instance of the FirecrawlApp class.
    * @param config - Configuration options for the FirecrawlApp instance.
@@ -73,6 +159,7 @@ var FirecrawlApp = class {
     }
     this.apiKey = apiKey || "";
     this.apiUrl = baseUrl;
+    this.init();
   }
   /**
    * Scrapes a URL using the Firecrawl API.
@@ -85,7 +172,7 @@ var FirecrawlApp = class {
       "Content-Type": "application/json",
       Authorization: `Bearer ${this.apiKey}`
     };
-    let jsonData = { url, ...params };
+    let jsonData = { url, ...params, origin: `js-sdk@${this.version}` };
     if (jsonData?.extract?.schema) {
       let schema = jsonData.extract.schema;
       try {
@@ -159,7 +246,7 @@ var FirecrawlApp = class {
       lang: params?.lang ?? "en",
       country: params?.country ?? "us",
       location: params?.location,
-      origin: params?.origin ?? "api",
+      origin: `js-sdk@${this.version}`,
       timeout: params?.timeout ?? 6e4,
       scrapeOptions: params?.scrapeOptions ?? { formats: [] }
     };
@@ -219,7 +306,7 @@ var FirecrawlApp = class {
    */
   async crawlUrl(url, params, pollInterval = 2, idempotencyKey) {
     const headers = this.prepareHeaders(idempotencyKey);
-    let jsonData = { url, ...params };
+    let jsonData = { url, ...params, origin: `js-sdk@${this.version}` };
     try {
       const response = await this.postRequest(
         this.apiUrl + `/v1/crawl`,
@@ -243,7 +330,7 @@ var FirecrawlApp = class {
   }
   async asyncCrawlUrl(url, params, idempotencyKey) {
     const headers = this.prepareHeaders(idempotencyKey);
-    let jsonData = { url, ...params };
+    let jsonData = { url, ...params, origin: `js-sdk@${this.version}` };
     try {
       const response = await this.postRequest(
         this.apiUrl + `/v1/crawl`,
@@ -402,7 +489,7 @@ var FirecrawlApp = class {
    */
   async mapUrl(url, params) {
     const headers = this.prepareHeaders();
-    let jsonData = { url, ...params };
+    let jsonData = { url, ...params, origin: `js-sdk@${this.version}` };
     try {
       const response = await this.postRequest(
         this.apiUrl + `/v1/map`,
@@ -431,7 +518,7 @@ var FirecrawlApp = class {
    */
   async batchScrapeUrls(urls, params, pollInterval = 2, idempotencyKey, webhook, ignoreInvalidURLs) {
     const headers = this.prepareHeaders(idempotencyKey);
-    let jsonData = { urls, webhook, ignoreInvalidURLs, ...params };
+    let jsonData = { urls, webhook, ignoreInvalidURLs, ...params, origin: `js-sdk@${this.version}` };
     if (jsonData?.extract?.schema) {
       let schema = jsonData.extract.schema;
       try {
@@ -483,7 +570,7 @@ var FirecrawlApp = class {
   }
   async asyncBatchScrapeUrls(urls, params, idempotencyKey, webhook, ignoreInvalidURLs) {
     const headers = this.prepareHeaders(idempotencyKey);
-    let jsonData = { urls, webhook, ignoreInvalidURLs, ...params ?? {} };
+    let jsonData = { urls, webhook, ignoreInvalidURLs, ...params, origin: `js-sdk@${this.version}` };
     try {
       const response = await this.postRequest(
         this.apiUrl + `/v1/batch/scrape`,
@@ -637,7 +724,7 @@ var FirecrawlApp = class {
     try {
       const response = await this.postRequest(
         this.apiUrl + `/v1/extract`,
-        { ...jsonData, schema: jsonSchema, origin: params?.origin || "api-sdk" },
+        { ...jsonData, schema: jsonSchema, origin: `js-sdk@${this.version}` },
         headers
       );
       if (response.status === 200) {
@@ -697,7 +784,7 @@ var FirecrawlApp = class {
     try {
       const response = await this.postRequest(
         this.apiUrl + `/v1/extract`,
-        { ...jsonData, schema: jsonSchema },
+        { ...jsonData, schema: jsonSchema, origin: `js-sdk@${this.version}` },
         headers
       );
       if (response.status === 200) {
@@ -849,6 +936,12 @@ var FirecrawlApp = class {
    * @param {string} action - The action being performed when the error occurred.
    */
   handleError(response, action) {
+    if (!response) {
+      throw new FirecrawlError(
+        `No response received while trying to ${action}. This may be a network error or the server is unreachable.`,
+        0
+      );
+    }
     if ([400, 402, 403, 408, 409, 500].includes(response.status)) {
       const errorMessage = response.data.error || "Unknown error occurred";
       const details = response.data.details ? ` - ${JSON.stringify(response.data.details)}` : "";
@@ -930,7 +1023,7 @@ var FirecrawlApp = class {
    */
   async asyncDeepResearch(query, params) {
     const headers = this.prepareHeaders();
-    let jsonData = { query, ...params };
+    let jsonData = { query, ...params, origin: `js-sdk@${this.version}` };
     if (jsonData?.jsonOptions?.schema) {
       let schema = jsonData.jsonOptions.schema;
       try {
@@ -1053,9 +1146,10 @@ var FirecrawlApp = class {
   async __asyncDeepResearch(topic, params) {
     const headers = this.prepareHeaders();
     try {
+      let jsonData = { topic, ...params, origin: `js-sdk@${this.version}` };
       const response = await this.postRequest(
         `${this.apiUrl}/v1/deep-research`,
-        { topic, ...params },
+        jsonData,
         headers
       );
       if (response.status === 200) {
@@ -1150,10 +1244,11 @@ var FirecrawlApp = class {
    */
   async asyncGenerateLLMsText(url, params) {
     const headers = this.prepareHeaders();
+    let jsonData = { url, ...params, origin: `js-sdk@${this.version}` };
     try {
       const response = await this.postRequest(
         `${this.apiUrl}/v1/llmstxt`,
-        { url, ...params },
+        jsonData,
         headers
       );
       if (response.status === 200) {

package/dist/index.d.cts

@@ -114,7 +114,8 @@ interface CrawlScrapeOptions {
     skipTlsVerification?: boolean;
     removeBase64Images?: boolean;
     blockAds?: boolean;
-    proxy?: "basic" | "stealth";
+    proxy?: "basic" | "stealth" | "auto";
+    storeInCache?: boolean;
 }
 type Action = {
     type: "wait";
@@ -158,6 +159,7 @@ interface ScrapeParams<LLMSchema extends zt.ZodSchema = any, ActionsSchema exten
         prompt?: string;
         schema?: any;
         modes?: ("json" | "git-diff")[];
+        tag?: string | null;
     };
     actions?: ActionsSchema;
     agent?: AgentOptions;
@@ -205,6 +207,11 @@ interface CrawlParams {
     deduplicateSimilarURLs?: boolean;
     ignoreQueryParameters?: boolean;
     regexOnFullURL?: boolean;
+    /**
+     * Delay in seconds between scrapes. This helps respect website rate limits.
+     * If not provided, the crawler may use the robots.txt crawl delay if available.
+     */
+    delay?: number;
 }
 /**
  * Response interface for crawling operations.
@@ -477,6 +484,11 @@ interface GenerateLLMsTextParams {
      * @default false
      */
     showFullText?: boolean;
+    /**
+     * Whether to use cached content if available
+     * @default true
+     */
+    cache?: boolean;
     /**
      * Experimental flag for streaming
      */
@@ -509,7 +521,10 @@ interface GenerateLLMsTextStatusResponse {
 declare class FirecrawlApp {
     apiKey: string;
     apiUrl: string;
+    version: string;
     private isCloudService;
+    private getVersion;
+    private init;
     /**
      * Initializes a new instance of the FirecrawlApp class.
      * @param config - Configuration options for the FirecrawlApp instance.

package/dist/index.d.ts

@@ -114,7 +114,8 @@ interface CrawlScrapeOptions {
     skipTlsVerification?: boolean;
     removeBase64Images?: boolean;
     blockAds?: boolean;
-    proxy?: "basic" | "stealth";
+    proxy?: "basic" | "stealth" | "auto";
+    storeInCache?: boolean;
 }
 type Action = {
     type: "wait";
@@ -158,6 +159,7 @@ interface ScrapeParams<LLMSchema extends zt.ZodSchema = any, ActionsSchema exten
         prompt?: string;
         schema?: any;
         modes?: ("json" | "git-diff")[];
+        tag?: string | null;
     };
     actions?: ActionsSchema;
     agent?: AgentOptions;
@@ -205,6 +207,11 @@ interface CrawlParams {
     deduplicateSimilarURLs?: boolean;
     ignoreQueryParameters?: boolean;
     regexOnFullURL?: boolean;
+    /**
+     * Delay in seconds between scrapes. This helps respect website rate limits.
+     * If not provided, the crawler may use the robots.txt crawl delay if available.
+     */
+    delay?: number;
 }
 /**
  * Response interface for crawling operations.
@@ -477,6 +484,11 @@ interface GenerateLLMsTextParams {
      * @default false
      */
     showFullText?: boolean;
+    /**
+     * Whether to use cached content if available
+     * @default true
+     */
+    cache?: boolean;
     /**
      * Experimental flag for streaming
      */
@@ -509,7 +521,10 @@ interface GenerateLLMsTextStatusResponse {
 declare class FirecrawlApp {
     apiKey: string;
     apiUrl: string;
+    version: string;
     private isCloudService;
+    private getVersion;
+    private init;
     /**
      * Initializes a new instance of the FirecrawlApp class.
      * @param config - Configuration options for the FirecrawlApp instance.

package/dist/index.js

@@ -23,9 +23,22 @@ var FirecrawlError = class extends Error {
 var FirecrawlApp = class {
   apiKey;
   apiUrl;
+  version = "1.25.1";
   isCloudService(url) {
     return url.includes("api.firecrawl.dev");
   }
+  async getVersion() {
+    try {
+      const packageJson = await import("./package-SG22PRGT.js");
+      return packageJson.default.version;
+    } catch (error) {
+      console.error("Error getting version:", error);
+      return "1.25.1";
+    }
+  }
+  async init() {
+    this.version = await this.getVersion();
+  }
   /**
    * Initializes a new instance of the FirecrawlApp class.
    * @param config - Configuration options for the FirecrawlApp instance.
@@ -37,6 +50,7 @@ var FirecrawlApp = class {
     }
     this.apiKey = apiKey || "";
     this.apiUrl = baseUrl;
+    this.init();
   }
   /**
    * Scrapes a URL using the Firecrawl API.
@@ -49,7 +63,7 @@ var FirecrawlApp = class {
       "Content-Type": "application/json",
       Authorization: `Bearer ${this.apiKey}`
     };
-    let jsonData = { url, ...params };
+    let jsonData = { url, ...params, origin: `js-sdk@${this.version}` };
     if (jsonData?.extract?.schema) {
       let schema = jsonData.extract.schema;
       try {
@@ -123,7 +137,7 @@ var FirecrawlApp = class {
       lang: params?.lang ?? "en",
       country: params?.country ?? "us",
       location: params?.location,
-      origin: params?.origin ?? "api",
+      origin: `js-sdk@${this.version}`,
       timeout: params?.timeout ?? 6e4,
       scrapeOptions: params?.scrapeOptions ?? { formats: [] }
     };
@@ -183,7 +197,7 @@ var FirecrawlApp = class {
    */
   async crawlUrl(url, params, pollInterval = 2, idempotencyKey) {
     const headers = this.prepareHeaders(idempotencyKey);
-    let jsonData = { url, ...params };
+    let jsonData = { url, ...params, origin: `js-sdk@${this.version}` };
     try {
       const response = await this.postRequest(
         this.apiUrl + `/v1/crawl`,
@@ -207,7 +221,7 @@ var FirecrawlApp = class {
   }
   async asyncCrawlUrl(url, params, idempotencyKey) {
     const headers = this.prepareHeaders(idempotencyKey);
-    let jsonData = { url, ...params };
+    let jsonData = { url, ...params, origin: `js-sdk@${this.version}` };
     try {
       const response = await this.postRequest(
         this.apiUrl + `/v1/crawl`,
@@ -366,7 +380,7 @@ var FirecrawlApp = class {
    */
   async mapUrl(url, params) {
     const headers = this.prepareHeaders();
-    let jsonData = { url, ...params };
+    let jsonData = { url, ...params, origin: `js-sdk@${this.version}` };
     try {
       const response = await this.postRequest(
         this.apiUrl + `/v1/map`,
@@ -395,7 +409,7 @@ var FirecrawlApp = class {
    */
   async batchScrapeUrls(urls, params, pollInterval = 2, idempotencyKey, webhook, ignoreInvalidURLs) {
     const headers = this.prepareHeaders(idempotencyKey);
-    let jsonData = { urls, webhook, ignoreInvalidURLs, ...params };
+    let jsonData = { urls, webhook, ignoreInvalidURLs, ...params, origin: `js-sdk@${this.version}` };
     if (jsonData?.extract?.schema) {
       let schema = jsonData.extract.schema;
       try {
@@ -447,7 +461,7 @@ var FirecrawlApp = class {
   }
   async asyncBatchScrapeUrls(urls, params, idempotencyKey, webhook, ignoreInvalidURLs) {
     const headers = this.prepareHeaders(idempotencyKey);
-    let jsonData = { urls, webhook, ignoreInvalidURLs, ...params ?? {} };
+    let jsonData = { urls, webhook, ignoreInvalidURLs, ...params, origin: `js-sdk@${this.version}` };
     try {
       const response = await this.postRequest(
         this.apiUrl + `/v1/batch/scrape`,
@@ -601,7 +615,7 @@ var FirecrawlApp = class {
     try {
       const response = await this.postRequest(
         this.apiUrl + `/v1/extract`,
-        { ...jsonData, schema: jsonSchema, origin: params?.origin || "api-sdk" },
+        { ...jsonData, schema: jsonSchema, origin: `js-sdk@${this.version}` },
         headers
       );
       if (response.status === 200) {
@@ -661,7 +675,7 @@ var FirecrawlApp = class {
     try {
       const response = await this.postRequest(
         this.apiUrl + `/v1/extract`,
-        { ...jsonData, schema: jsonSchema },
+        { ...jsonData, schema: jsonSchema, origin: `js-sdk@${this.version}` },
         headers
       );
       if (response.status === 200) {
@@ -813,6 +827,12 @@ var FirecrawlApp = class {
    * @param {string} action - The action being performed when the error occurred.
    */
   handleError(response, action) {
+    if (!response) {
+      throw new FirecrawlError(
+        `No response received while trying to ${action}. This may be a network error or the server is unreachable.`,
+        0
+      );
+    }
     if ([400, 402, 403, 408, 409, 500].includes(response.status)) {
       const errorMessage = response.data.error || "Unknown error occurred";
       const details = response.data.details ? ` - ${JSON.stringify(response.data.details)}` : "";
@@ -894,7 +914,7 @@ var FirecrawlApp = class {
    */
   async asyncDeepResearch(query, params) {
     const headers = this.prepareHeaders();
-    let jsonData = { query, ...params };
+    let jsonData = { query, ...params, origin: `js-sdk@${this.version}` };
     if (jsonData?.jsonOptions?.schema) {
       let schema = jsonData.jsonOptions.schema;
       try {
@@ -1017,9 +1037,10 @@ var FirecrawlApp = class {
   async __asyncDeepResearch(topic, params) {
     const headers = this.prepareHeaders();
     try {
+      let jsonData = { topic, ...params, origin: `js-sdk@${this.version}` };
       const response = await this.postRequest(
         `${this.apiUrl}/v1/deep-research`,
-        { topic, ...params },
+        jsonData,
         headers
       );
       if (response.status === 200) {
@@ -1114,10 +1135,11 @@ var FirecrawlApp = class {
    */
   async asyncGenerateLLMsText(url, params) {
     const headers = this.prepareHeaders();
+    let jsonData = { url, ...params, origin: `js-sdk@${this.version}` };
     try {
       const response = await this.postRequest(
         `${this.apiUrl}/v1/llmstxt`,
-        { url, ...params },
+        jsonData,
         headers
       );
       if (response.status === 200) {

package/dist/package-SG22PRGT.js

@@ -0,0 +1,102 @@
+// package.json
+var name = "@mendable/firecrawl-js";
+var version = "1.25.4";
+var description = "JavaScript SDK for Firecrawl API";
+var main = "dist/index.js";
+var types = "dist/index.d.ts";
+var exports = {
+  "./package.json": "./package.json",
+  ".": {
+    import: "./dist/index.js",
+    default: "./dist/index.cjs"
+  }
+};
+var type = "module";
+var scripts = {
+  build: "tsup",
+  "build-and-publish": "npm run build && npm publish --access public",
+  "publish-beta": "npm run build && npm publish --access public --tag beta",
+  test: "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/v1/**/*.test.ts"
+};
+var repository = {
+  type: "git",
+  url: "git+https://github.com/mendableai/firecrawl.git"
+};
+var author = "Mendable.ai";
+var license = "MIT";
+var dependencies = {
+  "typescript-event-target": "^1.1.1",
+  zod: "^3.23.8",
+  "zod-to-json-schema": "^3.23.0",
+  axios: "^1.6.8"
+};
+var bugs = {
+  url: "https://github.com/mendableai/firecrawl/issues"
+};
+var homepage = "https://github.com/mendableai/firecrawl#readme";
+var devDependencies = {
+  "@jest/globals": "^29.7.0",
+  "@types/axios": "^0.14.0",
+  "@types/dotenv": "^8.2.0",
+  "@types/jest": "^29.5.14",
+  "@types/mocha": "^10.0.6",
+  "@types/node": "^20.12.12",
+  "@types/uuid": "^9.0.8",
+  dotenv: "^16.4.5",
+  jest: "^29.7.0",
+  "ts-jest": "^29.2.2",
+  tsup: "^8.2.4",
+  typescript: "^5.4.5",
+  uuid: "^9.0.1"
+};
+var keywords = [
+  "firecrawl",
+  "mendable",
+  "crawler",
+  "web",
+  "scraper",
+  "api",
+  "sdk"
+];
+var engines = {
+  node: ">=22.0.0"
+};
+var package_default = {
+  name,
+  version,
+  description,
+  main,
+  types,
+  exports,
+  type,
+  scripts,
+  repository,
+  author,
+  license,
+  dependencies,
+  bugs,
+  homepage,
+  devDependencies,
+  keywords,
+  engines
+};
+export {
+  author,
+  bugs,
+  package_default as default,
+  dependencies,
+  description,
+  devDependencies,
+  engines,
+  exports,
+  homepage,
+  keywords,
+  license,
+  main,
+  name,
+  repository,
+  scripts,
+  type,
+  types,
+  version
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mendable/firecrawl",
-  "version": "1.24.0",
+  "version": "1.25.4",
   "description": "JavaScript SDK for Firecrawl API",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

package/src/index.ts

@@ -119,7 +119,8 @@ export interface CrawlScrapeOptions {
   skipTlsVerification?: boolean;
   removeBase64Images?: boolean;
   blockAds?: boolean;
-  proxy?: "basic" | "stealth";
+  proxy?: "basic" | "stealth" | "auto";
+  storeInCache?: boolean;
 }
 
 export type Action = {
@@ -165,6 +166,7 @@ export interface ScrapeParams<LLMSchema extends zt.ZodSchema = any, ActionsSchem
     prompt?: string;
     schema?: any;
     modes?: ("json" | "git-diff")[];
+    tag?: string | null;
   }
   actions?: ActionsSchema;
   agent?: AgentOptions;
@@ -215,6 +217,11 @@ export interface CrawlParams {
   deduplicateSimilarURLs?: boolean;
   ignoreQueryParameters?: boolean;
   regexOnFullURL?: boolean;
+  /**
+   * Delay in seconds between scrapes. This helps respect website rate limits.
+   * If not provided, the crawler may use the robots.txt crawl delay if available.
+   */
+  delay?: number;
 }
 
 /**
@@ -515,6 +522,11 @@ export interface GenerateLLMsTextParams {
    * @default false
    */
   showFullText?: boolean;
+  /**
+   * Whether to use cached content if available
+   * @default true
+   */
+  cache?: boolean;
   /**
    * Experimental flag for streaming
    */
@@ -550,11 +562,26 @@ export interface GenerateLLMsTextStatusResponse {
 export default class FirecrawlApp {
   public apiKey: string;
   public apiUrl: string;
-
+  public version: string =  "1.25.1";
+  
   private isCloudService(url: string): boolean {
     return url.includes('api.firecrawl.dev');
   }
 
+  private async getVersion(): Promise<string> {
+    try {
+      const packageJson = await import('../package.json', { assert: { type: 'json' } });
+      return packageJson.default.version;
+    } catch (error) {
+      console.error("Error getting version:", error);
+      return  "1.25.1";
+    }
+  }
+
+  private async init() {
+    this.version = await this.getVersion();
+  }
+
   /**
    * Initializes a new instance of the FirecrawlApp class.
    * @param config - Configuration options for the FirecrawlApp instance.
@@ -568,6 +595,7 @@ export default class FirecrawlApp {
 
     this.apiKey = apiKey || '';
     this.apiUrl = baseUrl;
+    this.init();
   }
 
   /**
@@ -584,7 +612,7 @@ export default class FirecrawlApp {
       "Content-Type": "application/json",
       Authorization: `Bearer ${this.apiKey}`,
     } as AxiosRequestHeaders;
-    let jsonData: any = { url, ...params };
+    let jsonData: any = { url, ...params, origin: `js-sdk@${this.version}` };
     if (jsonData?.extract?.schema) {
       let schema = jsonData.extract.schema;
 
@@ -666,7 +694,7 @@ export default class FirecrawlApp {
       lang: params?.lang ?? "en",
       country: params?.country ?? "us",
       location: params?.location,
-      origin: params?.origin ?? "api",
+      origin: `js-sdk@${this.version}`,
       timeout: params?.timeout ?? 60000,
       scrapeOptions: params?.scrapeOptions ?? { formats: [] },
     };
@@ -738,7 +766,7 @@ export default class FirecrawlApp {
     idempotencyKey?: string
   ): Promise<CrawlStatusResponse | ErrorResponse> {
     const headers = this.prepareHeaders(idempotencyKey);
-    let jsonData: any = { url, ...params };
+    let jsonData: any = { url, ...params, origin: `js-sdk@${this.version}` };
     try {
       const response: AxiosResponse = await this.postRequest(
         this.apiUrl + `/v1/crawl`,
@@ -767,7 +795,7 @@ export default class FirecrawlApp {
     idempotencyKey?: string
   ): Promise<CrawlResponse | ErrorResponse> {
     const headers = this.prepareHeaders(idempotencyKey);
-    let jsonData: any = { url, ...params };
+    let jsonData: any = { url, ...params, origin: `js-sdk@${this.version}` };
     try {
       const response: AxiosResponse = await this.postRequest(
         this.apiUrl + `/v1/crawl`,
@@ -943,7 +971,7 @@ export default class FirecrawlApp {
    */
   async mapUrl(url: string, params?: MapParams): Promise<MapResponse | ErrorResponse> {
     const headers = this.prepareHeaders();
-    let jsonData: { url: string } & MapParams = { url, ...params };
+    let jsonData: any = { url, ...params, origin: `js-sdk@${this.version}` };
 
     try {
       const response: AxiosResponse = await this.postRequest(
@@ -981,7 +1009,7 @@ export default class FirecrawlApp {
     ignoreInvalidURLs?: boolean,
   ): Promise<BatchScrapeStatusResponse | ErrorResponse> {
     const headers = this.prepareHeaders(idempotencyKey);
-    let jsonData: any = { urls, webhook, ignoreInvalidURLs, ...params };
+    let jsonData: any = { urls, webhook, ignoreInvalidURLs, ...params, origin: `js-sdk@${this.version}` };
     if (jsonData?.extract?.schema) {
       let schema = jsonData.extract.schema;
 
@@ -1046,7 +1074,7 @@ export default class FirecrawlApp {
     ignoreInvalidURLs?: boolean,
   ): Promise<BatchScrapeResponse | ErrorResponse> {
     const headers = this.prepareHeaders(idempotencyKey);
-    let jsonData: any = { urls, webhook, ignoreInvalidURLs, ...(params ?? {}) };
+    let jsonData: any = { urls, webhook, ignoreInvalidURLs, ...params, origin: `js-sdk@${this.version}` };
     try {
       const response: AxiosResponse = await this.postRequest(
         this.apiUrl + `/v1/batch/scrape`,
@@ -1220,7 +1248,7 @@ export default class FirecrawlApp {
     try {
       const response: AxiosResponse = await this.postRequest(
         this.apiUrl + `/v1/extract`,
-        { ...jsonData, schema: jsonSchema, origin: params?.origin || "api-sdk" },
+        { ...jsonData, schema: jsonSchema, origin: `js-sdk@${this.version}` },
         headers
       );
 
@@ -1288,7 +1316,7 @@ export default class FirecrawlApp {
     try {
       const response: AxiosResponse = await this.postRequest(
         this.apiUrl + `/v1/extract`,
-        { ...jsonData, schema: jsonSchema },
+        { ...jsonData, schema: jsonSchema, origin: `js-sdk@${this.version}` },
         headers
       );
 
@@ -1465,6 +1493,13 @@ export default class FirecrawlApp {
    * @param {string} action - The action being performed when the error occurred.
    */
   handleError(response: AxiosResponse, action: string): void {
+    if (!response) {
+      throw new FirecrawlError(
+        `No response received while trying to ${action}. This may be a network error or the server is unreachable.`,
+        0
+      );
+    }
+
     if ([400, 402, 403, 408, 409, 500].includes(response.status)) {
       const errorMessage: string =
         response.data.error || "Unknown error occurred";
@@ -1579,7 +1614,7 @@ export default class FirecrawlApp {
    */
   async asyncDeepResearch(query: string, params: DeepResearchParams<zt.ZodSchema>): Promise<DeepResearchResponse | ErrorResponse> {
     const headers = this.prepareHeaders();
-    let jsonData: any = { query, ...params };
+    let jsonData: any = { query, ...params, origin: `js-sdk@${this.version}` };
 
     if (jsonData?.jsonOptions?.schema) {
       let schema = jsonData.jsonOptions.schema;
@@ -1587,7 +1622,7 @@ export default class FirecrawlApp {
       try {
         schema = zodToJsonSchema(schema);
       } catch (error) {
-        
+        // Ignore error if schema can't be parsed as Zod
       }
       jsonData = {
         ...jsonData,
@@ -1733,9 +1768,10 @@ export default class FirecrawlApp {
   async __asyncDeepResearch(topic: string, params: DeepResearchParams): Promise<DeepResearchResponse | ErrorResponse> {
     const headers = this.prepareHeaders();
     try {
+      let jsonData: any = { topic, ...params, origin: `js-sdk@${this.version}` };
       const response: AxiosResponse = await this.postRequest(
         `${this.apiUrl}/v1/deep-research`,
-        { topic, ...params },
+        jsonData,
         headers
       );
 
@@ -1845,10 +1881,11 @@ export default class FirecrawlApp {
    */
   async asyncGenerateLLMsText(url: string, params?: GenerateLLMsTextParams): Promise<GenerateLLMsTextResponse | ErrorResponse> {
     const headers = this.prepareHeaders();
+    let jsonData: any = { url, ...params, origin: `js-sdk@${this.version}` };
     try {
       const response: AxiosResponse = await this.postRequest(
         `${this.apiUrl}/v1/llmstxt`,
-        { url, ...params },
+        jsonData,
         headers
       );