@librechat/agents 2.4.62 → 2.4.63

package/dist/cjs/tools/search/firecrawl.cjs

@@ -14,15 +14,45 @@ class FirecrawlScraper {
     defaultFormats;
     timeout;
     logger;
+    includeTags;
+    excludeTags;
+    waitFor;
+    maxAge;
+    mobile;
+    skipTlsVerification;
+    blockAds;
+    removeBase64Images;
+    parsePDF;
+    storeInCache;
+    zeroDataRetention;
+    headers;
+    location;
+    onlyMainContent;
+    changeTrackingOptions;
     constructor(config = {}) {
         this.apiKey = config.apiKey ?? process.env.FIRECRAWL_API_KEY ?? '';
         const baseUrl = config.apiUrl ??
             process.env.FIRECRAWL_BASE_URL ??
             'https://api.firecrawl.dev';
         this.apiUrl = `${baseUrl.replace(/\/+$/, '')}/v1/scrape`;
-        this.defaultFormats = config.formats ?? ['markdown', 'html'];
+        this.defaultFormats = config.formats ?? ['markdown', 'rawHtml'];
         this.timeout = config.timeout ?? 7500;
         this.logger = config.logger || utils.createDefaultLogger();
+        this.includeTags = config.includeTags;
+        this.excludeTags = config.excludeTags;
+        this.waitFor = config.waitFor;
+        this.maxAge = config.maxAge;
+        this.mobile = config.mobile;
+        this.skipTlsVerification = config.skipTlsVerification;
+        this.blockAds = config.blockAds;
+        this.removeBase64Images = config.removeBase64Images;
+        this.parsePDF = config.parsePDF;
+        this.storeInCache = config.storeInCache;
+        this.zeroDataRetention = config.zeroDataRetention;
+        this.headers = config.headers;
+        this.location = config.location;
+        this.onlyMainContent = config.onlyMainContent;
+        this.changeTrackingOptions = config.changeTrackingOptions;
         if (!this.apiKey) {
             this.logger.warn('FIRECRAWL_API_KEY is not set. Scraping will not work.');
         }
@@ -45,15 +75,27 @@ class FirecrawlScraper {
             ];
         }
         try {
-            const response = await axios.post(this.apiUrl, {
+            const payload = omitUndefined({
                 url,
-                formats: options.formats || this.defaultFormats,
-                includeTags: options.includeTags,
-                excludeTags: options.excludeTags,
-                headers: options.headers,
-                waitFor: options.waitFor,
+                formats: options.formats ?? this.defaultFormats,
+                includeTags: options.includeTags ?? this.includeTags,
+                excludeTags: options.excludeTags ?? this.excludeTags,
+                headers: options.headers ?? this.headers,
+                waitFor: options.waitFor ?? this.waitFor,
                 timeout: options.timeout ?? this.timeout,
-            }, {
+                onlyMainContent: options.onlyMainContent ?? this.onlyMainContent,
+                maxAge: options.maxAge ?? this.maxAge,
+                mobile: options.mobile ?? this.mobile,
+                skipTlsVerification: options.skipTlsVerification ?? this.skipTlsVerification,
+                parsePDF: options.parsePDF ?? this.parsePDF,
+                location: options.location ?? this.location,
+                removeBase64Images: options.removeBase64Images ?? this.removeBase64Images,
+                blockAds: options.blockAds ?? this.blockAds,
+                storeInCache: options.storeInCache ?? this.storeInCache,
+                zeroDataRetention: options.zeroDataRetention ?? this.zeroDataRetention,
+                changeTrackingOptions: options.changeTrackingOptions ?? this.changeTrackingOptions,
+            });
+            const response = await axios.post(this.apiUrl, payload, {
                 headers: {
                     'Content-Type': 'application/json',
                     Authorization: `Bearer ${this.apiKey}`,
@@ -125,6 +167,10 @@ class FirecrawlScraper {
 const createFirecrawlScraper = (config = {}) => {
     return new FirecrawlScraper(config);
 };
+// Helper function to clean up payload for firecrawl
+function omitUndefined(obj) {
+    return Object.fromEntries(Object.entries(obj).filter(([, v]) => v !== undefined));
+}
 
 exports.FirecrawlScraper = FirecrawlScraper;
 exports.createFirecrawlScraper = createFirecrawlScraper;

package/dist/cjs/tools/search/firecrawl.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"firecrawl.cjs","sources":["../../../../src/tools/search/firecrawl.ts"],"sourcesContent":["import axios from 'axios';\nimport { processContent } from './content';\nimport type * as t from './types';\nimport { createDefaultLogger } from './utils';\n\n/**\n * Firecrawl scraper implementation\n * Uses the Firecrawl API to scrape web pages\n */\nexport class FirecrawlScraper {\n  private apiKey: string;\n  private apiUrl: string;\n  private defaultFormats: string[];\n  private timeout: number;\n  private logger: t.Logger;\n\n  constructor(config: t.FirecrawlScraperConfig = {}) {\n    this.apiKey = config.apiKey ?? process.env.FIRECRAWL_API_KEY ?? '';\n\n    const baseUrl =\n      config.apiUrl ??\n      process.env.FIRECRAWL_BASE_URL ??\n      'https://api.firecrawl.dev';\n    this.apiUrl = `${baseUrl.replace(/\\/+$/, '')}/v1/scrape`;\n\n    this.defaultFormats = config.formats ?? ['markdown', 'html'];\n    this.timeout = config.timeout ?? 7500;\n\n    this.logger = config.logger || createDefaultLogger();\n\n    if (!this.apiKey) {\n      this.logger.warn('FIRECRAWL_API_KEY is not set. Scraping will not work.');\n    }\n\n    this.logger.debug(\n      `Firecrawl scraper initialized with API URL: ${this.apiUrl}`\n    );\n  }\n\n  /**\n   * Scrape a single URL\n   * @param url URL to scrape\n   * @param options Scrape options\n   * @returns Scrape response\n   */\n  async scrapeUrl(\n    url: string,\n    options: t.FirecrawlScrapeOptions = {}\n  ): Promise<[string, t.FirecrawlScrapeResponse]> {\n    if (!this.apiKey) {\n      return [\n        url,\n        {\n          success: false,\n          error: 'FIRECRAWL_API_KEY is not set',\n        },\n      ];\n    }\n\n    try {\n      const response = await axios.post(\n        this.apiUrl,\n        {\n          url,\n          formats: options.formats || this.defaultFormats,\n          includeTags: options.includeTags,\n          excludeTags: options.excludeTags,\n          headers: options.headers,\n          waitFor: options.waitFor,\n          timeout: options.timeout ?? this.timeout,\n        },\n        {\n          headers: {\n            'Content-Type': 'application/json',\n            Authorization: `Bearer ${this.apiKey}`,\n          },\n          timeout: this.timeout,\n        }\n      );\n\n      return [url, response.data];\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      return [\n        url,\n        {\n          success: false,\n          error: `Firecrawl API request failed: ${errorMessage}`,\n        },\n      ];\n    }\n  }\n\n  /**\n   * Extract content from scrape response\n   * @param response Scrape response\n   * @returns Extracted content or empty string if not available\n   */\n  extractContent(\n    response: t.FirecrawlScrapeResponse\n  ): [string, undefined | t.References] {\n    if (!response.success || !response.data) {\n      return ['', undefined];\n    }\n\n    if (response.data.markdown != null && response.data.html != null) {\n      try {\n        const { markdown, ...rest } = processContent(\n          response.data.html,\n          response.data.markdown\n        );\n        return [markdown, rest];\n      } catch (error) {\n        this.logger.error('Error processing content:', error);\n        return [response.data.markdown, undefined];\n      }\n    } else if (response.data.markdown != null) {\n      return [response.data.markdown, undefined];\n    }\n\n    // Fall back to HTML content\n    if (response.data.html != null) {\n      return [response.data.html, undefined];\n    }\n\n    // Fall back to raw HTML content\n    if (response.data.rawHtml != null) {\n      return [response.data.rawHtml, undefined];\n    }\n\n    return ['', undefined];\n  }\n\n  /**\n   * Extract metadata from scrape response\n   * @param response Scrape response\n   * @returns Metadata object\n   */\n  extractMetadata(response: t.FirecrawlScrapeResponse): t.ScrapeMetadata {\n    if (!response.success || !response.data || !response.data.metadata) {\n      return {};\n    }\n\n    return response.data.metadata;\n  }\n}\n\n/**\n * Create a Firecrawl scraper instance\n * @param config Scraper configuration\n * @returns Firecrawl scraper instance\n */\nexport const createFirecrawlScraper = (\n  config: t.FirecrawlScraperConfig = {}\n): FirecrawlScraper => {\n  return new FirecrawlScraper(config);\n};\n"],"names":["createDefaultLogger","processContent"],"mappings":";;;;;;AAKA;;;AAGG;MACU,gBAAgB,CAAA;AACnB,IAAA,MAAM;AACN,IAAA,MAAM;AACN,IAAA,cAAc;AACd,IAAA,OAAO;AACP,IAAA,MAAM;AAEd,IAAA,WAAA,CAAY,SAAmC,EAAE,EAAA;AAC/C,QAAA,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,EAAE;AAElE,QAAA,MAAM,OAAO,GACX,MAAM,CAAC,MAAM;YACb,OAAO,CAAC,GAAG,CAAC,kBAAkB;AAC9B,YAAA,2BAA2B;AAC7B,QAAA,IAAI,CAAC,MAAM,GAAG,CAAA,EAAG,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,YAAY;AAExD,QAAA,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,OAAO,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC;QAC5D,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,IAAI;QAErC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,IAAIA,yBAAmB,EAAE;AAEpD,QAAA,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;AAChB,YAAA,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,uDAAuD,CAAC;;QAG3E,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,CAA+C,4CAAA,EAAA,IAAI,CAAC,MAAM,CAAE,CAAA,CAC7D;;AAGH;;;;;AAKG;AACH,IAAA,MAAM,SAAS,CACb,GAAW,EACX,UAAoC,EAAE,EAAA;AAEtC,QAAA,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YAChB,OAAO;gBACL,GAAG;AACH,gBAAA;AACE,oBAAA,OAAO,EAAE,KAAK;AACd,oBAAA,KAAK,EAAE,8BAA8B;AACtC,iBAAA;aACF;;AAGH,QAAA,IAAI;YACF,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAC/B,IAAI,CAAC,MAAM,EACX;gBACE,GAAG;AACH,gBAAA,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,cAAc;gBAC/C,WAAW,EAAE,OAAO,CAAC,WAAW;gBAChC,WAAW,EAAE,OAAO,CAAC,WAAW;gBAChC,OAAO,EAAE,OAAO,CAAC,OAAO;gBACxB,OAAO,EAAE,OAAO,CAAC,OAAO;AACxB,gBAAA,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO;aACzC,EACD;AACE,gBAAA,OAAO,EAAE;AACP,oBAAA,cAAc,EAAE,kBAAkB;AAClC,oBAAA,aAAa,EAAE,CAAA,OAAA,EAAU,IAAI,CAAC,MAAM,CAAE,CAAA;AACvC,iBAAA;gBACD,OAAO,EAAE,IAAI,CAAC,OAAO;AACtB,aAAA,CACF;AAED,YAAA,OAAO,CAAC,GAAG,EAAE,QAAQ,CAAC,IAAI,CAAC;;QAC3B,OAAO,KAAK,EAAE;AACd,YAAA,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;YACxD,OAAO;gBACL,GAAG;AACH,gBAAA;AACE,oBAAA,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,CAAiC,8BAAA,EAAA,YAAY,CAAE,CAAA;AACvD,iBAAA;aACF;;;AAIL;;;;AAIG;AACH,IAAA,cAAc,CACZ,QAAmC,EAAA;QAEnC,IAAI,CAAC,QAAQ,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE;AACvC,YAAA,OAAO,CAAC,EAAE,EAAE,SAAS,CAAC;;AAGxB,QAAA,IAAI,QAAQ,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,EAAE;AAChE,YAAA,IAAI;gBACF,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAI,EAAE,GAAGC,sBAAc,CAC1C,QAAQ,CAAC,IAAI,CAAC,IAAI,EAClB,QAAQ,CAAC,IAAI,CAAC,QAAQ,CACvB;AACD,gBAAA,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC;;YACvB,OAAO,KAAK,EAAE;gBACd,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,2BAA2B,EAAE,KAAK,CAAC;gBACrD,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC;;;aAEvC,IAAI,QAAQ,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,EAAE;YACzC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC;;;QAI5C,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,EAAE;YAC9B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,SAAS,CAAC;;;QAIxC,IAAI,QAAQ,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,EAAE;YACjC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC;;AAG3C,QAAA,OAAO,CAAC,EAAE,EAAE,SAAS,CAAC;;AAGxB;;;;AAIG;AACH,IAAA,eAAe,CAAC,QAAmC,EAAA;AACjD,QAAA,IAAI,CAAC,QAAQ,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE;AAClE,YAAA,OAAO,EAAE;;AAGX,QAAA,OAAO,QAAQ,CAAC,IAAI,CAAC,QAAQ;;AAEhC;AAED;;;;AAIG;MACU,sBAAsB,GAAG,CACpC,MAAmC,GAAA,EAAE,KACjB;AACpB,IAAA,OAAO,IAAI,gBAAgB,CAAC,MAAM,CAAC;AACrC;;;;;"}
+{"version":3,"file":"firecrawl.cjs","sources":["../../../../src/tools/search/firecrawl.ts"],"sourcesContent":["import axios from 'axios';\nimport { processContent } from './content';\nimport type * as t from './types';\nimport { createDefaultLogger } from './utils';\n\n/**\n * Firecrawl scraper implementation\n * Uses the Firecrawl API to scrape web pages\n */\nexport class FirecrawlScraper {\n  private apiKey: string;\n  private apiUrl: string;\n  private defaultFormats: string[];\n  private timeout: number;\n  private logger: t.Logger;\n  private includeTags?: string[];\n  private excludeTags?: string[];\n  private waitFor?: number;\n  private maxAge?: number;\n  private mobile?: boolean;\n  private skipTlsVerification?: boolean;\n  private blockAds?: boolean;\n  private removeBase64Images?: boolean;\n  private parsePDF?: boolean;\n  private storeInCache?: boolean;\n  private zeroDataRetention?: boolean;\n  private headers?: Record<string, string>;\n  private location?: { country?: string; languages?: string[] };\n  private onlyMainContent?: boolean;\n  private changeTrackingOptions?: object;\n\n  constructor(config: t.FirecrawlScraperConfig = {}) {\n    this.apiKey = config.apiKey ?? process.env.FIRECRAWL_API_KEY ?? '';\n\n    const baseUrl =\n      config.apiUrl ??\n      process.env.FIRECRAWL_BASE_URL ??\n      'https://api.firecrawl.dev';\n    this.apiUrl = `${baseUrl.replace(/\\/+$/, '')}/v1/scrape`;\n\n    this.defaultFormats = config.formats ?? ['markdown', 'rawHtml'];\n    this.timeout = config.timeout ?? 7500;\n\n    this.logger = config.logger || createDefaultLogger();\n\n    this.includeTags = config.includeTags;\n    this.excludeTags = config.excludeTags;\n    this.waitFor = config.waitFor;\n    this.maxAge = config.maxAge;\n    this.mobile = config.mobile;\n    this.skipTlsVerification = config.skipTlsVerification;\n    this.blockAds = config.blockAds;\n    this.removeBase64Images = config.removeBase64Images;\n    this.parsePDF = config.parsePDF;\n    this.storeInCache = config.storeInCache;\n    this.zeroDataRetention = config.zeroDataRetention;\n    this.headers = config.headers;\n    this.location = config.location;\n    this.onlyMainContent = config.onlyMainContent;\n    this.changeTrackingOptions = config.changeTrackingOptions;\n\n    if (!this.apiKey) {\n      this.logger.warn('FIRECRAWL_API_KEY is not set. Scraping will not work.');\n    }\n\n    this.logger.debug(\n      `Firecrawl scraper initialized with API URL: ${this.apiUrl}`\n    );\n  }\n\n  /**\n   * Scrape a single URL\n   * @param url URL to scrape\n   * @param options Scrape options\n   * @returns Scrape response\n   */\n  async scrapeUrl(\n    url: string,\n    options: t.FirecrawlScrapeOptions = {}\n  ): Promise<[string, t.FirecrawlScrapeResponse]> {\n    if (!this.apiKey) {\n      return [\n        url,\n        {\n          success: false,\n          error: 'FIRECRAWL_API_KEY is not set',\n        },\n      ];\n    }\n\n    try {\n      const payload = omitUndefined({\n        url,\n        formats: options.formats ?? this.defaultFormats,\n        includeTags: options.includeTags ?? this.includeTags,\n        excludeTags: options.excludeTags ?? this.excludeTags,\n        headers: options.headers ?? this.headers,\n        waitFor: options.waitFor ?? this.waitFor,\n        timeout: options.timeout ?? this.timeout,\n        onlyMainContent: options.onlyMainContent ?? this.onlyMainContent,\n        maxAge: options.maxAge ?? this.maxAge,\n        mobile: options.mobile ?? this.mobile,\n        skipTlsVerification:\n          options.skipTlsVerification ?? this.skipTlsVerification,\n        parsePDF: options.parsePDF ?? this.parsePDF,\n        location: options.location ?? this.location,\n        removeBase64Images:\n          options.removeBase64Images ?? this.removeBase64Images,\n        blockAds: options.blockAds ?? this.blockAds,\n        storeInCache: options.storeInCache ?? this.storeInCache,\n        zeroDataRetention: options.zeroDataRetention ?? this.zeroDataRetention,\n        changeTrackingOptions:\n          options.changeTrackingOptions ?? this.changeTrackingOptions,\n      });\n      const response = await axios.post(this.apiUrl, payload, {\n        headers: {\n          'Content-Type': 'application/json',\n          Authorization: `Bearer ${this.apiKey}`,\n        },\n        timeout: this.timeout,\n      });\n\n      return [url, response.data];\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      return [\n        url,\n        {\n          success: false,\n          error: `Firecrawl API request failed: ${errorMessage}`,\n        },\n      ];\n    }\n  }\n\n  /**\n   * Extract content from scrape response\n   * @param response Scrape response\n   * @returns Extracted content or empty string if not available\n   */\n  extractContent(\n    response: t.FirecrawlScrapeResponse\n  ): [string, undefined | t.References] {\n    if (!response.success || !response.data) {\n      return ['', undefined];\n    }\n\n    if (response.data.markdown != null && response.data.html != null) {\n      try {\n        const { markdown, ...rest } = processContent(\n          response.data.html,\n          response.data.markdown\n        );\n        return [markdown, rest];\n      } catch (error) {\n        this.logger.error('Error processing content:', error);\n        return [response.data.markdown, undefined];\n      }\n    } else if (response.data.markdown != null) {\n      return [response.data.markdown, undefined];\n    }\n\n    // Fall back to HTML content\n    if (response.data.html != null) {\n      return [response.data.html, undefined];\n    }\n\n    // Fall back to raw HTML content\n    if (response.data.rawHtml != null) {\n      return [response.data.rawHtml, undefined];\n    }\n\n    return ['', undefined];\n  }\n\n  /**\n   * Extract metadata from scrape response\n   * @param response Scrape response\n   * @returns Metadata object\n   */\n  extractMetadata(response: t.FirecrawlScrapeResponse): t.ScrapeMetadata {\n    if (!response.success || !response.data || !response.data.metadata) {\n      return {};\n    }\n\n    return response.data.metadata;\n  }\n}\n\n/**\n * Create a Firecrawl scraper instance\n * @param config Scraper configuration\n * @returns Firecrawl scraper instance\n */\nexport const createFirecrawlScraper = (\n  config: t.FirecrawlScraperConfig = {}\n): FirecrawlScraper => {\n  return new FirecrawlScraper(config);\n};\n\n// Helper function to clean up payload for firecrawl\nfunction omitUndefined<T extends object>(obj: T): Partial<T> {\n  return Object.fromEntries(\n    Object.entries(obj).filter(([, v]) => v !== undefined)\n  ) as Partial<T>;\n}\n"],"names":["createDefaultLogger","processContent"],"mappings":";;;;;;AAKA;;;AAGG;MACU,gBAAgB,CAAA;AACnB,IAAA,MAAM;AACN,IAAA,MAAM;AACN,IAAA,cAAc;AACd,IAAA,OAAO;AACP,IAAA,MAAM;AACN,IAAA,WAAW;AACX,IAAA,WAAW;AACX,IAAA,OAAO;AACP,IAAA,MAAM;AACN,IAAA,MAAM;AACN,IAAA,mBAAmB;AACnB,IAAA,QAAQ;AACR,IAAA,kBAAkB;AAClB,IAAA,QAAQ;AACR,IAAA,YAAY;AACZ,IAAA,iBAAiB;AACjB,IAAA,OAAO;AACP,IAAA,QAAQ;AACR,IAAA,eAAe;AACf,IAAA,qBAAqB;AAE7B,IAAA,WAAA,CAAY,SAAmC,EAAE,EAAA;AAC/C,QAAA,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,EAAE;AAElE,QAAA,MAAM,OAAO,GACX,MAAM,CAAC,MAAM;YACb,OAAO,CAAC,GAAG,CAAC,kBAAkB;AAC9B,YAAA,2BAA2B;AAC7B,QAAA,IAAI,CAAC,MAAM,GAAG,CAAA,EAAG,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,YAAY;AAExD,QAAA,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,OAAO,IAAI,CAAC,UAAU,EAAE,SAAS,CAAC;QAC/D,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,IAAI;QAErC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,IAAIA,yBAAmB,EAAE;AAEpD,QAAA,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW;AACrC,QAAA,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW;AACrC,QAAA,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO;AAC7B,QAAA,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM;AAC3B,QAAA,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM;AAC3B,QAAA,IAAI,CAAC,mBAAmB,GAAG,MAAM,CAAC,mBAAmB;AACrD,QAAA,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ;AAC/B,QAAA,IAAI,CAAC,kBAAkB,GAAG,MAAM,CAAC,kBAAkB;AACnD,QAAA,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ;AAC/B,QAAA,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC,YAAY;AACvC,QAAA,IAAI,CAAC,iBAAiB,GAAG,MAAM,CAAC,iBAAiB;AACjD,QAAA,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO;AAC7B,QAAA,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ;AAC/B,QAAA,IAAI,CAAC,eAAe,GAAG,MAAM,CAAC,eAAe;AAC7C,QAAA,IAAI,CAAC,qBAAqB,GAAG,MAAM,CAAC,qBAAqB;AAEzD,QAAA,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;AAChB,YAAA,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,uDAAuD,CAAC;;QAG3E,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,CAA+C,4CAAA,EAAA,IAAI,CAAC,MAAM,CAAE,CAAA,CAC7D;;AAGH;;;;;AAKG;AACH,IAAA,MAAM,SAAS,CACb,GAAW,EACX,UAAoC,EAAE,EAAA;AAEtC,QAAA,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YAChB,OAAO;gBACL,GAAG;AACH,gBAAA;AACE,oBAAA,OAAO,EAAE,KAAK;AACd,oBAAA,KAAK,EAAE,8BAA8B;AACtC,iBAAA;aACF;;AAGH,QAAA,IAAI;YACF,MAAM,OAAO,GAAG,aAAa,CAAC;gBAC5B,GAAG;AACH,gBAAA,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,cAAc;AAC/C,gBAAA,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,IAAI,CAAC,WAAW;AACpD,gBAAA,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,IAAI,CAAC,WAAW;AACpD,gBAAA,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO;AACxC,gBAAA,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO;AACxC,gBAAA,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO;AACxC,gBAAA,eAAe,EAAE,OAAO,CAAC,eAAe,IAAI,IAAI,CAAC,eAAe;AAChE,gBAAA,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM;AACrC,gBAAA,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM;AACrC,gBAAA,mBAAmB,EACjB,OAAO,CAAC,mBAAmB,IAAI,IAAI,CAAC,mBAAmB;AACzD,gBAAA,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ;AAC3C,gBAAA,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ;AAC3C,gBAAA,kBAAkB,EAChB,OAAO,CAAC,kBAAkB,IAAI,IAAI,CAAC,kBAAkB;AACvD,gBAAA,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ;AAC3C,gBAAA,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,IAAI,CAAC,YAAY;AACvD,gBAAA,iBAAiB,EAAE,OAAO,CAAC,iBAAiB,IAAI,IAAI,CAAC,iBAAiB;AACtE,gBAAA,qBAAqB,EACnB,OAAO,CAAC,qBAAqB,IAAI,IAAI,CAAC,qBAAqB;AAC9D,aAAA,CAAC;AACF,YAAA,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE;AACtD,gBAAA,OAAO,EAAE;AACP,oBAAA,cAAc,EAAE,kBAAkB;AAClC,oBAAA,aAAa,EAAE,CAAA,OAAA,EAAU,IAAI,CAAC,MAAM,CAAE,CAAA;AACvC,iBAAA;gBACD,OAAO,EAAE,IAAI,CAAC,OAAO;AACtB,aAAA,CAAC;AAEF,YAAA,OAAO,CAAC,GAAG,EAAE,QAAQ,CAAC,IAAI,CAAC;;QAC3B,OAAO,KAAK,EAAE;AACd,YAAA,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;YACxD,OAAO;gBACL,GAAG;AACH,gBAAA;AACE,oBAAA,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,CAAiC,8BAAA,EAAA,YAAY,CAAE,CAAA;AACvD,iBAAA;aACF;;;AAIL;;;;AAIG;AACH,IAAA,cAAc,CACZ,QAAmC,EAAA;QAEnC,IAAI,CAAC,QAAQ,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE;AACvC,YAAA,OAAO,CAAC,EAAE,EAAE,SAAS,CAAC;;AAGxB,QAAA,IAAI,QAAQ,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,EAAE;AAChE,YAAA,IAAI;gBACF,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAI,EAAE,GAAGC,sBAAc,CAC1C,QAAQ,CAAC,IAAI,CAAC,IAAI,EAClB,QAAQ,CAAC,IAAI,CAAC,QAAQ,CACvB;AACD,gBAAA,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC;;YACvB,OAAO,KAAK,EAAE;gBACd,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,2BAA2B,EAAE,KAAK,CAAC;gBACrD,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC;;;aAEvC,IAAI,QAAQ,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,EAAE;YACzC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC;;;QAI5C,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,EAAE;YAC9B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,SAAS,CAAC;;;QAIxC,IAAI,QAAQ,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,EAAE;YACjC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC;;AAG3C,QAAA,OAAO,CAAC,EAAE,EAAE,SAAS,CAAC;;AAGxB;;;;AAIG;AACH,IAAA,eAAe,CAAC,QAAmC,EAAA;AACjD,QAAA,IAAI,CAAC,QAAQ,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE;AAClE,YAAA,OAAO,EAAE;;AAGX,QAAA,OAAO,QAAQ,CAAC,IAAI,CAAC,QAAQ;;AAEhC;AAED;;;;AAIG;MACU,sBAAsB,GAAG,CACpC,MAAmC,GAAA,EAAE,KACjB;AACpB,IAAA,OAAO,IAAI,gBAAgB,CAAC,MAAM,CAAC;AACrC;AAEA;AACA,SAAS,aAAa,CAAmB,GAAM,EAAA;IAC7C,OAAO,MAAM,CAAC,WAAW,CACvB,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,KAAK,SAAS,CAAC,CACzC;AACjB;;;;;"}

package/dist/cjs/tools/search/tool.cjs

@@ -236,7 +236,7 @@ Use anchor marker(s) immediately after the statement:
  * @returns A DynamicStructuredTool with a schema that depends on the searchProvider
  */
 const createSearchTool = (config = {}) => {
-    const { searchProvider = 'serper', serperApiKey, searxngInstanceUrl, searxngApiKey, rerankerType = 'cohere', topResults = 5, strategies = ['no_extraction'], filterContent = true, safeSearch = 1, firecrawlApiKey, firecrawlApiUrl, firecrawlFormats = ['markdown', 'html'], scraperTimeout, jinaApiKey, cohereApiKey, onSearchResults: _onSearchResults, onGetHighlights, } = config;
+    const { searchProvider = 'serper', serperApiKey, searxngInstanceUrl, searxngApiKey, rerankerType = 'cohere', topResults = 5, strategies = ['no_extraction'], filterContent = true, safeSearch = 1, firecrawlApiKey, firecrawlApiUrl, firecrawlOptions, scraperTimeout, jinaApiKey, cohereApiKey, onSearchResults: _onSearchResults, onGetHighlights, } = config;
     const logger = config.logger || utils.createDefaultLogger();
     const schemaObject = {
         query: schema.querySchema,
@@ -256,10 +256,11 @@ const createSearchTool = (config = {}) => {
         searxngApiKey,
     });
     const firecrawlScraper = firecrawl.createFirecrawlScraper({
+        ...firecrawlOptions,
         apiKey: firecrawlApiKey ?? process.env.FIRECRAWL_API_KEY,
         apiUrl: firecrawlApiUrl,
-        timeout: scraperTimeout,
-        formats: firecrawlFormats,
+        timeout: scraperTimeout ?? firecrawlOptions?.timeout,
+        formats: firecrawlOptions?.formats ?? ['markdown', 'rawHtml'],
     });
     const selectedReranker = rerankers.createReranker({
         rerankerType,

package/dist/cjs/tools/search/tool.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"tool.cjs","sources":["../../../../src/tools/search/tool.ts"],"sourcesContent":["import { z } from 'zod';\nimport { tool, DynamicStructuredTool } from '@langchain/core/tools';\nimport type { RunnableConfig } from '@langchain/core/runnables';\nimport type * as t from './types';\nimport {\n  DATE_RANGE,\n  querySchema,\n  dateSchema,\n  countrySchema,\n  imagesSchema,\n  videosSchema,\n  newsSchema,\n} from './schema';\nimport { createSearchAPI, createSourceProcessor } from './search';\nimport { createFirecrawlScraper } from './firecrawl';\nimport { expandHighlights } from './highlights';\nimport { formatResultsForLLM } from './format';\nimport { createDefaultLogger } from './utils';\nimport { createReranker } from './rerankers';\nimport { Constants } from '@/common';\n\n/**\n * Executes parallel searches and merges the results\n */\nasync function executeParallelSearches({\n  searchAPI,\n  query,\n  date,\n  country,\n  safeSearch,\n  images,\n  videos,\n  news,\n  logger,\n}: {\n  searchAPI: ReturnType<typeof createSearchAPI>;\n  query: string;\n  date?: DATE_RANGE;\n  country?: string;\n  safeSearch: t.SearchToolConfig['safeSearch'];\n  images: boolean;\n  videos: boolean;\n  news: boolean;\n  logger: t.Logger;\n}): Promise<t.SearchResult> {\n  // Prepare all search tasks to run in parallel\n  const searchTasks: Promise<t.SearchResult>[] = [\n    // Main search\n    searchAPI.getSources({\n      query,\n      date,\n      country,\n      safeSearch,\n    }),\n  ];\n\n  if (images) {\n    searchTasks.push(\n      searchAPI\n        .getSources({\n          query,\n          date,\n          country,\n          safeSearch,\n          type: 'images',\n        })\n        .catch((error) => {\n          logger.error('Error fetching images:', error);\n          return {\n            success: false,\n            error: `Images search failed: ${error instanceof Error ? error.message : String(error)}`,\n          };\n        })\n    );\n  }\n  if (videos) {\n    searchTasks.push(\n      searchAPI\n        .getSources({\n          query,\n          date,\n          country,\n          safeSearch,\n          type: 'videos',\n        })\n        .catch((error) => {\n          logger.error('Error fetching videos:', error);\n          return {\n            success: false,\n            error: `Videos search failed: ${error instanceof Error ? error.message : String(error)}`,\n          };\n        })\n    );\n  }\n  if (news) {\n    searchTasks.push(\n      searchAPI\n        .getSources({\n          query,\n          date,\n          country,\n          safeSearch,\n          type: 'news',\n        })\n        .catch((error) => {\n          logger.error('Error fetching news:', error);\n          return {\n            success: false,\n            error: `News search failed: ${error instanceof Error ? error.message : String(error)}`,\n          };\n        })\n    );\n  }\n\n  // Run all searches in parallel\n  const results = await Promise.all(searchTasks);\n\n  // Get the main search result (first result)\n  const mainResult = results[0];\n  if (!mainResult.success) {\n    throw new Error(mainResult.error ?? 'Search failed');\n  }\n\n  // Merge additional results with the main results\n  const mergedResults = { ...mainResult.data };\n\n  // Convert existing news to topStories if present\n  if (mergedResults.news !== undefined && mergedResults.news.length > 0) {\n    const existingNewsAsTopStories = mergedResults.news\n      .filter((newsItem) => newsItem.link !== undefined && newsItem.link !== '')\n      .map((newsItem) => ({\n        title: newsItem.title ?? '',\n        link: newsItem.link ?? '',\n        source: newsItem.source ?? '',\n        date: newsItem.date ?? '',\n        imageUrl: newsItem.imageUrl ?? '',\n        processed: false,\n      }));\n    mergedResults.topStories = [\n      ...(mergedResults.topStories ?? []),\n      ...existingNewsAsTopStories,\n    ];\n    delete mergedResults.news;\n  }\n\n  results.slice(1).forEach((result) => {\n    if (result.success && result.data !== undefined) {\n      if (result.data.images !== undefined && result.data.images.length > 0) {\n        mergedResults.images = [\n          ...(mergedResults.images ?? []),\n          ...result.data.images,\n        ];\n      }\n      if (result.data.videos !== undefined && result.data.videos.length > 0) {\n        mergedResults.videos = [\n          ...(mergedResults.videos ?? []),\n          ...result.data.videos,\n        ];\n      }\n      if (result.data.news !== undefined && result.data.news.length > 0) {\n        const newsAsTopStories = result.data.news.map((newsItem) => ({\n          ...newsItem,\n          link: newsItem.link ?? '',\n        }));\n        mergedResults.topStories = [\n          ...(mergedResults.topStories ?? []),\n          ...newsAsTopStories,\n        ];\n      }\n    }\n  });\n\n  return { success: true, data: mergedResults };\n}\n\nfunction createSearchProcessor({\n  searchAPI,\n  safeSearch,\n  sourceProcessor,\n  onGetHighlights,\n  logger,\n}: {\n  safeSearch: t.SearchToolConfig['safeSearch'];\n  searchAPI: ReturnType<typeof createSearchAPI>;\n  sourceProcessor: ReturnType<typeof createSourceProcessor>;\n  onGetHighlights: t.SearchToolConfig['onGetHighlights'];\n  logger: t.Logger;\n}) {\n  return async function ({\n    query,\n    date,\n    country,\n    proMode = true,\n    maxSources = 5,\n    onSearchResults,\n    images = false,\n    videos = false,\n    news = false,\n  }: {\n    query: string;\n    country?: string;\n    date?: DATE_RANGE;\n    proMode?: boolean;\n    maxSources?: number;\n    onSearchResults: t.SearchToolConfig['onSearchResults'];\n    images?: boolean;\n    videos?: boolean;\n    news?: boolean;\n  }): Promise<t.SearchResultData> {\n    try {\n      // Execute parallel searches and merge results\n      const searchResult = await executeParallelSearches({\n        searchAPI,\n        query,\n        date,\n        country,\n        safeSearch,\n        images,\n        videos,\n        news,\n        logger,\n      });\n\n      onSearchResults?.(searchResult);\n\n      const processedSources = await sourceProcessor.processSources({\n        query,\n        news,\n        result: searchResult,\n        proMode,\n        onGetHighlights,\n        numElements: maxSources,\n      });\n\n      return expandHighlights(processedSources);\n    } catch (error) {\n      logger.error('Error in search:', error);\n      return {\n        organic: [],\n        topStories: [],\n        images: [],\n        videos: [],\n        news: [],\n        relatedSearches: [],\n        error: error instanceof Error ? error.message : String(error),\n      };\n    }\n  };\n}\n\nfunction createOnSearchResults({\n  runnableConfig,\n  onSearchResults,\n}: {\n  runnableConfig: RunnableConfig;\n  onSearchResults: t.SearchToolConfig['onSearchResults'];\n}) {\n  return function (results: t.SearchResult): void {\n    if (!onSearchResults) {\n      return;\n    }\n    onSearchResults(results, runnableConfig);\n  };\n}\n\nfunction createTool({\n  schema,\n  search,\n  onSearchResults: _onSearchResults,\n}: {\n  schema: t.SearchToolSchema;\n  search: ReturnType<typeof createSearchProcessor>;\n  onSearchResults: t.SearchToolConfig['onSearchResults'];\n}): DynamicStructuredTool<typeof schema> {\n  return tool<typeof schema>(\n    async (params, runnableConfig) => {\n      const { query, date, country: _c, images, videos, news } = params;\n      const country = typeof _c === 'string' && _c ? _c : undefined;\n      const searchResult = await search({\n        query,\n        date,\n        country,\n        images,\n        videos,\n        news,\n        onSearchResults: createOnSearchResults({\n          runnableConfig,\n          onSearchResults: _onSearchResults,\n        }),\n      });\n      const turn = runnableConfig.toolCall?.turn ?? 0;\n      const { output, references } = formatResultsForLLM(turn, searchResult);\n      const data: t.SearchResultData = { turn, ...searchResult, references };\n      return [output, { [Constants.WEB_SEARCH]: data }];\n    },\n    {\n      name: Constants.WEB_SEARCH,\n      description: `Real-time search. Results have required citation anchors.\n\nNote: Use ONCE per reply unless instructed otherwise.\n\nAnchors:\n- \\\\ue202turnXtypeY\n- X = turn idx, type = 'search' | 'news' | 'image' | 'ref', Y = item idx\n\nSpecial Markers:\n- \\\\ue203...\\\\ue204 — highlight start/end of cited text (for Standalone or Group citations)\n- \\\\ue200...\\\\ue201 — group block (e.g. \\\\ue200\\\\ue202turn0search1\\\\ue202turn0news2\\\\ue201)\n\n**CITE EVERY NON-OBVIOUS FACT/QUOTE:**\nUse anchor marker(s) immediately after the statement:\n- Standalone: \"Pure functions produce same output. \\\\ue202turn0search0\"\n- Standalone (multiple): \"Today's News \\\\ue202turn0search0\\\\ue202turn0news0\"\n- Highlight: \"\\\\ue203Highlight text.\\\\ue204\\\\ue202turn0news1\"\n- Group: \"Sources. \\\\ue200\\\\ue202turn0search0\\\\ue202turn0news1\\\\ue201\"\n- Group Highlight: \"\\\\ue203Highlight for group.\\\\ue204 \\\\ue200\\\\ue202turn0search0\\\\ue202turn0news1\\\\ue201\"\n- Image: \"See photo \\\\ue202turn0image0.\"\n\n**NEVER use markdown links, [1], or footnotes. CITE ONLY with anchors provided.**\n`.trim(),\n      schema: schema,\n      responseFormat: Constants.CONTENT_AND_ARTIFACT,\n    }\n  );\n}\n\n/**\n * Creates a search tool with a schema that dynamically includes the country field\n * only when the searchProvider is 'serper'.\n *\n * @param config - The search tool configuration\n * @returns A DynamicStructuredTool with a schema that depends on the searchProvider\n */\nexport const createSearchTool = (\n  config: t.SearchToolConfig = {}\n): DynamicStructuredTool<typeof toolSchema> => {\n  const {\n    searchProvider = 'serper',\n    serperApiKey,\n    searxngInstanceUrl,\n    searxngApiKey,\n    rerankerType = 'cohere',\n    topResults = 5,\n    strategies = ['no_extraction'],\n    filterContent = true,\n    safeSearch = 1,\n    firecrawlApiKey,\n    firecrawlApiUrl,\n    firecrawlFormats = ['markdown', 'html'],\n    scraperTimeout,\n    jinaApiKey,\n    cohereApiKey,\n    onSearchResults: _onSearchResults,\n    onGetHighlights,\n  } = config;\n\n  const logger = config.logger || createDefaultLogger();\n\n  const schemaObject: {\n    query: z.ZodString;\n    date: z.ZodOptional<z.ZodNativeEnum<typeof DATE_RANGE>>;\n    country?: z.ZodOptional<z.ZodString>;\n    images: z.ZodOptional<z.ZodBoolean>;\n    videos: z.ZodOptional<z.ZodBoolean>;\n    news: z.ZodOptional<z.ZodBoolean>;\n  } = {\n    query: querySchema,\n    date: dateSchema,\n    images: imagesSchema,\n    videos: videosSchema,\n    news: newsSchema,\n  };\n\n  if (searchProvider === 'serper') {\n    schemaObject.country = countrySchema;\n  }\n\n  const toolSchema = z.object(schemaObject);\n\n  const searchAPI = createSearchAPI({\n    searchProvider,\n    serperApiKey,\n    searxngInstanceUrl,\n    searxngApiKey,\n  });\n\n  const firecrawlScraper = createFirecrawlScraper({\n    apiKey: firecrawlApiKey ?? process.env.FIRECRAWL_API_KEY,\n    apiUrl: firecrawlApiUrl,\n    timeout: scraperTimeout,\n    formats: firecrawlFormats,\n  });\n\n  const selectedReranker = createReranker({\n    rerankerType,\n    jinaApiKey,\n    cohereApiKey,\n    logger,\n  });\n\n  if (!selectedReranker) {\n    logger.warn('No reranker selected. Using default ranking.');\n  }\n\n  const sourceProcessor = createSourceProcessor(\n    {\n      reranker: selectedReranker,\n      topResults,\n      strategies,\n      filterContent,\n      logger,\n    },\n    firecrawlScraper\n  );\n\n  const search = createSearchProcessor({\n    searchAPI,\n    safeSearch,\n    sourceProcessor,\n    onGetHighlights,\n    logger,\n  });\n\n  return createTool({\n    search,\n    schema: toolSchema,\n    onSearchResults: _onSearchResults,\n  });\n};\n"],"names":["expandHighlights","tool","formatResultsForLLM","Constants","createDefaultLogger","querySchema","dateSchema","imagesSchema","videosSchema","newsSchema","countrySchema","z","createSearchAPI","createFirecrawlScraper","createReranker","createSourceProcessor","search"],"mappings":";;;;;;;;;;;;;AAqBA;;AAEG;AACH,eAAe,uBAAuB,CAAC,EACrC,SAAS,EACT,KAAK,EACL,IAAI,EACJ,OAAO,EACP,UAAU,EACV,MAAM,EACN,MAAM,EACN,IAAI,EACJ,MAAM,GAWP,EAAA;;AAEC,IAAA,MAAM,WAAW,GAA8B;;QAE7C,SAAS,CAAC,UAAU,CAAC;YACnB,KAAK;YACL,IAAI;YACJ,OAAO;YACP,UAAU;SACX,CAAC;KACH;IAED,IAAI,MAAM,EAAE;QACV,WAAW,CAAC,IAAI,CACd;AACG,aAAA,UAAU,CAAC;YACV,KAAK;YACL,IAAI;YACJ,OAAO;YACP,UAAU;AACV,YAAA,IAAI,EAAE,QAAQ;SACf;AACA,aAAA,KAAK,CAAC,CAAC,KAAK,KAAI;AACf,YAAA,MAAM,CAAC,KAAK,CAAC,wBAAwB,EAAE,KAAK,CAAC;YAC7C,OAAO;AACL,gBAAA,OAAO,EAAE,KAAK;AACd,gBAAA,KAAK,EAAE,CAAyB,sBAAA,EAAA,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,CAAE,CAAA;aACzF;SACF,CAAC,CACL;;IAEH,IAAI,MAAM,EAAE;QACV,WAAW,CAAC,IAAI,CACd;AACG,aAAA,UAAU,CAAC;YACV,KAAK;YACL,IAAI;YACJ,OAAO;YACP,UAAU;AACV,YAAA,IAAI,EAAE,QAAQ;SACf;AACA,aAAA,KAAK,CAAC,CAAC,KAAK,KAAI;AACf,YAAA,MAAM,CAAC,KAAK,CAAC,wBAAwB,EAAE,KAAK,CAAC;YAC7C,OAAO;AACL,gBAAA,OAAO,EAAE,KAAK;AACd,gBAAA,KAAK,EAAE,CAAyB,sBAAA,EAAA,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,CAAE,CAAA;aACzF;SACF,CAAC,CACL;;IAEH,IAAI,IAAI,EAAE;QACR,WAAW,CAAC,IAAI,CACd;AACG,aAAA,UAAU,CAAC;YACV,KAAK;YACL,IAAI;YACJ,OAAO;YACP,UAAU;AACV,YAAA,IAAI,EAAE,MAAM;SACb;AACA,aAAA,KAAK,CAAC,CAAC,KAAK,KAAI;AACf,YAAA,MAAM,CAAC,KAAK,CAAC,sBAAsB,EAAE,KAAK,CAAC;YAC3C,OAAO;AACL,gBAAA,OAAO,EAAE,KAAK;AACd,gBAAA,KAAK,EAAE,CAAuB,oBAAA,EAAA,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,CAAE,CAAA;aACvF;SACF,CAAC,CACL;;;IAIH,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC;;AAG9C,IAAA,MAAM,UAAU,GAAG,OAAO,CAAC,CAAC,CAAC;AAC7B,IAAA,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE;QACvB,MAAM,IAAI,KAAK,CAAC,UAAU,CAAC,KAAK,IAAI,eAAe,CAAC;;;IAItD,MAAM,aAAa,GAAG,EAAE,GAAG,UAAU,CAAC,IAAI,EAAE;;AAG5C,IAAA,IAAI,aAAa,CAAC,IAAI,KAAK,SAAS,IAAI,aAAa,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE;AACrE,QAAA,MAAM,wBAAwB,GAAG,aAAa,CAAC;AAC5C,aAAA,MAAM,CAAC,CAAC,QAAQ,KAAK,QAAQ,CAAC,IAAI,KAAK,SAAS,IAAI,QAAQ,CAAC,IAAI,KAAK,EAAE;AACxE,aAAA,GAAG,CAAC,CAAC,QAAQ,MAAM;AAClB,YAAA,KAAK,EAAE,QAAQ,CAAC,KAAK,IAAI,EAAE;AAC3B,YAAA,IAAI,EAAE,QAAQ,CAAC,IAAI,IAAI,EAAE;AACzB,YAAA,MAAM,EAAE,QAAQ,CAAC,MAAM,IAAI,EAAE;AAC7B,YAAA,IAAI,EAAE,QAAQ,CAAC,IAAI,IAAI,EAAE;AACzB,YAAA,QAAQ,EAAE,QAAQ,CAAC,QAAQ,IAAI,EAAE;AACjC,YAAA,SAAS,EAAE,KAAK;AACjB,SAAA,CAAC,CAAC;QACL,aAAa,CAAC,UAAU,GAAG;AACzB,YAAA,IAAI,aAAa,CAAC,UAAU,IAAI,EAAE,CAAC;AACnC,YAAA,GAAG,wBAAwB;SAC5B;QACD,OAAO,aAAa,CAAC,IAAI;;IAG3B,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,KAAI;QAClC,IAAI,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE;AAC/C,YAAA,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;gBACrE,aAAa,CAAC,MAAM,GAAG;AACrB,oBAAA,IAAI,aAAa,CAAC,MAAM,IAAI,EAAE,CAAC;AAC/B,oBAAA,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM;iBACtB;;AAEH,YAAA,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;gBACrE,aAAa,CAAC,MAAM,GAAG;AACrB,oBAAA,IAAI,aAAa,CAAC,MAAM,IAAI,EAAE,CAAC;AAC/B,oBAAA,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM;iBACtB;;AAEH,YAAA,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,KAAK,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE;AACjE,gBAAA,MAAM,gBAAgB,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,QAAQ,MAAM;AAC3D,oBAAA,GAAG,QAAQ;AACX,oBAAA,IAAI,EAAE,QAAQ,CAAC,IAAI,IAAI,EAAE;AAC1B,iBAAA,CAAC,CAAC;gBACH,aAAa,CAAC,UAAU,GAAG;AACzB,oBAAA,IAAI,aAAa,CAAC,UAAU,IAAI,EAAE,CAAC;AACnC,oBAAA,GAAG,gBAAgB;iBACpB;;;AAGP,KAAC,CAAC;IAEF,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,aAAa,EAAE;AAC/C;AAEA,SAAS,qBAAqB,CAAC,EAC7B,SAAS,EACT,UAAU,EACV,eAAe,EACf,eAAe,EACf,MAAM,GAOP,EAAA;AACC,IAAA,OAAO,gBAAgB,EACrB,KAAK,EACL,IAAI,EACJ,OAAO,EACP,OAAO,GAAG,IAAI,EACd,UAAU,GAAG,CAAC,EACd,eAAe,EACf,MAAM,GAAG,KAAK,EACd,MAAM,GAAG,KAAK,EACd,IAAI,GAAG,KAAK,GAWb,EAAA;AACC,QAAA,IAAI;;AAEF,YAAA,MAAM,YAAY,GAAG,MAAM,uBAAuB,CAAC;gBACjD,SAAS;gBACT,KAAK;gBACL,IAAI;gBACJ,OAAO;gBACP,UAAU;gBACV,MAAM;gBACN,MAAM;gBACN,IAAI;gBACJ,MAAM;AACP,aAAA,CAAC;AAEF,YAAA,eAAe,GAAG,YAAY,CAAC;AAE/B,YAAA,MAAM,gBAAgB,GAAG,MAAM,eAAe,CAAC,cAAc,CAAC;gBAC5D,KAAK;gBACL,IAAI;AACJ,gBAAA,MAAM,EAAE,YAAY;gBACpB,OAAO;gBACP,eAAe;AACf,gBAAA,WAAW,EAAE,UAAU;AACxB,aAAA,CAAC;AAEF,YAAA,OAAOA,2BAAgB,CAAC,gBAAgB,CAAC;;QACzC,OAAO,KAAK,EAAE;AACd,YAAA,MAAM,CAAC,KAAK,CAAC,kBAAkB,EAAE,KAAK,CAAC;YACvC,OAAO;AACL,gBAAA,OAAO,EAAE,EAAE;AACX,gBAAA,UAAU,EAAE,EAAE;AACd,gBAAA,MAAM,EAAE,EAAE;AACV,gBAAA,MAAM,EAAE,EAAE;AACV,gBAAA,IAAI,EAAE,EAAE;AACR,gBAAA,eAAe,EAAE,EAAE;AACnB,gBAAA,KAAK,EAAE,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;aAC9D;;AAEL,KAAC;AACH;AAEA,SAAS,qBAAqB,CAAC,EAC7B,cAAc,EACd,eAAe,GAIhB,EAAA;AACC,IAAA,OAAO,UAAU,OAAuB,EAAA;QACtC,IAAI,CAAC,eAAe,EAAE;YACpB;;AAEF,QAAA,eAAe,CAAC,OAAO,EAAE,cAAc,CAAC;AAC1C,KAAC;AACH;AAEA,SAAS,UAAU,CAAC,EAClB,MAAM,EACN,MAAM,EACN,eAAe,EAAE,gBAAgB,GAKlC,EAAA;IACC,OAAOC,UAAI,CACT,OAAO,MAAM,EAAE,cAAc,KAAI;AAC/B,QAAA,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM;AACjE,QAAA,MAAM,OAAO,GAAG,OAAO,EAAE,KAAK,QAAQ,IAAI,EAAE,GAAG,EAAE,GAAG,SAAS;AAC7D,QAAA,MAAM,YAAY,GAAG,MAAM,MAAM,CAAC;YAChC,KAAK;YACL,IAAI;YACJ,OAAO;YACP,MAAM;YACN,MAAM;YACN,IAAI;YACJ,eAAe,EAAE,qBAAqB,CAAC;gBACrC,cAAc;AACd,gBAAA,eAAe,EAAE,gBAAgB;aAClC,CAAC;AACH,SAAA,CAAC;QACF,MAAM,IAAI,GAAG,cAAc,CAAC,QAAQ,EAAE,IAAI,IAAI,CAAC;AAC/C,QAAA,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,GAAGC,0BAAmB,CAAC,IAAI,EAAE,YAAY,CAAC;QACtE,MAAM,IAAI,GAAuB,EAAE,IAAI,EAAE,GAAG,YAAY,EAAE,UAAU,EAAE;AACtE,QAAA,OAAO,CAAC,MAAM,EAAE,EAAE,CAACC,eAAS,CAAC,UAAU,GAAG,IAAI,EAAE,CAAC;AACnD,KAAC,EACD;QACE,IAAI,EAAEA,eAAS,CAAC,UAAU;AAC1B,QAAA,WAAW,EAAE,CAAA;;;;;;;;;;;;;;;;;;;;;;AAsBlB,CAAA,CAAC,IAAI,EAAE;AACF,QAAA,MAAM,EAAE,MAAM;QACd,cAAc,EAAEA,eAAS,CAAC,oBAAoB;AAC/C,KAAA,CACF;AACH;AAEA;;;;;;AAMG;MACU,gBAAgB,GAAG,CAC9B,MAA6B,GAAA,EAAE,KACa;IAC5C,MAAM,EACJ,cAAc,GAAG,QAAQ,EACzB,YAAY,EACZ,kBAAkB,EAClB,aAAa,EACb,YAAY,GAAG,QAAQ,EACvB,UAAU,GAAG,CAAC,EACd,UAAU,GAAG,CAAC,eAAe,CAAC,EAC9B,aAAa,GAAG,IAAI,EACpB,UAAU,GAAG,CAAC,EACd,eAAe,EACf,eAAe,EACf,gBAAgB,GAAG,CAAC,UAAU,EAAE,MAAM,CAAC,EACvC,cAAc,EACd,UAAU,EACV,YAAY,EACZ,eAAe,EAAE,gBAAgB,EACjC,eAAe,GAChB,GAAG,MAAM;IAEV,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,IAAIC,yBAAmB,EAAE;AAErD,IAAA,MAAM,YAAY,GAOd;AACF,QAAA,KAAK,EAAEC,kBAAW;AAClB,QAAA,IAAI,EAAEC,iBAAU;AAChB,QAAA,MAAM,EAAEC,mBAAY;AACpB,QAAA,MAAM,EAAEC,mBAAY;AACpB,QAAA,IAAI,EAAEC,iBAAU;KACjB;AAED,IAAA,IAAI,cAAc,KAAK,QAAQ,EAAE;AAC/B,QAAA,YAAY,CAAC,OAAO,GAAGC,oBAAa;;IAGtC,MAAM,UAAU,GAAGC,KAAC,CAAC,MAAM,CAAC,YAAY,CAAC;IAEzC,MAAM,SAAS,GAAGC,sBAAe,CAAC;QAChC,cAAc;QACd,YAAY;QACZ,kBAAkB;QAClB,aAAa;AACd,KAAA,CAAC;IAEF,MAAM,gBAAgB,GAAGC,gCAAsB,CAAC;AAC9C,QAAA,MAAM,EAAE,eAAe,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB;AACxD,QAAA,MAAM,EAAE,eAAe;AACvB,QAAA,OAAO,EAAE,cAAc;AACvB,QAAA,OAAO,EAAE,gBAAgB;AAC1B,KAAA,CAAC;IAEF,MAAM,gBAAgB,GAAGC,wBAAc,CAAC;QACtC,YAAY;QACZ,UAAU;QACV,YAAY;QACZ,MAAM;AACP,KAAA,CAAC;IAEF,IAAI,CAAC,gBAAgB,EAAE;AACrB,QAAA,MAAM,CAAC,IAAI,CAAC,8CAA8C,CAAC;;IAG7D,MAAM,eAAe,GAAGC,4BAAqB,CAC3C;AACE,QAAA,QAAQ,EAAE,gBAAgB;QAC1B,UAAU;QAGV,MAAM;KACP,EACD,gBAAgB,CACjB;IAED,MAAMC,QAAM,GAAG,qBAAqB,CAAC;QACnC,SAAS;QACT,UAAU;QACV,eAAe;QACf,eAAe;QACf,MAAM;AACP,KAAA,CAAC;AAEF,IAAA,OAAO,UAAU,CAAC;gBAChBA,QAAM;AACN,QAAA,MAAM,EAAE,UAAU;AAClB,QAAA,eAAe,EAAE,gBAAgB;AAClC,KAAA,CAAC;AACJ;;;;"}
+{"version":3,"file":"tool.cjs","sources":["../../../../src/tools/search/tool.ts"],"sourcesContent":["import { z } from 'zod';\nimport { tool, DynamicStructuredTool } from '@langchain/core/tools';\nimport type { RunnableConfig } from '@langchain/core/runnables';\nimport type * as t from './types';\nimport {\n  DATE_RANGE,\n  querySchema,\n  dateSchema,\n  countrySchema,\n  imagesSchema,\n  videosSchema,\n  newsSchema,\n} from './schema';\nimport { createSearchAPI, createSourceProcessor } from './search';\nimport { createFirecrawlScraper } from './firecrawl';\nimport { expandHighlights } from './highlights';\nimport { formatResultsForLLM } from './format';\nimport { createDefaultLogger } from './utils';\nimport { createReranker } from './rerankers';\nimport { Constants } from '@/common';\n\n/**\n * Executes parallel searches and merges the results\n */\nasync function executeParallelSearches({\n  searchAPI,\n  query,\n  date,\n  country,\n  safeSearch,\n  images,\n  videos,\n  news,\n  logger,\n}: {\n  searchAPI: ReturnType<typeof createSearchAPI>;\n  query: string;\n  date?: DATE_RANGE;\n  country?: string;\n  safeSearch: t.SearchToolConfig['safeSearch'];\n  images: boolean;\n  videos: boolean;\n  news: boolean;\n  logger: t.Logger;\n}): Promise<t.SearchResult> {\n  // Prepare all search tasks to run in parallel\n  const searchTasks: Promise<t.SearchResult>[] = [\n    // Main search\n    searchAPI.getSources({\n      query,\n      date,\n      country,\n      safeSearch,\n    }),\n  ];\n\n  if (images) {\n    searchTasks.push(\n      searchAPI\n        .getSources({\n          query,\n          date,\n          country,\n          safeSearch,\n          type: 'images',\n        })\n        .catch((error) => {\n          logger.error('Error fetching images:', error);\n          return {\n            success: false,\n            error: `Images search failed: ${error instanceof Error ? error.message : String(error)}`,\n          };\n        })\n    );\n  }\n  if (videos) {\n    searchTasks.push(\n      searchAPI\n        .getSources({\n          query,\n          date,\n          country,\n          safeSearch,\n          type: 'videos',\n        })\n        .catch((error) => {\n          logger.error('Error fetching videos:', error);\n          return {\n            success: false,\n            error: `Videos search failed: ${error instanceof Error ? error.message : String(error)}`,\n          };\n        })\n    );\n  }\n  if (news) {\n    searchTasks.push(\n      searchAPI\n        .getSources({\n          query,\n          date,\n          country,\n          safeSearch,\n          type: 'news',\n        })\n        .catch((error) => {\n          logger.error('Error fetching news:', error);\n          return {\n            success: false,\n            error: `News search failed: ${error instanceof Error ? error.message : String(error)}`,\n          };\n        })\n    );\n  }\n\n  // Run all searches in parallel\n  const results = await Promise.all(searchTasks);\n\n  // Get the main search result (first result)\n  const mainResult = results[0];\n  if (!mainResult.success) {\n    throw new Error(mainResult.error ?? 'Search failed');\n  }\n\n  // Merge additional results with the main results\n  const mergedResults = { ...mainResult.data };\n\n  // Convert existing news to topStories if present\n  if (mergedResults.news !== undefined && mergedResults.news.length > 0) {\n    const existingNewsAsTopStories = mergedResults.news\n      .filter((newsItem) => newsItem.link !== undefined && newsItem.link !== '')\n      .map((newsItem) => ({\n        title: newsItem.title ?? '',\n        link: newsItem.link ?? '',\n        source: newsItem.source ?? '',\n        date: newsItem.date ?? '',\n        imageUrl: newsItem.imageUrl ?? '',\n        processed: false,\n      }));\n    mergedResults.topStories = [\n      ...(mergedResults.topStories ?? []),\n      ...existingNewsAsTopStories,\n    ];\n    delete mergedResults.news;\n  }\n\n  results.slice(1).forEach((result) => {\n    if (result.success && result.data !== undefined) {\n      if (result.data.images !== undefined && result.data.images.length > 0) {\n        mergedResults.images = [\n          ...(mergedResults.images ?? []),\n          ...result.data.images,\n        ];\n      }\n      if (result.data.videos !== undefined && result.data.videos.length > 0) {\n        mergedResults.videos = [\n          ...(mergedResults.videos ?? []),\n          ...result.data.videos,\n        ];\n      }\n      if (result.data.news !== undefined && result.data.news.length > 0) {\n        const newsAsTopStories = result.data.news.map((newsItem) => ({\n          ...newsItem,\n          link: newsItem.link ?? '',\n        }));\n        mergedResults.topStories = [\n          ...(mergedResults.topStories ?? []),\n          ...newsAsTopStories,\n        ];\n      }\n    }\n  });\n\n  return { success: true, data: mergedResults };\n}\n\nfunction createSearchProcessor({\n  searchAPI,\n  safeSearch,\n  sourceProcessor,\n  onGetHighlights,\n  logger,\n}: {\n  safeSearch: t.SearchToolConfig['safeSearch'];\n  searchAPI: ReturnType<typeof createSearchAPI>;\n  sourceProcessor: ReturnType<typeof createSourceProcessor>;\n  onGetHighlights: t.SearchToolConfig['onGetHighlights'];\n  logger: t.Logger;\n}) {\n  return async function ({\n    query,\n    date,\n    country,\n    proMode = true,\n    maxSources = 5,\n    onSearchResults,\n    images = false,\n    videos = false,\n    news = false,\n  }: {\n    query: string;\n    country?: string;\n    date?: DATE_RANGE;\n    proMode?: boolean;\n    maxSources?: number;\n    onSearchResults: t.SearchToolConfig['onSearchResults'];\n    images?: boolean;\n    videos?: boolean;\n    news?: boolean;\n  }): Promise<t.SearchResultData> {\n    try {\n      // Execute parallel searches and merge results\n      const searchResult = await executeParallelSearches({\n        searchAPI,\n        query,\n        date,\n        country,\n        safeSearch,\n        images,\n        videos,\n        news,\n        logger,\n      });\n\n      onSearchResults?.(searchResult);\n\n      const processedSources = await sourceProcessor.processSources({\n        query,\n        news,\n        result: searchResult,\n        proMode,\n        onGetHighlights,\n        numElements: maxSources,\n      });\n\n      return expandHighlights(processedSources);\n    } catch (error) {\n      logger.error('Error in search:', error);\n      return {\n        organic: [],\n        topStories: [],\n        images: [],\n        videos: [],\n        news: [],\n        relatedSearches: [],\n        error: error instanceof Error ? error.message : String(error),\n      };\n    }\n  };\n}\n\nfunction createOnSearchResults({\n  runnableConfig,\n  onSearchResults,\n}: {\n  runnableConfig: RunnableConfig;\n  onSearchResults: t.SearchToolConfig['onSearchResults'];\n}) {\n  return function (results: t.SearchResult): void {\n    if (!onSearchResults) {\n      return;\n    }\n    onSearchResults(results, runnableConfig);\n  };\n}\n\nfunction createTool({\n  schema,\n  search,\n  onSearchResults: _onSearchResults,\n}: {\n  schema: t.SearchToolSchema;\n  search: ReturnType<typeof createSearchProcessor>;\n  onSearchResults: t.SearchToolConfig['onSearchResults'];\n}): DynamicStructuredTool<typeof schema> {\n  return tool<typeof schema>(\n    async (params, runnableConfig) => {\n      const { query, date, country: _c, images, videos, news } = params;\n      const country = typeof _c === 'string' && _c ? _c : undefined;\n      const searchResult = await search({\n        query,\n        date,\n        country,\n        images,\n        videos,\n        news,\n        onSearchResults: createOnSearchResults({\n          runnableConfig,\n          onSearchResults: _onSearchResults,\n        }),\n      });\n      const turn = runnableConfig.toolCall?.turn ?? 0;\n      const { output, references } = formatResultsForLLM(turn, searchResult);\n      const data: t.SearchResultData = { turn, ...searchResult, references };\n      return [output, { [Constants.WEB_SEARCH]: data }];\n    },\n    {\n      name: Constants.WEB_SEARCH,\n      description: `Real-time search. Results have required citation anchors.\n\nNote: Use ONCE per reply unless instructed otherwise.\n\nAnchors:\n- \\\\ue202turnXtypeY\n- X = turn idx, type = 'search' | 'news' | 'image' | 'ref', Y = item idx\n\nSpecial Markers:\n- \\\\ue203...\\\\ue204 — highlight start/end of cited text (for Standalone or Group citations)\n- \\\\ue200...\\\\ue201 — group block (e.g. \\\\ue200\\\\ue202turn0search1\\\\ue202turn0news2\\\\ue201)\n\n**CITE EVERY NON-OBVIOUS FACT/QUOTE:**\nUse anchor marker(s) immediately after the statement:\n- Standalone: \"Pure functions produce same output. \\\\ue202turn0search0\"\n- Standalone (multiple): \"Today's News \\\\ue202turn0search0\\\\ue202turn0news0\"\n- Highlight: \"\\\\ue203Highlight text.\\\\ue204\\\\ue202turn0news1\"\n- Group: \"Sources. \\\\ue200\\\\ue202turn0search0\\\\ue202turn0news1\\\\ue201\"\n- Group Highlight: \"\\\\ue203Highlight for group.\\\\ue204 \\\\ue200\\\\ue202turn0search0\\\\ue202turn0news1\\\\ue201\"\n- Image: \"See photo \\\\ue202turn0image0.\"\n\n**NEVER use markdown links, [1], or footnotes. CITE ONLY with anchors provided.**\n`.trim(),\n      schema: schema,\n      responseFormat: Constants.CONTENT_AND_ARTIFACT,\n    }\n  );\n}\n\n/**\n * Creates a search tool with a schema that dynamically includes the country field\n * only when the searchProvider is 'serper'.\n *\n * @param config - The search tool configuration\n * @returns A DynamicStructuredTool with a schema that depends on the searchProvider\n */\nexport const createSearchTool = (\n  config: t.SearchToolConfig = {}\n): DynamicStructuredTool<typeof toolSchema> => {\n  const {\n    searchProvider = 'serper',\n    serperApiKey,\n    searxngInstanceUrl,\n    searxngApiKey,\n    rerankerType = 'cohere',\n    topResults = 5,\n    strategies = ['no_extraction'],\n    filterContent = true,\n    safeSearch = 1,\n    firecrawlApiKey,\n    firecrawlApiUrl,\n    firecrawlOptions,\n    scraperTimeout,\n    jinaApiKey,\n    cohereApiKey,\n    onSearchResults: _onSearchResults,\n    onGetHighlights,\n  } = config;\n\n  const logger = config.logger || createDefaultLogger();\n\n  const schemaObject: {\n    query: z.ZodString;\n    date: z.ZodOptional<z.ZodNativeEnum<typeof DATE_RANGE>>;\n    country?: z.ZodOptional<z.ZodString>;\n    images: z.ZodOptional<z.ZodBoolean>;\n    videos: z.ZodOptional<z.ZodBoolean>;\n    news: z.ZodOptional<z.ZodBoolean>;\n  } = {\n    query: querySchema,\n    date: dateSchema,\n    images: imagesSchema,\n    videos: videosSchema,\n    news: newsSchema,\n  };\n\n  if (searchProvider === 'serper') {\n    schemaObject.country = countrySchema;\n  }\n\n  const toolSchema = z.object(schemaObject);\n\n  const searchAPI = createSearchAPI({\n    searchProvider,\n    serperApiKey,\n    searxngInstanceUrl,\n    searxngApiKey,\n  });\n\n  const firecrawlScraper = createFirecrawlScraper({\n    ...firecrawlOptions,\n    apiKey: firecrawlApiKey ?? process.env.FIRECRAWL_API_KEY,\n    apiUrl: firecrawlApiUrl,\n    timeout: scraperTimeout ?? firecrawlOptions?.timeout,\n    formats: firecrawlOptions?.formats ?? ['markdown', 'rawHtml'],\n  });\n\n  const selectedReranker = createReranker({\n    rerankerType,\n    jinaApiKey,\n    cohereApiKey,\n    logger,\n  });\n\n  if (!selectedReranker) {\n    logger.warn('No reranker selected. Using default ranking.');\n  }\n\n  const sourceProcessor = createSourceProcessor(\n    {\n      reranker: selectedReranker,\n      topResults,\n      strategies,\n      filterContent,\n      logger,\n    },\n    firecrawlScraper\n  );\n\n  const search = createSearchProcessor({\n    searchAPI,\n    safeSearch,\n    sourceProcessor,\n    onGetHighlights,\n    logger,\n  });\n\n  return createTool({\n    search,\n    schema: toolSchema,\n    onSearchResults: _onSearchResults,\n  });\n};\n"],"names":["expandHighlights","tool","formatResultsForLLM","Constants","createDefaultLogger","querySchema","dateSchema","imagesSchema","videosSchema","newsSchema","countrySchema","z","createSearchAPI","createFirecrawlScraper","createReranker","createSourceProcessor","search"],"mappings":";;;;;;;;;;;;;AAqBA;;AAEG;AACH,eAAe,uBAAuB,CAAC,EACrC,SAAS,EACT,KAAK,EACL,IAAI,EACJ,OAAO,EACP,UAAU,EACV,MAAM,EACN,MAAM,EACN,IAAI,EACJ,MAAM,GAWP,EAAA;;AAEC,IAAA,MAAM,WAAW,GAA8B;;QAE7C,SAAS,CAAC,UAAU,CAAC;YACnB,KAAK;YACL,IAAI;YACJ,OAAO;YACP,UAAU;SACX,CAAC;KACH;IAED,IAAI,MAAM,EAAE;QACV,WAAW,CAAC,IAAI,CACd;AACG,aAAA,UAAU,CAAC;YACV,KAAK;YACL,IAAI;YACJ,OAAO;YACP,UAAU;AACV,YAAA,IAAI,EAAE,QAAQ;SACf;AACA,aAAA,KAAK,CAAC,CAAC,KAAK,KAAI;AACf,YAAA,MAAM,CAAC,KAAK,CAAC,wBAAwB,EAAE,KAAK,CAAC;YAC7C,OAAO;AACL,gBAAA,OAAO,EAAE,KAAK;AACd,gBAAA,KAAK,EAAE,CAAyB,sBAAA,EAAA,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,CAAE,CAAA;aACzF;SACF,CAAC,CACL;;IAEH,IAAI,MAAM,EAAE;QACV,WAAW,CAAC,IAAI,CACd;AACG,aAAA,UAAU,CAAC;YACV,KAAK;YACL,IAAI;YACJ,OAAO;YACP,UAAU;AACV,YAAA,IAAI,EAAE,QAAQ;SACf;AACA,aAAA,KAAK,CAAC,CAAC,KAAK,KAAI;AACf,YAAA,MAAM,CAAC,KAAK,CAAC,wBAAwB,EAAE,KAAK,CAAC;YAC7C,OAAO;AACL,gBAAA,OAAO,EAAE,KAAK;AACd,gBAAA,KAAK,EAAE,CAAyB,sBAAA,EAAA,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,CAAE,CAAA;aACzF;SACF,CAAC,CACL;;IAEH,IAAI,IAAI,EAAE;QACR,WAAW,CAAC,IAAI,CACd;AACG,aAAA,UAAU,CAAC;YACV,KAAK;YACL,IAAI;YACJ,OAAO;YACP,UAAU;AACV,YAAA,IAAI,EAAE,MAAM;SACb;AACA,aAAA,KAAK,CAAC,CAAC,KAAK,KAAI;AACf,YAAA,MAAM,CAAC,KAAK,CAAC,sBAAsB,EAAE,KAAK,CAAC;YAC3C,OAAO;AACL,gBAAA,OAAO,EAAE,KAAK;AACd,gBAAA,KAAK,EAAE,CAAuB,oBAAA,EAAA,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,CAAE,CAAA;aACvF;SACF,CAAC,CACL;;;IAIH,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC;;AAG9C,IAAA,MAAM,UAAU,GAAG,OAAO,CAAC,CAAC,CAAC;AAC7B,IAAA,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE;QACvB,MAAM,IAAI,KAAK,CAAC,UAAU,CAAC,KAAK,IAAI,eAAe,CAAC;;;IAItD,MAAM,aAAa,GAAG,EAAE,GAAG,UAAU,CAAC,IAAI,EAAE;;AAG5C,IAAA,IAAI,aAAa,CAAC,IAAI,KAAK,SAAS,IAAI,aAAa,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE;AACrE,QAAA,MAAM,wBAAwB,GAAG,aAAa,CAAC;AAC5C,aAAA,MAAM,CAAC,CAAC,QAAQ,KAAK,QAAQ,CAAC,IAAI,KAAK,SAAS,IAAI,QAAQ,CAAC,IAAI,KAAK,EAAE;AACxE,aAAA,GAAG,CAAC,CAAC,QAAQ,MAAM;AAClB,YAAA,KAAK,EAAE,QAAQ,CAAC,KAAK,IAAI,EAAE;AAC3B,YAAA,IAAI,EAAE,QAAQ,CAAC,IAAI,IAAI,EAAE;AACzB,YAAA,MAAM,EAAE,QAAQ,CAAC,MAAM,IAAI,EAAE;AAC7B,YAAA,IAAI,EAAE,QAAQ,CAAC,IAAI,IAAI,EAAE;AACzB,YAAA,QAAQ,EAAE,QAAQ,CAAC,QAAQ,IAAI,EAAE;AACjC,YAAA,SAAS,EAAE,KAAK;AACjB,SAAA,CAAC,CAAC;QACL,aAAa,CAAC,UAAU,GAAG;AACzB,YAAA,IAAI,aAAa,CAAC,UAAU,IAAI,EAAE,CAAC;AACnC,YAAA,GAAG,wBAAwB;SAC5B;QACD,OAAO,aAAa,CAAC,IAAI;;IAG3B,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,KAAI;QAClC,IAAI,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE;AAC/C,YAAA,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;gBACrE,aAAa,CAAC,MAAM,GAAG;AACrB,oBAAA,IAAI,aAAa,CAAC,MAAM,IAAI,EAAE,CAAC;AAC/B,oBAAA,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM;iBACtB;;AAEH,YAAA,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;gBACrE,aAAa,CAAC,MAAM,GAAG;AACrB,oBAAA,IAAI,aAAa,CAAC,MAAM,IAAI,EAAE,CAAC;AAC/B,oBAAA,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM;iBACtB;;AAEH,YAAA,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,KAAK,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE;AACjE,gBAAA,MAAM,gBAAgB,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,QAAQ,MAAM;AAC3D,oBAAA,GAAG,QAAQ;AACX,oBAAA,IAAI,EAAE,QAAQ,CAAC,IAAI,IAAI,EAAE;AAC1B,iBAAA,CAAC,CAAC;gBACH,aAAa,CAAC,UAAU,GAAG;AACzB,oBAAA,IAAI,aAAa,CAAC,UAAU,IAAI,EAAE,CAAC;AACnC,oBAAA,GAAG,gBAAgB;iBACpB;;;AAGP,KAAC,CAAC;IAEF,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,aAAa,EAAE;AAC/C;AAEA,SAAS,qBAAqB,CAAC,EAC7B,SAAS,EACT,UAAU,EACV,eAAe,EACf,eAAe,EACf,MAAM,GAOP,EAAA;AACC,IAAA,OAAO,gBAAgB,EACrB,KAAK,EACL,IAAI,EACJ,OAAO,EACP,OAAO,GAAG,IAAI,EACd,UAAU,GAAG,CAAC,EACd,eAAe,EACf,MAAM,GAAG,KAAK,EACd,MAAM,GAAG,KAAK,EACd,IAAI,GAAG,KAAK,GAWb,EAAA;AACC,QAAA,IAAI;;AAEF,YAAA,MAAM,YAAY,GAAG,MAAM,uBAAuB,CAAC;gBACjD,SAAS;gBACT,KAAK;gBACL,IAAI;gBACJ,OAAO;gBACP,UAAU;gBACV,MAAM;gBACN,MAAM;gBACN,IAAI;gBACJ,MAAM;AACP,aAAA,CAAC;AAEF,YAAA,eAAe,GAAG,YAAY,CAAC;AAE/B,YAAA,MAAM,gBAAgB,GAAG,MAAM,eAAe,CAAC,cAAc,CAAC;gBAC5D,KAAK;gBACL,IAAI;AACJ,gBAAA,MAAM,EAAE,YAAY;gBACpB,OAAO;gBACP,eAAe;AACf,gBAAA,WAAW,EAAE,UAAU;AACxB,aAAA,CAAC;AAEF,YAAA,OAAOA,2BAAgB,CAAC,gBAAgB,CAAC;;QACzC,OAAO,KAAK,EAAE;AACd,YAAA,MAAM,CAAC,KAAK,CAAC,kBAAkB,EAAE,KAAK,CAAC;YACvC,OAAO;AACL,gBAAA,OAAO,EAAE,EAAE;AACX,gBAAA,UAAU,EAAE,EAAE;AACd,gBAAA,MAAM,EAAE,EAAE;AACV,gBAAA,MAAM,EAAE,EAAE;AACV,gBAAA,IAAI,EAAE,EAAE;AACR,gBAAA,eAAe,EAAE,EAAE;AACnB,gBAAA,KAAK,EAAE,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;aAC9D;;AAEL,KAAC;AACH;AAEA,SAAS,qBAAqB,CAAC,EAC7B,cAAc,EACd,eAAe,GAIhB,EAAA;AACC,IAAA,OAAO,UAAU,OAAuB,EAAA;QACtC,IAAI,CAAC,eAAe,EAAE;YACpB;;AAEF,QAAA,eAAe,CAAC,OAAO,EAAE,cAAc,CAAC;AAC1C,KAAC;AACH;AAEA,SAAS,UAAU,CAAC,EAClB,MAAM,EACN,MAAM,EACN,eAAe,EAAE,gBAAgB,GAKlC,EAAA;IACC,OAAOC,UAAI,CACT,OAAO,MAAM,EAAE,cAAc,KAAI;AAC/B,QAAA,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM;AACjE,QAAA,MAAM,OAAO,GAAG,OAAO,EAAE,KAAK,QAAQ,IAAI,EAAE,GAAG,EAAE,GAAG,SAAS;AAC7D,QAAA,MAAM,YAAY,GAAG,MAAM,MAAM,CAAC;YAChC,KAAK;YACL,IAAI;YACJ,OAAO;YACP,MAAM;YACN,MAAM;YACN,IAAI;YACJ,eAAe,EAAE,qBAAqB,CAAC;gBACrC,cAAc;AACd,gBAAA,eAAe,EAAE,gBAAgB;aAClC,CAAC;AACH,SAAA,CAAC;QACF,MAAM,IAAI,GAAG,cAAc,CAAC,QAAQ,EAAE,IAAI,IAAI,CAAC;AAC/C,QAAA,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,GAAGC,0BAAmB,CAAC,IAAI,EAAE,YAAY,CAAC;QACtE,MAAM,IAAI,GAAuB,EAAE,IAAI,EAAE,GAAG,YAAY,EAAE,UAAU,EAAE;AACtE,QAAA,OAAO,CAAC,MAAM,EAAE,EAAE,CAACC,eAAS,CAAC,UAAU,GAAG,IAAI,EAAE,CAAC;AACnD,KAAC,EACD;QACE,IAAI,EAAEA,eAAS,CAAC,UAAU;AAC1B,QAAA,WAAW,EAAE,CAAA;;;;;;;;;;;;;;;;;;;;;;AAsBlB,CAAA,CAAC,IAAI,EAAE;AACF,QAAA,MAAM,EAAE,MAAM;QACd,cAAc,EAAEA,eAAS,CAAC,oBAAoB;AAC/C,KAAA,CACF;AACH;AAEA;;;;;;AAMG;MACU,gBAAgB,GAAG,CAC9B,MAA6B,GAAA,EAAE,KACa;IAC5C,MAAM,EACJ,cAAc,GAAG,QAAQ,EACzB,YAAY,EACZ,kBAAkB,EAClB,aAAa,EACb,YAAY,GAAG,QAAQ,EACvB,UAAU,GAAG,CAAC,EACd,UAAU,GAAG,CAAC,eAAe,CAAC,EAC9B,aAAa,GAAG,IAAI,EACpB,UAAU,GAAG,CAAC,EACd,eAAe,EACf,eAAe,EACf,gBAAgB,EAChB,cAAc,EACd,UAAU,EACV,YAAY,EACZ,eAAe,EAAE,gBAAgB,EACjC,eAAe,GAChB,GAAG,MAAM;IAEV,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,IAAIC,yBAAmB,EAAE;AAErD,IAAA,MAAM,YAAY,GAOd;AACF,QAAA,KAAK,EAAEC,kBAAW;AAClB,QAAA,IAAI,EAAEC,iBAAU;AAChB,QAAA,MAAM,EAAEC,mBAAY;AACpB,QAAA,MAAM,EAAEC,mBAAY;AACpB,QAAA,IAAI,EAAEC,iBAAU;KACjB;AAED,IAAA,IAAI,cAAc,KAAK,QAAQ,EAAE;AAC/B,QAAA,YAAY,CAAC,OAAO,GAAGC,oBAAa;;IAGtC,MAAM,UAAU,GAAGC,KAAC,CAAC,MAAM,CAAC,YAAY,CAAC;IAEzC,MAAM,SAAS,GAAGC,sBAAe,CAAC;QAChC,cAAc;QACd,YAAY;QACZ,kBAAkB;QAClB,aAAa;AACd,KAAA,CAAC;IAEF,MAAM,gBAAgB,GAAGC,gCAAsB,CAAC;AAC9C,QAAA,GAAG,gBAAgB;AACnB,QAAA,MAAM,EAAE,eAAe,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB;AACxD,QAAA,MAAM,EAAE,eAAe;AACvB,QAAA,OAAO,EAAE,cAAc,IAAI,gBAAgB,EAAE,OAAO;QACpD,OAAO,EAAE,gBAAgB,EAAE,OAAO,IAAI,CAAC,UAAU,EAAE,SAAS,CAAC;AAC9D,KAAA,CAAC;IAEF,MAAM,gBAAgB,GAAGC,wBAAc,CAAC;QACtC,YAAY;QACZ,UAAU;QACV,YAAY;QACZ,MAAM;AACP,KAAA,CAAC;IAEF,IAAI,CAAC,gBAAgB,EAAE;AACrB,QAAA,MAAM,CAAC,IAAI,CAAC,8CAA8C,CAAC;;IAG7D,MAAM,eAAe,GAAGC,4BAAqB,CAC3C;AACE,QAAA,QAAQ,EAAE,gBAAgB;QAC1B,UAAU;QAGV,MAAM;KACP,EACD,gBAAgB,CACjB;IAED,MAAMC,QAAM,GAAG,qBAAqB,CAAC;QACnC,SAAS;QACT,UAAU;QACV,eAAe;QACf,eAAe;QACf,MAAM;AACP,KAAA,CAAC;AAEF,IAAA,OAAO,UAAU,CAAC;gBAChBA,QAAM;AACN,QAAA,MAAM,EAAE,UAAU;AAClB,QAAA,eAAe,EAAE,gBAAgB;AAClC,KAAA,CAAC;AACJ;;;;"}

package/dist/esm/tools/search/firecrawl.mjs

@@ -12,15 +12,45 @@ class FirecrawlScraper {
     defaultFormats;
     timeout;
     logger;
+    includeTags;
+    excludeTags;
+    waitFor;
+    maxAge;
+    mobile;
+    skipTlsVerification;
+    blockAds;
+    removeBase64Images;
+    parsePDF;
+    storeInCache;
+    zeroDataRetention;
+    headers;
+    location;
+    onlyMainContent;
+    changeTrackingOptions;
     constructor(config = {}) {
         this.apiKey = config.apiKey ?? process.env.FIRECRAWL_API_KEY ?? '';
         const baseUrl = config.apiUrl ??
             process.env.FIRECRAWL_BASE_URL ??
             'https://api.firecrawl.dev';
         this.apiUrl = `${baseUrl.replace(/\/+$/, '')}/v1/scrape`;
-        this.defaultFormats = config.formats ?? ['markdown', 'html'];
+        this.defaultFormats = config.formats ?? ['markdown', 'rawHtml'];
         this.timeout = config.timeout ?? 7500;
         this.logger = config.logger || createDefaultLogger();
+        this.includeTags = config.includeTags;
+        this.excludeTags = config.excludeTags;
+        this.waitFor = config.waitFor;
+        this.maxAge = config.maxAge;
+        this.mobile = config.mobile;
+        this.skipTlsVerification = config.skipTlsVerification;
+        this.blockAds = config.blockAds;
+        this.removeBase64Images = config.removeBase64Images;
+        this.parsePDF = config.parsePDF;
+        this.storeInCache = config.storeInCache;
+        this.zeroDataRetention = config.zeroDataRetention;
+        this.headers = config.headers;
+        this.location = config.location;
+        this.onlyMainContent = config.onlyMainContent;
+        this.changeTrackingOptions = config.changeTrackingOptions;
         if (!this.apiKey) {
             this.logger.warn('FIRECRAWL_API_KEY is not set. Scraping will not work.');
         }
@@ -43,15 +73,27 @@ class FirecrawlScraper {
             ];
         }
         try {
-            const response = await axios.post(this.apiUrl, {
+            const payload = omitUndefined({
                 url,
-                formats: options.formats || this.defaultFormats,
-                includeTags: options.includeTags,
-                excludeTags: options.excludeTags,
-                headers: options.headers,
-                waitFor: options.waitFor,
+                formats: options.formats ?? this.defaultFormats,
+                includeTags: options.includeTags ?? this.includeTags,
+                excludeTags: options.excludeTags ?? this.excludeTags,
+                headers: options.headers ?? this.headers,
+                waitFor: options.waitFor ?? this.waitFor,
                 timeout: options.timeout ?? this.timeout,
-            }, {
+                onlyMainContent: options.onlyMainContent ?? this.onlyMainContent,
+                maxAge: options.maxAge ?? this.maxAge,
+                mobile: options.mobile ?? this.mobile,
+                skipTlsVerification: options.skipTlsVerification ?? this.skipTlsVerification,
+                parsePDF: options.parsePDF ?? this.parsePDF,
+                location: options.location ?? this.location,
+                removeBase64Images: options.removeBase64Images ?? this.removeBase64Images,
+                blockAds: options.blockAds ?? this.blockAds,
+                storeInCache: options.storeInCache ?? this.storeInCache,
+                zeroDataRetention: options.zeroDataRetention ?? this.zeroDataRetention,
+                changeTrackingOptions: options.changeTrackingOptions ?? this.changeTrackingOptions,
+            });
+            const response = await axios.post(this.apiUrl, payload, {
                 headers: {
                     'Content-Type': 'application/json',
                     Authorization: `Bearer ${this.apiKey}`,
@@ -123,6 +165,10 @@ class FirecrawlScraper {
 const createFirecrawlScraper = (config = {}) => {
     return new FirecrawlScraper(config);
 };
+// Helper function to clean up payload for firecrawl
+function omitUndefined(obj) {
+    return Object.fromEntries(Object.entries(obj).filter(([, v]) => v !== undefined));
+}
 
 export { FirecrawlScraper, createFirecrawlScraper };
 //# sourceMappingURL=firecrawl.mjs.map

package/dist/esm/tools/search/firecrawl.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"firecrawl.mjs","sources":["../../../../src/tools/search/firecrawl.ts"],"sourcesContent":["import axios from 'axios';\nimport { processContent } from './content';\nimport type * as t from './types';\nimport { createDefaultLogger } from './utils';\n\n/**\n * Firecrawl scraper implementation\n * Uses the Firecrawl API to scrape web pages\n */\nexport class FirecrawlScraper {\n  private apiKey: string;\n  private apiUrl: string;\n  private defaultFormats: string[];\n  private timeout: number;\n  private logger: t.Logger;\n\n  constructor(config: t.FirecrawlScraperConfig = {}) {\n    this.apiKey = config.apiKey ?? process.env.FIRECRAWL_API_KEY ?? '';\n\n    const baseUrl =\n      config.apiUrl ??\n      process.env.FIRECRAWL_BASE_URL ??\n      'https://api.firecrawl.dev';\n    this.apiUrl = `${baseUrl.replace(/\\/+$/, '')}/v1/scrape`;\n\n    this.defaultFormats = config.formats ?? ['markdown', 'html'];\n    this.timeout = config.timeout ?? 7500;\n\n    this.logger = config.logger || createDefaultLogger();\n\n    if (!this.apiKey) {\n      this.logger.warn('FIRECRAWL_API_KEY is not set. Scraping will not work.');\n    }\n\n    this.logger.debug(\n      `Firecrawl scraper initialized with API URL: ${this.apiUrl}`\n    );\n  }\n\n  /**\n   * Scrape a single URL\n   * @param url URL to scrape\n   * @param options Scrape options\n   * @returns Scrape response\n   */\n  async scrapeUrl(\n    url: string,\n    options: t.FirecrawlScrapeOptions = {}\n  ): Promise<[string, t.FirecrawlScrapeResponse]> {\n    if (!this.apiKey) {\n      return [\n        url,\n        {\n          success: false,\n          error: 'FIRECRAWL_API_KEY is not set',\n        },\n      ];\n    }\n\n    try {\n      const response = await axios.post(\n        this.apiUrl,\n        {\n          url,\n          formats: options.formats || this.defaultFormats,\n          includeTags: options.includeTags,\n          excludeTags: options.excludeTags,\n          headers: options.headers,\n          waitFor: options.waitFor,\n          timeout: options.timeout ?? this.timeout,\n        },\n        {\n          headers: {\n            'Content-Type': 'application/json',\n            Authorization: `Bearer ${this.apiKey}`,\n          },\n          timeout: this.timeout,\n        }\n      );\n\n      return [url, response.data];\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      return [\n        url,\n        {\n          success: false,\n          error: `Firecrawl API request failed: ${errorMessage}`,\n        },\n      ];\n    }\n  }\n\n  /**\n   * Extract content from scrape response\n   * @param response Scrape response\n   * @returns Extracted content or empty string if not available\n   */\n  extractContent(\n    response: t.FirecrawlScrapeResponse\n  ): [string, undefined | t.References] {\n    if (!response.success || !response.data) {\n      return ['', undefined];\n    }\n\n    if (response.data.markdown != null && response.data.html != null) {\n      try {\n        const { markdown, ...rest } = processContent(\n          response.data.html,\n          response.data.markdown\n        );\n        return [markdown, rest];\n      } catch (error) {\n        this.logger.error('Error processing content:', error);\n        return [response.data.markdown, undefined];\n      }\n    } else if (response.data.markdown != null) {\n      return [response.data.markdown, undefined];\n    }\n\n    // Fall back to HTML content\n    if (response.data.html != null) {\n      return [response.data.html, undefined];\n    }\n\n    // Fall back to raw HTML content\n    if (response.data.rawHtml != null) {\n      return [response.data.rawHtml, undefined];\n    }\n\n    return ['', undefined];\n  }\n\n  /**\n   * Extract metadata from scrape response\n   * @param response Scrape response\n   * @returns Metadata object\n   */\n  extractMetadata(response: t.FirecrawlScrapeResponse): t.ScrapeMetadata {\n    if (!response.success || !response.data || !response.data.metadata) {\n      return {};\n    }\n\n    return response.data.metadata;\n  }\n}\n\n/**\n * Create a Firecrawl scraper instance\n * @param config Scraper configuration\n * @returns Firecrawl scraper instance\n */\nexport const createFirecrawlScraper = (\n  config: t.FirecrawlScraperConfig = {}\n): FirecrawlScraper => {\n  return new FirecrawlScraper(config);\n};\n"],"names":[],"mappings":";;;;AAKA;;;AAGG;MACU,gBAAgB,CAAA;AACnB,IAAA,MAAM;AACN,IAAA,MAAM;AACN,IAAA,cAAc;AACd,IAAA,OAAO;AACP,IAAA,MAAM;AAEd,IAAA,WAAA,CAAY,SAAmC,EAAE,EAAA;AAC/C,QAAA,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,EAAE;AAElE,QAAA,MAAM,OAAO,GACX,MAAM,CAAC,MAAM;YACb,OAAO,CAAC,GAAG,CAAC,kBAAkB;AAC9B,YAAA,2BAA2B;AAC7B,QAAA,IAAI,CAAC,MAAM,GAAG,CAAA,EAAG,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,YAAY;AAExD,QAAA,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,OAAO,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC;QAC5D,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,IAAI;QAErC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,mBAAmB,EAAE;AAEpD,QAAA,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;AAChB,YAAA,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,uDAAuD,CAAC;;QAG3E,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,CAA+C,4CAAA,EAAA,IAAI,CAAC,MAAM,CAAE,CAAA,CAC7D;;AAGH;;;;;AAKG;AACH,IAAA,MAAM,SAAS,CACb,GAAW,EACX,UAAoC,EAAE,EAAA;AAEtC,QAAA,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YAChB,OAAO;gBACL,GAAG;AACH,gBAAA;AACE,oBAAA,OAAO,EAAE,KAAK;AACd,oBAAA,KAAK,EAAE,8BAA8B;AACtC,iBAAA;aACF;;AAGH,QAAA,IAAI;YACF,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAC/B,IAAI,CAAC,MAAM,EACX;gBACE,GAAG;AACH,gBAAA,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,cAAc;gBAC/C,WAAW,EAAE,OAAO,CAAC,WAAW;gBAChC,WAAW,EAAE,OAAO,CAAC,WAAW;gBAChC,OAAO,EAAE,OAAO,CAAC,OAAO;gBACxB,OAAO,EAAE,OAAO,CAAC,OAAO;AACxB,gBAAA,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO;aACzC,EACD;AACE,gBAAA,OAAO,EAAE;AACP,oBAAA,cAAc,EAAE,kBAAkB;AAClC,oBAAA,aAAa,EAAE,CAAA,OAAA,EAAU,IAAI,CAAC,MAAM,CAAE,CAAA;AACvC,iBAAA;gBACD,OAAO,EAAE,IAAI,CAAC,OAAO;AACtB,aAAA,CACF;AAED,YAAA,OAAO,CAAC,GAAG,EAAE,QAAQ,CAAC,IAAI,CAAC;;QAC3B,OAAO,KAAK,EAAE;AACd,YAAA,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;YACxD,OAAO;gBACL,GAAG;AACH,gBAAA;AACE,oBAAA,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,CAAiC,8BAAA,EAAA,YAAY,CAAE,CAAA;AACvD,iBAAA;aACF;;;AAIL;;;;AAIG;AACH,IAAA,cAAc,CACZ,QAAmC,EAAA;QAEnC,IAAI,CAAC,QAAQ,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE;AACvC,YAAA,OAAO,CAAC,EAAE,EAAE,SAAS,CAAC;;AAGxB,QAAA,IAAI,QAAQ,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,EAAE;AAChE,YAAA,IAAI;gBACF,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAI,EAAE,GAAG,cAAc,CAC1C,QAAQ,CAAC,IAAI,CAAC,IAAI,EAClB,QAAQ,CAAC,IAAI,CAAC,QAAQ,CACvB;AACD,gBAAA,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC;;YACvB,OAAO,KAAK,EAAE;gBACd,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,2BAA2B,EAAE,KAAK,CAAC;gBACrD,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC;;;aAEvC,IAAI,QAAQ,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,EAAE;YACzC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC;;;QAI5C,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,EAAE;YAC9B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,SAAS,CAAC;;;QAIxC,IAAI,QAAQ,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,EAAE;YACjC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC;;AAG3C,QAAA,OAAO,CAAC,EAAE,EAAE,SAAS,CAAC;;AAGxB;;;;AAIG;AACH,IAAA,eAAe,CAAC,QAAmC,EAAA;AACjD,QAAA,IAAI,CAAC,QAAQ,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE;AAClE,YAAA,OAAO,EAAE;;AAGX,QAAA,OAAO,QAAQ,CAAC,IAAI,CAAC,QAAQ;;AAEhC;AAED;;;;AAIG;MACU,sBAAsB,GAAG,CACpC,MAAmC,GAAA,EAAE,KACjB;AACpB,IAAA,OAAO,IAAI,gBAAgB,CAAC,MAAM,CAAC;AACrC;;;;"}
+{"version":3,"file":"firecrawl.mjs","sources":["../../../../src/tools/search/firecrawl.ts"],"sourcesContent":["import axios from 'axios';\nimport { processContent } from './content';\nimport type * as t from './types';\nimport { createDefaultLogger } from './utils';\n\n/**\n * Firecrawl scraper implementation\n * Uses the Firecrawl API to scrape web pages\n */\nexport class FirecrawlScraper {\n  private apiKey: string;\n  private apiUrl: string;\n  private defaultFormats: string[];\n  private timeout: number;\n  private logger: t.Logger;\n  private includeTags?: string[];\n  private excludeTags?: string[];\n  private waitFor?: number;\n  private maxAge?: number;\n  private mobile?: boolean;\n  private skipTlsVerification?: boolean;\n  private blockAds?: boolean;\n  private removeBase64Images?: boolean;\n  private parsePDF?: boolean;\n  private storeInCache?: boolean;\n  private zeroDataRetention?: boolean;\n  private headers?: Record<string, string>;\n  private location?: { country?: string; languages?: string[] };\n  private onlyMainContent?: boolean;\n  private changeTrackingOptions?: object;\n\n  constructor(config: t.FirecrawlScraperConfig = {}) {\n    this.apiKey = config.apiKey ?? process.env.FIRECRAWL_API_KEY ?? '';\n\n    const baseUrl =\n      config.apiUrl ??\n      process.env.FIRECRAWL_BASE_URL ??\n      'https://api.firecrawl.dev';\n    this.apiUrl = `${baseUrl.replace(/\\/+$/, '')}/v1/scrape`;\n\n    this.defaultFormats = config.formats ?? ['markdown', 'rawHtml'];\n    this.timeout = config.timeout ?? 7500;\n\n    this.logger = config.logger || createDefaultLogger();\n\n    this.includeTags = config.includeTags;\n    this.excludeTags = config.excludeTags;\n    this.waitFor = config.waitFor;\n    this.maxAge = config.maxAge;\n    this.mobile = config.mobile;\n    this.skipTlsVerification = config.skipTlsVerification;\n    this.blockAds = config.blockAds;\n    this.removeBase64Images = config.removeBase64Images;\n    this.parsePDF = config.parsePDF;\n    this.storeInCache = config.storeInCache;\n    this.zeroDataRetention = config.zeroDataRetention;\n    this.headers = config.headers;\n    this.location = config.location;\n    this.onlyMainContent = config.onlyMainContent;\n    this.changeTrackingOptions = config.changeTrackingOptions;\n\n    if (!this.apiKey) {\n      this.logger.warn('FIRECRAWL_API_KEY is not set. Scraping will not work.');\n    }\n\n    this.logger.debug(\n      `Firecrawl scraper initialized with API URL: ${this.apiUrl}`\n    );\n  }\n\n  /**\n   * Scrape a single URL\n   * @param url URL to scrape\n   * @param options Scrape options\n   * @returns Scrape response\n   */\n  async scrapeUrl(\n    url: string,\n    options: t.FirecrawlScrapeOptions = {}\n  ): Promise<[string, t.FirecrawlScrapeResponse]> {\n    if (!this.apiKey) {\n      return [\n        url,\n        {\n          success: false,\n          error: 'FIRECRAWL_API_KEY is not set',\n        },\n      ];\n    }\n\n    try {\n      const payload = omitUndefined({\n        url,\n        formats: options.formats ?? this.defaultFormats,\n        includeTags: options.includeTags ?? this.includeTags,\n        excludeTags: options.excludeTags ?? this.excludeTags,\n        headers: options.headers ?? this.headers,\n        waitFor: options.waitFor ?? this.waitFor,\n        timeout: options.timeout ?? this.timeout,\n        onlyMainContent: options.onlyMainContent ?? this.onlyMainContent,\n        maxAge: options.maxAge ?? this.maxAge,\n        mobile: options.mobile ?? this.mobile,\n        skipTlsVerification:\n          options.skipTlsVerification ?? this.skipTlsVerification,\n        parsePDF: options.parsePDF ?? this.parsePDF,\n        location: options.location ?? this.location,\n        removeBase64Images:\n          options.removeBase64Images ?? this.removeBase64Images,\n        blockAds: options.blockAds ?? this.blockAds,\n        storeInCache: options.storeInCache ?? this.storeInCache,\n        zeroDataRetention: options.zeroDataRetention ?? this.zeroDataRetention,\n        changeTrackingOptions:\n          options.changeTrackingOptions ?? this.changeTrackingOptions,\n      });\n      const response = await axios.post(this.apiUrl, payload, {\n        headers: {\n          'Content-Type': 'application/json',\n          Authorization: `Bearer ${this.apiKey}`,\n        },\n        timeout: this.timeout,\n      });\n\n      return [url, response.data];\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      return [\n        url,\n        {\n          success: false,\n          error: `Firecrawl API request failed: ${errorMessage}`,\n        },\n      ];\n    }\n  }\n\n  /**\n   * Extract content from scrape response\n   * @param response Scrape response\n   * @returns Extracted content or empty string if not available\n   */\n  extractContent(\n    response: t.FirecrawlScrapeResponse\n  ): [string, undefined | t.References] {\n    if (!response.success || !response.data) {\n      return ['', undefined];\n    }\n\n    if (response.data.markdown != null && response.data.html != null) {\n      try {\n        const { markdown, ...rest } = processContent(\n          response.data.html,\n          response.data.markdown\n        );\n        return [markdown, rest];\n      } catch (error) {\n        this.logger.error('Error processing content:', error);\n        return [response.data.markdown, undefined];\n      }\n    } else if (response.data.markdown != null) {\n      return [response.data.markdown, undefined];\n    }\n\n    // Fall back to HTML content\n    if (response.data.html != null) {\n      return [response.data.html, undefined];\n    }\n\n    // Fall back to raw HTML content\n    if (response.data.rawHtml != null) {\n      return [response.data.rawHtml, undefined];\n    }\n\n    return ['', undefined];\n  }\n\n  /**\n   * Extract metadata from scrape response\n   * @param response Scrape response\n   * @returns Metadata object\n   */\n  extractMetadata(response: t.FirecrawlScrapeResponse): t.ScrapeMetadata {\n    if (!response.success || !response.data || !response.data.metadata) {\n      return {};\n    }\n\n    return response.data.metadata;\n  }\n}\n\n/**\n * Create a Firecrawl scraper instance\n * @param config Scraper configuration\n * @returns Firecrawl scraper instance\n */\nexport const createFirecrawlScraper = (\n  config: t.FirecrawlScraperConfig = {}\n): FirecrawlScraper => {\n  return new FirecrawlScraper(config);\n};\n\n// Helper function to clean up payload for firecrawl\nfunction omitUndefined<T extends object>(obj: T): Partial<T> {\n  return Object.fromEntries(\n    Object.entries(obj).filter(([, v]) => v !== undefined)\n  ) as Partial<T>;\n}\n"],"names":[],"mappings":";;;;AAKA;;;AAGG;MACU,gBAAgB,CAAA;AACnB,IAAA,MAAM;AACN,IAAA,MAAM;AACN,IAAA,cAAc;AACd,IAAA,OAAO;AACP,IAAA,MAAM;AACN,IAAA,WAAW;AACX,IAAA,WAAW;AACX,IAAA,OAAO;AACP,IAAA,MAAM;AACN,IAAA,MAAM;AACN,IAAA,mBAAmB;AACnB,IAAA,QAAQ;AACR,IAAA,kBAAkB;AAClB,IAAA,QAAQ;AACR,IAAA,YAAY;AACZ,IAAA,iBAAiB;AACjB,IAAA,OAAO;AACP,IAAA,QAAQ;AACR,IAAA,eAAe;AACf,IAAA,qBAAqB;AAE7B,IAAA,WAAA,CAAY,SAAmC,EAAE,EAAA;AAC/C,QAAA,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,EAAE;AAElE,QAAA,MAAM,OAAO,GACX,MAAM,CAAC,MAAM;YACb,OAAO,CAAC,GAAG,CAAC,kBAAkB;AAC9B,YAAA,2BAA2B;AAC7B,QAAA,IAAI,CAAC,MAAM,GAAG,CAAA,EAAG,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,YAAY;AAExD,QAAA,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,OAAO,IAAI,CAAC,UAAU,EAAE,SAAS,CAAC;QAC/D,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,IAAI;QAErC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,mBAAmB,EAAE;AAEpD,QAAA,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW;AACrC,QAAA,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW;AACrC,QAAA,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO;AAC7B,QAAA,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM;AAC3B,QAAA,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM;AAC3B,QAAA,IAAI,CAAC,mBAAmB,GAAG,MAAM,CAAC,mBAAmB;AACrD,QAAA,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ;AAC/B,QAAA,IAAI,CAAC,kBAAkB,GAAG,MAAM,CAAC,kBAAkB;AACnD,QAAA,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ;AAC/B,QAAA,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC,YAAY;AACvC,QAAA,IAAI,CAAC,iBAAiB,GAAG,MAAM,CAAC,iBAAiB;AACjD,QAAA,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO;AAC7B,QAAA,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ;AAC/B,QAAA,IAAI,CAAC,eAAe,GAAG,MAAM,CAAC,eAAe;AAC7C,QAAA,IAAI,CAAC,qBAAqB,GAAG,MAAM,CAAC,qBAAqB;AAEzD,QAAA,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;AAChB,YAAA,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,uDAAuD,CAAC;;QAG3E,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,CAA+C,4CAAA,EAAA,IAAI,CAAC,MAAM,CAAE,CAAA,CAC7D;;AAGH;;;;;AAKG;AACH,IAAA,MAAM,SAAS,CACb,GAAW,EACX,UAAoC,EAAE,EAAA;AAEtC,QAAA,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YAChB,OAAO;gBACL,GAAG;AACH,gBAAA;AACE,oBAAA,OAAO,EAAE,KAAK;AACd,oBAAA,KAAK,EAAE,8BAA8B;AACtC,iBAAA;aACF;;AAGH,QAAA,IAAI;YACF,MAAM,OAAO,GAAG,aAAa,CAAC;gBAC5B,GAAG;AACH,gBAAA,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,cAAc;AAC/C,gBAAA,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,IAAI,CAAC,WAAW;AACpD,gBAAA,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,IAAI,CAAC,WAAW;AACpD,gBAAA,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO;AACxC,gBAAA,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO;AACxC,gBAAA,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO;AACxC,gBAAA,eAAe,EAAE,OAAO,CAAC,eAAe,IAAI,IAAI,CAAC,eAAe;AAChE,gBAAA,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM;AACrC,gBAAA,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM;AACrC,gBAAA,mBAAmB,EACjB,OAAO,CAAC,mBAAmB,IAAI,IAAI,CAAC,mBAAmB;AACzD,gBAAA,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ;AAC3C,gBAAA,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ;AAC3C,gBAAA,kBAAkB,EAChB,OAAO,CAAC,kBAAkB,IAAI,IAAI,CAAC,kBAAkB;AACvD,gBAAA,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ;AAC3C,gBAAA,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,IAAI,CAAC,YAAY;AACvD,gBAAA,iBAAiB,EAAE,OAAO,CAAC,iBAAiB,IAAI,IAAI,CAAC,iBAAiB;AACtE,gBAAA,qBAAqB,EACnB,OAAO,CAAC,qBAAqB,IAAI,IAAI,CAAC,qBAAqB;AAC9D,aAAA,CAAC;AACF,YAAA,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE;AACtD,gBAAA,OAAO,EAAE;AACP,oBAAA,cAAc,EAAE,kBAAkB;AAClC,oBAAA,aAAa,EAAE,CAAA,OAAA,EAAU,IAAI,CAAC,MAAM,CAAE,CAAA;AACvC,iBAAA;gBACD,OAAO,EAAE,IAAI,CAAC,OAAO;AACtB,aAAA,CAAC;AAEF,YAAA,OAAO,CAAC,GAAG,EAAE,QAAQ,CAAC,IAAI,CAAC;;QAC3B,OAAO,KAAK,EAAE;AACd,YAAA,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;YACxD,OAAO;gBACL,GAAG;AACH,gBAAA;AACE,oBAAA,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,CAAiC,8BAAA,EAAA,YAAY,CAAE,CAAA;AACvD,iBAAA;aACF;;;AAIL;;;;AAIG;AACH,IAAA,cAAc,CACZ,QAAmC,EAAA;QAEnC,IAAI,CAAC,QAAQ,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE;AACvC,YAAA,OAAO,CAAC,EAAE,EAAE,SAAS,CAAC;;AAGxB,QAAA,IAAI,QAAQ,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,EAAE;AAChE,YAAA,IAAI;gBACF,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAI,EAAE,GAAG,cAAc,CAC1C,QAAQ,CAAC,IAAI,CAAC,IAAI,EAClB,QAAQ,CAAC,IAAI,CAAC,QAAQ,CACvB;AACD,gBAAA,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC;;YACvB,OAAO,KAAK,EAAE;gBACd,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,2BAA2B,EAAE,KAAK,CAAC;gBACrD,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC;;;aAEvC,IAAI,QAAQ,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,EAAE;YACzC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC;;;QAI5C,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,EAAE;YAC9B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,SAAS,CAAC;;;QAIxC,IAAI,QAAQ,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,EAAE;YACjC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC;;AAG3C,QAAA,OAAO,CAAC,EAAE,EAAE,SAAS,CAAC;;AAGxB;;;;AAIG;AACH,IAAA,eAAe,CAAC,QAAmC,EAAA;AACjD,QAAA,IAAI,CAAC,QAAQ,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE;AAClE,YAAA,OAAO,EAAE;;AAGX,QAAA,OAAO,QAAQ,CAAC,IAAI,CAAC,QAAQ;;AAEhC;AAED;;;;AAIG;MACU,sBAAsB,GAAG,CACpC,MAAmC,GAAA,EAAE,KACjB;AACpB,IAAA,OAAO,IAAI,gBAAgB,CAAC,MAAM,CAAC;AACrC;AAEA;AACA,SAAS,aAAa,CAAmB,GAAM,EAAA;IAC7C,OAAO,MAAM,CAAC,WAAW,CACvB,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,KAAK,SAAS,CAAC,CACzC;AACjB;;;;"}

package/dist/esm/tools/search/tool.mjs

@@ -234,7 +234,7 @@ Use anchor marker(s) immediately after the statement:
  * @returns A DynamicStructuredTool with a schema that depends on the searchProvider
  */
 const createSearchTool = (config = {}) => {
-    const { searchProvider = 'serper', serperApiKey, searxngInstanceUrl, searxngApiKey, rerankerType = 'cohere', topResults = 5, strategies = ['no_extraction'], filterContent = true, safeSearch = 1, firecrawlApiKey, firecrawlApiUrl, firecrawlFormats = ['markdown', 'html'], scraperTimeout, jinaApiKey, cohereApiKey, onSearchResults: _onSearchResults, onGetHighlights, } = config;
+    const { searchProvider = 'serper', serperApiKey, searxngInstanceUrl, searxngApiKey, rerankerType = 'cohere', topResults = 5, strategies = ['no_extraction'], filterContent = true, safeSearch = 1, firecrawlApiKey, firecrawlApiUrl, firecrawlOptions, scraperTimeout, jinaApiKey, cohereApiKey, onSearchResults: _onSearchResults, onGetHighlights, } = config;
     const logger = config.logger || createDefaultLogger();
     const schemaObject = {
         query: querySchema,
@@ -254,10 +254,11 @@ const createSearchTool = (config = {}) => {
         searxngApiKey,
     });
     const firecrawlScraper = createFirecrawlScraper({
+        ...firecrawlOptions,
         apiKey: firecrawlApiKey ?? process.env.FIRECRAWL_API_KEY,
         apiUrl: firecrawlApiUrl,
-        timeout: scraperTimeout,
-        formats: firecrawlFormats,
+        timeout: scraperTimeout ?? firecrawlOptions?.timeout,
+        formats: firecrawlOptions?.formats ?? ['markdown', 'rawHtml'],
     });
     const selectedReranker = createReranker({
         rerankerType,

package/dist/esm/tools/search/tool.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"tool.mjs","sources":["../../../../src/tools/search/tool.ts"],"sourcesContent":["import { z } from 'zod';\nimport { tool, DynamicStructuredTool } from '@langchain/core/tools';\nimport type { RunnableConfig } from '@langchain/core/runnables';\nimport type * as t from './types';\nimport {\n  DATE_RANGE,\n  querySchema,\n  dateSchema,\n  countrySchema,\n  imagesSchema,\n  videosSchema,\n  newsSchema,\n} from './schema';\nimport { createSearchAPI, createSourceProcessor } from './search';\nimport { createFirecrawlScraper } from './firecrawl';\nimport { expandHighlights } from './highlights';\nimport { formatResultsForLLM } from './format';\nimport { createDefaultLogger } from './utils';\nimport { createReranker } from './rerankers';\nimport { Constants } from '@/common';\n\n/**\n * Executes parallel searches and merges the results\n */\nasync function executeParallelSearches({\n  searchAPI,\n  query,\n  date,\n  country,\n  safeSearch,\n  images,\n  videos,\n  news,\n  logger,\n}: {\n  searchAPI: ReturnType<typeof createSearchAPI>;\n  query: string;\n  date?: DATE_RANGE;\n  country?: string;\n  safeSearch: t.SearchToolConfig['safeSearch'];\n  images: boolean;\n  videos: boolean;\n  news: boolean;\n  logger: t.Logger;\n}): Promise<t.SearchResult> {\n  // Prepare all search tasks to run in parallel\n  const searchTasks: Promise<t.SearchResult>[] = [\n    // Main search\n    searchAPI.getSources({\n      query,\n      date,\n      country,\n      safeSearch,\n    }),\n  ];\n\n  if (images) {\n    searchTasks.push(\n      searchAPI\n        .getSources({\n          query,\n          date,\n          country,\n          safeSearch,\n          type: 'images',\n        })\n        .catch((error) => {\n          logger.error('Error fetching images:', error);\n          return {\n            success: false,\n            error: `Images search failed: ${error instanceof Error ? error.message : String(error)}`,\n          };\n        })\n    );\n  }\n  if (videos) {\n    searchTasks.push(\n      searchAPI\n        .getSources({\n          query,\n          date,\n          country,\n          safeSearch,\n          type: 'videos',\n        })\n        .catch((error) => {\n          logger.error('Error fetching videos:', error);\n          return {\n            success: false,\n            error: `Videos search failed: ${error instanceof Error ? error.message : String(error)}`,\n          };\n        })\n    );\n  }\n  if (news) {\n    searchTasks.push(\n      searchAPI\n        .getSources({\n          query,\n          date,\n          country,\n          safeSearch,\n          type: 'news',\n        })\n        .catch((error) => {\n          logger.error('Error fetching news:', error);\n          return {\n            success: false,\n            error: `News search failed: ${error instanceof Error ? error.message : String(error)}`,\n          };\n        })\n    );\n  }\n\n  // Run all searches in parallel\n  const results = await Promise.all(searchTasks);\n\n  // Get the main search result (first result)\n  const mainResult = results[0];\n  if (!mainResult.success) {\n    throw new Error(mainResult.error ?? 'Search failed');\n  }\n\n  // Merge additional results with the main results\n  const mergedResults = { ...mainResult.data };\n\n  // Convert existing news to topStories if present\n  if (mergedResults.news !== undefined && mergedResults.news.length > 0) {\n    const existingNewsAsTopStories = mergedResults.news\n      .filter((newsItem) => newsItem.link !== undefined && newsItem.link !== '')\n      .map((newsItem) => ({\n        title: newsItem.title ?? '',\n        link: newsItem.link ?? '',\n        source: newsItem.source ?? '',\n        date: newsItem.date ?? '',\n        imageUrl: newsItem.imageUrl ?? '',\n        processed: false,\n      }));\n    mergedResults.topStories = [\n      ...(mergedResults.topStories ?? []),\n      ...existingNewsAsTopStories,\n    ];\n    delete mergedResults.news;\n  }\n\n  results.slice(1).forEach((result) => {\n    if (result.success && result.data !== undefined) {\n      if (result.data.images !== undefined && result.data.images.length > 0) {\n        mergedResults.images = [\n          ...(mergedResults.images ?? []),\n          ...result.data.images,\n        ];\n      }\n      if (result.data.videos !== undefined && result.data.videos.length > 0) {\n        mergedResults.videos = [\n          ...(mergedResults.videos ?? []),\n          ...result.data.videos,\n        ];\n      }\n      if (result.data.news !== undefined && result.data.news.length > 0) {\n        const newsAsTopStories = result.data.news.map((newsItem) => ({\n          ...newsItem,\n          link: newsItem.link ?? '',\n        }));\n        mergedResults.topStories = [\n          ...(mergedResults.topStories ?? []),\n          ...newsAsTopStories,\n        ];\n      }\n    }\n  });\n\n  return { success: true, data: mergedResults };\n}\n\nfunction createSearchProcessor({\n  searchAPI,\n  safeSearch,\n  sourceProcessor,\n  onGetHighlights,\n  logger,\n}: {\n  safeSearch: t.SearchToolConfig['safeSearch'];\n  searchAPI: ReturnType<typeof createSearchAPI>;\n  sourceProcessor: ReturnType<typeof createSourceProcessor>;\n  onGetHighlights: t.SearchToolConfig['onGetHighlights'];\n  logger: t.Logger;\n}) {\n  return async function ({\n    query,\n    date,\n    country,\n    proMode = true,\n    maxSources = 5,\n    onSearchResults,\n    images = false,\n    videos = false,\n    news = false,\n  }: {\n    query: string;\n    country?: string;\n    date?: DATE_RANGE;\n    proMode?: boolean;\n    maxSources?: number;\n    onSearchResults: t.SearchToolConfig['onSearchResults'];\n    images?: boolean;\n    videos?: boolean;\n    news?: boolean;\n  }): Promise<t.SearchResultData> {\n    try {\n      // Execute parallel searches and merge results\n      const searchResult = await executeParallelSearches({\n        searchAPI,\n        query,\n        date,\n        country,\n        safeSearch,\n        images,\n        videos,\n        news,\n        logger,\n      });\n\n      onSearchResults?.(searchResult);\n\n      const processedSources = await sourceProcessor.processSources({\n        query,\n        news,\n        result: searchResult,\n        proMode,\n        onGetHighlights,\n        numElements: maxSources,\n      });\n\n      return expandHighlights(processedSources);\n    } catch (error) {\n      logger.error('Error in search:', error);\n      return {\n        organic: [],\n        topStories: [],\n        images: [],\n        videos: [],\n        news: [],\n        relatedSearches: [],\n        error: error instanceof Error ? error.message : String(error),\n      };\n    }\n  };\n}\n\nfunction createOnSearchResults({\n  runnableConfig,\n  onSearchResults,\n}: {\n  runnableConfig: RunnableConfig;\n  onSearchResults: t.SearchToolConfig['onSearchResults'];\n}) {\n  return function (results: t.SearchResult): void {\n    if (!onSearchResults) {\n      return;\n    }\n    onSearchResults(results, runnableConfig);\n  };\n}\n\nfunction createTool({\n  schema,\n  search,\n  onSearchResults: _onSearchResults,\n}: {\n  schema: t.SearchToolSchema;\n  search: ReturnType<typeof createSearchProcessor>;\n  onSearchResults: t.SearchToolConfig['onSearchResults'];\n}): DynamicStructuredTool<typeof schema> {\n  return tool<typeof schema>(\n    async (params, runnableConfig) => {\n      const { query, date, country: _c, images, videos, news } = params;\n      const country = typeof _c === 'string' && _c ? _c : undefined;\n      const searchResult = await search({\n        query,\n        date,\n        country,\n        images,\n        videos,\n        news,\n        onSearchResults: createOnSearchResults({\n          runnableConfig,\n          onSearchResults: _onSearchResults,\n        }),\n      });\n      const turn = runnableConfig.toolCall?.turn ?? 0;\n      const { output, references } = formatResultsForLLM(turn, searchResult);\n      const data: t.SearchResultData = { turn, ...searchResult, references };\n      return [output, { [Constants.WEB_SEARCH]: data }];\n    },\n    {\n      name: Constants.WEB_SEARCH,\n      description: `Real-time search. Results have required citation anchors.\n\nNote: Use ONCE per reply unless instructed otherwise.\n\nAnchors:\n- \\\\ue202turnXtypeY\n- X = turn idx, type = 'search' | 'news' | 'image' | 'ref', Y = item idx\n\nSpecial Markers:\n- \\\\ue203...\\\\ue204 — highlight start/end of cited text (for Standalone or Group citations)\n- \\\\ue200...\\\\ue201 — group block (e.g. \\\\ue200\\\\ue202turn0search1\\\\ue202turn0news2\\\\ue201)\n\n**CITE EVERY NON-OBVIOUS FACT/QUOTE:**\nUse anchor marker(s) immediately after the statement:\n- Standalone: \"Pure functions produce same output. \\\\ue202turn0search0\"\n- Standalone (multiple): \"Today's News \\\\ue202turn0search0\\\\ue202turn0news0\"\n- Highlight: \"\\\\ue203Highlight text.\\\\ue204\\\\ue202turn0news1\"\n- Group: \"Sources. \\\\ue200\\\\ue202turn0search0\\\\ue202turn0news1\\\\ue201\"\n- Group Highlight: \"\\\\ue203Highlight for group.\\\\ue204 \\\\ue200\\\\ue202turn0search0\\\\ue202turn0news1\\\\ue201\"\n- Image: \"See photo \\\\ue202turn0image0.\"\n\n**NEVER use markdown links, [1], or footnotes. CITE ONLY with anchors provided.**\n`.trim(),\n      schema: schema,\n      responseFormat: Constants.CONTENT_AND_ARTIFACT,\n    }\n  );\n}\n\n/**\n * Creates a search tool with a schema that dynamically includes the country field\n * only when the searchProvider is 'serper'.\n *\n * @param config - The search tool configuration\n * @returns A DynamicStructuredTool with a schema that depends on the searchProvider\n */\nexport const createSearchTool = (\n  config: t.SearchToolConfig = {}\n): DynamicStructuredTool<typeof toolSchema> => {\n  const {\n    searchProvider = 'serper',\n    serperApiKey,\n    searxngInstanceUrl,\n    searxngApiKey,\n    rerankerType = 'cohere',\n    topResults = 5,\n    strategies = ['no_extraction'],\n    filterContent = true,\n    safeSearch = 1,\n    firecrawlApiKey,\n    firecrawlApiUrl,\n    firecrawlFormats = ['markdown', 'html'],\n    scraperTimeout,\n    jinaApiKey,\n    cohereApiKey,\n    onSearchResults: _onSearchResults,\n    onGetHighlights,\n  } = config;\n\n  const logger = config.logger || createDefaultLogger();\n\n  const schemaObject: {\n    query: z.ZodString;\n    date: z.ZodOptional<z.ZodNativeEnum<typeof DATE_RANGE>>;\n    country?: z.ZodOptional<z.ZodString>;\n    images: z.ZodOptional<z.ZodBoolean>;\n    videos: z.ZodOptional<z.ZodBoolean>;\n    news: z.ZodOptional<z.ZodBoolean>;\n  } = {\n    query: querySchema,\n    date: dateSchema,\n    images: imagesSchema,\n    videos: videosSchema,\n    news: newsSchema,\n  };\n\n  if (searchProvider === 'serper') {\n    schemaObject.country = countrySchema;\n  }\n\n  const toolSchema = z.object(schemaObject);\n\n  const searchAPI = createSearchAPI({\n    searchProvider,\n    serperApiKey,\n    searxngInstanceUrl,\n    searxngApiKey,\n  });\n\n  const firecrawlScraper = createFirecrawlScraper({\n    apiKey: firecrawlApiKey ?? process.env.FIRECRAWL_API_KEY,\n    apiUrl: firecrawlApiUrl,\n    timeout: scraperTimeout,\n    formats: firecrawlFormats,\n  });\n\n  const selectedReranker = createReranker({\n    rerankerType,\n    jinaApiKey,\n    cohereApiKey,\n    logger,\n  });\n\n  if (!selectedReranker) {\n    logger.warn('No reranker selected. Using default ranking.');\n  }\n\n  const sourceProcessor = createSourceProcessor(\n    {\n      reranker: selectedReranker,\n      topResults,\n      strategies,\n      filterContent,\n      logger,\n    },\n    firecrawlScraper\n  );\n\n  const search = createSearchProcessor({\n    searchAPI,\n    safeSearch,\n    sourceProcessor,\n    onGetHighlights,\n    logger,\n  });\n\n  return createTool({\n    search,\n    schema: toolSchema,\n    onSearchResults: _onSearchResults,\n  });\n};\n"],"names":[],"mappings":";;;;;;;;;;;AAqBA;;AAEG;AACH,eAAe,uBAAuB,CAAC,EACrC,SAAS,EACT,KAAK,EACL,IAAI,EACJ,OAAO,EACP,UAAU,EACV,MAAM,EACN,MAAM,EACN,IAAI,EACJ,MAAM,GAWP,EAAA;;AAEC,IAAA,MAAM,WAAW,GAA8B;;QAE7C,SAAS,CAAC,UAAU,CAAC;YACnB,KAAK;YACL,IAAI;YACJ,OAAO;YACP,UAAU;SACX,CAAC;KACH;IAED,IAAI,MAAM,EAAE;QACV,WAAW,CAAC,IAAI,CACd;AACG,aAAA,UAAU,CAAC;YACV,KAAK;YACL,IAAI;YACJ,OAAO;YACP,UAAU;AACV,YAAA,IAAI,EAAE,QAAQ;SACf;AACA,aAAA,KAAK,CAAC,CAAC,KAAK,KAAI;AACf,YAAA,MAAM,CAAC,KAAK,CAAC,wBAAwB,EAAE,KAAK,CAAC;YAC7C,OAAO;AACL,gBAAA,OAAO,EAAE,KAAK;AACd,gBAAA,KAAK,EAAE,CAAyB,sBAAA,EAAA,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,CAAE,CAAA;aACzF;SACF,CAAC,CACL;;IAEH,IAAI,MAAM,EAAE;QACV,WAAW,CAAC,IAAI,CACd;AACG,aAAA,UAAU,CAAC;YACV,KAAK;YACL,IAAI;YACJ,OAAO;YACP,UAAU;AACV,YAAA,IAAI,EAAE,QAAQ;SACf;AACA,aAAA,KAAK,CAAC,CAAC,KAAK,KAAI;AACf,YAAA,MAAM,CAAC,KAAK,CAAC,wBAAwB,EAAE,KAAK,CAAC;YAC7C,OAAO;AACL,gBAAA,OAAO,EAAE,KAAK;AACd,gBAAA,KAAK,EAAE,CAAyB,sBAAA,EAAA,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,CAAE,CAAA;aACzF;SACF,CAAC,CACL;;IAEH,IAAI,IAAI,EAAE;QACR,WAAW,CAAC,IAAI,CACd;AACG,aAAA,UAAU,CAAC;YACV,KAAK;YACL,IAAI;YACJ,OAAO;YACP,UAAU;AACV,YAAA,IAAI,EAAE,MAAM;SACb;AACA,aAAA,KAAK,CAAC,CAAC,KAAK,KAAI;AACf,YAAA,MAAM,CAAC,KAAK,CAAC,sBAAsB,EAAE,KAAK,CAAC;YAC3C,OAAO;AACL,gBAAA,OAAO,EAAE,KAAK;AACd,gBAAA,KAAK,EAAE,CAAuB,oBAAA,EAAA,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,CAAE,CAAA;aACvF;SACF,CAAC,CACL;;;IAIH,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC;;AAG9C,IAAA,MAAM,UAAU,GAAG,OAAO,CAAC,CAAC,CAAC;AAC7B,IAAA,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE;QACvB,MAAM,IAAI,KAAK,CAAC,UAAU,CAAC,KAAK,IAAI,eAAe,CAAC;;;IAItD,MAAM,aAAa,GAAG,EAAE,GAAG,UAAU,CAAC,IAAI,EAAE;;AAG5C,IAAA,IAAI,aAAa,CAAC,IAAI,KAAK,SAAS,IAAI,aAAa,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE;AACrE,QAAA,MAAM,wBAAwB,GAAG,aAAa,CAAC;AAC5C,aAAA,MAAM,CAAC,CAAC,QAAQ,KAAK,QAAQ,CAAC,IAAI,KAAK,SAAS,IAAI,QAAQ,CAAC,IAAI,KAAK,EAAE;AACxE,aAAA,GAAG,CAAC,CAAC,QAAQ,MAAM;AAClB,YAAA,KAAK,EAAE,QAAQ,CAAC,KAAK,IAAI,EAAE;AAC3B,YAAA,IAAI,EAAE,QAAQ,CAAC,IAAI,IAAI,EAAE;AACzB,YAAA,MAAM,EAAE,QAAQ,CAAC,MAAM,IAAI,EAAE;AAC7B,YAAA,IAAI,EAAE,QAAQ,CAAC,IAAI,IAAI,EAAE;AACzB,YAAA,QAAQ,EAAE,QAAQ,CAAC,QAAQ,IAAI,EAAE;AACjC,YAAA,SAAS,EAAE,KAAK;AACjB,SAAA,CAAC,CAAC;QACL,aAAa,CAAC,UAAU,GAAG;AACzB,YAAA,IAAI,aAAa,CAAC,UAAU,IAAI,EAAE,CAAC;AACnC,YAAA,GAAG,wBAAwB;SAC5B;QACD,OAAO,aAAa,CAAC,IAAI;;IAG3B,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,KAAI;QAClC,IAAI,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE;AAC/C,YAAA,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;gBACrE,aAAa,CAAC,MAAM,GAAG;AACrB,oBAAA,IAAI,aAAa,CAAC,MAAM,IAAI,EAAE,CAAC;AAC/B,oBAAA,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM;iBACtB;;AAEH,YAAA,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;gBACrE,aAAa,CAAC,MAAM,GAAG;AACrB,oBAAA,IAAI,aAAa,CAAC,MAAM,IAAI,EAAE,CAAC;AAC/B,oBAAA,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM;iBACtB;;AAEH,YAAA,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,KAAK,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE;AACjE,gBAAA,MAAM,gBAAgB,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,QAAQ,MAAM;AAC3D,oBAAA,GAAG,QAAQ;AACX,oBAAA,IAAI,EAAE,QAAQ,CAAC,IAAI,IAAI,EAAE;AAC1B,iBAAA,CAAC,CAAC;gBACH,aAAa,CAAC,UAAU,GAAG;AACzB,oBAAA,IAAI,aAAa,CAAC,UAAU,IAAI,EAAE,CAAC;AACnC,oBAAA,GAAG,gBAAgB;iBACpB;;;AAGP,KAAC,CAAC;IAEF,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,aAAa,EAAE;AAC/C;AAEA,SAAS,qBAAqB,CAAC,EAC7B,SAAS,EACT,UAAU,EACV,eAAe,EACf,eAAe,EACf,MAAM,GAOP,EAAA;AACC,IAAA,OAAO,gBAAgB,EACrB,KAAK,EACL,IAAI,EACJ,OAAO,EACP,OAAO,GAAG,IAAI,EACd,UAAU,GAAG,CAAC,EACd,eAAe,EACf,MAAM,GAAG,KAAK,EACd,MAAM,GAAG,KAAK,EACd,IAAI,GAAG,KAAK,GAWb,EAAA;AACC,QAAA,IAAI;;AAEF,YAAA,MAAM,YAAY,GAAG,MAAM,uBAAuB,CAAC;gBACjD,SAAS;gBACT,KAAK;gBACL,IAAI;gBACJ,OAAO;gBACP,UAAU;gBACV,MAAM;gBACN,MAAM;gBACN,IAAI;gBACJ,MAAM;AACP,aAAA,CAAC;AAEF,YAAA,eAAe,GAAG,YAAY,CAAC;AAE/B,YAAA,MAAM,gBAAgB,GAAG,MAAM,eAAe,CAAC,cAAc,CAAC;gBAC5D,KAAK;gBACL,IAAI;AACJ,gBAAA,MAAM,EAAE,YAAY;gBACpB,OAAO;gBACP,eAAe;AACf,gBAAA,WAAW,EAAE,UAAU;AACxB,aAAA,CAAC;AAEF,YAAA,OAAO,gBAAgB,CAAC,gBAAgB,CAAC;;QACzC,OAAO,KAAK,EAAE;AACd,YAAA,MAAM,CAAC,KAAK,CAAC,kBAAkB,EAAE,KAAK,CAAC;YACvC,OAAO;AACL,gBAAA,OAAO,EAAE,EAAE;AACX,gBAAA,UAAU,EAAE,EAAE;AACd,gBAAA,MAAM,EAAE,EAAE;AACV,gBAAA,MAAM,EAAE,EAAE;AACV,gBAAA,IAAI,EAAE,EAAE;AACR,gBAAA,eAAe,EAAE,EAAE;AACnB,gBAAA,KAAK,EAAE,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;aAC9D;;AAEL,KAAC;AACH;AAEA,SAAS,qBAAqB,CAAC,EAC7B,cAAc,EACd,eAAe,GAIhB,EAAA;AACC,IAAA,OAAO,UAAU,OAAuB,EAAA;QACtC,IAAI,CAAC,eAAe,EAAE;YACpB;;AAEF,QAAA,eAAe,CAAC,OAAO,EAAE,cAAc,CAAC;AAC1C,KAAC;AACH;AAEA,SAAS,UAAU,CAAC,EAClB,MAAM,EACN,MAAM,EACN,eAAe,EAAE,gBAAgB,GAKlC,EAAA;IACC,OAAO,IAAI,CACT,OAAO,MAAM,EAAE,cAAc,KAAI;AAC/B,QAAA,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM;AACjE,QAAA,MAAM,OAAO,GAAG,OAAO,EAAE,KAAK,QAAQ,IAAI,EAAE,GAAG,EAAE,GAAG,SAAS;AAC7D,QAAA,MAAM,YAAY,GAAG,MAAM,MAAM,CAAC;YAChC,KAAK;YACL,IAAI;YACJ,OAAO;YACP,MAAM;YACN,MAAM;YACN,IAAI;YACJ,eAAe,EAAE,qBAAqB,CAAC;gBACrC,cAAc;AACd,gBAAA,eAAe,EAAE,gBAAgB;aAClC,CAAC;AACH,SAAA,CAAC;QACF,MAAM,IAAI,GAAG,cAAc,CAAC,QAAQ,EAAE,IAAI,IAAI,CAAC;AAC/C,QAAA,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,mBAAmB,CAAC,IAAI,EAAE,YAAY,CAAC;QACtE,MAAM,IAAI,GAAuB,EAAE,IAAI,EAAE,GAAG,YAAY,EAAE,UAAU,EAAE;AACtE,QAAA,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,SAAS,CAAC,UAAU,GAAG,IAAI,EAAE,CAAC;AACnD,KAAC,EACD;QACE,IAAI,EAAE,SAAS,CAAC,UAAU;AAC1B,QAAA,WAAW,EAAE,CAAA;;;;;;;;;;;;;;;;;;;;;;AAsBlB,CAAA,CAAC,IAAI,EAAE;AACF,QAAA,MAAM,EAAE,MAAM;QACd,cAAc,EAAE,SAAS,CAAC,oBAAoB;AAC/C,KAAA,CACF;AACH;AAEA;;;;;;AAMG;MACU,gBAAgB,GAAG,CAC9B,MAA6B,GAAA,EAAE,KACa;IAC5C,MAAM,EACJ,cAAc,GAAG,QAAQ,EACzB,YAAY,EACZ,kBAAkB,EAClB,aAAa,EACb,YAAY,GAAG,QAAQ,EACvB,UAAU,GAAG,CAAC,EACd,UAAU,GAAG,CAAC,eAAe,CAAC,EAC9B,aAAa,GAAG,IAAI,EACpB,UAAU,GAAG,CAAC,EACd,eAAe,EACf,eAAe,EACf,gBAAgB,GAAG,CAAC,UAAU,EAAE,MAAM,CAAC,EACvC,cAAc,EACd,UAAU,EACV,YAAY,EACZ,eAAe,EAAE,gBAAgB,EACjC,eAAe,GAChB,GAAG,MAAM;IAEV,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,mBAAmB,EAAE;AAErD,IAAA,MAAM,YAAY,GAOd;AACF,QAAA,KAAK,EAAE,WAAW;AAClB,QAAA,IAAI,EAAE,UAAU;AAChB,QAAA,MAAM,EAAE,YAAY;AACpB,QAAA,MAAM,EAAE,YAAY;AACpB,QAAA,IAAI,EAAE,UAAU;KACjB;AAED,IAAA,IAAI,cAAc,KAAK,QAAQ,EAAE;AAC/B,QAAA,YAAY,CAAC,OAAO,GAAG,aAAa;;IAGtC,MAAM,UAAU,GAAG,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC;IAEzC,MAAM,SAAS,GAAG,eAAe,CAAC;QAChC,cAAc;QACd,YAAY;QACZ,kBAAkB;QAClB,aAAa;AACd,KAAA,CAAC;IAEF,MAAM,gBAAgB,GAAG,sBAAsB,CAAC;AAC9C,QAAA,MAAM,EAAE,eAAe,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB;AACxD,QAAA,MAAM,EAAE,eAAe;AACvB,QAAA,OAAO,EAAE,cAAc;AACvB,QAAA,OAAO,EAAE,gBAAgB;AAC1B,KAAA,CAAC;IAEF,MAAM,gBAAgB,GAAG,cAAc,CAAC;QACtC,YAAY;QACZ,UAAU;QACV,YAAY;QACZ,MAAM;AACP,KAAA,CAAC;IAEF,IAAI,CAAC,gBAAgB,EAAE;AACrB,QAAA,MAAM,CAAC,IAAI,CAAC,8CAA8C,CAAC;;IAG7D,MAAM,eAAe,GAAG,qBAAqB,CAC3C;AACE,QAAA,QAAQ,EAAE,gBAAgB;QAC1B,UAAU;QAGV,MAAM;KACP,EACD,gBAAgB,CACjB;IAED,MAAM,MAAM,GAAG,qBAAqB,CAAC;QACnC,SAAS;QACT,UAAU;QACV,eAAe;QACf,eAAe;QACf,MAAM;AACP,KAAA,CAAC;AAEF,IAAA,OAAO,UAAU,CAAC;QAChB,MAAM;AACN,QAAA,MAAM,EAAE,UAAU;AAClB,QAAA,eAAe,EAAE,gBAAgB;AAClC,KAAA,CAAC;AACJ;;;;"}
+{"version":3,"file":"tool.mjs","sources":["../../../../src/tools/search/tool.ts"],"sourcesContent":["import { z } from 'zod';\nimport { tool, DynamicStructuredTool } from '@langchain/core/tools';\nimport type { RunnableConfig } from '@langchain/core/runnables';\nimport type * as t from './types';\nimport {\n  DATE_RANGE,\n  querySchema,\n  dateSchema,\n  countrySchema,\n  imagesSchema,\n  videosSchema,\n  newsSchema,\n} from './schema';\nimport { createSearchAPI, createSourceProcessor } from './search';\nimport { createFirecrawlScraper } from './firecrawl';\nimport { expandHighlights } from './highlights';\nimport { formatResultsForLLM } from './format';\nimport { createDefaultLogger } from './utils';\nimport { createReranker } from './rerankers';\nimport { Constants } from '@/common';\n\n/**\n * Executes parallel searches and merges the results\n */\nasync function executeParallelSearches({\n  searchAPI,\n  query,\n  date,\n  country,\n  safeSearch,\n  images,\n  videos,\n  news,\n  logger,\n}: {\n  searchAPI: ReturnType<typeof createSearchAPI>;\n  query: string;\n  date?: DATE_RANGE;\n  country?: string;\n  safeSearch: t.SearchToolConfig['safeSearch'];\n  images: boolean;\n  videos: boolean;\n  news: boolean;\n  logger: t.Logger;\n}): Promise<t.SearchResult> {\n  // Prepare all search tasks to run in parallel\n  const searchTasks: Promise<t.SearchResult>[] = [\n    // Main search\n    searchAPI.getSources({\n      query,\n      date,\n      country,\n      safeSearch,\n    }),\n  ];\n\n  if (images) {\n    searchTasks.push(\n      searchAPI\n        .getSources({\n          query,\n          date,\n          country,\n          safeSearch,\n          type: 'images',\n        })\n        .catch((error) => {\n          logger.error('Error fetching images:', error);\n          return {\n            success: false,\n            error: `Images search failed: ${error instanceof Error ? error.message : String(error)}`,\n          };\n        })\n    );\n  }\n  if (videos) {\n    searchTasks.push(\n      searchAPI\n        .getSources({\n          query,\n          date,\n          country,\n          safeSearch,\n          type: 'videos',\n        })\n        .catch((error) => {\n          logger.error('Error fetching videos:', error);\n          return {\n            success: false,\n            error: `Videos search failed: ${error instanceof Error ? error.message : String(error)}`,\n          };\n        })\n    );\n  }\n  if (news) {\n    searchTasks.push(\n      searchAPI\n        .getSources({\n          query,\n          date,\n          country,\n          safeSearch,\n          type: 'news',\n        })\n        .catch((error) => {\n          logger.error('Error fetching news:', error);\n          return {\n            success: false,\n            error: `News search failed: ${error instanceof Error ? error.message : String(error)}`,\n          };\n        })\n    );\n  }\n\n  // Run all searches in parallel\n  const results = await Promise.all(searchTasks);\n\n  // Get the main search result (first result)\n  const mainResult = results[0];\n  if (!mainResult.success) {\n    throw new Error(mainResult.error ?? 'Search failed');\n  }\n\n  // Merge additional results with the main results\n  const mergedResults = { ...mainResult.data };\n\n  // Convert existing news to topStories if present\n  if (mergedResults.news !== undefined && mergedResults.news.length > 0) {\n    const existingNewsAsTopStories = mergedResults.news\n      .filter((newsItem) => newsItem.link !== undefined && newsItem.link !== '')\n      .map((newsItem) => ({\n        title: newsItem.title ?? '',\n        link: newsItem.link ?? '',\n        source: newsItem.source ?? '',\n        date: newsItem.date ?? '',\n        imageUrl: newsItem.imageUrl ?? '',\n        processed: false,\n      }));\n    mergedResults.topStories = [\n      ...(mergedResults.topStories ?? []),\n      ...existingNewsAsTopStories,\n    ];\n    delete mergedResults.news;\n  }\n\n  results.slice(1).forEach((result) => {\n    if (result.success && result.data !== undefined) {\n      if (result.data.images !== undefined && result.data.images.length > 0) {\n        mergedResults.images = [\n          ...(mergedResults.images ?? []),\n          ...result.data.images,\n        ];\n      }\n      if (result.data.videos !== undefined && result.data.videos.length > 0) {\n        mergedResults.videos = [\n          ...(mergedResults.videos ?? []),\n          ...result.data.videos,\n        ];\n      }\n      if (result.data.news !== undefined && result.data.news.length > 0) {\n        const newsAsTopStories = result.data.news.map((newsItem) => ({\n          ...newsItem,\n          link: newsItem.link ?? '',\n        }));\n        mergedResults.topStories = [\n          ...(mergedResults.topStories ?? []),\n          ...newsAsTopStories,\n        ];\n      }\n    }\n  });\n\n  return { success: true, data: mergedResults };\n}\n\nfunction createSearchProcessor({\n  searchAPI,\n  safeSearch,\n  sourceProcessor,\n  onGetHighlights,\n  logger,\n}: {\n  safeSearch: t.SearchToolConfig['safeSearch'];\n  searchAPI: ReturnType<typeof createSearchAPI>;\n  sourceProcessor: ReturnType<typeof createSourceProcessor>;\n  onGetHighlights: t.SearchToolConfig['onGetHighlights'];\n  logger: t.Logger;\n}) {\n  return async function ({\n    query,\n    date,\n    country,\n    proMode = true,\n    maxSources = 5,\n    onSearchResults,\n    images = false,\n    videos = false,\n    news = false,\n  }: {\n    query: string;\n    country?: string;\n    date?: DATE_RANGE;\n    proMode?: boolean;\n    maxSources?: number;\n    onSearchResults: t.SearchToolConfig['onSearchResults'];\n    images?: boolean;\n    videos?: boolean;\n    news?: boolean;\n  }): Promise<t.SearchResultData> {\n    try {\n      // Execute parallel searches and merge results\n      const searchResult = await executeParallelSearches({\n        searchAPI,\n        query,\n        date,\n        country,\n        safeSearch,\n        images,\n        videos,\n        news,\n        logger,\n      });\n\n      onSearchResults?.(searchResult);\n\n      const processedSources = await sourceProcessor.processSources({\n        query,\n        news,\n        result: searchResult,\n        proMode,\n        onGetHighlights,\n        numElements: maxSources,\n      });\n\n      return expandHighlights(processedSources);\n    } catch (error) {\n      logger.error('Error in search:', error);\n      return {\n        organic: [],\n        topStories: [],\n        images: [],\n        videos: [],\n        news: [],\n        relatedSearches: [],\n        error: error instanceof Error ? error.message : String(error),\n      };\n    }\n  };\n}\n\nfunction createOnSearchResults({\n  runnableConfig,\n  onSearchResults,\n}: {\n  runnableConfig: RunnableConfig;\n  onSearchResults: t.SearchToolConfig['onSearchResults'];\n}) {\n  return function (results: t.SearchResult): void {\n    if (!onSearchResults) {\n      return;\n    }\n    onSearchResults(results, runnableConfig);\n  };\n}\n\nfunction createTool({\n  schema,\n  search,\n  onSearchResults: _onSearchResults,\n}: {\n  schema: t.SearchToolSchema;\n  search: ReturnType<typeof createSearchProcessor>;\n  onSearchResults: t.SearchToolConfig['onSearchResults'];\n}): DynamicStructuredTool<typeof schema> {\n  return tool<typeof schema>(\n    async (params, runnableConfig) => {\n      const { query, date, country: _c, images, videos, news } = params;\n      const country = typeof _c === 'string' && _c ? _c : undefined;\n      const searchResult = await search({\n        query,\n        date,\n        country,\n        images,\n        videos,\n        news,\n        onSearchResults: createOnSearchResults({\n          runnableConfig,\n          onSearchResults: _onSearchResults,\n        }),\n      });\n      const turn = runnableConfig.toolCall?.turn ?? 0;\n      const { output, references } = formatResultsForLLM(turn, searchResult);\n      const data: t.SearchResultData = { turn, ...searchResult, references };\n      return [output, { [Constants.WEB_SEARCH]: data }];\n    },\n    {\n      name: Constants.WEB_SEARCH,\n      description: `Real-time search. Results have required citation anchors.\n\nNote: Use ONCE per reply unless instructed otherwise.\n\nAnchors:\n- \\\\ue202turnXtypeY\n- X = turn idx, type = 'search' | 'news' | 'image' | 'ref', Y = item idx\n\nSpecial Markers:\n- \\\\ue203...\\\\ue204 — highlight start/end of cited text (for Standalone or Group citations)\n- \\\\ue200...\\\\ue201 — group block (e.g. \\\\ue200\\\\ue202turn0search1\\\\ue202turn0news2\\\\ue201)\n\n**CITE EVERY NON-OBVIOUS FACT/QUOTE:**\nUse anchor marker(s) immediately after the statement:\n- Standalone: \"Pure functions produce same output. \\\\ue202turn0search0\"\n- Standalone (multiple): \"Today's News \\\\ue202turn0search0\\\\ue202turn0news0\"\n- Highlight: \"\\\\ue203Highlight text.\\\\ue204\\\\ue202turn0news1\"\n- Group: \"Sources. \\\\ue200\\\\ue202turn0search0\\\\ue202turn0news1\\\\ue201\"\n- Group Highlight: \"\\\\ue203Highlight for group.\\\\ue204 \\\\ue200\\\\ue202turn0search0\\\\ue202turn0news1\\\\ue201\"\n- Image: \"See photo \\\\ue202turn0image0.\"\n\n**NEVER use markdown links, [1], or footnotes. CITE ONLY with anchors provided.**\n`.trim(),\n      schema: schema,\n      responseFormat: Constants.CONTENT_AND_ARTIFACT,\n    }\n  );\n}\n\n/**\n * Creates a search tool with a schema that dynamically includes the country field\n * only when the searchProvider is 'serper'.\n *\n * @param config - The search tool configuration\n * @returns A DynamicStructuredTool with a schema that depends on the searchProvider\n */\nexport const createSearchTool = (\n  config: t.SearchToolConfig = {}\n): DynamicStructuredTool<typeof toolSchema> => {\n  const {\n    searchProvider = 'serper',\n    serperApiKey,\n    searxngInstanceUrl,\n    searxngApiKey,\n    rerankerType = 'cohere',\n    topResults = 5,\n    strategies = ['no_extraction'],\n    filterContent = true,\n    safeSearch = 1,\n    firecrawlApiKey,\n    firecrawlApiUrl,\n    firecrawlOptions,\n    scraperTimeout,\n    jinaApiKey,\n    cohereApiKey,\n    onSearchResults: _onSearchResults,\n    onGetHighlights,\n  } = config;\n\n  const logger = config.logger || createDefaultLogger();\n\n  const schemaObject: {\n    query: z.ZodString;\n    date: z.ZodOptional<z.ZodNativeEnum<typeof DATE_RANGE>>;\n    country?: z.ZodOptional<z.ZodString>;\n    images: z.ZodOptional<z.ZodBoolean>;\n    videos: z.ZodOptional<z.ZodBoolean>;\n    news: z.ZodOptional<z.ZodBoolean>;\n  } = {\n    query: querySchema,\n    date: dateSchema,\n    images: imagesSchema,\n    videos: videosSchema,\n    news: newsSchema,\n  };\n\n  if (searchProvider === 'serper') {\n    schemaObject.country = countrySchema;\n  }\n\n  const toolSchema = z.object(schemaObject);\n\n  const searchAPI = createSearchAPI({\n    searchProvider,\n    serperApiKey,\n    searxngInstanceUrl,\n    searxngApiKey,\n  });\n\n  const firecrawlScraper = createFirecrawlScraper({\n    ...firecrawlOptions,\n    apiKey: firecrawlApiKey ?? process.env.FIRECRAWL_API_KEY,\n    apiUrl: firecrawlApiUrl,\n    timeout: scraperTimeout ?? firecrawlOptions?.timeout,\n    formats: firecrawlOptions?.formats ?? ['markdown', 'rawHtml'],\n  });\n\n  const selectedReranker = createReranker({\n    rerankerType,\n    jinaApiKey,\n    cohereApiKey,\n    logger,\n  });\n\n  if (!selectedReranker) {\n    logger.warn('No reranker selected. Using default ranking.');\n  }\n\n  const sourceProcessor = createSourceProcessor(\n    {\n      reranker: selectedReranker,\n      topResults,\n      strategies,\n      filterContent,\n      logger,\n    },\n    firecrawlScraper\n  );\n\n  const search = createSearchProcessor({\n    searchAPI,\n    safeSearch,\n    sourceProcessor,\n    onGetHighlights,\n    logger,\n  });\n\n  return createTool({\n    search,\n    schema: toolSchema,\n    onSearchResults: _onSearchResults,\n  });\n};\n"],"names":[],"mappings":";;;;;;;;;;;AAqBA;;AAEG;AACH,eAAe,uBAAuB,CAAC,EACrC,SAAS,EACT,KAAK,EACL,IAAI,EACJ,OAAO,EACP,UAAU,EACV,MAAM,EACN,MAAM,EACN,IAAI,EACJ,MAAM,GAWP,EAAA;;AAEC,IAAA,MAAM,WAAW,GAA8B;;QAE7C,SAAS,CAAC,UAAU,CAAC;YACnB,KAAK;YACL,IAAI;YACJ,OAAO;YACP,UAAU;SACX,CAAC;KACH;IAED,IAAI,MAAM,EAAE;QACV,WAAW,CAAC,IAAI,CACd;AACG,aAAA,UAAU,CAAC;YACV,KAAK;YACL,IAAI;YACJ,OAAO;YACP,UAAU;AACV,YAAA,IAAI,EAAE,QAAQ;SACf;AACA,aAAA,KAAK,CAAC,CAAC,KAAK,KAAI;AACf,YAAA,MAAM,CAAC,KAAK,CAAC,wBAAwB,EAAE,KAAK,CAAC;YAC7C,OAAO;AACL,gBAAA,OAAO,EAAE,KAAK;AACd,gBAAA,KAAK,EAAE,CAAyB,sBAAA,EAAA,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,CAAE,CAAA;aACzF;SACF,CAAC,CACL;;IAEH,IAAI,MAAM,EAAE;QACV,WAAW,CAAC,IAAI,CACd;AACG,aAAA,UAAU,CAAC;YACV,KAAK;YACL,IAAI;YACJ,OAAO;YACP,UAAU;AACV,YAAA,IAAI,EAAE,QAAQ;SACf;AACA,aAAA,KAAK,CAAC,CAAC,KAAK,KAAI;AACf,YAAA,MAAM,CAAC,KAAK,CAAC,wBAAwB,EAAE,KAAK,CAAC;YAC7C,OAAO;AACL,gBAAA,OAAO,EAAE,KAAK;AACd,gBAAA,KAAK,EAAE,CAAyB,sBAAA,EAAA,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,CAAE,CAAA;aACzF;SACF,CAAC,CACL;;IAEH,IAAI,IAAI,EAAE;QACR,WAAW,CAAC,IAAI,CACd;AACG,aAAA,UAAU,CAAC;YACV,KAAK;YACL,IAAI;YACJ,OAAO;YACP,UAAU;AACV,YAAA,IAAI,EAAE,MAAM;SACb;AACA,aAAA,KAAK,CAAC,CAAC,KAAK,KAAI;AACf,YAAA,MAAM,CAAC,KAAK,CAAC,sBAAsB,EAAE,KAAK,CAAC;YAC3C,OAAO;AACL,gBAAA,OAAO,EAAE,KAAK;AACd,gBAAA,KAAK,EAAE,CAAuB,oBAAA,EAAA,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,CAAE,CAAA;aACvF;SACF,CAAC,CACL;;;IAIH,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC;;AAG9C,IAAA,MAAM,UAAU,GAAG,OAAO,CAAC,CAAC,CAAC;AAC7B,IAAA,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE;QACvB,MAAM,IAAI,KAAK,CAAC,UAAU,CAAC,KAAK,IAAI,eAAe,CAAC;;;IAItD,MAAM,aAAa,GAAG,EAAE,GAAG,UAAU,CAAC,IAAI,EAAE;;AAG5C,IAAA,IAAI,aAAa,CAAC,IAAI,KAAK,SAAS,IAAI,aAAa,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE;AACrE,QAAA,MAAM,wBAAwB,GAAG,aAAa,CAAC;AAC5C,aAAA,MAAM,CAAC,CAAC,QAAQ,KAAK,QAAQ,CAAC,IAAI,KAAK,SAAS,IAAI,QAAQ,CAAC,IAAI,KAAK,EAAE;AACxE,aAAA,GAAG,CAAC,CAAC,QAAQ,MAAM;AAClB,YAAA,KAAK,EAAE,QAAQ,CAAC,KAAK,IAAI,EAAE;AAC3B,YAAA,IAAI,EAAE,QAAQ,CAAC,IAAI,IAAI,EAAE;AACzB,YAAA,MAAM,EAAE,QAAQ,CAAC,MAAM,IAAI,EAAE;AAC7B,YAAA,IAAI,EAAE,QAAQ,CAAC,IAAI,IAAI,EAAE;AACzB,YAAA,QAAQ,EAAE,QAAQ,CAAC,QAAQ,IAAI,EAAE;AACjC,YAAA,SAAS,EAAE,KAAK;AACjB,SAAA,CAAC,CAAC;QACL,aAAa,CAAC,UAAU,GAAG;AACzB,YAAA,IAAI,aAAa,CAAC,UAAU,IAAI,EAAE,CAAC;AACnC,YAAA,GAAG,wBAAwB;SAC5B;QACD,OAAO,aAAa,CAAC,IAAI;;IAG3B,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,KAAI;QAClC,IAAI,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE;AAC/C,YAAA,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;gBACrE,aAAa,CAAC,MAAM,GAAG;AACrB,oBAAA,IAAI,aAAa,CAAC,MAAM,IAAI,EAAE,CAAC;AAC/B,oBAAA,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM;iBACtB;;AAEH,YAAA,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;gBACrE,aAAa,CAAC,MAAM,GAAG;AACrB,oBAAA,IAAI,aAAa,CAAC,MAAM,IAAI,EAAE,CAAC;AAC/B,oBAAA,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM;iBACtB;;AAEH,YAAA,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,KAAK,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE;AACjE,gBAAA,MAAM,gBAAgB,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,QAAQ,MAAM;AAC3D,oBAAA,GAAG,QAAQ;AACX,oBAAA,IAAI,EAAE,QAAQ,CAAC,IAAI,IAAI,EAAE;AAC1B,iBAAA,CAAC,CAAC;gBACH,aAAa,CAAC,UAAU,GAAG;AACzB,oBAAA,IAAI,aAAa,CAAC,UAAU,IAAI,EAAE,CAAC;AACnC,oBAAA,GAAG,gBAAgB;iBACpB;;;AAGP,KAAC,CAAC;IAEF,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,aAAa,EAAE;AAC/C;AAEA,SAAS,qBAAqB,CAAC,EAC7B,SAAS,EACT,UAAU,EACV,eAAe,EACf,eAAe,EACf,MAAM,GAOP,EAAA;AACC,IAAA,OAAO,gBAAgB,EACrB,KAAK,EACL,IAAI,EACJ,OAAO,EACP,OAAO,GAAG,IAAI,EACd,UAAU,GAAG,CAAC,EACd,eAAe,EACf,MAAM,GAAG,KAAK,EACd,MAAM,GAAG,KAAK,EACd,IAAI,GAAG,KAAK,GAWb,EAAA;AACC,QAAA,IAAI;;AAEF,YAAA,MAAM,YAAY,GAAG,MAAM,uBAAuB,CAAC;gBACjD,SAAS;gBACT,KAAK;gBACL,IAAI;gBACJ,OAAO;gBACP,UAAU;gBACV,MAAM;gBACN,MAAM;gBACN,IAAI;gBACJ,MAAM;AACP,aAAA,CAAC;AAEF,YAAA,eAAe,GAAG,YAAY,CAAC;AAE/B,YAAA,MAAM,gBAAgB,GAAG,MAAM,eAAe,CAAC,cAAc,CAAC;gBAC5D,KAAK;gBACL,IAAI;AACJ,gBAAA,MAAM,EAAE,YAAY;gBACpB,OAAO;gBACP,eAAe;AACf,gBAAA,WAAW,EAAE,UAAU;AACxB,aAAA,CAAC;AAEF,YAAA,OAAO,gBAAgB,CAAC,gBAAgB,CAAC;;QACzC,OAAO,KAAK,EAAE;AACd,YAAA,MAAM,CAAC,KAAK,CAAC,kBAAkB,EAAE,KAAK,CAAC;YACvC,OAAO;AACL,gBAAA,OAAO,EAAE,EAAE;AACX,gBAAA,UAAU,EAAE,EAAE;AACd,gBAAA,MAAM,EAAE,EAAE;AACV,gBAAA,MAAM,EAAE,EAAE;AACV,gBAAA,IAAI,EAAE,EAAE;AACR,gBAAA,eAAe,EAAE,EAAE;AACnB,gBAAA,KAAK,EAAE,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;aAC9D;;AAEL,KAAC;AACH;AAEA,SAAS,qBAAqB,CAAC,EAC7B,cAAc,EACd,eAAe,GAIhB,EAAA;AACC,IAAA,OAAO,UAAU,OAAuB,EAAA;QACtC,IAAI,CAAC,eAAe,EAAE;YACpB;;AAEF,QAAA,eAAe,CAAC,OAAO,EAAE,cAAc,CAAC;AAC1C,KAAC;AACH;AAEA,SAAS,UAAU,CAAC,EAClB,MAAM,EACN,MAAM,EACN,eAAe,EAAE,gBAAgB,GAKlC,EAAA;IACC,OAAO,IAAI,CACT,OAAO,MAAM,EAAE,cAAc,KAAI;AAC/B,QAAA,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM;AACjE,QAAA,MAAM,OAAO,GAAG,OAAO,EAAE,KAAK,QAAQ,IAAI,EAAE,GAAG,EAAE,GAAG,SAAS;AAC7D,QAAA,MAAM,YAAY,GAAG,MAAM,MAAM,CAAC;YAChC,KAAK;YACL,IAAI;YACJ,OAAO;YACP,MAAM;YACN,MAAM;YACN,IAAI;YACJ,eAAe,EAAE,qBAAqB,CAAC;gBACrC,cAAc;AACd,gBAAA,eAAe,EAAE,gBAAgB;aAClC,CAAC;AACH,SAAA,CAAC;QACF,MAAM,IAAI,GAAG,cAAc,CAAC,QAAQ,EAAE,IAAI,IAAI,CAAC;AAC/C,QAAA,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,mBAAmB,CAAC,IAAI,EAAE,YAAY,CAAC;QACtE,MAAM,IAAI,GAAuB,EAAE,IAAI,EAAE,GAAG,YAAY,EAAE,UAAU,EAAE;AACtE,QAAA,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,SAAS,CAAC,UAAU,GAAG,IAAI,EAAE,CAAC;AACnD,KAAC,EACD;QACE,IAAI,EAAE,SAAS,CAAC,UAAU;AAC1B,QAAA,WAAW,EAAE,CAAA;;;;;;;;;;;;;;;;;;;;;;AAsBlB,CAAA,CAAC,IAAI,EAAE;AACF,QAAA,MAAM,EAAE,MAAM;QACd,cAAc,EAAE,SAAS,CAAC,oBAAoB;AAC/C,KAAA,CACF;AACH;AAEA;;;;;;AAMG;MACU,gBAAgB,GAAG,CAC9B,MAA6B,GAAA,EAAE,KACa;IAC5C,MAAM,EACJ,cAAc,GAAG,QAAQ,EACzB,YAAY,EACZ,kBAAkB,EAClB,aAAa,EACb,YAAY,GAAG,QAAQ,EACvB,UAAU,GAAG,CAAC,EACd,UAAU,GAAG,CAAC,eAAe,CAAC,EAC9B,aAAa,GAAG,IAAI,EACpB,UAAU,GAAG,CAAC,EACd,eAAe,EACf,eAAe,EACf,gBAAgB,EAChB,cAAc,EACd,UAAU,EACV,YAAY,EACZ,eAAe,EAAE,gBAAgB,EACjC,eAAe,GAChB,GAAG,MAAM;IAEV,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,mBAAmB,EAAE;AAErD,IAAA,MAAM,YAAY,GAOd;AACF,QAAA,KAAK,EAAE,WAAW;AAClB,QAAA,IAAI,EAAE,UAAU;AAChB,QAAA,MAAM,EAAE,YAAY;AACpB,QAAA,MAAM,EAAE,YAAY;AACpB,QAAA,IAAI,EAAE,UAAU;KACjB;AAED,IAAA,IAAI,cAAc,KAAK,QAAQ,EAAE;AAC/B,QAAA,YAAY,CAAC,OAAO,GAAG,aAAa;;IAGtC,MAAM,UAAU,GAAG,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC;IAEzC,MAAM,SAAS,GAAG,eAAe,CAAC;QAChC,cAAc;QACd,YAAY;QACZ,kBAAkB;QAClB,aAAa;AACd,KAAA,CAAC;IAEF,MAAM,gBAAgB,GAAG,sBAAsB,CAAC;AAC9C,QAAA,GAAG,gBAAgB;AACnB,QAAA,MAAM,EAAE,eAAe,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB;AACxD,QAAA,MAAM,EAAE,eAAe;AACvB,QAAA,OAAO,EAAE,cAAc,IAAI,gBAAgB,EAAE,OAAO;QACpD,OAAO,EAAE,gBAAgB,EAAE,OAAO,IAAI,CAAC,UAAU,EAAE,SAAS,CAAC;AAC9D,KAAA,CAAC;IAEF,MAAM,gBAAgB,GAAG,cAAc,CAAC;QACtC,YAAY;QACZ,UAAU;QACV,YAAY;QACZ,MAAM;AACP,KAAA,CAAC;IAEF,IAAI,CAAC,gBAAgB,EAAE;AACrB,QAAA,MAAM,CAAC,IAAI,CAAC,8CAA8C,CAAC;;IAG7D,MAAM,eAAe,GAAG,qBAAqB,CAC3C;AACE,QAAA,QAAQ,EAAE,gBAAgB;QAC1B,UAAU;QAGV,MAAM;KACP,EACD,gBAAgB,CACjB;IAED,MAAM,MAAM,GAAG,qBAAqB,CAAC;QACnC,SAAS;QACT,UAAU;QACV,eAAe;QACf,eAAe;QACf,MAAM;AACP,KAAA,CAAC;AAEF,IAAA,OAAO,UAAU,CAAC;QAChB,MAAM;AACN,QAAA,MAAM,EAAE,UAAU;AAClB,QAAA,eAAe,EAAE,gBAAgB;AAClC,KAAA,CAAC;AACJ;;;;"}

package/dist/types/tools/search/firecrawl.d.ts

@@ -9,6 +9,21 @@ export declare class FirecrawlScraper {
     private defaultFormats;
     private timeout;
     private logger;
+    private includeTags?;
+    private excludeTags?;
+    private waitFor?;
+    private maxAge?;
+    private mobile?;
+    private skipTlsVerification?;
+    private blockAds?;
+    private removeBase64Images?;
+    private parsePDF?;
+    private storeInCache?;
+    private zeroDataRetention?;
+    private headers?;
+    private location?;
+    private onlyMainContent?;
+    private changeTrackingOptions?;
     constructor(config?: t.FirecrawlScraperConfig);
     /**
      * Scrape a single URL

package/dist/types/tools/search/types.d.ts

@@ -85,7 +85,7 @@ export interface ProcessSourcesConfig {
 export interface FirecrawlConfig {
     firecrawlApiKey?: string;
     firecrawlApiUrl?: string;
-    firecrawlFormats?: string[];
+    firecrawlOptions?: FirecrawlScraperConfig;
 }
 export interface ScraperContentResult {
     content: string;
@@ -147,14 +147,7 @@ export type UsedReferences = {
     reference: MediaReference;
 }[];
 /** Firecrawl */
-export interface FirecrawlScrapeOptions {
-    formats?: string[];
-    includeTags?: string[];
-    excludeTags?: string[];
-    headers?: Record<string, string>;
-    waitFor?: number;
-    timeout?: number;
-}
+export type FirecrawlScrapeOptions = Omit<FirecrawlScraperConfig, 'apiKey' | 'apiUrl' | 'logger'>;
 export interface ScrapeMetadata {
     sourceURL?: string;
     url?: string;
@@ -226,6 +219,24 @@ export interface FirecrawlScraperConfig {
     formats?: string[];
     timeout?: number;
     logger?: Logger;
+    includeTags?: string[];
+    excludeTags?: string[];
+    waitFor?: number;
+    maxAge?: number;
+    mobile?: boolean;
+    skipTlsVerification?: boolean;
+    blockAds?: boolean;
+    removeBase64Images?: boolean;
+    parsePDF?: boolean;
+    storeInCache?: boolean;
+    zeroDataRetention?: boolean;
+    headers?: Record<string, string>;
+    location?: {
+        country?: string;
+        languages?: string[];
+    };
+    onlyMainContent?: boolean;
+    changeTrackingOptions?: object;
 }
 export type GetSourcesParams = {
     query: string;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@librechat/agents",
-  "version": "2.4.62",
+  "version": "2.4.63",
   "main": "./dist/cjs/main.cjs",
   "module": "./dist/esm/main.mjs",
   "types": "./dist/types/index.d.ts",
@@ -77,7 +77,7 @@
   },
   "dependencies": {
     "@langchain/anthropic": "^0.3.24",
-    "@langchain/aws": "^0.1.11",
+    "@langchain/aws": "^0.1.12",
     "@langchain/community": "^0.3.47",
     "@langchain/core": "^0.3.62",
     "@langchain/deepseek": "^0.0.2",
@@ -112,7 +112,7 @@
     "@types/yargs-parser": "^21.0.3",
     "@typescript-eslint/eslint-plugin": "^8.24.0",
     "@typescript-eslint/parser": "^8.24.0",
-    "eslint": "^9.22.0",
+    "eslint": "^9.31.0",
     "eslint-import-resolver-typescript": "^3.7.0",
     "eslint-plugin-import": "^2.31.0",
     "husky": "^9.1.7",

package/src/tools/search/firecrawl.ts

@@ -13,6 +13,21 @@ export class FirecrawlScraper {
   private defaultFormats: string[];
   private timeout: number;
   private logger: t.Logger;
+  private includeTags?: string[];
+  private excludeTags?: string[];
+  private waitFor?: number;
+  private maxAge?: number;
+  private mobile?: boolean;
+  private skipTlsVerification?: boolean;
+  private blockAds?: boolean;
+  private removeBase64Images?: boolean;
+  private parsePDF?: boolean;
+  private storeInCache?: boolean;
+  private zeroDataRetention?: boolean;
+  private headers?: Record<string, string>;
+  private location?: { country?: string; languages?: string[] };
+  private onlyMainContent?: boolean;
+  private changeTrackingOptions?: object;
 
   constructor(config: t.FirecrawlScraperConfig = {}) {
     this.apiKey = config.apiKey ?? process.env.FIRECRAWL_API_KEY ?? '';
@@ -23,11 +38,27 @@ export class FirecrawlScraper {
       'https://api.firecrawl.dev';
     this.apiUrl = `${baseUrl.replace(/\/+$/, '')}/v1/scrape`;
 
-    this.defaultFormats = config.formats ?? ['markdown', 'html'];
+    this.defaultFormats = config.formats ?? ['markdown', 'rawHtml'];
     this.timeout = config.timeout ?? 7500;
 
     this.logger = config.logger || createDefaultLogger();
 
+    this.includeTags = config.includeTags;
+    this.excludeTags = config.excludeTags;
+    this.waitFor = config.waitFor;
+    this.maxAge = config.maxAge;
+    this.mobile = config.mobile;
+    this.skipTlsVerification = config.skipTlsVerification;
+    this.blockAds = config.blockAds;
+    this.removeBase64Images = config.removeBase64Images;
+    this.parsePDF = config.parsePDF;
+    this.storeInCache = config.storeInCache;
+    this.zeroDataRetention = config.zeroDataRetention;
+    this.headers = config.headers;
+    this.location = config.location;
+    this.onlyMainContent = config.onlyMainContent;
+    this.changeTrackingOptions = config.changeTrackingOptions;
+
     if (!this.apiKey) {
       this.logger.warn('FIRECRAWL_API_KEY is not set. Scraping will not work.');
     }
@@ -58,25 +89,36 @@ export class FirecrawlScraper {
     }
 
     try {
-      const response = await axios.post(
-        this.apiUrl,
-        {
-          url,
-          formats: options.formats || this.defaultFormats,
-          includeTags: options.includeTags,
-          excludeTags: options.excludeTags,
-          headers: options.headers,
-          waitFor: options.waitFor,
-          timeout: options.timeout ?? this.timeout,
+      const payload = omitUndefined({
+        url,
+        formats: options.formats ?? this.defaultFormats,
+        includeTags: options.includeTags ?? this.includeTags,
+        excludeTags: options.excludeTags ?? this.excludeTags,
+        headers: options.headers ?? this.headers,
+        waitFor: options.waitFor ?? this.waitFor,
+        timeout: options.timeout ?? this.timeout,
+        onlyMainContent: options.onlyMainContent ?? this.onlyMainContent,
+        maxAge: options.maxAge ?? this.maxAge,
+        mobile: options.mobile ?? this.mobile,
+        skipTlsVerification:
+          options.skipTlsVerification ?? this.skipTlsVerification,
+        parsePDF: options.parsePDF ?? this.parsePDF,
+        location: options.location ?? this.location,
+        removeBase64Images:
+          options.removeBase64Images ?? this.removeBase64Images,
+        blockAds: options.blockAds ?? this.blockAds,
+        storeInCache: options.storeInCache ?? this.storeInCache,
+        zeroDataRetention: options.zeroDataRetention ?? this.zeroDataRetention,
+        changeTrackingOptions:
+          options.changeTrackingOptions ?? this.changeTrackingOptions,
+      });
+      const response = await axios.post(this.apiUrl, payload, {
+        headers: {
+          'Content-Type': 'application/json',
+          Authorization: `Bearer ${this.apiKey}`,
         },
-        {
-          headers: {
-            'Content-Type': 'application/json',
-            Authorization: `Bearer ${this.apiKey}`,
-          },
-          timeout: this.timeout,
-        }
-      );
+        timeout: this.timeout,
+      });
 
       return [url, response.data];
     } catch (error) {
@@ -156,3 +198,10 @@ export const createFirecrawlScraper = (
 ): FirecrawlScraper => {
   return new FirecrawlScraper(config);
 };
+
+// Helper function to clean up payload for firecrawl
+function omitUndefined<T extends object>(obj: T): Partial<T> {
+  return Object.fromEntries(
+    Object.entries(obj).filter(([, v]) => v !== undefined)
+  ) as Partial<T>;
+}

package/src/tools/search/tool.ts

@@ -346,7 +346,7 @@ export const createSearchTool = (
     safeSearch = 1,
     firecrawlApiKey,
     firecrawlApiUrl,
-    firecrawlFormats = ['markdown', 'html'],
+    firecrawlOptions,
     scraperTimeout,
     jinaApiKey,
     cohereApiKey,
@@ -385,10 +385,11 @@ export const createSearchTool = (
   });
 
   const firecrawlScraper = createFirecrawlScraper({
+    ...firecrawlOptions,
     apiKey: firecrawlApiKey ?? process.env.FIRECRAWL_API_KEY,
     apiUrl: firecrawlApiUrl,
-    timeout: scraperTimeout,
-    formats: firecrawlFormats,
+    timeout: scraperTimeout ?? firecrawlOptions?.timeout,
+    formats: firecrawlOptions?.formats ?? ['markdown', 'rawHtml'],
   });
 
   const selectedReranker = createReranker({

package/src/tools/search/types.ts

@@ -94,7 +94,7 @@ export interface ProcessSourcesConfig {
 export interface FirecrawlConfig {
   firecrawlApiKey?: string;
   firecrawlApiUrl?: string;
-  firecrawlFormats?: string[];
+  firecrawlOptions?: FirecrawlScraperConfig;
 }
 
 export interface ScraperContentResult {
@@ -170,15 +170,10 @@ export type UsedReferences = {
 }[];
 
 /** Firecrawl */
-
-export interface FirecrawlScrapeOptions {
-  formats?: string[];
-  includeTags?: string[];
-  excludeTags?: string[];
-  headers?: Record<string, string>;
-  waitFor?: number;
-  timeout?: number;
-}
+export type FirecrawlScrapeOptions = Omit<
+  FirecrawlScraperConfig,
+  'apiKey' | 'apiUrl' | 'logger'
+>;
 
 export interface ScrapeMetadata {
   // Core source information
@@ -261,6 +256,21 @@ export interface FirecrawlScraperConfig {
   formats?: string[];
   timeout?: number;
   logger?: Logger;
+  includeTags?: string[];
+  excludeTags?: string[];
+  waitFor?: number;
+  maxAge?: number;
+  mobile?: boolean;
+  skipTlsVerification?: boolean;
+  blockAds?: boolean;
+  removeBase64Images?: boolean;
+  parsePDF?: boolean;
+  storeInCache?: boolean;
+  zeroDataRetention?: boolean;
+  headers?: Record<string, string>;
+  location?: { country?: string; languages?: string[] };
+  onlyMainContent?: boolean;
+  changeTrackingOptions?: object;
 }
 
 export type GetSourcesParams = {