@librechat/agents 2.4.31 → 2.4.311

package/dist/cjs/tools/search/firecrawl.cjs

@@ -48,7 +48,7 @@ class FirecrawlScraper {
             'https://api.firecrawl.dev';
         this.apiUrl = `${baseUrl.replace(/\/+$/, '')}/v1/scrape`;
         this.defaultFormats = config.formats ?? ['markdown', 'html'];
-        this.timeout = config.timeout ?? 30000;
+        this.timeout = config.timeout ?? 15000;
         if (!this.apiKey) {
             console.warn('FIRECRAWL_API_KEY is not set. Scraping will not work.');
         }

package/dist/cjs/tools/search/firecrawl.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"firecrawl.cjs","sources":["../../../../src/tools/search/firecrawl.ts"],"sourcesContent":["/* eslint-disable no-console */\nimport axios from 'axios';\n\nexport interface FirecrawlScrapeOptions {\n  formats?: string[];\n  includeTags?: string[];\n  excludeTags?: string[];\n  headers?: Record<string, string>;\n  waitFor?: number;\n  timeout?: number;\n}\n\ninterface ScrapeMetadata {\n  // Core source information\n  sourceURL?: string;\n  url?: string;\n  scrapeId?: string;\n  statusCode?: number;\n  // Basic metadata\n  title?: string;\n  description?: string;\n  language?: string;\n  favicon?: string;\n  viewport?: string;\n  robots?: string;\n  'theme-color'?: string;\n  // Open Graph metadata\n  'og:url'?: string;\n  'og:title'?: string;\n  'og:description'?: string;\n  'og:type'?: string;\n  'og:image'?: string;\n  'og:image:width'?: string;\n  'og:image:height'?: string;\n  'og:site_name'?: string;\n  ogUrl?: string;\n  ogTitle?: string;\n  ogDescription?: string;\n  ogImage?: string;\n  ogSiteName?: string;\n  // Article metadata\n  'article:author'?: string;\n  'article:published_time'?: string;\n  'article:modified_time'?: string;\n  'article:section'?: string;\n  'article:tag'?: string;\n  'article:publisher'?: string;\n  publishedTime?: string;\n  modifiedTime?: string;\n  // Twitter metadata\n  'twitter:site'?: string;\n  'twitter:creator'?: string;\n  'twitter:card'?: string;\n  'twitter:image'?: string;\n  'twitter:dnt'?: string;\n  'twitter:app:name:iphone'?: string;\n  'twitter:app:id:iphone'?: string;\n  'twitter:app:url:iphone'?: string;\n  'twitter:app:name:ipad'?: string;\n  'twitter:app:id:ipad'?: string;\n  'twitter:app:url:ipad'?: string;\n  'twitter:app:name:googleplay'?: string;\n  'twitter:app:id:googleplay'?: string;\n  'twitter:app:url:googleplay'?: string;\n  // Facebook metadata\n  'fb:app_id'?: string;\n  // App links\n  'al:ios:url'?: string;\n  'al:ios:app_name'?: string;\n  'al:ios:app_store_id'?: string;\n  // Allow for additional properties that might be present\n  [key: string]: string | number | boolean | null | undefined;\n}\n\nexport interface FirecrawlScrapeResponse {\n  success: boolean;\n  data?: {\n    markdown?: string;\n    html?: string;\n    rawHtml?: string;\n    screenshot?: string;\n    links?: string[];\n    metadata?: ScrapeMetadata;\n  };\n  error?: string;\n}\n\nexport interface FirecrawlScraperConfig {\n  apiKey?: string;\n  apiUrl?: string;\n  formats?: string[];\n  timeout?: number;\n}\nconst getDomainName = (\n  link: string,\n  metadata?: ScrapeMetadata\n): string | undefined => {\n  try {\n    const url = metadata?.sourceURL ?? metadata?.url ?? (link || '');\n    const domain = new URL(url).hostname.replace(/^www\\./, '');\n    if (domain) {\n      return domain;\n    }\n  } catch (e) {\n    // URL parsing failed\n    console.error('Error parsing URL:', e);\n  }\n\n  return;\n};\n\nexport function getAttribution(\n  link: string,\n  metadata?: ScrapeMetadata\n): string | undefined {\n  if (!metadata) return getDomainName(link, metadata);\n\n  const possibleAttributions = [\n    metadata.ogSiteName,\n    metadata['og:site_name'],\n    metadata.title?.split('|').pop()?.trim(),\n    metadata['twitter:site']?.replace(/^@/, ''),\n  ];\n\n  const attribution = possibleAttributions.find(\n    (attr) => attr != null && typeof attr === 'string' && attr.trim() !== ''\n  );\n  if (attribution != null) {\n    return attribution;\n  }\n\n  return getDomainName(link, metadata);\n}\n\n/**\n * Firecrawl scraper implementation\n * Uses the Firecrawl API to scrape web pages\n */\nexport class FirecrawlScraper {\n  private apiKey: string;\n  private apiUrl: string;\n  private defaultFormats: string[];\n  private timeout: number;\n\n  constructor(config: FirecrawlScraperConfig = {}) {\n    this.apiKey = config.apiKey ?? process.env.FIRECRAWL_API_KEY ?? '';\n\n    const baseUrl =\n      config.apiUrl ??\n      process.env.FIRECRAWL_BASE_URL ??\n      'https://api.firecrawl.dev';\n    this.apiUrl = `${baseUrl.replace(/\\/+$/, '')}/v1/scrape`;\n\n    this.defaultFormats = config.formats ?? ['markdown', 'html'];\n    this.timeout = config.timeout ?? 30000;\n\n    if (!this.apiKey) {\n      console.warn('FIRECRAWL_API_KEY is not set. Scraping will not work.');\n    }\n\n    console.log(`Firecrawl scraper initialized with API URL: ${this.apiUrl}`);\n  }\n\n  /**\n   * Scrape a single URL\n   * @param url URL to scrape\n   * @param options Scrape options\n   * @returns Scrape response\n   */\n  async scrapeUrl(\n    url: string,\n    options: FirecrawlScrapeOptions = {}\n  ): Promise<[string, FirecrawlScrapeResponse]> {\n    if (!this.apiKey) {\n      return [\n        url,\n        {\n          success: false,\n          error: 'FIRECRAWL_API_KEY is not set',\n        },\n      ];\n    }\n\n    try {\n      const response = await axios.post(\n        this.apiUrl,\n        {\n          url,\n          formats: options.formats || this.defaultFormats,\n          includeTags: options.includeTags,\n          excludeTags: options.excludeTags,\n          headers: options.headers,\n          waitFor: options.waitFor,\n          timeout: options.timeout ?? this.timeout,\n        },\n        {\n          headers: {\n            'Content-Type': 'application/json',\n            Authorization: `Bearer ${this.apiKey}`,\n          },\n          timeout: this.timeout,\n        }\n      );\n\n      return [url, response.data];\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      return [\n        url,\n        {\n          success: false,\n          error: `Firecrawl API request failed: ${errorMessage}`,\n        },\n      ];\n    }\n  }\n\n  /**\n   * Extract content from scrape response\n   * @param response Scrape response\n   * @returns Extracted content or empty string if not available\n   */\n  extractContent(response: FirecrawlScrapeResponse): string {\n    if (!response.success || !response.data) {\n      return '';\n    }\n\n    // Prefer markdown content if available\n    if (response.data.markdown != null) {\n      return response.data.markdown;\n    }\n\n    // Fall back to HTML content\n    if (response.data.html != null) {\n      return response.data.html;\n    }\n\n    // Fall back to raw HTML content\n    if (response.data.rawHtml != null) {\n      return response.data.rawHtml;\n    }\n\n    return '';\n  }\n\n  /**\n   * Extract metadata from scrape response\n   * @param response Scrape response\n   * @returns Metadata object\n   */\n  extractMetadata(response: FirecrawlScrapeResponse): ScrapeMetadata {\n    if (!response.success || !response.data || !response.data.metadata) {\n      return {};\n    }\n\n    return response.data.metadata;\n  }\n}\n\n/**\n * Create a Firecrawl scraper instance\n * @param config Scraper configuration\n * @returns Firecrawl scraper instance\n */\nexport const createFirecrawlScraper = (\n  config: FirecrawlScraperConfig = {}\n): FirecrawlScraper => {\n  return new FirecrawlScraper(config);\n};\n"],"names":[],"mappings":";;;;AAAA;AA6FA,MAAM,aAAa,GAAG,CACpB,IAAY,EACZ,QAAyB,KACH;AACtB,IAAA,IAAI;AACF,QAAA,MAAM,GAAG,GAAG,QAAQ,EAAE,SAAS,IAAI,QAAQ,EAAE,GAAG,KAAK,IAAI,IAAI,EAAE,CAAC;AAChE,QAAA,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC;QAC1D,IAAI,MAAM,EAAE;AACV,YAAA,OAAO,MAAM;;;IAEf,OAAO,CAAC,EAAE;;AAEV,QAAA,OAAO,CAAC,KAAK,CAAC,oBAAoB,EAAE,CAAC,CAAC;;IAGxC;AACF,CAAC;AAEe,SAAA,cAAc,CAC5B,IAAY,EACZ,QAAyB,EAAA;AAEzB,IAAA,IAAI,CAAC,QAAQ;AAAE,QAAA,OAAO,aAAa,CAAC,IAAI,EAAE,QAAQ,CAAC;AAEnD,IAAA,MAAM,oBAAoB,GAAG;AAC3B,QAAA,QAAQ,CAAC,UAAU;QACnB,QAAQ,CAAC,cAAc,CAAC;AACxB,QAAA,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE;QACxC,QAAQ,CAAC,cAAc,CAAC,EAAE,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;KAC5C;IAED,MAAM,WAAW,GAAG,oBAAoB,CAAC,IAAI,CAC3C,CAAC,IAAI,KAAK,IAAI,IAAI,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CACzE;AACD,IAAA,IAAI,WAAW,IAAI,IAAI,EAAE;AACvB,QAAA,OAAO,WAAW;;AAGpB,IAAA,OAAO,aAAa,CAAC,IAAI,EAAE,QAAQ,CAAC;AACtC;AAEA;;;AAGG;MACU,gBAAgB,CAAA;AACnB,IAAA,MAAM;AACN,IAAA,MAAM;AACN,IAAA,cAAc;AACd,IAAA,OAAO;AAEf,IAAA,WAAA,CAAY,SAAiC,EAAE,EAAA;AAC7C,QAAA,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,EAAE;AAElE,QAAA,MAAM,OAAO,GACX,MAAM,CAAC,MAAM;YACb,OAAO,CAAC,GAAG,CAAC,kBAAkB;AAC9B,YAAA,2BAA2B;AAC7B,QAAA,IAAI,CAAC,MAAM,GAAG,CAAA,EAAG,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,YAAY;AAExD,QAAA,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,OAAO,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC;QAC5D,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,KAAK;AAEtC,QAAA,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;AAChB,YAAA,OAAO,CAAC,IAAI,CAAC,uDAAuD,CAAC;;QAGvE,OAAO,CAAC,GAAG,CAAC,CAAA,4CAAA,EAA+C,IAAI,CAAC,MAAM,CAAE,CAAA,CAAC;;AAG3E;;;;;AAKG;AACH,IAAA,MAAM,SAAS,CACb,GAAW,EACX,UAAkC,EAAE,EAAA;AAEpC,QAAA,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YAChB,OAAO;gBACL,GAAG;AACH,gBAAA;AACE,oBAAA,OAAO,EAAE,KAAK;AACd,oBAAA,KAAK,EAAE,8BAA8B;AACtC,iBAAA;aACF;;AAGH,QAAA,IAAI;YACF,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAC/B,IAAI,CAAC,MAAM,EACX;gBACE,GAAG;AACH,gBAAA,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,cAAc;gBAC/C,WAAW,EAAE,OAAO,CAAC,WAAW;gBAChC,WAAW,EAAE,OAAO,CAAC,WAAW;gBAChC,OAAO,EAAE,OAAO,CAAC,OAAO;gBACxB,OAAO,EAAE,OAAO,CAAC,OAAO;AACxB,gBAAA,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO;aACzC,EACD;AACE,gBAAA,OAAO,EAAE;AACP,oBAAA,cAAc,EAAE,kBAAkB;AAClC,oBAAA,aAAa,EAAE,CAAA,OAAA,EAAU,IAAI,CAAC,MAAM,CAAE,CAAA;AACvC,iBAAA;gBACD,OAAO,EAAE,IAAI,CAAC,OAAO;AACtB,aAAA,CACF;AAED,YAAA,OAAO,CAAC,GAAG,EAAE,QAAQ,CAAC,IAAI,CAAC;;QAC3B,OAAO,KAAK,EAAE;AACd,YAAA,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;YACxD,OAAO;gBACL,GAAG;AACH,gBAAA;AACE,oBAAA,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,CAAiC,8BAAA,EAAA,YAAY,CAAE,CAAA;AACvD,iBAAA;aACF;;;AAIL;;;;AAIG;AACH,IAAA,cAAc,CAAC,QAAiC,EAAA;QAC9C,IAAI,CAAC,QAAQ,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE;AACvC,YAAA,OAAO,EAAE;;;QAIX,IAAI,QAAQ,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,EAAE;AAClC,YAAA,OAAO,QAAQ,CAAC,IAAI,CAAC,QAAQ;;;QAI/B,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,EAAE;AAC9B,YAAA,OAAO,QAAQ,CAAC,IAAI,CAAC,IAAI;;;QAI3B,IAAI,QAAQ,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,EAAE;AACjC,YAAA,OAAO,QAAQ,CAAC,IAAI,CAAC,OAAO;;AAG9B,QAAA,OAAO,EAAE;;AAGX;;;;AAIG;AACH,IAAA,eAAe,CAAC,QAAiC,EAAA;AAC/C,QAAA,IAAI,CAAC,QAAQ,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE;AAClE,YAAA,OAAO,EAAE;;AAGX,QAAA,OAAO,QAAQ,CAAC,IAAI,CAAC,QAAQ;;AAEhC;AAED;;;;AAIG;MACU,sBAAsB,GAAG,CACpC,MAAiC,GAAA,EAAE,KACf;AACpB,IAAA,OAAO,IAAI,gBAAgB,CAAC,MAAM,CAAC;AACrC;;;;;;"}
+{"version":3,"file":"firecrawl.cjs","sources":["../../../../src/tools/search/firecrawl.ts"],"sourcesContent":["/* eslint-disable no-console */\nimport axios from 'axios';\n\nexport interface FirecrawlScrapeOptions {\n  formats?: string[];\n  includeTags?: string[];\n  excludeTags?: string[];\n  headers?: Record<string, string>;\n  waitFor?: number;\n  timeout?: number;\n}\n\ninterface ScrapeMetadata {\n  // Core source information\n  sourceURL?: string;\n  url?: string;\n  scrapeId?: string;\n  statusCode?: number;\n  // Basic metadata\n  title?: string;\n  description?: string;\n  language?: string;\n  favicon?: string;\n  viewport?: string;\n  robots?: string;\n  'theme-color'?: string;\n  // Open Graph metadata\n  'og:url'?: string;\n  'og:title'?: string;\n  'og:description'?: string;\n  'og:type'?: string;\n  'og:image'?: string;\n  'og:image:width'?: string;\n  'og:image:height'?: string;\n  'og:site_name'?: string;\n  ogUrl?: string;\n  ogTitle?: string;\n  ogDescription?: string;\n  ogImage?: string;\n  ogSiteName?: string;\n  // Article metadata\n  'article:author'?: string;\n  'article:published_time'?: string;\n  'article:modified_time'?: string;\n  'article:section'?: string;\n  'article:tag'?: string;\n  'article:publisher'?: string;\n  publishedTime?: string;\n  modifiedTime?: string;\n  // Twitter metadata\n  'twitter:site'?: string;\n  'twitter:creator'?: string;\n  'twitter:card'?: string;\n  'twitter:image'?: string;\n  'twitter:dnt'?: string;\n  'twitter:app:name:iphone'?: string;\n  'twitter:app:id:iphone'?: string;\n  'twitter:app:url:iphone'?: string;\n  'twitter:app:name:ipad'?: string;\n  'twitter:app:id:ipad'?: string;\n  'twitter:app:url:ipad'?: string;\n  'twitter:app:name:googleplay'?: string;\n  'twitter:app:id:googleplay'?: string;\n  'twitter:app:url:googleplay'?: string;\n  // Facebook metadata\n  'fb:app_id'?: string;\n  // App links\n  'al:ios:url'?: string;\n  'al:ios:app_name'?: string;\n  'al:ios:app_store_id'?: string;\n  // Allow for additional properties that might be present\n  [key: string]: string | number | boolean | null | undefined;\n}\n\nexport interface FirecrawlScrapeResponse {\n  success: boolean;\n  data?: {\n    markdown?: string;\n    html?: string;\n    rawHtml?: string;\n    screenshot?: string;\n    links?: string[];\n    metadata?: ScrapeMetadata;\n  };\n  error?: string;\n}\n\nexport interface FirecrawlScraperConfig {\n  apiKey?: string;\n  apiUrl?: string;\n  formats?: string[];\n  timeout?: number;\n}\nconst getDomainName = (\n  link: string,\n  metadata?: ScrapeMetadata\n): string | undefined => {\n  try {\n    const url = metadata?.sourceURL ?? metadata?.url ?? (link || '');\n    const domain = new URL(url).hostname.replace(/^www\\./, '');\n    if (domain) {\n      return domain;\n    }\n  } catch (e) {\n    // URL parsing failed\n    console.error('Error parsing URL:', e);\n  }\n\n  return;\n};\n\nexport function getAttribution(\n  link: string,\n  metadata?: ScrapeMetadata\n): string | undefined {\n  if (!metadata) return getDomainName(link, metadata);\n\n  const possibleAttributions = [\n    metadata.ogSiteName,\n    metadata['og:site_name'],\n    metadata.title?.split('|').pop()?.trim(),\n    metadata['twitter:site']?.replace(/^@/, ''),\n  ];\n\n  const attribution = possibleAttributions.find(\n    (attr) => attr != null && typeof attr === 'string' && attr.trim() !== ''\n  );\n  if (attribution != null) {\n    return attribution;\n  }\n\n  return getDomainName(link, metadata);\n}\n\n/**\n * Firecrawl scraper implementation\n * Uses the Firecrawl API to scrape web pages\n */\nexport class FirecrawlScraper {\n  private apiKey: string;\n  private apiUrl: string;\n  private defaultFormats: string[];\n  private timeout: number;\n\n  constructor(config: FirecrawlScraperConfig = {}) {\n    this.apiKey = config.apiKey ?? process.env.FIRECRAWL_API_KEY ?? '';\n\n    const baseUrl =\n      config.apiUrl ??\n      process.env.FIRECRAWL_BASE_URL ??\n      'https://api.firecrawl.dev';\n    this.apiUrl = `${baseUrl.replace(/\\/+$/, '')}/v1/scrape`;\n\n    this.defaultFormats = config.formats ?? ['markdown', 'html'];\n    this.timeout = config.timeout ?? 15000;\n\n    if (!this.apiKey) {\n      console.warn('FIRECRAWL_API_KEY is not set. Scraping will not work.');\n    }\n\n    console.log(`Firecrawl scraper initialized with API URL: ${this.apiUrl}`);\n  }\n\n  /**\n   * Scrape a single URL\n   * @param url URL to scrape\n   * @param options Scrape options\n   * @returns Scrape response\n   */\n  async scrapeUrl(\n    url: string,\n    options: FirecrawlScrapeOptions = {}\n  ): Promise<[string, FirecrawlScrapeResponse]> {\n    if (!this.apiKey) {\n      return [\n        url,\n        {\n          success: false,\n          error: 'FIRECRAWL_API_KEY is not set',\n        },\n      ];\n    }\n\n    try {\n      const response = await axios.post(\n        this.apiUrl,\n        {\n          url,\n          formats: options.formats || this.defaultFormats,\n          includeTags: options.includeTags,\n          excludeTags: options.excludeTags,\n          headers: options.headers,\n          waitFor: options.waitFor,\n          timeout: options.timeout ?? this.timeout,\n        },\n        {\n          headers: {\n            'Content-Type': 'application/json',\n            Authorization: `Bearer ${this.apiKey}`,\n          },\n          timeout: this.timeout,\n        }\n      );\n\n      return [url, response.data];\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      return [\n        url,\n        {\n          success: false,\n          error: `Firecrawl API request failed: ${errorMessage}`,\n        },\n      ];\n    }\n  }\n\n  /**\n   * Extract content from scrape response\n   * @param response Scrape response\n   * @returns Extracted content or empty string if not available\n   */\n  extractContent(response: FirecrawlScrapeResponse): string {\n    if (!response.success || !response.data) {\n      return '';\n    }\n\n    // Prefer markdown content if available\n    if (response.data.markdown != null) {\n      return response.data.markdown;\n    }\n\n    // Fall back to HTML content\n    if (response.data.html != null) {\n      return response.data.html;\n    }\n\n    // Fall back to raw HTML content\n    if (response.data.rawHtml != null) {\n      return response.data.rawHtml;\n    }\n\n    return '';\n  }\n\n  /**\n   * Extract metadata from scrape response\n   * @param response Scrape response\n   * @returns Metadata object\n   */\n  extractMetadata(response: FirecrawlScrapeResponse): ScrapeMetadata {\n    if (!response.success || !response.data || !response.data.metadata) {\n      return {};\n    }\n\n    return response.data.metadata;\n  }\n}\n\n/**\n * Create a Firecrawl scraper instance\n * @param config Scraper configuration\n * @returns Firecrawl scraper instance\n */\nexport const createFirecrawlScraper = (\n  config: FirecrawlScraperConfig = {}\n): FirecrawlScraper => {\n  return new FirecrawlScraper(config);\n};\n"],"names":[],"mappings":";;;;AAAA;AA6FA,MAAM,aAAa,GAAG,CACpB,IAAY,EACZ,QAAyB,KACH;AACtB,IAAA,IAAI;AACF,QAAA,MAAM,GAAG,GAAG,QAAQ,EAAE,SAAS,IAAI,QAAQ,EAAE,GAAG,KAAK,IAAI,IAAI,EAAE,CAAC;AAChE,QAAA,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC;QAC1D,IAAI,MAAM,EAAE;AACV,YAAA,OAAO,MAAM;;;IAEf,OAAO,CAAC,EAAE;;AAEV,QAAA,OAAO,CAAC,KAAK,CAAC,oBAAoB,EAAE,CAAC,CAAC;;IAGxC;AACF,CAAC;AAEe,SAAA,cAAc,CAC5B,IAAY,EACZ,QAAyB,EAAA;AAEzB,IAAA,IAAI,CAAC,QAAQ;AAAE,QAAA,OAAO,aAAa,CAAC,IAAI,EAAE,QAAQ,CAAC;AAEnD,IAAA,MAAM,oBAAoB,GAAG;AAC3B,QAAA,QAAQ,CAAC,UAAU;QACnB,QAAQ,CAAC,cAAc,CAAC;AACxB,QAAA,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE;QACxC,QAAQ,CAAC,cAAc,CAAC,EAAE,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;KAC5C;IAED,MAAM,WAAW,GAAG,oBAAoB,CAAC,IAAI,CAC3C,CAAC,IAAI,KAAK,IAAI,IAAI,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CACzE;AACD,IAAA,IAAI,WAAW,IAAI,IAAI,EAAE;AACvB,QAAA,OAAO,WAAW;;AAGpB,IAAA,OAAO,aAAa,CAAC,IAAI,EAAE,QAAQ,CAAC;AACtC;AAEA;;;AAGG;MACU,gBAAgB,CAAA;AACnB,IAAA,MAAM;AACN,IAAA,MAAM;AACN,IAAA,cAAc;AACd,IAAA,OAAO;AAEf,IAAA,WAAA,CAAY,SAAiC,EAAE,EAAA;AAC7C,QAAA,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,EAAE;AAElE,QAAA,MAAM,OAAO,GACX,MAAM,CAAC,MAAM;YACb,OAAO,CAAC,GAAG,CAAC,kBAAkB;AAC9B,YAAA,2BAA2B;AAC7B,QAAA,IAAI,CAAC,MAAM,GAAG,CAAA,EAAG,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,YAAY;AAExD,QAAA,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,OAAO,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC;QAC5D,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,KAAK;AAEtC,QAAA,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;AAChB,YAAA,OAAO,CAAC,IAAI,CAAC,uDAAuD,CAAC;;QAGvE,OAAO,CAAC,GAAG,CAAC,CAAA,4CAAA,EAA+C,IAAI,CAAC,MAAM,CAAE,CAAA,CAAC;;AAG3E;;;;;AAKG;AACH,IAAA,MAAM,SAAS,CACb,GAAW,EACX,UAAkC,EAAE,EAAA;AAEpC,QAAA,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YAChB,OAAO;gBACL,GAAG;AACH,gBAAA;AACE,oBAAA,OAAO,EAAE,KAAK;AACd,oBAAA,KAAK,EAAE,8BAA8B;AACtC,iBAAA;aACF;;AAGH,QAAA,IAAI;YACF,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAC/B,IAAI,CAAC,MAAM,EACX;gBACE,GAAG;AACH,gBAAA,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,cAAc;gBAC/C,WAAW,EAAE,OAAO,CAAC,WAAW;gBAChC,WAAW,EAAE,OAAO,CAAC,WAAW;gBAChC,OAAO,EAAE,OAAO,CAAC,OAAO;gBACxB,OAAO,EAAE,OAAO,CAAC,OAAO;AACxB,gBAAA,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO;aACzC,EACD;AACE,gBAAA,OAAO,EAAE;AACP,oBAAA,cAAc,EAAE,kBAAkB;AAClC,oBAAA,aAAa,EAAE,CAAA,OAAA,EAAU,IAAI,CAAC,MAAM,CAAE,CAAA;AACvC,iBAAA;gBACD,OAAO,EAAE,IAAI,CAAC,OAAO;AACtB,aAAA,CACF;AAED,YAAA,OAAO,CAAC,GAAG,EAAE,QAAQ,CAAC,IAAI,CAAC;;QAC3B,OAAO,KAAK,EAAE;AACd,YAAA,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;YACxD,OAAO;gBACL,GAAG;AACH,gBAAA;AACE,oBAAA,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,CAAiC,8BAAA,EAAA,YAAY,CAAE,CAAA;AACvD,iBAAA;aACF;;;AAIL;;;;AAIG;AACH,IAAA,cAAc,CAAC,QAAiC,EAAA;QAC9C,IAAI,CAAC,QAAQ,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE;AACvC,YAAA,OAAO,EAAE;;;QAIX,IAAI,QAAQ,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,EAAE;AAClC,YAAA,OAAO,QAAQ,CAAC,IAAI,CAAC,QAAQ;;;QAI/B,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,EAAE;AAC9B,YAAA,OAAO,QAAQ,CAAC,IAAI,CAAC,IAAI;;;QAI3B,IAAI,QAAQ,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,EAAE;AACjC,YAAA,OAAO,QAAQ,CAAC,IAAI,CAAC,OAAO;;AAG9B,QAAA,OAAO,EAAE;;AAGX;;;;AAIG;AACH,IAAA,eAAe,CAAC,QAAiC,EAAA;AAC/C,QAAA,IAAI,CAAC,QAAQ,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE;AAClE,YAAA,OAAO,EAAE;;AAGX,QAAA,OAAO,QAAQ,CAAC,IAAI,CAAC,QAAQ;;AAEhC;AAED;;;;AAIG;MACU,sBAAsB,GAAG,CACpC,MAAiC,GAAA,EAAE,KACf;AACpB,IAAA,OAAO,IAAI,gBAAgB,CAAC,MAAM,CAAC;AACrC;;;;;;"}

package/dist/cjs/tools/search/format.cjs

@@ -11,8 +11,8 @@ function formatResultsForLLM(results) {
         addSection('Web Results');
         organic.forEach((r, i) => {
             output += [
-                `Source ${i + 1}: ${r.title ?? '(no title)'}`,
-                `Citation Anchor: \\ue202turn0search${i + 1}`,
+                `Source ${i}: ${r.title ?? '(no title)'}`,
+                `Citation Anchor: \\ue202turn0search${i}`,
                 `URL: ${r.link}`,
                 r.snippet != null ? `Summary: ${r.snippet}` : '',
                 r.date != null ? `Date: ${r.date}` : '',
@@ -35,7 +35,7 @@ function formatResultsForLLM(results) {
     //   addSection('News Results');
     //   topStores.forEach((r, i) => {
     //     output += [
-    //       `Anchor: \ue202turn0news${i + 1}`,
+    //       `Anchor: \ue202turn0news${i}`,
     //       `Title: ${r.title ?? '(no title)'}`,
     //       `URL: ${r.link}`,
     //       r.snippet != null ? `Snippet: ${r.snippet}` : '',
@@ -51,7 +51,7 @@ function formatResultsForLLM(results) {
     //   addSection('Image Results');
     //   images.forEach((img, i) => {
     //     output += [
-    //       `Anchor: \ue202turn0image${i + 1}`,
+    //       `Anchor: \ue202turn0image${i}`,
     //       `Title: ${img.title ?? '(no title)'}`,
     //       `Image URL: ${img.imageUrl}`,
     //       ''

package/dist/cjs/tools/search/format.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"format.cjs","sources":["../../../../src/tools/search/format.ts"],"sourcesContent":["import type * as t from './types';\n\nexport function formatResultsForLLM(results: t.SearchResultData): string {\n  let output = '';\n\n  const addSection = (title: string): void => {\n    output += `\\n=== ${title} ===\\n`;\n  };\n\n  // Organic (web) results\n  const organic = results.organic ?? [];\n  if (organic.length) {\n    addSection('Web Results');\n    organic.forEach((r, i) => {\n      output += [\n        `Source ${i + 1}: ${r.title ?? '(no title)'}`,\n        `Citation Anchor: \\\\ue202turn0search${i + 1}`,\n        `URL: ${r.link}`,\n        r.snippet != null ? `Summary: ${r.snippet}` : '',\n        r.date != null ? `Date: ${r.date}` : '',\n        r.attribution != null ? `Source: ${r.attribution}` : '',\n        '',\n        '--- Content Highlights ---',\n        ...(r.highlights ?? [])\n          .filter((h) => h.text.trim().length > 0)\n          .map((h) => `[Relevance: ${h.score.toFixed(2)}]\\n${h.text.trim()}`),\n        '',\n      ]\n        .filter(Boolean)\n        .join('\\n');\n    });\n  }\n\n  // Ignoring these sections for now\n  // // Top stories (news)\n  // const topStores = results.topStories ?? [];\n  // if (topStores.length) {\n  //   addSection('News Results');\n  //   topStores.forEach((r, i) => {\n  //     output += [\n  //       `Anchor: \\ue202turn0news${i + 1}`,\n  //       `Title: ${r.title ?? '(no title)'}`,\n  //       `URL: ${r.link}`,\n  //       r.snippet != null ? `Snippet: ${r.snippet}` : '',\n  //       r.date != null ? `Date: ${r.date}` : '',\n  //       r.attribution != null ? `Source: ${r.attribution}` : '',\n  //       ''\n  //     ].filter(Boolean).join('\\n');\n  //   });\n  // }\n\n  // // Images\n  // const images = results.images ?? [];\n  // if (images.length) {\n  //   addSection('Image Results');\n  //   images.forEach((img, i) => {\n  //     output += [\n  //       `Anchor: \\ue202turn0image${i + 1}`,\n  //       `Title: ${img.title ?? '(no title)'}`,\n  //       `Image URL: ${img.imageUrl}`,\n  //       ''\n  //     ].join('\\n');\n  //   });\n  // }\n\n  // Knowledge Graph\n  if (results.knowledgeGraph != null) {\n    addSection('Knowledge Graph');\n    output += [\n      `Title: ${results.knowledgeGraph.title ?? '(no title)'}`,\n      results.knowledgeGraph.description != null\n        ? `Description: ${results.knowledgeGraph.description}`\n        : '',\n      results.knowledgeGraph.type != null\n        ? `Type: ${results.knowledgeGraph.type}`\n        : '',\n      results.knowledgeGraph.imageUrl != null\n        ? `Image URL: ${results.knowledgeGraph.imageUrl}`\n        : '',\n      results.knowledgeGraph.attributes != null\n        ? `Attributes: ${JSON.stringify(results.knowledgeGraph.attributes, null, 2)}`\n        : '',\n      '',\n    ]\n      .filter(Boolean)\n      .join('\\n');\n  }\n\n  // Answer Box\n  if (results.answerBox != null) {\n    addSection('Answer Box');\n    output += [\n      results.answerBox.title != null\n        ? `Title: ${results.answerBox.title}`\n        : '',\n      results.answerBox.answer != null\n        ? `Answer: ${results.answerBox.answer}`\n        : '',\n      results.answerBox.snippet != null\n        ? `Snippet: ${results.answerBox.snippet}`\n        : '',\n      results.answerBox.date != null ? `Date: ${results.answerBox.date}` : '',\n      '',\n    ]\n      .filter(Boolean)\n      .join('\\n');\n  }\n\n  // People also ask\n  const peopleAlsoAsk = results.peopleAlsoAsk ?? [];\n  if (peopleAlsoAsk.length) {\n    addSection('People Also Ask');\n    peopleAlsoAsk.forEach((p, _i) => {\n      output += [`Q: ${p.question}`, `A: ${p.answer}`, '']\n        .filter(Boolean)\n        .join('\\n');\n    });\n  }\n\n  return output.trim();\n}\n"],"names":[],"mappings":";;AAEM,SAAU,mBAAmB,CAAC,OAA2B,EAAA;IAC7D,IAAI,MAAM,GAAG,EAAE;AAEf,IAAA,MAAM,UAAU,GAAG,CAAC,KAAa,KAAU;AACzC,QAAA,MAAM,IAAI,CAAA,MAAA,EAAS,KAAK,CAAA,MAAA,CAAQ;AAClC,KAAC;;AAGD,IAAA,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,EAAE;AACrC,IAAA,IAAI,OAAO,CAAC,MAAM,EAAE;QAClB,UAAU,CAAC,aAAa,CAAC;QACzB,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,KAAI;AACvB,YAAA,MAAM,IAAI;gBACR,CAAU,OAAA,EAAA,CAAC,GAAG,CAAC,CAAA,EAAA,EAAK,CAAC,CAAC,KAAK,IAAI,YAAY,CAAE,CAAA;gBAC7C,CAAsC,mCAAA,EAAA,CAAC,GAAG,CAAC,CAAE,CAAA;gBAC7C,CAAQ,KAAA,EAAA,CAAC,CAAC,IAAI,CAAE,CAAA;AAChB,gBAAA,CAAC,CAAC,OAAO,IAAI,IAAI,GAAG,CAAY,SAAA,EAAA,CAAC,CAAC,OAAO,CAAA,CAAE,GAAG,EAAE;AAChD,gBAAA,CAAC,CAAC,IAAI,IAAI,IAAI,GAAG,CAAS,MAAA,EAAA,CAAC,CAAC,IAAI,CAAA,CAAE,GAAG,EAAE;AACvC,gBAAA,CAAC,CAAC,WAAW,IAAI,IAAI,GAAG,CAAW,QAAA,EAAA,CAAC,CAAC,WAAW,CAAA,CAAE,GAAG,EAAE;gBACvD,EAAE;gBACF,4BAA4B;AAC5B,gBAAA,GAAG,CAAC,CAAC,CAAC,UAAU,IAAI,EAAE;AACnB,qBAAA,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC;qBACtC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAA,YAAA,EAAe,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA,GAAA,EAAM,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAA,CAAE,CAAC;gBACrE,EAAE;AACH;iBACE,MAAM,CAAC,OAAO;iBACd,IAAI,CAAC,IAAI,CAAC;AACf,SAAC,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAoCJ,IAAA,IAAI,OAAO,CAAC,cAAc,IAAI,IAAI,EAAE;QAClC,UAAU,CAAC,iBAAiB,CAAC;AAC7B,QAAA,MAAM,IAAI;AACR,YAAA,CAAA,OAAA,EAAU,OAAO,CAAC,cAAc,CAAC,KAAK,IAAI,YAAY,CAAE,CAAA;AACxD,YAAA,OAAO,CAAC,cAAc,CAAC,WAAW,IAAI;AACpC,kBAAE,CAAgB,aAAA,EAAA,OAAO,CAAC,cAAc,CAAC,WAAW,CAAE;AACtD,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,cAAc,CAAC,IAAI,IAAI;AAC7B,kBAAE,CAAS,MAAA,EAAA,OAAO,CAAC,cAAc,CAAC,IAAI,CAAE;AACxC,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,cAAc,CAAC,QAAQ,IAAI;AACjC,kBAAE,CAAc,WAAA,EAAA,OAAO,CAAC,cAAc,CAAC,QAAQ,CAAE;AACjD,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,cAAc,CAAC,UAAU,IAAI;AACnC,kBAAE,CAAe,YAAA,EAAA,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,cAAc,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,CAAE;AAC7E,kBAAE,EAAE;YACN,EAAE;AACH;aACE,MAAM,CAAC,OAAO;aACd,IAAI,CAAC,IAAI,CAAC;;;AAIf,IAAA,IAAI,OAAO,CAAC,SAAS,IAAI,IAAI,EAAE;QAC7B,UAAU,CAAC,YAAY,CAAC;AACxB,QAAA,MAAM,IAAI;AACR,YAAA,OAAO,CAAC,SAAS,CAAC,KAAK,IAAI;AACzB,kBAAE,CAAU,OAAA,EAAA,OAAO,CAAC,SAAS,CAAC,KAAK,CAAE;AACrC,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,SAAS,CAAC,MAAM,IAAI;AAC1B,kBAAE,CAAW,QAAA,EAAA,OAAO,CAAC,SAAS,CAAC,MAAM,CAAE;AACvC,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,SAAS,CAAC,OAAO,IAAI;AAC3B,kBAAE,CAAY,SAAA,EAAA,OAAO,CAAC,SAAS,CAAC,OAAO,CAAE;AACzC,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,SAAS,CAAC,IAAI,IAAI,IAAI,GAAG,SAAS,OAAO,CAAC,SAAS,CAAC,IAAI,EAAE,GAAG,EAAE;YACvE,EAAE;AACH;aACE,MAAM,CAAC,OAAO;aACd,IAAI,CAAC,IAAI,CAAC;;;AAIf,IAAA,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,EAAE;AACjD,IAAA,IAAI,aAAa,CAAC,MAAM,EAAE;QACxB,UAAU,CAAC,iBAAiB,CAAC;QAC7B,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,KAAI;AAC9B,YAAA,MAAM,IAAI,CAAC,CAAM,GAAA,EAAA,CAAC,CAAC,QAAQ,CAAA,CAAE,EAAE,CAAA,GAAA,EAAM,CAAC,CAAC,MAAM,CAAE,CAAA,EAAE,EAAE;iBAChD,MAAM,CAAC,OAAO;iBACd,IAAI,CAAC,IAAI,CAAC;AACf,SAAC,CAAC;;AAGJ,IAAA,OAAO,MAAM,CAAC,IAAI,EAAE;AACtB;;;;"}
+{"version":3,"file":"format.cjs","sources":["../../../../src/tools/search/format.ts"],"sourcesContent":["import type * as t from './types';\n\nexport function formatResultsForLLM(results: t.SearchResultData): string {\n  let output = '';\n\n  const addSection = (title: string): void => {\n    output += `\\n=== ${title} ===\\n`;\n  };\n\n  // Organic (web) results\n  const organic = results.organic ?? [];\n  if (organic.length) {\n    addSection('Web Results');\n    organic.forEach((r, i) => {\n      output += [\n        `Source ${i}: ${r.title ?? '(no title)'}`,\n        `Citation Anchor: \\\\ue202turn0search${i}`,\n        `URL: ${r.link}`,\n        r.snippet != null ? `Summary: ${r.snippet}` : '',\n        r.date != null ? `Date: ${r.date}` : '',\n        r.attribution != null ? `Source: ${r.attribution}` : '',\n        '',\n        '--- Content Highlights ---',\n        ...(r.highlights ?? [])\n          .filter((h) => h.text.trim().length > 0)\n          .map((h) => `[Relevance: ${h.score.toFixed(2)}]\\n${h.text.trim()}`),\n        '',\n      ]\n        .filter(Boolean)\n        .join('\\n');\n    });\n  }\n\n  // Ignoring these sections for now\n  // // Top stories (news)\n  // const topStores = results.topStories ?? [];\n  // if (topStores.length) {\n  //   addSection('News Results');\n  //   topStores.forEach((r, i) => {\n  //     output += [\n  //       `Anchor: \\ue202turn0news${i}`,\n  //       `Title: ${r.title ?? '(no title)'}`,\n  //       `URL: ${r.link}`,\n  //       r.snippet != null ? `Snippet: ${r.snippet}` : '',\n  //       r.date != null ? `Date: ${r.date}` : '',\n  //       r.attribution != null ? `Source: ${r.attribution}` : '',\n  //       ''\n  //     ].filter(Boolean).join('\\n');\n  //   });\n  // }\n\n  // // Images\n  // const images = results.images ?? [];\n  // if (images.length) {\n  //   addSection('Image Results');\n  //   images.forEach((img, i) => {\n  //     output += [\n  //       `Anchor: \\ue202turn0image${i}`,\n  //       `Title: ${img.title ?? '(no title)'}`,\n  //       `Image URL: ${img.imageUrl}`,\n  //       ''\n  //     ].join('\\n');\n  //   });\n  // }\n\n  // Knowledge Graph\n  if (results.knowledgeGraph != null) {\n    addSection('Knowledge Graph');\n    output += [\n      `Title: ${results.knowledgeGraph.title ?? '(no title)'}`,\n      results.knowledgeGraph.description != null\n        ? `Description: ${results.knowledgeGraph.description}`\n        : '',\n      results.knowledgeGraph.type != null\n        ? `Type: ${results.knowledgeGraph.type}`\n        : '',\n      results.knowledgeGraph.imageUrl != null\n        ? `Image URL: ${results.knowledgeGraph.imageUrl}`\n        : '',\n      results.knowledgeGraph.attributes != null\n        ? `Attributes: ${JSON.stringify(results.knowledgeGraph.attributes, null, 2)}`\n        : '',\n      '',\n    ]\n      .filter(Boolean)\n      .join('\\n');\n  }\n\n  // Answer Box\n  if (results.answerBox != null) {\n    addSection('Answer Box');\n    output += [\n      results.answerBox.title != null\n        ? `Title: ${results.answerBox.title}`\n        : '',\n      results.answerBox.answer != null\n        ? `Answer: ${results.answerBox.answer}`\n        : '',\n      results.answerBox.snippet != null\n        ? `Snippet: ${results.answerBox.snippet}`\n        : '',\n      results.answerBox.date != null ? `Date: ${results.answerBox.date}` : '',\n      '',\n    ]\n      .filter(Boolean)\n      .join('\\n');\n  }\n\n  // People also ask\n  const peopleAlsoAsk = results.peopleAlsoAsk ?? [];\n  if (peopleAlsoAsk.length) {\n    addSection('People Also Ask');\n    peopleAlsoAsk.forEach((p, _i) => {\n      output += [`Q: ${p.question}`, `A: ${p.answer}`, '']\n        .filter(Boolean)\n        .join('\\n');\n    });\n  }\n\n  return output.trim();\n}\n"],"names":[],"mappings":";;AAEM,SAAU,mBAAmB,CAAC,OAA2B,EAAA;IAC7D,IAAI,MAAM,GAAG,EAAE;AAEf,IAAA,MAAM,UAAU,GAAG,CAAC,KAAa,KAAU;AACzC,QAAA,MAAM,IAAI,CAAA,MAAA,EAAS,KAAK,CAAA,MAAA,CAAQ;AAClC,KAAC;;AAGD,IAAA,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,EAAE;AACrC,IAAA,IAAI,OAAO,CAAC,MAAM,EAAE;QAClB,UAAU,CAAC,aAAa,CAAC;QACzB,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,KAAI;AACvB,YAAA,MAAM,IAAI;AACR,gBAAA,CAAA,OAAA,EAAU,CAAC,CAAK,EAAA,EAAA,CAAC,CAAC,KAAK,IAAI,YAAY,CAAE,CAAA;AACzC,gBAAA,CAAA,mCAAA,EAAsC,CAAC,CAAE,CAAA;gBACzC,CAAQ,KAAA,EAAA,CAAC,CAAC,IAAI,CAAE,CAAA;AAChB,gBAAA,CAAC,CAAC,OAAO,IAAI,IAAI,GAAG,CAAY,SAAA,EAAA,CAAC,CAAC,OAAO,CAAA,CAAE,GAAG,EAAE;AAChD,gBAAA,CAAC,CAAC,IAAI,IAAI,IAAI,GAAG,CAAS,MAAA,EAAA,CAAC,CAAC,IAAI,CAAA,CAAE,GAAG,EAAE;AACvC,gBAAA,CAAC,CAAC,WAAW,IAAI,IAAI,GAAG,CAAW,QAAA,EAAA,CAAC,CAAC,WAAW,CAAA,CAAE,GAAG,EAAE;gBACvD,EAAE;gBACF,4BAA4B;AAC5B,gBAAA,GAAG,CAAC,CAAC,CAAC,UAAU,IAAI,EAAE;AACnB,qBAAA,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC;qBACtC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAA,YAAA,EAAe,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA,GAAA,EAAM,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAA,CAAE,CAAC;gBACrE,EAAE;AACH;iBACE,MAAM,CAAC,OAAO;iBACd,IAAI,CAAC,IAAI,CAAC;AACf,SAAC,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAoCJ,IAAA,IAAI,OAAO,CAAC,cAAc,IAAI,IAAI,EAAE;QAClC,UAAU,CAAC,iBAAiB,CAAC;AAC7B,QAAA,MAAM,IAAI;AACR,YAAA,CAAA,OAAA,EAAU,OAAO,CAAC,cAAc,CAAC,KAAK,IAAI,YAAY,CAAE,CAAA;AACxD,YAAA,OAAO,CAAC,cAAc,CAAC,WAAW,IAAI;AACpC,kBAAE,CAAgB,aAAA,EAAA,OAAO,CAAC,cAAc,CAAC,WAAW,CAAE;AACtD,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,cAAc,CAAC,IAAI,IAAI;AAC7B,kBAAE,CAAS,MAAA,EAAA,OAAO,CAAC,cAAc,CAAC,IAAI,CAAE;AACxC,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,cAAc,CAAC,QAAQ,IAAI;AACjC,kBAAE,CAAc,WAAA,EAAA,OAAO,CAAC,cAAc,CAAC,QAAQ,CAAE;AACjD,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,cAAc,CAAC,UAAU,IAAI;AACnC,kBAAE,CAAe,YAAA,EAAA,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,cAAc,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,CAAE;AAC7E,kBAAE,EAAE;YACN,EAAE;AACH;aACE,MAAM,CAAC,OAAO;aACd,IAAI,CAAC,IAAI,CAAC;;;AAIf,IAAA,IAAI,OAAO,CAAC,SAAS,IAAI,IAAI,EAAE;QAC7B,UAAU,CAAC,YAAY,CAAC;AACxB,QAAA,MAAM,IAAI;AACR,YAAA,OAAO,CAAC,SAAS,CAAC,KAAK,IAAI;AACzB,kBAAE,CAAU,OAAA,EAAA,OAAO,CAAC,SAAS,CAAC,KAAK,CAAE;AACrC,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,SAAS,CAAC,MAAM,IAAI;AAC1B,kBAAE,CAAW,QAAA,EAAA,OAAO,CAAC,SAAS,CAAC,MAAM,CAAE;AACvC,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,SAAS,CAAC,OAAO,IAAI;AAC3B,kBAAE,CAAY,SAAA,EAAA,OAAO,CAAC,SAAS,CAAC,OAAO,CAAE;AACzC,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,SAAS,CAAC,IAAI,IAAI,IAAI,GAAG,SAAS,OAAO,CAAC,SAAS,CAAC,IAAI,EAAE,GAAG,EAAE;YACvE,EAAE;AACH;aACE,MAAM,CAAC,OAAO;aACd,IAAI,CAAC,IAAI,CAAC;;;AAIf,IAAA,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,EAAE;AACjD,IAAA,IAAI,aAAa,CAAC,MAAM,EAAE;QACxB,UAAU,CAAC,iBAAiB,CAAC;QAC7B,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,KAAI;AAC9B,YAAA,MAAM,IAAI,CAAC,CAAM,GAAA,EAAA,CAAC,CAAC,QAAQ,CAAA,CAAE,EAAE,CAAA,GAAA,EAAM,CAAC,CAAC,MAAM,CAAE,CAAA,EAAE,EAAE;iBAChD,MAAM,CAAC,OAAO;iBACd,IAAI,CAAC,IAAI,CAAC;AACf,SAAC,CAAC;;AAGJ,IAAA,OAAO,MAAM,CAAC,IAAI,EAAE;AACtB;;;;"}

package/dist/cjs/tools/search/highlights.cjs

@@ -182,6 +182,7 @@ function expandHighlights(searchResults, mainExpandBy = 300, separatorExpandBy =
                     score: highlight.score,
                 });
             }
+            delete resultCopy.content;
             resultCopy.highlights = highlights;
             return resultCopy;
         });

package/dist/cjs/tools/search/highlights.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"highlights.cjs","sources":["../../../../src/tools/search/highlights.ts"],"sourcesContent":["import type * as t from './types';\n\n// 2. Pre-compile all regular expressions (only do this once)\n// Group patterns by priority for early returns\nconst priorityPatterns = [\n  // High priority patterns (structural)\n  [\n    { regex: /\\n\\n/g }, // Double newline (paragraph break)\n    { regex: /\\n/g }, // Single newline\n    { regex: /={3,}\\s*\\n|-{3,}\\s*\\n/g }, // Section separators\n  ],\n  // Medium priority (semantic)\n  [\n    { regex: /[.!?][\")\\]]?\\s/g }, // End of sentence\n    { regex: /;\\s/g }, // Semicolon\n    { regex: /:\\s/g }, // Colon\n  ],\n  // Low priority (any breaks)\n  [\n    { regex: /,\\s/g }, // Comma\n    { regex: /\\s-\\s/g }, // Dash surrounded by spaces\n    { regex: /\\s/g }, // Any space\n  ],\n];\n\nfunction findFirstMatch(text: string, regex: RegExp): number {\n  // Reset regex\n  regex.lastIndex = 0;\n\n  // For very long texts, try chunking\n  if (text.length > 10000) {\n    const chunkSize = 2000;\n    let position = 0;\n\n    while (position < text.length) {\n      const chunk = text.substring(position, position + chunkSize);\n      regex.lastIndex = 0;\n\n      const match = regex.exec(chunk);\n      if (match) {\n        return position + match.index;\n      }\n\n      // Move to next chunk with some overlap\n      position += chunkSize - 100;\n      if (position >= text.length) break;\n    }\n    return -1;\n  }\n\n  // For shorter texts, normal regex search\n  const match = regex.exec(text);\n  return match ? match.index : -1;\n}\n\n// 3. Optimized boundary finding functions\nfunction findLastMatch(text: string, regex: RegExp): number {\n  // Reset regex state\n  regex.lastIndex = 0;\n\n  let lastIndex = -1;\n  let lastLength = 0;\n  let match;\n\n  // For very long texts, use a different approach to avoid regex engine slowdowns\n  if (text.length > 10000) {\n    // Try dividing the text into chunks for faster processing\n    const chunkSize = 2000;\n    let startPosition = Math.max(0, text.length - chunkSize);\n\n    while (startPosition >= 0) {\n      const chunk = text.substring(startPosition, startPosition + chunkSize);\n      regex.lastIndex = 0;\n\n      let chunkLastIndex = -1;\n      let chunkLastLength = 0;\n\n      while ((match = regex.exec(chunk)) !== null) {\n        chunkLastIndex = match.index;\n        chunkLastLength = match[0].length;\n      }\n\n      if (chunkLastIndex !== -1) {\n        return startPosition + chunkLastIndex + chunkLastLength;\n      }\n\n      // Move to previous chunk with some overlap\n      startPosition = Math.max(0, startPosition - chunkSize + 100) - 1;\n      if (startPosition <= 0) break;\n    }\n    return -1;\n  }\n\n  // For shorter texts, normal regex search\n  while ((match = regex.exec(text)) !== null) {\n    lastIndex = match.index;\n    lastLength = match[0].length;\n  }\n\n  return lastIndex === -1 ? -1 : lastIndex + lastLength;\n}\n\n// 4. Find the best boundary with priority groups\nfunction findBestBoundary(text: string, direction = 'backward'): number {\n  if (!text || text.length === 0) return 0;\n\n  // Try each priority group\n  for (const patternGroup of priorityPatterns) {\n    for (const pattern of patternGroup) {\n      const position =\n        direction === 'backward'\n          ? findLastMatch(text, pattern.regex)\n          : findFirstMatch(text, pattern.regex);\n\n      if (position !== -1) {\n        return position;\n      }\n    }\n  }\n\n  // No match found, use character boundary\n  return direction === 'backward' ? text.length : 0;\n}\n\n/**\n * Expand highlights in search results using smart boundary detection.\n *\n * This implementation finds natural text boundaries like paragraphs, sentences,\n * and phrases to provide context while maintaining readability.\n *\n * @param searchResults - Search results object\n * @param mainExpandBy - Primary expansion size on each side (default: 300)\n * @param separatorExpandBy - Additional range to look for separators (default: 150)\n * @returns Copy of search results with expanded highlights\n */\nexport function expandHighlights(\n  searchResults: t.SearchResultData,\n  mainExpandBy = 300,\n  separatorExpandBy = 150\n): t.SearchResultData {\n  // 1. Avoid full deep copy - only copy what we modify\n  const resultCopy = { ...searchResults };\n\n  // Only deep copy the relevant arrays\n  if (resultCopy.organic) {\n    resultCopy.organic = [...resultCopy.organic];\n  }\n  if (resultCopy.topStories) {\n    resultCopy.topStories = [...resultCopy.topStories];\n  }\n\n  // 5. Process the results efficiently\n  const processResultTypes = ['organic', 'topStories'] as const;\n\n  for (const resultType of processResultTypes) {\n    if (!resultCopy[resultType as 'organic' | 'topStories']) continue;\n\n    // Map results to new array with modified highlights\n    resultCopy[resultType] = resultCopy[resultType]?.map((result) => {\n      if (\n        result.content == null ||\n        result.content === '' ||\n        !result.highlights ||\n        result.highlights.length === 0\n      ) {\n        return result; // No modification needed\n      }\n\n      // Create a shallow copy with expanded highlights\n      const resultCopy = { ...result };\n      const content = result.content;\n      const highlights = [];\n\n      // Process each highlight\n      for (const highlight of result.highlights) {\n        const highlightText = highlight.text;\n\n        let startPos = content.indexOf(highlightText);\n        let highlightLen = highlightText.length;\n\n        if (startPos === -1) {\n          // Try with stripped whitespace\n          const strippedHighlight = highlightText.trim();\n          startPos = content.indexOf(strippedHighlight);\n\n          if (startPos === -1) {\n            highlights.push({\n              text: highlight.text,\n              score: highlight.score,\n            });\n            continue;\n          }\n          highlightLen = strippedHighlight.length;\n        }\n\n        // Calculate boundaries\n        const mainStart = Math.max(0, startPos - mainExpandBy);\n        const mainEnd = Math.min(\n          content.length,\n          startPos + highlightLen + mainExpandBy\n        );\n\n        const separatorStart = Math.max(0, mainStart - separatorExpandBy);\n        const separatorEnd = Math.min(\n          content.length,\n          mainEnd + separatorExpandBy\n        );\n\n        // Extract text segments\n        const headText = content.substring(separatorStart, mainStart);\n        const tailText = content.substring(mainEnd, separatorEnd);\n\n        // Find natural boundaries\n        const bestHeadBoundary = findBestBoundary(headText, 'backward');\n        const bestTailBoundary = findBestBoundary(tailText, 'forward');\n\n        // Calculate final positions\n        const finalStart = separatorStart + bestHeadBoundary;\n        const finalEnd = mainEnd + bestTailBoundary;\n\n        // Extract the expanded highlight\n        const expandedHighlightText = content\n          .substring(finalStart, finalEnd)\n          .trim();\n        highlights.push({\n          text: expandedHighlightText,\n          score: highlight.score,\n        });\n      }\n\n      resultCopy.highlights = highlights;\n      return resultCopy;\n    });\n  }\n\n  return resultCopy;\n}\n"],"names":[],"mappings":";;AAEA;AACA;AACA,MAAM,gBAAgB,GAAG;;AAEvB,IAAA;AACE,QAAA,EAAE,KAAK,EAAE,OAAO,EAAE;AAClB,QAAA,EAAE,KAAK,EAAE,KAAK,EAAE;AAChB,QAAA,EAAE,KAAK,EAAE,wBAAwB,EAAE;AACpC,KAAA;;AAED,IAAA;AACE,QAAA,EAAE,KAAK,EAAE,iBAAiB,EAAE;AAC5B,QAAA,EAAE,KAAK,EAAE,MAAM,EAAE;AACjB,QAAA,EAAE,KAAK,EAAE,MAAM,EAAE;AAClB,KAAA;;AAED,IAAA;AACE,QAAA,EAAE,KAAK,EAAE,MAAM,EAAE;AACjB,QAAA,EAAE,KAAK,EAAE,QAAQ,EAAE;AACnB,QAAA,EAAE,KAAK,EAAE,KAAK,EAAE;AACjB,KAAA;CACF;AAED,SAAS,cAAc,CAAC,IAAY,EAAE,KAAa,EAAA;;AAEjD,IAAA,KAAK,CAAC,SAAS,GAAG,CAAC;;AAGnB,IAAA,IAAI,IAAI,CAAC,MAAM,GAAG,KAAK,EAAE;QACvB,MAAM,SAAS,GAAG,IAAI;QACtB,IAAI,QAAQ,GAAG,CAAC;AAEhB,QAAA,OAAO,QAAQ,GAAG,IAAI,CAAC,MAAM,EAAE;AAC7B,YAAA,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,QAAQ,GAAG,SAAS,CAAC;AAC5D,YAAA,KAAK,CAAC,SAAS,GAAG,CAAC;YAEnB,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC;YAC/B,IAAI,KAAK,EAAE;AACT,gBAAA,OAAO,QAAQ,GAAG,KAAK,CAAC,KAAK;;;AAI/B,YAAA,QAAQ,IAAI,SAAS,GAAG,GAAG;AAC3B,YAAA,IAAI,QAAQ,IAAI,IAAI,CAAC,MAAM;gBAAE;;QAE/B,OAAO,EAAE;;;IAIX,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC;AAC9B,IAAA,OAAO,KAAK,GAAG,KAAK,CAAC,KAAK,GAAG,EAAE;AACjC;AAEA;AACA,SAAS,aAAa,CAAC,IAAY,EAAE,KAAa,EAAA;;AAEhD,IAAA,KAAK,CAAC,SAAS,GAAG,CAAC;AAEnB,IAAA,IAAI,SAAS,GAAG,EAAE;IAClB,IAAI,UAAU,GAAG,CAAC;AAClB,IAAA,IAAI,KAAK;;AAGT,IAAA,IAAI,IAAI,CAAC,MAAM,GAAG,KAAK,EAAE;;QAEvB,MAAM,SAAS,GAAG,IAAI;AACtB,QAAA,IAAI,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,GAAG,SAAS,CAAC;AAExD,QAAA,OAAO,aAAa,IAAI,CAAC,EAAE;AACzB,YAAA,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,aAAa,EAAE,aAAa,GAAG,SAAS,CAAC;AACtE,YAAA,KAAK,CAAC,SAAS,GAAG,CAAC;AAEnB,YAAA,IAAI,cAAc,GAAG,EAAE;YACvB,IAAI,eAAe,GAAG,CAAC;AAEvB,YAAA,OAAO,CAAC,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,EAAE;AAC3C,gBAAA,cAAc,GAAG,KAAK,CAAC,KAAK;AAC5B,gBAAA,eAAe,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM;;AAGnC,YAAA,IAAI,cAAc,KAAK,EAAE,EAAE;AACzB,gBAAA,OAAO,aAAa,GAAG,cAAc,GAAG,eAAe;;;AAIzD,YAAA,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,aAAa,GAAG,SAAS,GAAG,GAAG,CAAC,GAAG,CAAC;YAChE,IAAI,aAAa,IAAI,CAAC;gBAAE;;QAE1B,OAAO,EAAE;;;AAIX,IAAA,OAAO,CAAC,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE;AAC1C,QAAA,SAAS,GAAG,KAAK,CAAC,KAAK;AACvB,QAAA,UAAU,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM;;AAG9B,IAAA,OAAO,SAAS,KAAK,EAAE,GAAG,EAAE,GAAG,SAAS,GAAG,UAAU;AACvD;AAEA;AACA,SAAS,gBAAgB,CAAC,IAAY,EAAE,SAAS,GAAG,UAAU,EAAA;AAC5D,IAAA,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;AAAE,QAAA,OAAO,CAAC;;AAGxC,IAAA,KAAK,MAAM,YAAY,IAAI,gBAAgB,EAAE;AAC3C,QAAA,KAAK,MAAM,OAAO,IAAI,YAAY,EAAE;AAClC,YAAA,MAAM,QAAQ,GACZ,SAAS,KAAK;kBACV,aAAa,CAAC,IAAI,EAAE,OAAO,CAAC,KAAK;kBACjC,cAAc,CAAC,IAAI,EAAE,OAAO,CAAC,KAAK,CAAC;AAEzC,YAAA,IAAI,QAAQ,KAAK,EAAE,EAAE;AACnB,gBAAA,OAAO,QAAQ;;;;;AAMrB,IAAA,OAAO,SAAS,KAAK,UAAU,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC;AACnD;AAEA;;;;;;;;;;AAUG;AACG,SAAU,gBAAgB,CAC9B,aAAiC,EACjC,YAAY,GAAG,GAAG,EAClB,iBAAiB,GAAG,GAAG,EAAA;;AAGvB,IAAA,MAAM,UAAU,GAAG,EAAE,GAAG,aAAa,EAAE;;AAGvC,IAAA,IAAI,UAAU,CAAC,OAAO,EAAE;QACtB,UAAU,CAAC,OAAO,GAAG,CAAC,GAAG,UAAU,CAAC,OAAO,CAAC;;AAE9C,IAAA,IAAI,UAAU,CAAC,UAAU,EAAE;QACzB,UAAU,CAAC,UAAU,GAAG,CAAC,GAAG,UAAU,CAAC,UAAU,CAAC;;;AAIpD,IAAA,MAAM,kBAAkB,GAAG,CAAC,SAAS,EAAE,YAAY,CAAU;AAE7D,IAAA,KAAK,MAAM,UAAU,IAAI,kBAAkB,EAAE;AAC3C,QAAA,IAAI,CAAC,UAAU,CAAC,UAAsC,CAAC;YAAE;;AAGzD,QAAA,UAAU,CAAC,UAAU,CAAC,GAAG,UAAU,CAAC,UAAU,CAAC,EAAE,GAAG,CAAC,CAAC,MAAM,KAAI;AAC9D,YAAA,IACE,MAAM,CAAC,OAAO,IAAI,IAAI;gBACtB,MAAM,CAAC,OAAO,KAAK,EAAE;gBACrB,CAAC,MAAM,CAAC,UAAU;AAClB,gBAAA,MAAM,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC,EAC9B;gBACA,OAAO,MAAM,CAAC;;;AAIhB,YAAA,MAAM,UAAU,GAAG,EAAE,GAAG,MAAM,EAAE;AAChC,YAAA,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO;YAC9B,MAAM,UAAU,GAAG,EAAE;;AAGrB,YAAA,KAAK,MAAM,SAAS,IAAI,MAAM,CAAC,UAAU,EAAE;AACzC,gBAAA,MAAM,aAAa,GAAG,SAAS,CAAC,IAAI;gBAEpC,IAAI,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,aAAa,CAAC;AAC7C,gBAAA,IAAI,YAAY,GAAG,aAAa,CAAC,MAAM;AAEvC,gBAAA,IAAI,QAAQ,KAAK,EAAE,EAAE;;AAEnB,oBAAA,MAAM,iBAAiB,GAAG,aAAa,CAAC,IAAI,EAAE;AAC9C,oBAAA,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,iBAAiB,CAAC;AAE7C,oBAAA,IAAI,QAAQ,KAAK,EAAE,EAAE;wBACnB,UAAU,CAAC,IAAI,CAAC;4BACd,IAAI,EAAE,SAAS,CAAC,IAAI;4BACpB,KAAK,EAAE,SAAS,CAAC,KAAK;AACvB,yBAAA,CAAC;wBACF;;AAEF,oBAAA,YAAY,GAAG,iBAAiB,CAAC,MAAM;;;AAIzC,gBAAA,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,GAAG,YAAY,CAAC;AACtD,gBAAA,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CACtB,OAAO,CAAC,MAAM,EACd,QAAQ,GAAG,YAAY,GAAG,YAAY,CACvC;AAED,gBAAA,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,GAAG,iBAAiB,CAAC;AACjE,gBAAA,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAC3B,OAAO,CAAC,MAAM,EACd,OAAO,GAAG,iBAAiB,CAC5B;;gBAGD,MAAM,QAAQ,GAAG,OAAO,CAAC,SAAS,CAAC,cAAc,EAAE,SAAS,CAAC;gBAC7D,MAAM,QAAQ,GAAG,OAAO,CAAC,SAAS,CAAC,OAAO,EAAE,YAAY,CAAC;;gBAGzD,MAAM,gBAAgB,GAAG,gBAAgB,CAAC,QAAQ,EAAE,UAAU,CAAC;gBAC/D,MAAM,gBAAgB,GAAG,gBAAgB,CAAC,QAAQ,EAAE,SAAS,CAAC;;AAG9D,gBAAA,MAAM,UAAU,GAAG,cAAc,GAAG,gBAAgB;AACpD,gBAAA,MAAM,QAAQ,GAAG,OAAO,GAAG,gBAAgB;;gBAG3C,MAAM,qBAAqB,GAAG;AAC3B,qBAAA,SAAS,CAAC,UAAU,EAAE,QAAQ;AAC9B,qBAAA,IAAI,EAAE;gBACT,UAAU,CAAC,IAAI,CAAC;AACd,oBAAA,IAAI,EAAE,qBAAqB;oBAC3B,KAAK,EAAE,SAAS,CAAC,KAAK;AACvB,iBAAA,CAAC;;AAGJ,YAAA,UAAU,CAAC,UAAU,GAAG,UAAU;AAClC,YAAA,OAAO,UAAU;AACnB,SAAC,CAAC;;AAGJ,IAAA,OAAO,UAAU;AACnB;;;;"}
+{"version":3,"file":"highlights.cjs","sources":["../../../../src/tools/search/highlights.ts"],"sourcesContent":["import type * as t from './types';\n\n// 2. Pre-compile all regular expressions (only do this once)\n// Group patterns by priority for early returns\nconst priorityPatterns = [\n  // High priority patterns (structural)\n  [\n    { regex: /\\n\\n/g }, // Double newline (paragraph break)\n    { regex: /\\n/g }, // Single newline\n    { regex: /={3,}\\s*\\n|-{3,}\\s*\\n/g }, // Section separators\n  ],\n  // Medium priority (semantic)\n  [\n    { regex: /[.!?][\")\\]]?\\s/g }, // End of sentence\n    { regex: /;\\s/g }, // Semicolon\n    { regex: /:\\s/g }, // Colon\n  ],\n  // Low priority (any breaks)\n  [\n    { regex: /,\\s/g }, // Comma\n    { regex: /\\s-\\s/g }, // Dash surrounded by spaces\n    { regex: /\\s/g }, // Any space\n  ],\n];\n\nfunction findFirstMatch(text: string, regex: RegExp): number {\n  // Reset regex\n  regex.lastIndex = 0;\n\n  // For very long texts, try chunking\n  if (text.length > 10000) {\n    const chunkSize = 2000;\n    let position = 0;\n\n    while (position < text.length) {\n      const chunk = text.substring(position, position + chunkSize);\n      regex.lastIndex = 0;\n\n      const match = regex.exec(chunk);\n      if (match) {\n        return position + match.index;\n      }\n\n      // Move to next chunk with some overlap\n      position += chunkSize - 100;\n      if (position >= text.length) break;\n    }\n    return -1;\n  }\n\n  // For shorter texts, normal regex search\n  const match = regex.exec(text);\n  return match ? match.index : -1;\n}\n\n// 3. Optimized boundary finding functions\nfunction findLastMatch(text: string, regex: RegExp): number {\n  // Reset regex state\n  regex.lastIndex = 0;\n\n  let lastIndex = -1;\n  let lastLength = 0;\n  let match;\n\n  // For very long texts, use a different approach to avoid regex engine slowdowns\n  if (text.length > 10000) {\n    // Try dividing the text into chunks for faster processing\n    const chunkSize = 2000;\n    let startPosition = Math.max(0, text.length - chunkSize);\n\n    while (startPosition >= 0) {\n      const chunk = text.substring(startPosition, startPosition + chunkSize);\n      regex.lastIndex = 0;\n\n      let chunkLastIndex = -1;\n      let chunkLastLength = 0;\n\n      while ((match = regex.exec(chunk)) !== null) {\n        chunkLastIndex = match.index;\n        chunkLastLength = match[0].length;\n      }\n\n      if (chunkLastIndex !== -1) {\n        return startPosition + chunkLastIndex + chunkLastLength;\n      }\n\n      // Move to previous chunk with some overlap\n      startPosition = Math.max(0, startPosition - chunkSize + 100) - 1;\n      if (startPosition <= 0) break;\n    }\n    return -1;\n  }\n\n  // For shorter texts, normal regex search\n  while ((match = regex.exec(text)) !== null) {\n    lastIndex = match.index;\n    lastLength = match[0].length;\n  }\n\n  return lastIndex === -1 ? -1 : lastIndex + lastLength;\n}\n\n// 4. Find the best boundary with priority groups\nfunction findBestBoundary(text: string, direction = 'backward'): number {\n  if (!text || text.length === 0) return 0;\n\n  // Try each priority group\n  for (const patternGroup of priorityPatterns) {\n    for (const pattern of patternGroup) {\n      const position =\n        direction === 'backward'\n          ? findLastMatch(text, pattern.regex)\n          : findFirstMatch(text, pattern.regex);\n\n      if (position !== -1) {\n        return position;\n      }\n    }\n  }\n\n  // No match found, use character boundary\n  return direction === 'backward' ? text.length : 0;\n}\n\n/**\n * Expand highlights in search results using smart boundary detection.\n *\n * This implementation finds natural text boundaries like paragraphs, sentences,\n * and phrases to provide context while maintaining readability.\n *\n * @param searchResults - Search results object\n * @param mainExpandBy - Primary expansion size on each side (default: 300)\n * @param separatorExpandBy - Additional range to look for separators (default: 150)\n * @returns Copy of search results with expanded highlights\n */\nexport function expandHighlights(\n  searchResults: t.SearchResultData,\n  mainExpandBy = 300,\n  separatorExpandBy = 150\n): t.SearchResultData {\n  // 1. Avoid full deep copy - only copy what we modify\n  const resultCopy = { ...searchResults };\n\n  // Only deep copy the relevant arrays\n  if (resultCopy.organic) {\n    resultCopy.organic = [...resultCopy.organic];\n  }\n  if (resultCopy.topStories) {\n    resultCopy.topStories = [...resultCopy.topStories];\n  }\n\n  // 5. Process the results efficiently\n  const processResultTypes = ['organic', 'topStories'] as const;\n\n  for (const resultType of processResultTypes) {\n    if (!resultCopy[resultType as 'organic' | 'topStories']) continue;\n\n    // Map results to new array with modified highlights\n    resultCopy[resultType] = resultCopy[resultType]?.map((result) => {\n      if (\n        result.content == null ||\n        result.content === '' ||\n        !result.highlights ||\n        result.highlights.length === 0\n      ) {\n        return result; // No modification needed\n      }\n\n      // Create a shallow copy with expanded highlights\n      const resultCopy = { ...result };\n      const content = result.content;\n      const highlights = [];\n\n      // Process each highlight\n      for (const highlight of result.highlights) {\n        const highlightText = highlight.text;\n\n        let startPos = content.indexOf(highlightText);\n        let highlightLen = highlightText.length;\n\n        if (startPos === -1) {\n          // Try with stripped whitespace\n          const strippedHighlight = highlightText.trim();\n          startPos = content.indexOf(strippedHighlight);\n\n          if (startPos === -1) {\n            highlights.push({\n              text: highlight.text,\n              score: highlight.score,\n            });\n            continue;\n          }\n          highlightLen = strippedHighlight.length;\n        }\n\n        // Calculate boundaries\n        const mainStart = Math.max(0, startPos - mainExpandBy);\n        const mainEnd = Math.min(\n          content.length,\n          startPos + highlightLen + mainExpandBy\n        );\n\n        const separatorStart = Math.max(0, mainStart - separatorExpandBy);\n        const separatorEnd = Math.min(\n          content.length,\n          mainEnd + separatorExpandBy\n        );\n\n        // Extract text segments\n        const headText = content.substring(separatorStart, mainStart);\n        const tailText = content.substring(mainEnd, separatorEnd);\n\n        // Find natural boundaries\n        const bestHeadBoundary = findBestBoundary(headText, 'backward');\n        const bestTailBoundary = findBestBoundary(tailText, 'forward');\n\n        // Calculate final positions\n        const finalStart = separatorStart + bestHeadBoundary;\n        const finalEnd = mainEnd + bestTailBoundary;\n\n        // Extract the expanded highlight\n        const expandedHighlightText = content\n          .substring(finalStart, finalEnd)\n          .trim();\n        highlights.push({\n          text: expandedHighlightText,\n          score: highlight.score,\n        });\n      }\n\n      delete resultCopy.content;\n      resultCopy.highlights = highlights;\n      return resultCopy;\n    });\n  }\n\n  return resultCopy;\n}\n"],"names":[],"mappings":";;AAEA;AACA;AACA,MAAM,gBAAgB,GAAG;;AAEvB,IAAA;AACE,QAAA,EAAE,KAAK,EAAE,OAAO,EAAE;AAClB,QAAA,EAAE,KAAK,EAAE,KAAK,EAAE;AAChB,QAAA,EAAE,KAAK,EAAE,wBAAwB,EAAE;AACpC,KAAA;;AAED,IAAA;AACE,QAAA,EAAE,KAAK,EAAE,iBAAiB,EAAE;AAC5B,QAAA,EAAE,KAAK,EAAE,MAAM,EAAE;AACjB,QAAA,EAAE,KAAK,EAAE,MAAM,EAAE;AAClB,KAAA;;AAED,IAAA;AACE,QAAA,EAAE,KAAK,EAAE,MAAM,EAAE;AACjB,QAAA,EAAE,KAAK,EAAE,QAAQ,EAAE;AACnB,QAAA,EAAE,KAAK,EAAE,KAAK,EAAE;AACjB,KAAA;CACF;AAED,SAAS,cAAc,CAAC,IAAY,EAAE,KAAa,EAAA;;AAEjD,IAAA,KAAK,CAAC,SAAS,GAAG,CAAC;;AAGnB,IAAA,IAAI,IAAI,CAAC,MAAM,GAAG,KAAK,EAAE;QACvB,MAAM,SAAS,GAAG,IAAI;QACtB,IAAI,QAAQ,GAAG,CAAC;AAEhB,QAAA,OAAO,QAAQ,GAAG,IAAI,CAAC,MAAM,EAAE;AAC7B,YAAA,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,QAAQ,GAAG,SAAS,CAAC;AAC5D,YAAA,KAAK,CAAC,SAAS,GAAG,CAAC;YAEnB,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC;YAC/B,IAAI,KAAK,EAAE;AACT,gBAAA,OAAO,QAAQ,GAAG,KAAK,CAAC,KAAK;;;AAI/B,YAAA,QAAQ,IAAI,SAAS,GAAG,GAAG;AAC3B,YAAA,IAAI,QAAQ,IAAI,IAAI,CAAC,MAAM;gBAAE;;QAE/B,OAAO,EAAE;;;IAIX,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC;AAC9B,IAAA,OAAO,KAAK,GAAG,KAAK,CAAC,KAAK,GAAG,EAAE;AACjC;AAEA;AACA,SAAS,aAAa,CAAC,IAAY,EAAE,KAAa,EAAA;;AAEhD,IAAA,KAAK,CAAC,SAAS,GAAG,CAAC;AAEnB,IAAA,IAAI,SAAS,GAAG,EAAE;IAClB,IAAI,UAAU,GAAG,CAAC;AAClB,IAAA,IAAI,KAAK;;AAGT,IAAA,IAAI,IAAI,CAAC,MAAM,GAAG,KAAK,EAAE;;QAEvB,MAAM,SAAS,GAAG,IAAI;AACtB,QAAA,IAAI,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,GAAG,SAAS,CAAC;AAExD,QAAA,OAAO,aAAa,IAAI,CAAC,EAAE;AACzB,YAAA,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,aAAa,EAAE,aAAa,GAAG,SAAS,CAAC;AACtE,YAAA,KAAK,CAAC,SAAS,GAAG,CAAC;AAEnB,YAAA,IAAI,cAAc,GAAG,EAAE;YACvB,IAAI,eAAe,GAAG,CAAC;AAEvB,YAAA,OAAO,CAAC,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,EAAE;AAC3C,gBAAA,cAAc,GAAG,KAAK,CAAC,KAAK;AAC5B,gBAAA,eAAe,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM;;AAGnC,YAAA,IAAI,cAAc,KAAK,EAAE,EAAE;AACzB,gBAAA,OAAO,aAAa,GAAG,cAAc,GAAG,eAAe;;;AAIzD,YAAA,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,aAAa,GAAG,SAAS,GAAG,GAAG,CAAC,GAAG,CAAC;YAChE,IAAI,aAAa,IAAI,CAAC;gBAAE;;QAE1B,OAAO,EAAE;;;AAIX,IAAA,OAAO,CAAC,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE;AAC1C,QAAA,SAAS,GAAG,KAAK,CAAC,KAAK;AACvB,QAAA,UAAU,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM;;AAG9B,IAAA,OAAO,SAAS,KAAK,EAAE,GAAG,EAAE,GAAG,SAAS,GAAG,UAAU;AACvD;AAEA;AACA,SAAS,gBAAgB,CAAC,IAAY,EAAE,SAAS,GAAG,UAAU,EAAA;AAC5D,IAAA,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;AAAE,QAAA,OAAO,CAAC;;AAGxC,IAAA,KAAK,MAAM,YAAY,IAAI,gBAAgB,EAAE;AAC3C,QAAA,KAAK,MAAM,OAAO,IAAI,YAAY,EAAE;AAClC,YAAA,MAAM,QAAQ,GACZ,SAAS,KAAK;kBACV,aAAa,CAAC,IAAI,EAAE,OAAO,CAAC,KAAK;kBACjC,cAAc,CAAC,IAAI,EAAE,OAAO,CAAC,KAAK,CAAC;AAEzC,YAAA,IAAI,QAAQ,KAAK,EAAE,EAAE;AACnB,gBAAA,OAAO,QAAQ;;;;;AAMrB,IAAA,OAAO,SAAS,KAAK,UAAU,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC;AACnD;AAEA;;;;;;;;;;AAUG;AACG,SAAU,gBAAgB,CAC9B,aAAiC,EACjC,YAAY,GAAG,GAAG,EAClB,iBAAiB,GAAG,GAAG,EAAA;;AAGvB,IAAA,MAAM,UAAU,GAAG,EAAE,GAAG,aAAa,EAAE;;AAGvC,IAAA,IAAI,UAAU,CAAC,OAAO,EAAE;QACtB,UAAU,CAAC,OAAO,GAAG,CAAC,GAAG,UAAU,CAAC,OAAO,CAAC;;AAE9C,IAAA,IAAI,UAAU,CAAC,UAAU,EAAE;QACzB,UAAU,CAAC,UAAU,GAAG,CAAC,GAAG,UAAU,CAAC,UAAU,CAAC;;;AAIpD,IAAA,MAAM,kBAAkB,GAAG,CAAC,SAAS,EAAE,YAAY,CAAU;AAE7D,IAAA,KAAK,MAAM,UAAU,IAAI,kBAAkB,EAAE;AAC3C,QAAA,IAAI,CAAC,UAAU,CAAC,UAAsC,CAAC;YAAE;;AAGzD,QAAA,UAAU,CAAC,UAAU,CAAC,GAAG,UAAU,CAAC,UAAU,CAAC,EAAE,GAAG,CAAC,CAAC,MAAM,KAAI;AAC9D,YAAA,IACE,MAAM,CAAC,OAAO,IAAI,IAAI;gBACtB,MAAM,CAAC,OAAO,KAAK,EAAE;gBACrB,CAAC,MAAM,CAAC,UAAU;AAClB,gBAAA,MAAM,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC,EAC9B;gBACA,OAAO,MAAM,CAAC;;;AAIhB,YAAA,MAAM,UAAU,GAAG,EAAE,GAAG,MAAM,EAAE;AAChC,YAAA,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO;YAC9B,MAAM,UAAU,GAAG,EAAE;;AAGrB,YAAA,KAAK,MAAM,SAAS,IAAI,MAAM,CAAC,UAAU,EAAE;AACzC,gBAAA,MAAM,aAAa,GAAG,SAAS,CAAC,IAAI;gBAEpC,IAAI,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,aAAa,CAAC;AAC7C,gBAAA,IAAI,YAAY,GAAG,aAAa,CAAC,MAAM;AAEvC,gBAAA,IAAI,QAAQ,KAAK,EAAE,EAAE;;AAEnB,oBAAA,MAAM,iBAAiB,GAAG,aAAa,CAAC,IAAI,EAAE;AAC9C,oBAAA,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,iBAAiB,CAAC;AAE7C,oBAAA,IAAI,QAAQ,KAAK,EAAE,EAAE;wBACnB,UAAU,CAAC,IAAI,CAAC;4BACd,IAAI,EAAE,SAAS,CAAC,IAAI;4BACpB,KAAK,EAAE,SAAS,CAAC,KAAK;AACvB,yBAAA,CAAC;wBACF;;AAEF,oBAAA,YAAY,GAAG,iBAAiB,CAAC,MAAM;;;AAIzC,gBAAA,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,GAAG,YAAY,CAAC;AACtD,gBAAA,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CACtB,OAAO,CAAC,MAAM,EACd,QAAQ,GAAG,YAAY,GAAG,YAAY,CACvC;AAED,gBAAA,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,GAAG,iBAAiB,CAAC;AACjE,gBAAA,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAC3B,OAAO,CAAC,MAAM,EACd,OAAO,GAAG,iBAAiB,CAC5B;;gBAGD,MAAM,QAAQ,GAAG,OAAO,CAAC,SAAS,CAAC,cAAc,EAAE,SAAS,CAAC;gBAC7D,MAAM,QAAQ,GAAG,OAAO,CAAC,SAAS,CAAC,OAAO,EAAE,YAAY,CAAC;;gBAGzD,MAAM,gBAAgB,GAAG,gBAAgB,CAAC,QAAQ,EAAE,UAAU,CAAC;gBAC/D,MAAM,gBAAgB,GAAG,gBAAgB,CAAC,QAAQ,EAAE,SAAS,CAAC;;AAG9D,gBAAA,MAAM,UAAU,GAAG,cAAc,GAAG,gBAAgB;AACpD,gBAAA,MAAM,QAAQ,GAAG,OAAO,GAAG,gBAAgB;;gBAG3C,MAAM,qBAAqB,GAAG;AAC3B,qBAAA,SAAS,CAAC,UAAU,EAAE,QAAQ;AAC9B,qBAAA,IAAI,EAAE;gBACT,UAAU,CAAC,IAAI,CAAC;AACd,oBAAA,IAAI,EAAE,qBAAqB;oBAC3B,KAAK,EAAE,SAAS,CAAC,KAAK;AACvB,iBAAA,CAAC;;YAGJ,OAAO,UAAU,CAAC,OAAO;AACzB,YAAA,UAAU,CAAC,UAAU,GAAG,UAAU;AAClC,YAAA,OAAO,UAAU;AACnB,SAAC,CAAC;;AAGJ,IAAA,OAAO,UAAU;AACnB;;;;"}

package/dist/cjs/tools/search/tool.cjs

@@ -70,7 +70,7 @@ const createSearchTool = (config = {}) => {
                 : undefined,
         });
         const output = format.formatResultsForLLM(searchResult);
-        return [output, searchResult];
+        return [output, { [_enum.Constants.WEB_SEARCH]: { ...searchResult } }];
     }, {
         name: _enum.Constants.WEB_SEARCH,
         description: `

package/dist/cjs/tools/search/tool.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"tool.cjs","sources":["../../../../src/tools/search/tool.ts"],"sourcesContent":["/* eslint-disable no-console */\nimport { z } from 'zod';\nimport { tool, DynamicStructuredTool } from '@langchain/core/tools';\nimport type * as t from './types';\nimport { createSearchAPI, createSourceProcessor } from './search';\nimport { createFirecrawlScraper } from './firecrawl';\nimport { expandHighlights } from './highlights';\nimport { formatResultsForLLM } from './format';\nimport { createReranker } from './rerankers';\nimport { Constants } from '@/common';\n\nconst SearchToolSchema = z.object({\n  query: z\n    .string()\n    .describe(\n      'The search query string that specifies what should be searched for.'\n    ),\n});\n\nexport const createSearchTool = (\n  config: t.SearchToolConfig = {}\n): DynamicStructuredTool<typeof SearchToolSchema> => {\n  const {\n    searchProvider = 'serper',\n    serperApiKey,\n    searxngInstanceUrl,\n    searxngApiKey,\n    rerankerType = 'cohere',\n    topResults = 5,\n    strategies = ['no_extraction'],\n    filterContent = true,\n    firecrawlApiKey,\n    firecrawlApiUrl,\n    firecrawlFormats = ['markdown', 'html'],\n    jinaApiKey,\n    cohereApiKey,\n    onSearchResults: _onSearchResults,\n  } = config;\n\n  const searchAPI = createSearchAPI({\n    searchProvider,\n    serperApiKey,\n    searxngInstanceUrl,\n    searxngApiKey,\n  });\n\n  const firecrawlScraper = createFirecrawlScraper({\n    apiKey: firecrawlApiKey ?? process.env.FIRECRAWL_API_KEY,\n    apiUrl: firecrawlApiUrl,\n    formats: firecrawlFormats,\n  });\n\n  const selectedReranker = createReranker({\n    rerankerType,\n    jinaApiKey,\n    cohereApiKey,\n  });\n\n  if (!selectedReranker) {\n    console.warn('No reranker selected. Using default ranking.');\n  }\n\n  const sourceProcessor = createSourceProcessor(\n    {\n      reranker: selectedReranker,\n      topResults,\n      strategies,\n      filterContent,\n    },\n    firecrawlScraper\n  );\n\n  const search = async ({\n    query,\n    proMode = true,\n    maxSources = 5,\n    onSearchResults,\n  }: {\n    query: string;\n    proMode?: boolean;\n    maxSources?: number;\n    onSearchResults?: (sources: t.SearchResult) => void;\n  }): Promise<t.SearchResultData> => {\n    try {\n      const sources = await searchAPI.getSources(query);\n      onSearchResults?.(sources);\n\n      if (!sources.success) {\n        throw new Error(sources.error ?? 'Search failed');\n      }\n\n      const processedSources = await sourceProcessor.processSources(\n        sources,\n        maxSources,\n        query,\n        proMode\n      );\n      return expandHighlights(processedSources);\n    } catch (error) {\n      console.error('Error in search:', error);\n      return {\n        organic: [],\n        topStories: [],\n        images: [],\n        relatedSearches: [],\n        error: error instanceof Error ? error.message : String(error),\n      };\n    }\n  };\n\n  return tool<typeof SearchToolSchema>(\n    async ({ query }, runnableConfig) => {\n      const searchResult = await search({\n        query,\n        onSearchResults: _onSearchResults\n          ? (result): void => {\n            _onSearchResults(result, runnableConfig);\n          }\n          : undefined,\n      });\n      const output = formatResultsForLLM(searchResult);\n      return [output, searchResult];\n    },\n    {\n      name: Constants.WEB_SEARCH,\n      description: `\nReal-time search. Results have required unique citation anchors.\n\nAnchors:\n- \\\\ue202turn0searchN (web), \\\\ue202turn0newsN (news), \\\\ue202turn0imageN (image)\n\nSpecial Markers:\n- \\\\ue203...\\\\ue204 — mark start/end of cited span\n- \\\\ue200...\\\\ue201 — composite/group block (e.g. \\\\ue200cite\\\\ue202turn0search1\\\\ue202turn0news2\\\\ue201)\n- \\\\ue206 — marks grouped/summary citation areas\n\n**CITE EVERY NON-OBVIOUS FACT/QUOTE:**\nInsert the anchor marker(s) immediately after the statement:\n- \"Pure functions produce same output \\\\ue202turn0search0.\"\n- Multiple: \"Benefits \\\\ue202turn0search0\\\\ue202turn0news0.\"\n- Span: \\\\ue203Key: first-class functions\\\\ue204\\\\ue202turn0news1\n- Group: \"Functional languages.\"\\\\ue206 or \\\\ue200cite\\\\ue202turn0search0\\\\ue202turn0news1\\\\ue201\n- Image: \"See photo \\\\ue202turn0image0.\"\n\n**NEVER use markdown links, [1], or footnotes. CITE ONLY with anchors provided.**\n`.trim(),\n      schema: SearchToolSchema,\n      responseFormat: Constants.CONTENT_AND_ARTIFACT,\n    }\n  );\n};\n"],"names":["z","createSearchAPI","createFirecrawlScraper","createReranker","createSourceProcessor","search","expandHighlights","tool","formatResultsForLLM","Constants"],"mappings":";;;;;;;;;;;AAAA;AAWA,MAAM,gBAAgB,GAAGA,KAAC,CAAC,MAAM,CAAC;AAChC,IAAA,KAAK,EAAEA;AACJ,SAAA,MAAM;SACN,QAAQ,CACP,qEAAqE,CACtE;AACJ,CAAA,CAAC;MAEW,gBAAgB,GAAG,CAC9B,MAA6B,GAAA,EAAE,KACmB;IAClD,MAAM,EACJ,cAAc,GAAG,QAAQ,EACzB,YAAY,EACZ,kBAAkB,EAClB,aAAa,EACb,YAAY,GAAG,QAAQ,EACvB,UAAU,GAAG,CAAC,EACd,UAAU,GAAG,CAAC,eAAe,CAAC,EAC9B,aAAa,GAAG,IAAI,EACpB,eAAe,EACf,eAAe,EACf,gBAAgB,GAAG,CAAC,UAAU,EAAE,MAAM,CAAC,EACvC,UAAU,EACV,YAAY,EACZ,eAAe,EAAE,gBAAgB,GAClC,GAAG,MAAM;IAEV,MAAM,SAAS,GAAGC,sBAAe,CAAC;QAChC,cAAc;QACd,YAAY;QACZ,kBAAkB;QAClB,aAAa;AACd,KAAA,CAAC;IAEF,MAAM,gBAAgB,GAAGC,gCAAsB,CAAC;AAC9C,QAAA,MAAM,EAAE,eAAe,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB;AACxD,QAAA,MAAM,EAAE,eAAe;AACvB,QAAA,OAAO,EAAE,gBAAgB;AAC1B,KAAA,CAAC;IAEF,MAAM,gBAAgB,GAAGC,wBAAc,CAAC;QACtC,YAAY;QACZ,UAAU;QACV,YAAY;AACb,KAAA,CAAC;IAEF,IAAI,CAAC,gBAAgB,EAAE;AACrB,QAAA,OAAO,CAAC,IAAI,CAAC,8CAA8C,CAAC;;IAG9D,MAAM,eAAe,GAAGC,4BAAqB,CAC3C;AACE,QAAA,QAAQ,EAAE,gBAAgB;QAC1B,WAGD,EACD,gBAAgB,CACjB;AAED,IAAA,MAAMC,QAAM,GAAG,OAAO,EACpB,KAAK,EACL,OAAO,GAAG,IAAI,EACd,UAAU,GAAG,CAAC,EACd,eAAe,GAMhB,KAAiC;AAChC,QAAA,IAAI;YACF,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,UAAU,CAAC,KAAK,CAAC;AACjD,YAAA,eAAe,GAAG,OAAO,CAAC;AAE1B,YAAA,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE;gBACpB,MAAM,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,IAAI,eAAe,CAAC;;AAGnD,YAAA,MAAM,gBAAgB,GAAG,MAAM,eAAe,CAAC,cAAc,CAC3D,OAAO,EACP,UAAU,EACV,KAAK,EACL,OAAO,CACR;AACD,YAAA,OAAOC,2BAAgB,CAAC,gBAAgB,CAAC;;QACzC,OAAO,KAAK,EAAE;AACd,YAAA,OAAO,CAAC,KAAK,CAAC,kBAAkB,EAAE,KAAK,CAAC;YACxC,OAAO;AACL,gBAAA,OAAO,EAAE,EAAE;AACX,gBAAA,UAAU,EAAE,EAAE;AACd,gBAAA,MAAM,EAAE,EAAE;AACV,gBAAA,eAAe,EAAE,EAAE;AACnB,gBAAA,KAAK,EAAE,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;aAC9D;;AAEL,KAAC;IAED,OAAOC,UAAI,CACT,OAAO,EAAE,KAAK,EAAE,EAAE,cAAc,KAAI;AAClC,QAAA,MAAM,YAAY,GAAG,MAAMF,QAAM,CAAC;YAChC,KAAK;AACL,YAAA,eAAe,EAAE;AACf,kBAAE,CAAC,MAAM,KAAU;AACjB,oBAAA,gBAAgB,CAAC,MAAM,EAAE,cAAc,CAAC;;AAE1C,kBAAE,SAAS;AACd,SAAA,CAAC;AACF,QAAA,MAAM,MAAM,GAAGG,0BAAmB,CAAC,YAAY,CAAC;AAChD,QAAA,OAAO,CAAC,MAAM,EAAE,YAAY,CAAC;AAC/B,KAAC,EACD;QACE,IAAI,EAAEC,eAAS,CAAC,UAAU;AAC1B,QAAA,WAAW,EAAE;;;;;;;;;;;;;;;;;;;;AAoBlB,CAAA,CAAC,IAAI,EAAE;AACF,QAAA,MAAM,EAAE,gBAAgB;QACxB,cAAc,EAAEA,eAAS,CAAC,oBAAoB;AAC/C,KAAA,CACF;AACH;;;;"}
+{"version":3,"file":"tool.cjs","sources":["../../../../src/tools/search/tool.ts"],"sourcesContent":["/* eslint-disable no-console */\nimport { z } from 'zod';\nimport { tool, DynamicStructuredTool } from '@langchain/core/tools';\nimport type * as t from './types';\nimport { createSearchAPI, createSourceProcessor } from './search';\nimport { createFirecrawlScraper } from './firecrawl';\nimport { expandHighlights } from './highlights';\nimport { formatResultsForLLM } from './format';\nimport { createReranker } from './rerankers';\nimport { Constants } from '@/common';\n\nconst SearchToolSchema = z.object({\n  query: z\n    .string()\n    .describe(\n      'The search query string that specifies what should be searched for.'\n    ),\n});\n\nexport const createSearchTool = (\n  config: t.SearchToolConfig = {}\n): DynamicStructuredTool<typeof SearchToolSchema> => {\n  const {\n    searchProvider = 'serper',\n    serperApiKey,\n    searxngInstanceUrl,\n    searxngApiKey,\n    rerankerType = 'cohere',\n    topResults = 5,\n    strategies = ['no_extraction'],\n    filterContent = true,\n    firecrawlApiKey,\n    firecrawlApiUrl,\n    firecrawlFormats = ['markdown', 'html'],\n    jinaApiKey,\n    cohereApiKey,\n    onSearchResults: _onSearchResults,\n  } = config;\n\n  const searchAPI = createSearchAPI({\n    searchProvider,\n    serperApiKey,\n    searxngInstanceUrl,\n    searxngApiKey,\n  });\n\n  const firecrawlScraper = createFirecrawlScraper({\n    apiKey: firecrawlApiKey ?? process.env.FIRECRAWL_API_KEY,\n    apiUrl: firecrawlApiUrl,\n    formats: firecrawlFormats,\n  });\n\n  const selectedReranker = createReranker({\n    rerankerType,\n    jinaApiKey,\n    cohereApiKey,\n  });\n\n  if (!selectedReranker) {\n    console.warn('No reranker selected. Using default ranking.');\n  }\n\n  const sourceProcessor = createSourceProcessor(\n    {\n      reranker: selectedReranker,\n      topResults,\n      strategies,\n      filterContent,\n    },\n    firecrawlScraper\n  );\n\n  const search = async ({\n    query,\n    proMode = true,\n    maxSources = 5,\n    onSearchResults,\n  }: {\n    query: string;\n    proMode?: boolean;\n    maxSources?: number;\n    onSearchResults?: (sources: t.SearchResult) => void;\n  }): Promise<t.SearchResultData> => {\n    try {\n      const sources = await searchAPI.getSources(query);\n      onSearchResults?.(sources);\n\n      if (!sources.success) {\n        throw new Error(sources.error ?? 'Search failed');\n      }\n\n      const processedSources = await sourceProcessor.processSources(\n        sources,\n        maxSources,\n        query,\n        proMode\n      );\n      return expandHighlights(processedSources);\n    } catch (error) {\n      console.error('Error in search:', error);\n      return {\n        organic: [],\n        topStories: [],\n        images: [],\n        relatedSearches: [],\n        error: error instanceof Error ? error.message : String(error),\n      };\n    }\n  };\n\n  return tool<typeof SearchToolSchema>(\n    async ({ query }, runnableConfig) => {\n      const searchResult = await search({\n        query,\n        onSearchResults: _onSearchResults\n          ? (result): void => {\n            _onSearchResults(result, runnableConfig);\n          }\n          : undefined,\n      });\n      const output = formatResultsForLLM(searchResult);\n      return [output, { [Constants.WEB_SEARCH]: { ...searchResult } }];\n    },\n    {\n      name: Constants.WEB_SEARCH,\n      description: `\nReal-time search. Results have required unique citation anchors.\n\nAnchors:\n- \\\\ue202turn0searchN (web), \\\\ue202turn0newsN (news), \\\\ue202turn0imageN (image)\n\nSpecial Markers:\n- \\\\ue203...\\\\ue204 — mark start/end of cited span\n- \\\\ue200...\\\\ue201 — composite/group block (e.g. \\\\ue200cite\\\\ue202turn0search1\\\\ue202turn0news2\\\\ue201)\n- \\\\ue206 — marks grouped/summary citation areas\n\n**CITE EVERY NON-OBVIOUS FACT/QUOTE:**\nInsert the anchor marker(s) immediately after the statement:\n- \"Pure functions produce same output \\\\ue202turn0search0.\"\n- Multiple: \"Benefits \\\\ue202turn0search0\\\\ue202turn0news0.\"\n- Span: \\\\ue203Key: first-class functions\\\\ue204\\\\ue202turn0news1\n- Group: \"Functional languages.\"\\\\ue206 or \\\\ue200cite\\\\ue202turn0search0\\\\ue202turn0news1\\\\ue201\n- Image: \"See photo \\\\ue202turn0image0.\"\n\n**NEVER use markdown links, [1], or footnotes. CITE ONLY with anchors provided.**\n`.trim(),\n      schema: SearchToolSchema,\n      responseFormat: Constants.CONTENT_AND_ARTIFACT,\n    }\n  );\n};\n"],"names":["z","createSearchAPI","createFirecrawlScraper","createReranker","createSourceProcessor","search","expandHighlights","tool","formatResultsForLLM","Constants"],"mappings":";;;;;;;;;;;AAAA;AAWA,MAAM,gBAAgB,GAAGA,KAAC,CAAC,MAAM,CAAC;AAChC,IAAA,KAAK,EAAEA;AACJ,SAAA,MAAM;SACN,QAAQ,CACP,qEAAqE,CACtE;AACJ,CAAA,CAAC;MAEW,gBAAgB,GAAG,CAC9B,MAA6B,GAAA,EAAE,KACmB;IAClD,MAAM,EACJ,cAAc,GAAG,QAAQ,EACzB,YAAY,EACZ,kBAAkB,EAClB,aAAa,EACb,YAAY,GAAG,QAAQ,EACvB,UAAU,GAAG,CAAC,EACd,UAAU,GAAG,CAAC,eAAe,CAAC,EAC9B,aAAa,GAAG,IAAI,EACpB,eAAe,EACf,eAAe,EACf,gBAAgB,GAAG,CAAC,UAAU,EAAE,MAAM,CAAC,EACvC,UAAU,EACV,YAAY,EACZ,eAAe,EAAE,gBAAgB,GAClC,GAAG,MAAM;IAEV,MAAM,SAAS,GAAGC,sBAAe,CAAC;QAChC,cAAc;QACd,YAAY;QACZ,kBAAkB;QAClB,aAAa;AACd,KAAA,CAAC;IAEF,MAAM,gBAAgB,GAAGC,gCAAsB,CAAC;AAC9C,QAAA,MAAM,EAAE,eAAe,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB;AACxD,QAAA,MAAM,EAAE,eAAe;AACvB,QAAA,OAAO,EAAE,gBAAgB;AAC1B,KAAA,CAAC;IAEF,MAAM,gBAAgB,GAAGC,wBAAc,CAAC;QACtC,YAAY;QACZ,UAAU;QACV,YAAY;AACb,KAAA,CAAC;IAEF,IAAI,CAAC,gBAAgB,EAAE;AACrB,QAAA,OAAO,CAAC,IAAI,CAAC,8CAA8C,CAAC;;IAG9D,MAAM,eAAe,GAAGC,4BAAqB,CAC3C;AACE,QAAA,QAAQ,EAAE,gBAAgB;QAC1B,WAGD,EACD,gBAAgB,CACjB;AAED,IAAA,MAAMC,QAAM,GAAG,OAAO,EACpB,KAAK,EACL,OAAO,GAAG,IAAI,EACd,UAAU,GAAG,CAAC,EACd,eAAe,GAMhB,KAAiC;AAChC,QAAA,IAAI;YACF,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,UAAU,CAAC,KAAK,CAAC;AACjD,YAAA,eAAe,GAAG,OAAO,CAAC;AAE1B,YAAA,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE;gBACpB,MAAM,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,IAAI,eAAe,CAAC;;AAGnD,YAAA,MAAM,gBAAgB,GAAG,MAAM,eAAe,CAAC,cAAc,CAC3D,OAAO,EACP,UAAU,EACV,KAAK,EACL,OAAO,CACR;AACD,YAAA,OAAOC,2BAAgB,CAAC,gBAAgB,CAAC;;QACzC,OAAO,KAAK,EAAE;AACd,YAAA,OAAO,CAAC,KAAK,CAAC,kBAAkB,EAAE,KAAK,CAAC;YACxC,OAAO;AACL,gBAAA,OAAO,EAAE,EAAE;AACX,gBAAA,UAAU,EAAE,EAAE;AACd,gBAAA,MAAM,EAAE,EAAE;AACV,gBAAA,eAAe,EAAE,EAAE;AACnB,gBAAA,KAAK,EAAE,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;aAC9D;;AAEL,KAAC;IAED,OAAOC,UAAI,CACT,OAAO,EAAE,KAAK,EAAE,EAAE,cAAc,KAAI;AAClC,QAAA,MAAM,YAAY,GAAG,MAAMF,QAAM,CAAC;YAChC,KAAK;AACL,YAAA,eAAe,EAAE;AACf,kBAAE,CAAC,MAAM,KAAU;AACjB,oBAAA,gBAAgB,CAAC,MAAM,EAAE,cAAc,CAAC;;AAE1C,kBAAE,SAAS;AACd,SAAA,CAAC;AACF,QAAA,MAAM,MAAM,GAAGG,0BAAmB,CAAC,YAAY,CAAC;AAChD,QAAA,OAAO,CAAC,MAAM,EAAE,EAAE,CAACC,eAAS,CAAC,UAAU,GAAG,EAAE,GAAG,YAAY,EAAE,EAAE,CAAC;AAClE,KAAC,EACD;QACE,IAAI,EAAEA,eAAS,CAAC,UAAU;AAC1B,QAAA,WAAW,EAAE;;;;;;;;;;;;;;;;;;;;AAoBlB,CAAA,CAAC,IAAI,EAAE;AACF,QAAA,MAAM,EAAE,gBAAgB;QACxB,cAAc,EAAEA,eAAS,CAAC,oBAAoB;AAC/C,KAAA,CACF;AACH;;;;"}

package/dist/esm/tools/search/firecrawl.mjs

@@ -46,7 +46,7 @@ class FirecrawlScraper {
             'https://api.firecrawl.dev';
         this.apiUrl = `${baseUrl.replace(/\/+$/, '')}/v1/scrape`;
         this.defaultFormats = config.formats ?? ['markdown', 'html'];
-        this.timeout = config.timeout ?? 30000;
+        this.timeout = config.timeout ?? 15000;
         if (!this.apiKey) {
             console.warn('FIRECRAWL_API_KEY is not set. Scraping will not work.');
         }

package/dist/esm/tools/search/firecrawl.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"firecrawl.mjs","sources":["../../../../src/tools/search/firecrawl.ts"],"sourcesContent":["/* eslint-disable no-console */\nimport axios from 'axios';\n\nexport interface FirecrawlScrapeOptions {\n  formats?: string[];\n  includeTags?: string[];\n  excludeTags?: string[];\n  headers?: Record<string, string>;\n  waitFor?: number;\n  timeout?: number;\n}\n\ninterface ScrapeMetadata {\n  // Core source information\n  sourceURL?: string;\n  url?: string;\n  scrapeId?: string;\n  statusCode?: number;\n  // Basic metadata\n  title?: string;\n  description?: string;\n  language?: string;\n  favicon?: string;\n  viewport?: string;\n  robots?: string;\n  'theme-color'?: string;\n  // Open Graph metadata\n  'og:url'?: string;\n  'og:title'?: string;\n  'og:description'?: string;\n  'og:type'?: string;\n  'og:image'?: string;\n  'og:image:width'?: string;\n  'og:image:height'?: string;\n  'og:site_name'?: string;\n  ogUrl?: string;\n  ogTitle?: string;\n  ogDescription?: string;\n  ogImage?: string;\n  ogSiteName?: string;\n  // Article metadata\n  'article:author'?: string;\n  'article:published_time'?: string;\n  'article:modified_time'?: string;\n  'article:section'?: string;\n  'article:tag'?: string;\n  'article:publisher'?: string;\n  publishedTime?: string;\n  modifiedTime?: string;\n  // Twitter metadata\n  'twitter:site'?: string;\n  'twitter:creator'?: string;\n  'twitter:card'?: string;\n  'twitter:image'?: string;\n  'twitter:dnt'?: string;\n  'twitter:app:name:iphone'?: string;\n  'twitter:app:id:iphone'?: string;\n  'twitter:app:url:iphone'?: string;\n  'twitter:app:name:ipad'?: string;\n  'twitter:app:id:ipad'?: string;\n  'twitter:app:url:ipad'?: string;\n  'twitter:app:name:googleplay'?: string;\n  'twitter:app:id:googleplay'?: string;\n  'twitter:app:url:googleplay'?: string;\n  // Facebook metadata\n  'fb:app_id'?: string;\n  // App links\n  'al:ios:url'?: string;\n  'al:ios:app_name'?: string;\n  'al:ios:app_store_id'?: string;\n  // Allow for additional properties that might be present\n  [key: string]: string | number | boolean | null | undefined;\n}\n\nexport interface FirecrawlScrapeResponse {\n  success: boolean;\n  data?: {\n    markdown?: string;\n    html?: string;\n    rawHtml?: string;\n    screenshot?: string;\n    links?: string[];\n    metadata?: ScrapeMetadata;\n  };\n  error?: string;\n}\n\nexport interface FirecrawlScraperConfig {\n  apiKey?: string;\n  apiUrl?: string;\n  formats?: string[];\n  timeout?: number;\n}\nconst getDomainName = (\n  link: string,\n  metadata?: ScrapeMetadata\n): string | undefined => {\n  try {\n    const url = metadata?.sourceURL ?? metadata?.url ?? (link || '');\n    const domain = new URL(url).hostname.replace(/^www\\./, '');\n    if (domain) {\n      return domain;\n    }\n  } catch (e) {\n    // URL parsing failed\n    console.error('Error parsing URL:', e);\n  }\n\n  return;\n};\n\nexport function getAttribution(\n  link: string,\n  metadata?: ScrapeMetadata\n): string | undefined {\n  if (!metadata) return getDomainName(link, metadata);\n\n  const possibleAttributions = [\n    metadata.ogSiteName,\n    metadata['og:site_name'],\n    metadata.title?.split('|').pop()?.trim(),\n    metadata['twitter:site']?.replace(/^@/, ''),\n  ];\n\n  const attribution = possibleAttributions.find(\n    (attr) => attr != null && typeof attr === 'string' && attr.trim() !== ''\n  );\n  if (attribution != null) {\n    return attribution;\n  }\n\n  return getDomainName(link, metadata);\n}\n\n/**\n * Firecrawl scraper implementation\n * Uses the Firecrawl API to scrape web pages\n */\nexport class FirecrawlScraper {\n  private apiKey: string;\n  private apiUrl: string;\n  private defaultFormats: string[];\n  private timeout: number;\n\n  constructor(config: FirecrawlScraperConfig = {}) {\n    this.apiKey = config.apiKey ?? process.env.FIRECRAWL_API_KEY ?? '';\n\n    const baseUrl =\n      config.apiUrl ??\n      process.env.FIRECRAWL_BASE_URL ??\n      'https://api.firecrawl.dev';\n    this.apiUrl = `${baseUrl.replace(/\\/+$/, '')}/v1/scrape`;\n\n    this.defaultFormats = config.formats ?? ['markdown', 'html'];\n    this.timeout = config.timeout ?? 30000;\n\n    if (!this.apiKey) {\n      console.warn('FIRECRAWL_API_KEY is not set. Scraping will not work.');\n    }\n\n    console.log(`Firecrawl scraper initialized with API URL: ${this.apiUrl}`);\n  }\n\n  /**\n   * Scrape a single URL\n   * @param url URL to scrape\n   * @param options Scrape options\n   * @returns Scrape response\n   */\n  async scrapeUrl(\n    url: string,\n    options: FirecrawlScrapeOptions = {}\n  ): Promise<[string, FirecrawlScrapeResponse]> {\n    if (!this.apiKey) {\n      return [\n        url,\n        {\n          success: false,\n          error: 'FIRECRAWL_API_KEY is not set',\n        },\n      ];\n    }\n\n    try {\n      const response = await axios.post(\n        this.apiUrl,\n        {\n          url,\n          formats: options.formats || this.defaultFormats,\n          includeTags: options.includeTags,\n          excludeTags: options.excludeTags,\n          headers: options.headers,\n          waitFor: options.waitFor,\n          timeout: options.timeout ?? this.timeout,\n        },\n        {\n          headers: {\n            'Content-Type': 'application/json',\n            Authorization: `Bearer ${this.apiKey}`,\n          },\n          timeout: this.timeout,\n        }\n      );\n\n      return [url, response.data];\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      return [\n        url,\n        {\n          success: false,\n          error: `Firecrawl API request failed: ${errorMessage}`,\n        },\n      ];\n    }\n  }\n\n  /**\n   * Extract content from scrape response\n   * @param response Scrape response\n   * @returns Extracted content or empty string if not available\n   */\n  extractContent(response: FirecrawlScrapeResponse): string {\n    if (!response.success || !response.data) {\n      return '';\n    }\n\n    // Prefer markdown content if available\n    if (response.data.markdown != null) {\n      return response.data.markdown;\n    }\n\n    // Fall back to HTML content\n    if (response.data.html != null) {\n      return response.data.html;\n    }\n\n    // Fall back to raw HTML content\n    if (response.data.rawHtml != null) {\n      return response.data.rawHtml;\n    }\n\n    return '';\n  }\n\n  /**\n   * Extract metadata from scrape response\n   * @param response Scrape response\n   * @returns Metadata object\n   */\n  extractMetadata(response: FirecrawlScrapeResponse): ScrapeMetadata {\n    if (!response.success || !response.data || !response.data.metadata) {\n      return {};\n    }\n\n    return response.data.metadata;\n  }\n}\n\n/**\n * Create a Firecrawl scraper instance\n * @param config Scraper configuration\n * @returns Firecrawl scraper instance\n */\nexport const createFirecrawlScraper = (\n  config: FirecrawlScraperConfig = {}\n): FirecrawlScraper => {\n  return new FirecrawlScraper(config);\n};\n"],"names":[],"mappings":";;AAAA;AA6FA,MAAM,aAAa,GAAG,CACpB,IAAY,EACZ,QAAyB,KACH;AACtB,IAAA,IAAI;AACF,QAAA,MAAM,GAAG,GAAG,QAAQ,EAAE,SAAS,IAAI,QAAQ,EAAE,GAAG,KAAK,IAAI,IAAI,EAAE,CAAC;AAChE,QAAA,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC;QAC1D,IAAI,MAAM,EAAE;AACV,YAAA,OAAO,MAAM;;;IAEf,OAAO,CAAC,EAAE;;AAEV,QAAA,OAAO,CAAC,KAAK,CAAC,oBAAoB,EAAE,CAAC,CAAC;;IAGxC;AACF,CAAC;AAEe,SAAA,cAAc,CAC5B,IAAY,EACZ,QAAyB,EAAA;AAEzB,IAAA,IAAI,CAAC,QAAQ;AAAE,QAAA,OAAO,aAAa,CAAC,IAAI,EAAE,QAAQ,CAAC;AAEnD,IAAA,MAAM,oBAAoB,GAAG;AAC3B,QAAA,QAAQ,CAAC,UAAU;QACnB,QAAQ,CAAC,cAAc,CAAC;AACxB,QAAA,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE;QACxC,QAAQ,CAAC,cAAc,CAAC,EAAE,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;KAC5C;IAED,MAAM,WAAW,GAAG,oBAAoB,CAAC,IAAI,CAC3C,CAAC,IAAI,KAAK,IAAI,IAAI,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CACzE;AACD,IAAA,IAAI,WAAW,IAAI,IAAI,EAAE;AACvB,QAAA,OAAO,WAAW;;AAGpB,IAAA,OAAO,aAAa,CAAC,IAAI,EAAE,QAAQ,CAAC;AACtC;AAEA;;;AAGG;MACU,gBAAgB,CAAA;AACnB,IAAA,MAAM;AACN,IAAA,MAAM;AACN,IAAA,cAAc;AACd,IAAA,OAAO;AAEf,IAAA,WAAA,CAAY,SAAiC,EAAE,EAAA;AAC7C,QAAA,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,EAAE;AAElE,QAAA,MAAM,OAAO,GACX,MAAM,CAAC,MAAM;YACb,OAAO,CAAC,GAAG,CAAC,kBAAkB;AAC9B,YAAA,2BAA2B;AAC7B,QAAA,IAAI,CAAC,MAAM,GAAG,CAAA,EAAG,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,YAAY;AAExD,QAAA,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,OAAO,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC;QAC5D,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,KAAK;AAEtC,QAAA,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;AAChB,YAAA,OAAO,CAAC,IAAI,CAAC,uDAAuD,CAAC;;QAGvE,OAAO,CAAC,GAAG,CAAC,CAAA,4CAAA,EAA+C,IAAI,CAAC,MAAM,CAAE,CAAA,CAAC;;AAG3E;;;;;AAKG;AACH,IAAA,MAAM,SAAS,CACb,GAAW,EACX,UAAkC,EAAE,EAAA;AAEpC,QAAA,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YAChB,OAAO;gBACL,GAAG;AACH,gBAAA;AACE,oBAAA,OAAO,EAAE,KAAK;AACd,oBAAA,KAAK,EAAE,8BAA8B;AACtC,iBAAA;aACF;;AAGH,QAAA,IAAI;YACF,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAC/B,IAAI,CAAC,MAAM,EACX;gBACE,GAAG;AACH,gBAAA,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,cAAc;gBAC/C,WAAW,EAAE,OAAO,CAAC,WAAW;gBAChC,WAAW,EAAE,OAAO,CAAC,WAAW;gBAChC,OAAO,EAAE,OAAO,CAAC,OAAO;gBACxB,OAAO,EAAE,OAAO,CAAC,OAAO;AACxB,gBAAA,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO;aACzC,EACD;AACE,gBAAA,OAAO,EAAE;AACP,oBAAA,cAAc,EAAE,kBAAkB;AAClC,oBAAA,aAAa,EAAE,CAAA,OAAA,EAAU,IAAI,CAAC,MAAM,CAAE,CAAA;AACvC,iBAAA;gBACD,OAAO,EAAE,IAAI,CAAC,OAAO;AACtB,aAAA,CACF;AAED,YAAA,OAAO,CAAC,GAAG,EAAE,QAAQ,CAAC,IAAI,CAAC;;QAC3B,OAAO,KAAK,EAAE;AACd,YAAA,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;YACxD,OAAO;gBACL,GAAG;AACH,gBAAA;AACE,oBAAA,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,CAAiC,8BAAA,EAAA,YAAY,CAAE,CAAA;AACvD,iBAAA;aACF;;;AAIL;;;;AAIG;AACH,IAAA,cAAc,CAAC,QAAiC,EAAA;QAC9C,IAAI,CAAC,QAAQ,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE;AACvC,YAAA,OAAO,EAAE;;;QAIX,IAAI,QAAQ,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,EAAE;AAClC,YAAA,OAAO,QAAQ,CAAC,IAAI,CAAC,QAAQ;;;QAI/B,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,EAAE;AAC9B,YAAA,OAAO,QAAQ,CAAC,IAAI,CAAC,IAAI;;;QAI3B,IAAI,QAAQ,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,EAAE;AACjC,YAAA,OAAO,QAAQ,CAAC,IAAI,CAAC,OAAO;;AAG9B,QAAA,OAAO,EAAE;;AAGX;;;;AAIG;AACH,IAAA,eAAe,CAAC,QAAiC,EAAA;AAC/C,QAAA,IAAI,CAAC,QAAQ,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE;AAClE,YAAA,OAAO,EAAE;;AAGX,QAAA,OAAO,QAAQ,CAAC,IAAI,CAAC,QAAQ;;AAEhC;AAED;;;;AAIG;MACU,sBAAsB,GAAG,CACpC,MAAiC,GAAA,EAAE,KACf;AACpB,IAAA,OAAO,IAAI,gBAAgB,CAAC,MAAM,CAAC;AACrC;;;;"}
+{"version":3,"file":"firecrawl.mjs","sources":["../../../../src/tools/search/firecrawl.ts"],"sourcesContent":["/* eslint-disable no-console */\nimport axios from 'axios';\n\nexport interface FirecrawlScrapeOptions {\n  formats?: string[];\n  includeTags?: string[];\n  excludeTags?: string[];\n  headers?: Record<string, string>;\n  waitFor?: number;\n  timeout?: number;\n}\n\ninterface ScrapeMetadata {\n  // Core source information\n  sourceURL?: string;\n  url?: string;\n  scrapeId?: string;\n  statusCode?: number;\n  // Basic metadata\n  title?: string;\n  description?: string;\n  language?: string;\n  favicon?: string;\n  viewport?: string;\n  robots?: string;\n  'theme-color'?: string;\n  // Open Graph metadata\n  'og:url'?: string;\n  'og:title'?: string;\n  'og:description'?: string;\n  'og:type'?: string;\n  'og:image'?: string;\n  'og:image:width'?: string;\n  'og:image:height'?: string;\n  'og:site_name'?: string;\n  ogUrl?: string;\n  ogTitle?: string;\n  ogDescription?: string;\n  ogImage?: string;\n  ogSiteName?: string;\n  // Article metadata\n  'article:author'?: string;\n  'article:published_time'?: string;\n  'article:modified_time'?: string;\n  'article:section'?: string;\n  'article:tag'?: string;\n  'article:publisher'?: string;\n  publishedTime?: string;\n  modifiedTime?: string;\n  // Twitter metadata\n  'twitter:site'?: string;\n  'twitter:creator'?: string;\n  'twitter:card'?: string;\n  'twitter:image'?: string;\n  'twitter:dnt'?: string;\n  'twitter:app:name:iphone'?: string;\n  'twitter:app:id:iphone'?: string;\n  'twitter:app:url:iphone'?: string;\n  'twitter:app:name:ipad'?: string;\n  'twitter:app:id:ipad'?: string;\n  'twitter:app:url:ipad'?: string;\n  'twitter:app:name:googleplay'?: string;\n  'twitter:app:id:googleplay'?: string;\n  'twitter:app:url:googleplay'?: string;\n  // Facebook metadata\n  'fb:app_id'?: string;\n  // App links\n  'al:ios:url'?: string;\n  'al:ios:app_name'?: string;\n  'al:ios:app_store_id'?: string;\n  // Allow for additional properties that might be present\n  [key: string]: string | number | boolean | null | undefined;\n}\n\nexport interface FirecrawlScrapeResponse {\n  success: boolean;\n  data?: {\n    markdown?: string;\n    html?: string;\n    rawHtml?: string;\n    screenshot?: string;\n    links?: string[];\n    metadata?: ScrapeMetadata;\n  };\n  error?: string;\n}\n\nexport interface FirecrawlScraperConfig {\n  apiKey?: string;\n  apiUrl?: string;\n  formats?: string[];\n  timeout?: number;\n}\nconst getDomainName = (\n  link: string,\n  metadata?: ScrapeMetadata\n): string | undefined => {\n  try {\n    const url = metadata?.sourceURL ?? metadata?.url ?? (link || '');\n    const domain = new URL(url).hostname.replace(/^www\\./, '');\n    if (domain) {\n      return domain;\n    }\n  } catch (e) {\n    // URL parsing failed\n    console.error('Error parsing URL:', e);\n  }\n\n  return;\n};\n\nexport function getAttribution(\n  link: string,\n  metadata?: ScrapeMetadata\n): string | undefined {\n  if (!metadata) return getDomainName(link, metadata);\n\n  const possibleAttributions = [\n    metadata.ogSiteName,\n    metadata['og:site_name'],\n    metadata.title?.split('|').pop()?.trim(),\n    metadata['twitter:site']?.replace(/^@/, ''),\n  ];\n\n  const attribution = possibleAttributions.find(\n    (attr) => attr != null && typeof attr === 'string' && attr.trim() !== ''\n  );\n  if (attribution != null) {\n    return attribution;\n  }\n\n  return getDomainName(link, metadata);\n}\n\n/**\n * Firecrawl scraper implementation\n * Uses the Firecrawl API to scrape web pages\n */\nexport class FirecrawlScraper {\n  private apiKey: string;\n  private apiUrl: string;\n  private defaultFormats: string[];\n  private timeout: number;\n\n  constructor(config: FirecrawlScraperConfig = {}) {\n    this.apiKey = config.apiKey ?? process.env.FIRECRAWL_API_KEY ?? '';\n\n    const baseUrl =\n      config.apiUrl ??\n      process.env.FIRECRAWL_BASE_URL ??\n      'https://api.firecrawl.dev';\n    this.apiUrl = `${baseUrl.replace(/\\/+$/, '')}/v1/scrape`;\n\n    this.defaultFormats = config.formats ?? ['markdown', 'html'];\n    this.timeout = config.timeout ?? 15000;\n\n    if (!this.apiKey) {\n      console.warn('FIRECRAWL_API_KEY is not set. Scraping will not work.');\n    }\n\n    console.log(`Firecrawl scraper initialized with API URL: ${this.apiUrl}`);\n  }\n\n  /**\n   * Scrape a single URL\n   * @param url URL to scrape\n   * @param options Scrape options\n   * @returns Scrape response\n   */\n  async scrapeUrl(\n    url: string,\n    options: FirecrawlScrapeOptions = {}\n  ): Promise<[string, FirecrawlScrapeResponse]> {\n    if (!this.apiKey) {\n      return [\n        url,\n        {\n          success: false,\n          error: 'FIRECRAWL_API_KEY is not set',\n        },\n      ];\n    }\n\n    try {\n      const response = await axios.post(\n        this.apiUrl,\n        {\n          url,\n          formats: options.formats || this.defaultFormats,\n          includeTags: options.includeTags,\n          excludeTags: options.excludeTags,\n          headers: options.headers,\n          waitFor: options.waitFor,\n          timeout: options.timeout ?? this.timeout,\n        },\n        {\n          headers: {\n            'Content-Type': 'application/json',\n            Authorization: `Bearer ${this.apiKey}`,\n          },\n          timeout: this.timeout,\n        }\n      );\n\n      return [url, response.data];\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      return [\n        url,\n        {\n          success: false,\n          error: `Firecrawl API request failed: ${errorMessage}`,\n        },\n      ];\n    }\n  }\n\n  /**\n   * Extract content from scrape response\n   * @param response Scrape response\n   * @returns Extracted content or empty string if not available\n   */\n  extractContent(response: FirecrawlScrapeResponse): string {\n    if (!response.success || !response.data) {\n      return '';\n    }\n\n    // Prefer markdown content if available\n    if (response.data.markdown != null) {\n      return response.data.markdown;\n    }\n\n    // Fall back to HTML content\n    if (response.data.html != null) {\n      return response.data.html;\n    }\n\n    // Fall back to raw HTML content\n    if (response.data.rawHtml != null) {\n      return response.data.rawHtml;\n    }\n\n    return '';\n  }\n\n  /**\n   * Extract metadata from scrape response\n   * @param response Scrape response\n   * @returns Metadata object\n   */\n  extractMetadata(response: FirecrawlScrapeResponse): ScrapeMetadata {\n    if (!response.success || !response.data || !response.data.metadata) {\n      return {};\n    }\n\n    return response.data.metadata;\n  }\n}\n\n/**\n * Create a Firecrawl scraper instance\n * @param config Scraper configuration\n * @returns Firecrawl scraper instance\n */\nexport const createFirecrawlScraper = (\n  config: FirecrawlScraperConfig = {}\n): FirecrawlScraper => {\n  return new FirecrawlScraper(config);\n};\n"],"names":[],"mappings":";;AAAA;AA6FA,MAAM,aAAa,GAAG,CACpB,IAAY,EACZ,QAAyB,KACH;AACtB,IAAA,IAAI;AACF,QAAA,MAAM,GAAG,GAAG,QAAQ,EAAE,SAAS,IAAI,QAAQ,EAAE,GAAG,KAAK,IAAI,IAAI,EAAE,CAAC;AAChE,QAAA,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC;QAC1D,IAAI,MAAM,EAAE;AACV,YAAA,OAAO,MAAM;;;IAEf,OAAO,CAAC,EAAE;;AAEV,QAAA,OAAO,CAAC,KAAK,CAAC,oBAAoB,EAAE,CAAC,CAAC;;IAGxC;AACF,CAAC;AAEe,SAAA,cAAc,CAC5B,IAAY,EACZ,QAAyB,EAAA;AAEzB,IAAA,IAAI,CAAC,QAAQ;AAAE,QAAA,OAAO,aAAa,CAAC,IAAI,EAAE,QAAQ,CAAC;AAEnD,IAAA,MAAM,oBAAoB,GAAG;AAC3B,QAAA,QAAQ,CAAC,UAAU;QACnB,QAAQ,CAAC,cAAc,CAAC;AACxB,QAAA,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE;QACxC,QAAQ,CAAC,cAAc,CAAC,EAAE,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;KAC5C;IAED,MAAM,WAAW,GAAG,oBAAoB,CAAC,IAAI,CAC3C,CAAC,IAAI,KAAK,IAAI,IAAI,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CACzE;AACD,IAAA,IAAI,WAAW,IAAI,IAAI,EAAE;AACvB,QAAA,OAAO,WAAW;;AAGpB,IAAA,OAAO,aAAa,CAAC,IAAI,EAAE,QAAQ,CAAC;AACtC;AAEA;;;AAGG;MACU,gBAAgB,CAAA;AACnB,IAAA,MAAM;AACN,IAAA,MAAM;AACN,IAAA,cAAc;AACd,IAAA,OAAO;AAEf,IAAA,WAAA,CAAY,SAAiC,EAAE,EAAA;AAC7C,QAAA,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,EAAE;AAElE,QAAA,MAAM,OAAO,GACX,MAAM,CAAC,MAAM;YACb,OAAO,CAAC,GAAG,CAAC,kBAAkB;AAC9B,YAAA,2BAA2B;AAC7B,QAAA,IAAI,CAAC,MAAM,GAAG,CAAA,EAAG,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,YAAY;AAExD,QAAA,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,OAAO,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC;QAC5D,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,KAAK;AAEtC,QAAA,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;AAChB,YAAA,OAAO,CAAC,IAAI,CAAC,uDAAuD,CAAC;;QAGvE,OAAO,CAAC,GAAG,CAAC,CAAA,4CAAA,EAA+C,IAAI,CAAC,MAAM,CAAE,CAAA,CAAC;;AAG3E;;;;;AAKG;AACH,IAAA,MAAM,SAAS,CACb,GAAW,EACX,UAAkC,EAAE,EAAA;AAEpC,QAAA,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YAChB,OAAO;gBACL,GAAG;AACH,gBAAA;AACE,oBAAA,OAAO,EAAE,KAAK;AACd,oBAAA,KAAK,EAAE,8BAA8B;AACtC,iBAAA;aACF;;AAGH,QAAA,IAAI;YACF,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAC/B,IAAI,CAAC,MAAM,EACX;gBACE,GAAG;AACH,gBAAA,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,cAAc;gBAC/C,WAAW,EAAE,OAAO,CAAC,WAAW;gBAChC,WAAW,EAAE,OAAO,CAAC,WAAW;gBAChC,OAAO,EAAE,OAAO,CAAC,OAAO;gBACxB,OAAO,EAAE,OAAO,CAAC,OAAO;AACxB,gBAAA,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO;aACzC,EACD;AACE,gBAAA,OAAO,EAAE;AACP,oBAAA,cAAc,EAAE,kBAAkB;AAClC,oBAAA,aAAa,EAAE,CAAA,OAAA,EAAU,IAAI,CAAC,MAAM,CAAE,CAAA;AACvC,iBAAA;gBACD,OAAO,EAAE,IAAI,CAAC,OAAO;AACtB,aAAA,CACF;AAED,YAAA,OAAO,CAAC,GAAG,EAAE,QAAQ,CAAC,IAAI,CAAC;;QAC3B,OAAO,KAAK,EAAE;AACd,YAAA,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;YACxD,OAAO;gBACL,GAAG;AACH,gBAAA;AACE,oBAAA,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,CAAiC,8BAAA,EAAA,YAAY,CAAE,CAAA;AACvD,iBAAA;aACF;;;AAIL;;;;AAIG;AACH,IAAA,cAAc,CAAC,QAAiC,EAAA;QAC9C,IAAI,CAAC,QAAQ,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE;AACvC,YAAA,OAAO,EAAE;;;QAIX,IAAI,QAAQ,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,EAAE;AAClC,YAAA,OAAO,QAAQ,CAAC,IAAI,CAAC,QAAQ;;;QAI/B,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,EAAE;AAC9B,YAAA,OAAO,QAAQ,CAAC,IAAI,CAAC,IAAI;;;QAI3B,IAAI,QAAQ,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,EAAE;AACjC,YAAA,OAAO,QAAQ,CAAC,IAAI,CAAC,OAAO;;AAG9B,QAAA,OAAO,EAAE;;AAGX;;;;AAIG;AACH,IAAA,eAAe,CAAC,QAAiC,EAAA;AAC/C,QAAA,IAAI,CAAC,QAAQ,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE;AAClE,YAAA,OAAO,EAAE;;AAGX,QAAA,OAAO,QAAQ,CAAC,IAAI,CAAC,QAAQ;;AAEhC;AAED;;;;AAIG;MACU,sBAAsB,GAAG,CACpC,MAAiC,GAAA,EAAE,KACf;AACpB,IAAA,OAAO,IAAI,gBAAgB,CAAC,MAAM,CAAC;AACrC;;;;"}

package/dist/esm/tools/search/format.mjs

@@ -9,8 +9,8 @@ function formatResultsForLLM(results) {
         addSection('Web Results');
         organic.forEach((r, i) => {
             output += [
-                `Source ${i + 1}: ${r.title ?? '(no title)'}`,
-                `Citation Anchor: \\ue202turn0search${i + 1}`,
+                `Source ${i}: ${r.title ?? '(no title)'}`,
+                `Citation Anchor: \\ue202turn0search${i}`,
                 `URL: ${r.link}`,
                 r.snippet != null ? `Summary: ${r.snippet}` : '',
                 r.date != null ? `Date: ${r.date}` : '',
@@ -33,7 +33,7 @@ function formatResultsForLLM(results) {
     //   addSection('News Results');
     //   topStores.forEach((r, i) => {
     //     output += [
-    //       `Anchor: \ue202turn0news${i + 1}`,
+    //       `Anchor: \ue202turn0news${i}`,
     //       `Title: ${r.title ?? '(no title)'}`,
     //       `URL: ${r.link}`,
     //       r.snippet != null ? `Snippet: ${r.snippet}` : '',
@@ -49,7 +49,7 @@ function formatResultsForLLM(results) {
     //   addSection('Image Results');
     //   images.forEach((img, i) => {
     //     output += [
-    //       `Anchor: \ue202turn0image${i + 1}`,
+    //       `Anchor: \ue202turn0image${i}`,
     //       `Title: ${img.title ?? '(no title)'}`,
     //       `Image URL: ${img.imageUrl}`,
     //       ''

package/dist/esm/tools/search/format.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"format.mjs","sources":["../../../../src/tools/search/format.ts"],"sourcesContent":["import type * as t from './types';\n\nexport function formatResultsForLLM(results: t.SearchResultData): string {\n  let output = '';\n\n  const addSection = (title: string): void => {\n    output += `\\n=== ${title} ===\\n`;\n  };\n\n  // Organic (web) results\n  const organic = results.organic ?? [];\n  if (organic.length) {\n    addSection('Web Results');\n    organic.forEach((r, i) => {\n      output += [\n        `Source ${i + 1}: ${r.title ?? '(no title)'}`,\n        `Citation Anchor: \\\\ue202turn0search${i + 1}`,\n        `URL: ${r.link}`,\n        r.snippet != null ? `Summary: ${r.snippet}` : '',\n        r.date != null ? `Date: ${r.date}` : '',\n        r.attribution != null ? `Source: ${r.attribution}` : '',\n        '',\n        '--- Content Highlights ---',\n        ...(r.highlights ?? [])\n          .filter((h) => h.text.trim().length > 0)\n          .map((h) => `[Relevance: ${h.score.toFixed(2)}]\\n${h.text.trim()}`),\n        '',\n      ]\n        .filter(Boolean)\n        .join('\\n');\n    });\n  }\n\n  // Ignoring these sections for now\n  // // Top stories (news)\n  // const topStores = results.topStories ?? [];\n  // if (topStores.length) {\n  //   addSection('News Results');\n  //   topStores.forEach((r, i) => {\n  //     output += [\n  //       `Anchor: \\ue202turn0news${i + 1}`,\n  //       `Title: ${r.title ?? '(no title)'}`,\n  //       `URL: ${r.link}`,\n  //       r.snippet != null ? `Snippet: ${r.snippet}` : '',\n  //       r.date != null ? `Date: ${r.date}` : '',\n  //       r.attribution != null ? `Source: ${r.attribution}` : '',\n  //       ''\n  //     ].filter(Boolean).join('\\n');\n  //   });\n  // }\n\n  // // Images\n  // const images = results.images ?? [];\n  // if (images.length) {\n  //   addSection('Image Results');\n  //   images.forEach((img, i) => {\n  //     output += [\n  //       `Anchor: \\ue202turn0image${i + 1}`,\n  //       `Title: ${img.title ?? '(no title)'}`,\n  //       `Image URL: ${img.imageUrl}`,\n  //       ''\n  //     ].join('\\n');\n  //   });\n  // }\n\n  // Knowledge Graph\n  if (results.knowledgeGraph != null) {\n    addSection('Knowledge Graph');\n    output += [\n      `Title: ${results.knowledgeGraph.title ?? '(no title)'}`,\n      results.knowledgeGraph.description != null\n        ? `Description: ${results.knowledgeGraph.description}`\n        : '',\n      results.knowledgeGraph.type != null\n        ? `Type: ${results.knowledgeGraph.type}`\n        : '',\n      results.knowledgeGraph.imageUrl != null\n        ? `Image URL: ${results.knowledgeGraph.imageUrl}`\n        : '',\n      results.knowledgeGraph.attributes != null\n        ? `Attributes: ${JSON.stringify(results.knowledgeGraph.attributes, null, 2)}`\n        : '',\n      '',\n    ]\n      .filter(Boolean)\n      .join('\\n');\n  }\n\n  // Answer Box\n  if (results.answerBox != null) {\n    addSection('Answer Box');\n    output += [\n      results.answerBox.title != null\n        ? `Title: ${results.answerBox.title}`\n        : '',\n      results.answerBox.answer != null\n        ? `Answer: ${results.answerBox.answer}`\n        : '',\n      results.answerBox.snippet != null\n        ? `Snippet: ${results.answerBox.snippet}`\n        : '',\n      results.answerBox.date != null ? `Date: ${results.answerBox.date}` : '',\n      '',\n    ]\n      .filter(Boolean)\n      .join('\\n');\n  }\n\n  // People also ask\n  const peopleAlsoAsk = results.peopleAlsoAsk ?? [];\n  if (peopleAlsoAsk.length) {\n    addSection('People Also Ask');\n    peopleAlsoAsk.forEach((p, _i) => {\n      output += [`Q: ${p.question}`, `A: ${p.answer}`, '']\n        .filter(Boolean)\n        .join('\\n');\n    });\n  }\n\n  return output.trim();\n}\n"],"names":[],"mappings":"AAEM,SAAU,mBAAmB,CAAC,OAA2B,EAAA;IAC7D,IAAI,MAAM,GAAG,EAAE;AAEf,IAAA,MAAM,UAAU,GAAG,CAAC,KAAa,KAAU;AACzC,QAAA,MAAM,IAAI,CAAA,MAAA,EAAS,KAAK,CAAA,MAAA,CAAQ;AAClC,KAAC;;AAGD,IAAA,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,EAAE;AACrC,IAAA,IAAI,OAAO,CAAC,MAAM,EAAE;QAClB,UAAU,CAAC,aAAa,CAAC;QACzB,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,KAAI;AACvB,YAAA,MAAM,IAAI;gBACR,CAAU,OAAA,EAAA,CAAC,GAAG,CAAC,CAAA,EAAA,EAAK,CAAC,CAAC,KAAK,IAAI,YAAY,CAAE,CAAA;gBAC7C,CAAsC,mCAAA,EAAA,CAAC,GAAG,CAAC,CAAE,CAAA;gBAC7C,CAAQ,KAAA,EAAA,CAAC,CAAC,IAAI,CAAE,CAAA;AAChB,gBAAA,CAAC,CAAC,OAAO,IAAI,IAAI,GAAG,CAAY,SAAA,EAAA,CAAC,CAAC,OAAO,CAAA,CAAE,GAAG,EAAE;AAChD,gBAAA,CAAC,CAAC,IAAI,IAAI,IAAI,GAAG,CAAS,MAAA,EAAA,CAAC,CAAC,IAAI,CAAA,CAAE,GAAG,EAAE;AACvC,gBAAA,CAAC,CAAC,WAAW,IAAI,IAAI,GAAG,CAAW,QAAA,EAAA,CAAC,CAAC,WAAW,CAAA,CAAE,GAAG,EAAE;gBACvD,EAAE;gBACF,4BAA4B;AAC5B,gBAAA,GAAG,CAAC,CAAC,CAAC,UAAU,IAAI,EAAE;AACnB,qBAAA,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC;qBACtC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAA,YAAA,EAAe,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA,GAAA,EAAM,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAA,CAAE,CAAC;gBACrE,EAAE;AACH;iBACE,MAAM,CAAC,OAAO;iBACd,IAAI,CAAC,IAAI,CAAC;AACf,SAAC,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAoCJ,IAAA,IAAI,OAAO,CAAC,cAAc,IAAI,IAAI,EAAE;QAClC,UAAU,CAAC,iBAAiB,CAAC;AAC7B,QAAA,MAAM,IAAI;AACR,YAAA,CAAA,OAAA,EAAU,OAAO,CAAC,cAAc,CAAC,KAAK,IAAI,YAAY,CAAE,CAAA;AACxD,YAAA,OAAO,CAAC,cAAc,CAAC,WAAW,IAAI;AACpC,kBAAE,CAAgB,aAAA,EAAA,OAAO,CAAC,cAAc,CAAC,WAAW,CAAE;AACtD,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,cAAc,CAAC,IAAI,IAAI;AAC7B,kBAAE,CAAS,MAAA,EAAA,OAAO,CAAC,cAAc,CAAC,IAAI,CAAE;AACxC,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,cAAc,CAAC,QAAQ,IAAI;AACjC,kBAAE,CAAc,WAAA,EAAA,OAAO,CAAC,cAAc,CAAC,QAAQ,CAAE;AACjD,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,cAAc,CAAC,UAAU,IAAI;AACnC,kBAAE,CAAe,YAAA,EAAA,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,cAAc,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,CAAE;AAC7E,kBAAE,EAAE;YACN,EAAE;AACH;aACE,MAAM,CAAC,OAAO;aACd,IAAI,CAAC,IAAI,CAAC;;;AAIf,IAAA,IAAI,OAAO,CAAC,SAAS,IAAI,IAAI,EAAE;QAC7B,UAAU,CAAC,YAAY,CAAC;AACxB,QAAA,MAAM,IAAI;AACR,YAAA,OAAO,CAAC,SAAS,CAAC,KAAK,IAAI;AACzB,kBAAE,CAAU,OAAA,EAAA,OAAO,CAAC,SAAS,CAAC,KAAK,CAAE;AACrC,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,SAAS,CAAC,MAAM,IAAI;AAC1B,kBAAE,CAAW,QAAA,EAAA,OAAO,CAAC,SAAS,CAAC,MAAM,CAAE;AACvC,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,SAAS,CAAC,OAAO,IAAI;AAC3B,kBAAE,CAAY,SAAA,EAAA,OAAO,CAAC,SAAS,CAAC,OAAO,CAAE;AACzC,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,SAAS,CAAC,IAAI,IAAI,IAAI,GAAG,SAAS,OAAO,CAAC,SAAS,CAAC,IAAI,EAAE,GAAG,EAAE;YACvE,EAAE;AACH;aACE,MAAM,CAAC,OAAO;aACd,IAAI,CAAC,IAAI,CAAC;;;AAIf,IAAA,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,EAAE;AACjD,IAAA,IAAI,aAAa,CAAC,MAAM,EAAE;QACxB,UAAU,CAAC,iBAAiB,CAAC;QAC7B,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,KAAI;AAC9B,YAAA,MAAM,IAAI,CAAC,CAAM,GAAA,EAAA,CAAC,CAAC,QAAQ,CAAA,CAAE,EAAE,CAAA,GAAA,EAAM,CAAC,CAAC,MAAM,CAAE,CAAA,EAAE,EAAE;iBAChD,MAAM,CAAC,OAAO;iBACd,IAAI,CAAC,IAAI,CAAC;AACf,SAAC,CAAC;;AAGJ,IAAA,OAAO,MAAM,CAAC,IAAI,EAAE;AACtB;;;;"}
+{"version":3,"file":"format.mjs","sources":["../../../../src/tools/search/format.ts"],"sourcesContent":["import type * as t from './types';\n\nexport function formatResultsForLLM(results: t.SearchResultData): string {\n  let output = '';\n\n  const addSection = (title: string): void => {\n    output += `\\n=== ${title} ===\\n`;\n  };\n\n  // Organic (web) results\n  const organic = results.organic ?? [];\n  if (organic.length) {\n    addSection('Web Results');\n    organic.forEach((r, i) => {\n      output += [\n        `Source ${i}: ${r.title ?? '(no title)'}`,\n        `Citation Anchor: \\\\ue202turn0search${i}`,\n        `URL: ${r.link}`,\n        r.snippet != null ? `Summary: ${r.snippet}` : '',\n        r.date != null ? `Date: ${r.date}` : '',\n        r.attribution != null ? `Source: ${r.attribution}` : '',\n        '',\n        '--- Content Highlights ---',\n        ...(r.highlights ?? [])\n          .filter((h) => h.text.trim().length > 0)\n          .map((h) => `[Relevance: ${h.score.toFixed(2)}]\\n${h.text.trim()}`),\n        '',\n      ]\n        .filter(Boolean)\n        .join('\\n');\n    });\n  }\n\n  // Ignoring these sections for now\n  // // Top stories (news)\n  // const topStores = results.topStories ?? [];\n  // if (topStores.length) {\n  //   addSection('News Results');\n  //   topStores.forEach((r, i) => {\n  //     output += [\n  //       `Anchor: \\ue202turn0news${i}`,\n  //       `Title: ${r.title ?? '(no title)'}`,\n  //       `URL: ${r.link}`,\n  //       r.snippet != null ? `Snippet: ${r.snippet}` : '',\n  //       r.date != null ? `Date: ${r.date}` : '',\n  //       r.attribution != null ? `Source: ${r.attribution}` : '',\n  //       ''\n  //     ].filter(Boolean).join('\\n');\n  //   });\n  // }\n\n  // // Images\n  // const images = results.images ?? [];\n  // if (images.length) {\n  //   addSection('Image Results');\n  //   images.forEach((img, i) => {\n  //     output += [\n  //       `Anchor: \\ue202turn0image${i}`,\n  //       `Title: ${img.title ?? '(no title)'}`,\n  //       `Image URL: ${img.imageUrl}`,\n  //       ''\n  //     ].join('\\n');\n  //   });\n  // }\n\n  // Knowledge Graph\n  if (results.knowledgeGraph != null) {\n    addSection('Knowledge Graph');\n    output += [\n      `Title: ${results.knowledgeGraph.title ?? '(no title)'}`,\n      results.knowledgeGraph.description != null\n        ? `Description: ${results.knowledgeGraph.description}`\n        : '',\n      results.knowledgeGraph.type != null\n        ? `Type: ${results.knowledgeGraph.type}`\n        : '',\n      results.knowledgeGraph.imageUrl != null\n        ? `Image URL: ${results.knowledgeGraph.imageUrl}`\n        : '',\n      results.knowledgeGraph.attributes != null\n        ? `Attributes: ${JSON.stringify(results.knowledgeGraph.attributes, null, 2)}`\n        : '',\n      '',\n    ]\n      .filter(Boolean)\n      .join('\\n');\n  }\n\n  // Answer Box\n  if (results.answerBox != null) {\n    addSection('Answer Box');\n    output += [\n      results.answerBox.title != null\n        ? `Title: ${results.answerBox.title}`\n        : '',\n      results.answerBox.answer != null\n        ? `Answer: ${results.answerBox.answer}`\n        : '',\n      results.answerBox.snippet != null\n        ? `Snippet: ${results.answerBox.snippet}`\n        : '',\n      results.answerBox.date != null ? `Date: ${results.answerBox.date}` : '',\n      '',\n    ]\n      .filter(Boolean)\n      .join('\\n');\n  }\n\n  // People also ask\n  const peopleAlsoAsk = results.peopleAlsoAsk ?? [];\n  if (peopleAlsoAsk.length) {\n    addSection('People Also Ask');\n    peopleAlsoAsk.forEach((p, _i) => {\n      output += [`Q: ${p.question}`, `A: ${p.answer}`, '']\n        .filter(Boolean)\n        .join('\\n');\n    });\n  }\n\n  return output.trim();\n}\n"],"names":[],"mappings":"AAEM,SAAU,mBAAmB,CAAC,OAA2B,EAAA;IAC7D,IAAI,MAAM,GAAG,EAAE;AAEf,IAAA,MAAM,UAAU,GAAG,CAAC,KAAa,KAAU;AACzC,QAAA,MAAM,IAAI,CAAA,MAAA,EAAS,KAAK,CAAA,MAAA,CAAQ;AAClC,KAAC;;AAGD,IAAA,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,EAAE;AACrC,IAAA,IAAI,OAAO,CAAC,MAAM,EAAE;QAClB,UAAU,CAAC,aAAa,CAAC;QACzB,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,KAAI;AACvB,YAAA,MAAM,IAAI;AACR,gBAAA,CAAA,OAAA,EAAU,CAAC,CAAK,EAAA,EAAA,CAAC,CAAC,KAAK,IAAI,YAAY,CAAE,CAAA;AACzC,gBAAA,CAAA,mCAAA,EAAsC,CAAC,CAAE,CAAA;gBACzC,CAAQ,KAAA,EAAA,CAAC,CAAC,IAAI,CAAE,CAAA;AAChB,gBAAA,CAAC,CAAC,OAAO,IAAI,IAAI,GAAG,CAAY,SAAA,EAAA,CAAC,CAAC,OAAO,CAAA,CAAE,GAAG,EAAE;AAChD,gBAAA,CAAC,CAAC,IAAI,IAAI,IAAI,GAAG,CAAS,MAAA,EAAA,CAAC,CAAC,IAAI,CAAA,CAAE,GAAG,EAAE;AACvC,gBAAA,CAAC,CAAC,WAAW,IAAI,IAAI,GAAG,CAAW,QAAA,EAAA,CAAC,CAAC,WAAW,CAAA,CAAE,GAAG,EAAE;gBACvD,EAAE;gBACF,4BAA4B;AAC5B,gBAAA,GAAG,CAAC,CAAC,CAAC,UAAU,IAAI,EAAE;AACnB,qBAAA,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC;qBACtC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAA,YAAA,EAAe,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA,GAAA,EAAM,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAA,CAAE,CAAC;gBACrE,EAAE;AACH;iBACE,MAAM,CAAC,OAAO;iBACd,IAAI,CAAC,IAAI,CAAC;AACf,SAAC,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAoCJ,IAAA,IAAI,OAAO,CAAC,cAAc,IAAI,IAAI,EAAE;QAClC,UAAU,CAAC,iBAAiB,CAAC;AAC7B,QAAA,MAAM,IAAI;AACR,YAAA,CAAA,OAAA,EAAU,OAAO,CAAC,cAAc,CAAC,KAAK,IAAI,YAAY,CAAE,CAAA;AACxD,YAAA,OAAO,CAAC,cAAc,CAAC,WAAW,IAAI;AACpC,kBAAE,CAAgB,aAAA,EAAA,OAAO,CAAC,cAAc,CAAC,WAAW,CAAE;AACtD,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,cAAc,CAAC,IAAI,IAAI;AAC7B,kBAAE,CAAS,MAAA,EAAA,OAAO,CAAC,cAAc,CAAC,IAAI,CAAE;AACxC,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,cAAc,CAAC,QAAQ,IAAI;AACjC,kBAAE,CAAc,WAAA,EAAA,OAAO,CAAC,cAAc,CAAC,QAAQ,CAAE;AACjD,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,cAAc,CAAC,UAAU,IAAI;AACnC,kBAAE,CAAe,YAAA,EAAA,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,cAAc,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,CAAE;AAC7E,kBAAE,EAAE;YACN,EAAE;AACH;aACE,MAAM,CAAC,OAAO;aACd,IAAI,CAAC,IAAI,CAAC;;;AAIf,IAAA,IAAI,OAAO,CAAC,SAAS,IAAI,IAAI,EAAE;QAC7B,UAAU,CAAC,YAAY,CAAC;AACxB,QAAA,MAAM,IAAI;AACR,YAAA,OAAO,CAAC,SAAS,CAAC,KAAK,IAAI;AACzB,kBAAE,CAAU,OAAA,EAAA,OAAO,CAAC,SAAS,CAAC,KAAK,CAAE;AACrC,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,SAAS,CAAC,MAAM,IAAI;AAC1B,kBAAE,CAAW,QAAA,EAAA,OAAO,CAAC,SAAS,CAAC,MAAM,CAAE;AACvC,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,SAAS,CAAC,OAAO,IAAI;AAC3B,kBAAE,CAAY,SAAA,EAAA,OAAO,CAAC,SAAS,CAAC,OAAO,CAAE;AACzC,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,SAAS,CAAC,IAAI,IAAI,IAAI,GAAG,SAAS,OAAO,CAAC,SAAS,CAAC,IAAI,EAAE,GAAG,EAAE;YACvE,EAAE;AACH;aACE,MAAM,CAAC,OAAO;aACd,IAAI,CAAC,IAAI,CAAC;;;AAIf,IAAA,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,EAAE;AACjD,IAAA,IAAI,aAAa,CAAC,MAAM,EAAE;QACxB,UAAU,CAAC,iBAAiB,CAAC;QAC7B,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,KAAI;AAC9B,YAAA,MAAM,IAAI,CAAC,CAAM,GAAA,EAAA,CAAC,CAAC,QAAQ,CAAA,CAAE,EAAE,CAAA,GAAA,EAAM,CAAC,CAAC,MAAM,CAAE,CAAA,EAAE,EAAE;iBAChD,MAAM,CAAC,OAAO;iBACd,IAAI,CAAC,IAAI,CAAC;AACf,SAAC,CAAC;;AAGJ,IAAA,OAAO,MAAM,CAAC,IAAI,EAAE;AACtB;;;;"}

package/dist/esm/tools/search/highlights.mjs

@@ -180,6 +180,7 @@ function expandHighlights(searchResults, mainExpandBy = 300, separatorExpandBy =
                     score: highlight.score,
                 });
             }
+            delete resultCopy.content;
             resultCopy.highlights = highlights;
             return resultCopy;
         });

package/dist/esm/tools/search/highlights.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"highlights.mjs","sources":["../../../../src/tools/search/highlights.ts"],"sourcesContent":["import type * as t from './types';\n\n// 2. Pre-compile all regular expressions (only do this once)\n// Group patterns by priority for early returns\nconst priorityPatterns = [\n  // High priority patterns (structural)\n  [\n    { regex: /\\n\\n/g }, // Double newline (paragraph break)\n    { regex: /\\n/g }, // Single newline\n    { regex: /={3,}\\s*\\n|-{3,}\\s*\\n/g }, // Section separators\n  ],\n  // Medium priority (semantic)\n  [\n    { regex: /[.!?][\")\\]]?\\s/g }, // End of sentence\n    { regex: /;\\s/g }, // Semicolon\n    { regex: /:\\s/g }, // Colon\n  ],\n  // Low priority (any breaks)\n  [\n    { regex: /,\\s/g }, // Comma\n    { regex: /\\s-\\s/g }, // Dash surrounded by spaces\n    { regex: /\\s/g }, // Any space\n  ],\n];\n\nfunction findFirstMatch(text: string, regex: RegExp): number {\n  // Reset regex\n  regex.lastIndex = 0;\n\n  // For very long texts, try chunking\n  if (text.length > 10000) {\n    const chunkSize = 2000;\n    let position = 0;\n\n    while (position < text.length) {\n      const chunk = text.substring(position, position + chunkSize);\n      regex.lastIndex = 0;\n\n      const match = regex.exec(chunk);\n      if (match) {\n        return position + match.index;\n      }\n\n      // Move to next chunk with some overlap\n      position += chunkSize - 100;\n      if (position >= text.length) break;\n    }\n    return -1;\n  }\n\n  // For shorter texts, normal regex search\n  const match = regex.exec(text);\n  return match ? match.index : -1;\n}\n\n// 3. Optimized boundary finding functions\nfunction findLastMatch(text: string, regex: RegExp): number {\n  // Reset regex state\n  regex.lastIndex = 0;\n\n  let lastIndex = -1;\n  let lastLength = 0;\n  let match;\n\n  // For very long texts, use a different approach to avoid regex engine slowdowns\n  if (text.length > 10000) {\n    // Try dividing the text into chunks for faster processing\n    const chunkSize = 2000;\n    let startPosition = Math.max(0, text.length - chunkSize);\n\n    while (startPosition >= 0) {\n      const chunk = text.substring(startPosition, startPosition + chunkSize);\n      regex.lastIndex = 0;\n\n      let chunkLastIndex = -1;\n      let chunkLastLength = 0;\n\n      while ((match = regex.exec(chunk)) !== null) {\n        chunkLastIndex = match.index;\n        chunkLastLength = match[0].length;\n      }\n\n      if (chunkLastIndex !== -1) {\n        return startPosition + chunkLastIndex + chunkLastLength;\n      }\n\n      // Move to previous chunk with some overlap\n      startPosition = Math.max(0, startPosition - chunkSize + 100) - 1;\n      if (startPosition <= 0) break;\n    }\n    return -1;\n  }\n\n  // For shorter texts, normal regex search\n  while ((match = regex.exec(text)) !== null) {\n    lastIndex = match.index;\n    lastLength = match[0].length;\n  }\n\n  return lastIndex === -1 ? -1 : lastIndex + lastLength;\n}\n\n// 4. Find the best boundary with priority groups\nfunction findBestBoundary(text: string, direction = 'backward'): number {\n  if (!text || text.length === 0) return 0;\n\n  // Try each priority group\n  for (const patternGroup of priorityPatterns) {\n    for (const pattern of patternGroup) {\n      const position =\n        direction === 'backward'\n          ? findLastMatch(text, pattern.regex)\n          : findFirstMatch(text, pattern.regex);\n\n      if (position !== -1) {\n        return position;\n      }\n    }\n  }\n\n  // No match found, use character boundary\n  return direction === 'backward' ? text.length : 0;\n}\n\n/**\n * Expand highlights in search results using smart boundary detection.\n *\n * This implementation finds natural text boundaries like paragraphs, sentences,\n * and phrases to provide context while maintaining readability.\n *\n * @param searchResults - Search results object\n * @param mainExpandBy - Primary expansion size on each side (default: 300)\n * @param separatorExpandBy - Additional range to look for separators (default: 150)\n * @returns Copy of search results with expanded highlights\n */\nexport function expandHighlights(\n  searchResults: t.SearchResultData,\n  mainExpandBy = 300,\n  separatorExpandBy = 150\n): t.SearchResultData {\n  // 1. Avoid full deep copy - only copy what we modify\n  const resultCopy = { ...searchResults };\n\n  // Only deep copy the relevant arrays\n  if (resultCopy.organic) {\n    resultCopy.organic = [...resultCopy.organic];\n  }\n  if (resultCopy.topStories) {\n    resultCopy.topStories = [...resultCopy.topStories];\n  }\n\n  // 5. Process the results efficiently\n  const processResultTypes = ['organic', 'topStories'] as const;\n\n  for (const resultType of processResultTypes) {\n    if (!resultCopy[resultType as 'organic' | 'topStories']) continue;\n\n    // Map results to new array with modified highlights\n    resultCopy[resultType] = resultCopy[resultType]?.map((result) => {\n      if (\n        result.content == null ||\n        result.content === '' ||\n        !result.highlights ||\n        result.highlights.length === 0\n      ) {\n        return result; // No modification needed\n      }\n\n      // Create a shallow copy with expanded highlights\n      const resultCopy = { ...result };\n      const content = result.content;\n      const highlights = [];\n\n      // Process each highlight\n      for (const highlight of result.highlights) {\n        const highlightText = highlight.text;\n\n        let startPos = content.indexOf(highlightText);\n        let highlightLen = highlightText.length;\n\n        if (startPos === -1) {\n          // Try with stripped whitespace\n          const strippedHighlight = highlightText.trim();\n          startPos = content.indexOf(strippedHighlight);\n\n          if (startPos === -1) {\n            highlights.push({\n              text: highlight.text,\n              score: highlight.score,\n            });\n            continue;\n          }\n          highlightLen = strippedHighlight.length;\n        }\n\n        // Calculate boundaries\n        const mainStart = Math.max(0, startPos - mainExpandBy);\n        const mainEnd = Math.min(\n          content.length,\n          startPos + highlightLen + mainExpandBy\n        );\n\n        const separatorStart = Math.max(0, mainStart - separatorExpandBy);\n        const separatorEnd = Math.min(\n          content.length,\n          mainEnd + separatorExpandBy\n        );\n\n        // Extract text segments\n        const headText = content.substring(separatorStart, mainStart);\n        const tailText = content.substring(mainEnd, separatorEnd);\n\n        // Find natural boundaries\n        const bestHeadBoundary = findBestBoundary(headText, 'backward');\n        const bestTailBoundary = findBestBoundary(tailText, 'forward');\n\n        // Calculate final positions\n        const finalStart = separatorStart + bestHeadBoundary;\n        const finalEnd = mainEnd + bestTailBoundary;\n\n        // Extract the expanded highlight\n        const expandedHighlightText = content\n          .substring(finalStart, finalEnd)\n          .trim();\n        highlights.push({\n          text: expandedHighlightText,\n          score: highlight.score,\n        });\n      }\n\n      resultCopy.highlights = highlights;\n      return resultCopy;\n    });\n  }\n\n  return resultCopy;\n}\n"],"names":[],"mappings":"AAEA;AACA;AACA,MAAM,gBAAgB,GAAG;;AAEvB,IAAA;AACE,QAAA,EAAE,KAAK,EAAE,OAAO,EAAE;AAClB,QAAA,EAAE,KAAK,EAAE,KAAK,EAAE;AAChB,QAAA,EAAE,KAAK,EAAE,wBAAwB,EAAE;AACpC,KAAA;;AAED,IAAA;AACE,QAAA,EAAE,KAAK,EAAE,iBAAiB,EAAE;AAC5B,QAAA,EAAE,KAAK,EAAE,MAAM,EAAE;AACjB,QAAA,EAAE,KAAK,EAAE,MAAM,EAAE;AAClB,KAAA;;AAED,IAAA;AACE,QAAA,EAAE,KAAK,EAAE,MAAM,EAAE;AACjB,QAAA,EAAE,KAAK,EAAE,QAAQ,EAAE;AACnB,QAAA,EAAE,KAAK,EAAE,KAAK,EAAE;AACjB,KAAA;CACF;AAED,SAAS,cAAc,CAAC,IAAY,EAAE,KAAa,EAAA;;AAEjD,IAAA,KAAK,CAAC,SAAS,GAAG,CAAC;;AAGnB,IAAA,IAAI,IAAI,CAAC,MAAM,GAAG,KAAK,EAAE;QACvB,MAAM,SAAS,GAAG,IAAI;QACtB,IAAI,QAAQ,GAAG,CAAC;AAEhB,QAAA,OAAO,QAAQ,GAAG,IAAI,CAAC,MAAM,EAAE;AAC7B,YAAA,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,QAAQ,GAAG,SAAS,CAAC;AAC5D,YAAA,KAAK,CAAC,SAAS,GAAG,CAAC;YAEnB,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC;YAC/B,IAAI,KAAK,EAAE;AACT,gBAAA,OAAO,QAAQ,GAAG,KAAK,CAAC,KAAK;;;AAI/B,YAAA,QAAQ,IAAI,SAAS,GAAG,GAAG;AAC3B,YAAA,IAAI,QAAQ,IAAI,IAAI,CAAC,MAAM;gBAAE;;QAE/B,OAAO,EAAE;;;IAIX,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC;AAC9B,IAAA,OAAO,KAAK,GAAG,KAAK,CAAC,KAAK,GAAG,EAAE;AACjC;AAEA;AACA,SAAS,aAAa,CAAC,IAAY,EAAE,KAAa,EAAA;;AAEhD,IAAA,KAAK,CAAC,SAAS,GAAG,CAAC;AAEnB,IAAA,IAAI,SAAS,GAAG,EAAE;IAClB,IAAI,UAAU,GAAG,CAAC;AAClB,IAAA,IAAI,KAAK;;AAGT,IAAA,IAAI,IAAI,CAAC,MAAM,GAAG,KAAK,EAAE;;QAEvB,MAAM,SAAS,GAAG,IAAI;AACtB,QAAA,IAAI,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,GAAG,SAAS,CAAC;AAExD,QAAA,OAAO,aAAa,IAAI,CAAC,EAAE;AACzB,YAAA,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,aAAa,EAAE,aAAa,GAAG,SAAS,CAAC;AACtE,YAAA,KAAK,CAAC,SAAS,GAAG,CAAC;AAEnB,YAAA,IAAI,cAAc,GAAG,EAAE;YACvB,IAAI,eAAe,GAAG,CAAC;AAEvB,YAAA,OAAO,CAAC,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,EAAE;AAC3C,gBAAA,cAAc,GAAG,KAAK,CAAC,KAAK;AAC5B,gBAAA,eAAe,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM;;AAGnC,YAAA,IAAI,cAAc,KAAK,EAAE,EAAE;AACzB,gBAAA,OAAO,aAAa,GAAG,cAAc,GAAG,eAAe;;;AAIzD,YAAA,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,aAAa,GAAG,SAAS,GAAG,GAAG,CAAC,GAAG,CAAC;YAChE,IAAI,aAAa,IAAI,CAAC;gBAAE;;QAE1B,OAAO,EAAE;;;AAIX,IAAA,OAAO,CAAC,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE;AAC1C,QAAA,SAAS,GAAG,KAAK,CAAC,KAAK;AACvB,QAAA,UAAU,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM;;AAG9B,IAAA,OAAO,SAAS,KAAK,EAAE,GAAG,EAAE,GAAG,SAAS,GAAG,UAAU;AACvD;AAEA;AACA,SAAS,gBAAgB,CAAC,IAAY,EAAE,SAAS,GAAG,UAAU,EAAA;AAC5D,IAAA,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;AAAE,QAAA,OAAO,CAAC;;AAGxC,IAAA,KAAK,MAAM,YAAY,IAAI,gBAAgB,EAAE;AAC3C,QAAA,KAAK,MAAM,OAAO,IAAI,YAAY,EAAE;AAClC,YAAA,MAAM,QAAQ,GACZ,SAAS,KAAK;kBACV,aAAa,CAAC,IAAI,EAAE,OAAO,CAAC,KAAK;kBACjC,cAAc,CAAC,IAAI,EAAE,OAAO,CAAC,KAAK,CAAC;AAEzC,YAAA,IAAI,QAAQ,KAAK,EAAE,EAAE;AACnB,gBAAA,OAAO,QAAQ;;;;;AAMrB,IAAA,OAAO,SAAS,KAAK,UAAU,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC;AACnD;AAEA;;;;;;;;;;AAUG;AACG,SAAU,gBAAgB,CAC9B,aAAiC,EACjC,YAAY,GAAG,GAAG,EAClB,iBAAiB,GAAG,GAAG,EAAA;;AAGvB,IAAA,MAAM,UAAU,GAAG,EAAE,GAAG,aAAa,EAAE;;AAGvC,IAAA,IAAI,UAAU,CAAC,OAAO,EAAE;QACtB,UAAU,CAAC,OAAO,GAAG,CAAC,GAAG,UAAU,CAAC,OAAO,CAAC;;AAE9C,IAAA,IAAI,UAAU,CAAC,UAAU,EAAE;QACzB,UAAU,CAAC,UAAU,GAAG,CAAC,GAAG,UAAU,CAAC,UAAU,CAAC;;;AAIpD,IAAA,MAAM,kBAAkB,GAAG,CAAC,SAAS,EAAE,YAAY,CAAU;AAE7D,IAAA,KAAK,MAAM,UAAU,IAAI,kBAAkB,EAAE;AAC3C,QAAA,IAAI,CAAC,UAAU,CAAC,UAAsC,CAAC;YAAE;;AAGzD,QAAA,UAAU,CAAC,UAAU,CAAC,GAAG,UAAU,CAAC,UAAU,CAAC,EAAE,GAAG,CAAC,CAAC,MAAM,KAAI;AAC9D,YAAA,IACE,MAAM,CAAC,OAAO,IAAI,IAAI;gBACtB,MAAM,CAAC,OAAO,KAAK,EAAE;gBACrB,CAAC,MAAM,CAAC,UAAU;AAClB,gBAAA,MAAM,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC,EAC9B;gBACA,OAAO,MAAM,CAAC;;;AAIhB,YAAA,MAAM,UAAU,GAAG,EAAE,GAAG,MAAM,EAAE;AAChC,YAAA,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO;YAC9B,MAAM,UAAU,GAAG,EAAE;;AAGrB,YAAA,KAAK,MAAM,SAAS,IAAI,MAAM,CAAC,UAAU,EAAE;AACzC,gBAAA,MAAM,aAAa,GAAG,SAAS,CAAC,IAAI;gBAEpC,IAAI,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,aAAa,CAAC;AAC7C,gBAAA,IAAI,YAAY,GAAG,aAAa,CAAC,MAAM;AAEvC,gBAAA,IAAI,QAAQ,KAAK,EAAE,EAAE;;AAEnB,oBAAA,MAAM,iBAAiB,GAAG,aAAa,CAAC,IAAI,EAAE;AAC9C,oBAAA,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,iBAAiB,CAAC;AAE7C,oBAAA,IAAI,QAAQ,KAAK,EAAE,EAAE;wBACnB,UAAU,CAAC,IAAI,CAAC;4BACd,IAAI,EAAE,SAAS,CAAC,IAAI;4BACpB,KAAK,EAAE,SAAS,CAAC,KAAK;AACvB,yBAAA,CAAC;wBACF;;AAEF,oBAAA,YAAY,GAAG,iBAAiB,CAAC,MAAM;;;AAIzC,gBAAA,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,GAAG,YAAY,CAAC;AACtD,gBAAA,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CACtB,OAAO,CAAC,MAAM,EACd,QAAQ,GAAG,YAAY,GAAG,YAAY,CACvC;AAED,gBAAA,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,GAAG,iBAAiB,CAAC;AACjE,gBAAA,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAC3B,OAAO,CAAC,MAAM,EACd,OAAO,GAAG,iBAAiB,CAC5B;;gBAGD,MAAM,QAAQ,GAAG,OAAO,CAAC,SAAS,CAAC,cAAc,EAAE,SAAS,CAAC;gBAC7D,MAAM,QAAQ,GAAG,OAAO,CAAC,SAAS,CAAC,OAAO,EAAE,YAAY,CAAC;;gBAGzD,MAAM,gBAAgB,GAAG,gBAAgB,CAAC,QAAQ,EAAE,UAAU,CAAC;gBAC/D,MAAM,gBAAgB,GAAG,gBAAgB,CAAC,QAAQ,EAAE,SAAS,CAAC;;AAG9D,gBAAA,MAAM,UAAU,GAAG,cAAc,GAAG,gBAAgB;AACpD,gBAAA,MAAM,QAAQ,GAAG,OAAO,GAAG,gBAAgB;;gBAG3C,MAAM,qBAAqB,GAAG;AAC3B,qBAAA,SAAS,CAAC,UAAU,EAAE,QAAQ;AAC9B,qBAAA,IAAI,EAAE;gBACT,UAAU,CAAC,IAAI,CAAC;AACd,oBAAA,IAAI,EAAE,qBAAqB;oBAC3B,KAAK,EAAE,SAAS,CAAC,KAAK;AACvB,iBAAA,CAAC;;AAGJ,YAAA,UAAU,CAAC,UAAU,GAAG,UAAU;AAClC,YAAA,OAAO,UAAU;AACnB,SAAC,CAAC;;AAGJ,IAAA,OAAO,UAAU;AACnB;;;;"}
+{"version":3,"file":"highlights.mjs","sources":["../../../../src/tools/search/highlights.ts"],"sourcesContent":["import type * as t from './types';\n\n// 2. Pre-compile all regular expressions (only do this once)\n// Group patterns by priority for early returns\nconst priorityPatterns = [\n  // High priority patterns (structural)\n  [\n    { regex: /\\n\\n/g }, // Double newline (paragraph break)\n    { regex: /\\n/g }, // Single newline\n    { regex: /={3,}\\s*\\n|-{3,}\\s*\\n/g }, // Section separators\n  ],\n  // Medium priority (semantic)\n  [\n    { regex: /[.!?][\")\\]]?\\s/g }, // End of sentence\n    { regex: /;\\s/g }, // Semicolon\n    { regex: /:\\s/g }, // Colon\n  ],\n  // Low priority (any breaks)\n  [\n    { regex: /,\\s/g }, // Comma\n    { regex: /\\s-\\s/g }, // Dash surrounded by spaces\n    { regex: /\\s/g }, // Any space\n  ],\n];\n\nfunction findFirstMatch(text: string, regex: RegExp): number {\n  // Reset regex\n  regex.lastIndex = 0;\n\n  // For very long texts, try chunking\n  if (text.length > 10000) {\n    const chunkSize = 2000;\n    let position = 0;\n\n    while (position < text.length) {\n      const chunk = text.substring(position, position + chunkSize);\n      regex.lastIndex = 0;\n\n      const match = regex.exec(chunk);\n      if (match) {\n        return position + match.index;\n      }\n\n      // Move to next chunk with some overlap\n      position += chunkSize - 100;\n      if (position >= text.length) break;\n    }\n    return -1;\n  }\n\n  // For shorter texts, normal regex search\n  const match = regex.exec(text);\n  return match ? match.index : -1;\n}\n\n// 3. Optimized boundary finding functions\nfunction findLastMatch(text: string, regex: RegExp): number {\n  // Reset regex state\n  regex.lastIndex = 0;\n\n  let lastIndex = -1;\n  let lastLength = 0;\n  let match;\n\n  // For very long texts, use a different approach to avoid regex engine slowdowns\n  if (text.length > 10000) {\n    // Try dividing the text into chunks for faster processing\n    const chunkSize = 2000;\n    let startPosition = Math.max(0, text.length - chunkSize);\n\n    while (startPosition >= 0) {\n      const chunk = text.substring(startPosition, startPosition + chunkSize);\n      regex.lastIndex = 0;\n\n      let chunkLastIndex = -1;\n      let chunkLastLength = 0;\n\n      while ((match = regex.exec(chunk)) !== null) {\n        chunkLastIndex = match.index;\n        chunkLastLength = match[0].length;\n      }\n\n      if (chunkLastIndex !== -1) {\n        return startPosition + chunkLastIndex + chunkLastLength;\n      }\n\n      // Move to previous chunk with some overlap\n      startPosition = Math.max(0, startPosition - chunkSize + 100) - 1;\n      if (startPosition <= 0) break;\n    }\n    return -1;\n  }\n\n  // For shorter texts, normal regex search\n  while ((match = regex.exec(text)) !== null) {\n    lastIndex = match.index;\n    lastLength = match[0].length;\n  }\n\n  return lastIndex === -1 ? -1 : lastIndex + lastLength;\n}\n\n// 4. Find the best boundary with priority groups\nfunction findBestBoundary(text: string, direction = 'backward'): number {\n  if (!text || text.length === 0) return 0;\n\n  // Try each priority group\n  for (const patternGroup of priorityPatterns) {\n    for (const pattern of patternGroup) {\n      const position =\n        direction === 'backward'\n          ? findLastMatch(text, pattern.regex)\n          : findFirstMatch(text, pattern.regex);\n\n      if (position !== -1) {\n        return position;\n      }\n    }\n  }\n\n  // No match found, use character boundary\n  return direction === 'backward' ? text.length : 0;\n}\n\n/**\n * Expand highlights in search results using smart boundary detection.\n *\n * This implementation finds natural text boundaries like paragraphs, sentences,\n * and phrases to provide context while maintaining readability.\n *\n * @param searchResults - Search results object\n * @param mainExpandBy - Primary expansion size on each side (default: 300)\n * @param separatorExpandBy - Additional range to look for separators (default: 150)\n * @returns Copy of search results with expanded highlights\n */\nexport function expandHighlights(\n  searchResults: t.SearchResultData,\n  mainExpandBy = 300,\n  separatorExpandBy = 150\n): t.SearchResultData {\n  // 1. Avoid full deep copy - only copy what we modify\n  const resultCopy = { ...searchResults };\n\n  // Only deep copy the relevant arrays\n  if (resultCopy.organic) {\n    resultCopy.organic = [...resultCopy.organic];\n  }\n  if (resultCopy.topStories) {\n    resultCopy.topStories = [...resultCopy.topStories];\n  }\n\n  // 5. Process the results efficiently\n  const processResultTypes = ['organic', 'topStories'] as const;\n\n  for (const resultType of processResultTypes) {\n    if (!resultCopy[resultType as 'organic' | 'topStories']) continue;\n\n    // Map results to new array with modified highlights\n    resultCopy[resultType] = resultCopy[resultType]?.map((result) => {\n      if (\n        result.content == null ||\n        result.content === '' ||\n        !result.highlights ||\n        result.highlights.length === 0\n      ) {\n        return result; // No modification needed\n      }\n\n      // Create a shallow copy with expanded highlights\n      const resultCopy = { ...result };\n      const content = result.content;\n      const highlights = [];\n\n      // Process each highlight\n      for (const highlight of result.highlights) {\n        const highlightText = highlight.text;\n\n        let startPos = content.indexOf(highlightText);\n        let highlightLen = highlightText.length;\n\n        if (startPos === -1) {\n          // Try with stripped whitespace\n          const strippedHighlight = highlightText.trim();\n          startPos = content.indexOf(strippedHighlight);\n\n          if (startPos === -1) {\n            highlights.push({\n              text: highlight.text,\n              score: highlight.score,\n            });\n            continue;\n          }\n          highlightLen = strippedHighlight.length;\n        }\n\n        // Calculate boundaries\n        const mainStart = Math.max(0, startPos - mainExpandBy);\n        const mainEnd = Math.min(\n          content.length,\n          startPos + highlightLen + mainExpandBy\n        );\n\n        const separatorStart = Math.max(0, mainStart - separatorExpandBy);\n        const separatorEnd = Math.min(\n          content.length,\n          mainEnd + separatorExpandBy\n        );\n\n        // Extract text segments\n        const headText = content.substring(separatorStart, mainStart);\n        const tailText = content.substring(mainEnd, separatorEnd);\n\n        // Find natural boundaries\n        const bestHeadBoundary = findBestBoundary(headText, 'backward');\n        const bestTailBoundary = findBestBoundary(tailText, 'forward');\n\n        // Calculate final positions\n        const finalStart = separatorStart + bestHeadBoundary;\n        const finalEnd = mainEnd + bestTailBoundary;\n\n        // Extract the expanded highlight\n        const expandedHighlightText = content\n          .substring(finalStart, finalEnd)\n          .trim();\n        highlights.push({\n          text: expandedHighlightText,\n          score: highlight.score,\n        });\n      }\n\n      delete resultCopy.content;\n      resultCopy.highlights = highlights;\n      return resultCopy;\n    });\n  }\n\n  return resultCopy;\n}\n"],"names":[],"mappings":"AAEA;AACA;AACA,MAAM,gBAAgB,GAAG;;AAEvB,IAAA;AACE,QAAA,EAAE,KAAK,EAAE,OAAO,EAAE;AAClB,QAAA,EAAE,KAAK,EAAE,KAAK,EAAE;AAChB,QAAA,EAAE,KAAK,EAAE,wBAAwB,EAAE;AACpC,KAAA;;AAED,IAAA;AACE,QAAA,EAAE,KAAK,EAAE,iBAAiB,EAAE;AAC5B,QAAA,EAAE,KAAK,EAAE,MAAM,EAAE;AACjB,QAAA,EAAE,KAAK,EAAE,MAAM,EAAE;AAClB,KAAA;;AAED,IAAA;AACE,QAAA,EAAE,KAAK,EAAE,MAAM,EAAE;AACjB,QAAA,EAAE,KAAK,EAAE,QAAQ,EAAE;AACnB,QAAA,EAAE,KAAK,EAAE,KAAK,EAAE;AACjB,KAAA;CACF;AAED,SAAS,cAAc,CAAC,IAAY,EAAE,KAAa,EAAA;;AAEjD,IAAA,KAAK,CAAC,SAAS,GAAG,CAAC;;AAGnB,IAAA,IAAI,IAAI,CAAC,MAAM,GAAG,KAAK,EAAE;QACvB,MAAM,SAAS,GAAG,IAAI;QACtB,IAAI,QAAQ,GAAG,CAAC;AAEhB,QAAA,OAAO,QAAQ,GAAG,IAAI,CAAC,MAAM,EAAE;AAC7B,YAAA,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,QAAQ,GAAG,SAAS,CAAC;AAC5D,YAAA,KAAK,CAAC,SAAS,GAAG,CAAC;YAEnB,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC;YAC/B,IAAI,KAAK,EAAE;AACT,gBAAA,OAAO,QAAQ,GAAG,KAAK,CAAC,KAAK;;;AAI/B,YAAA,QAAQ,IAAI,SAAS,GAAG,GAAG;AAC3B,YAAA,IAAI,QAAQ,IAAI,IAAI,CAAC,MAAM;gBAAE;;QAE/B,OAAO,EAAE;;;IAIX,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC;AAC9B,IAAA,OAAO,KAAK,GAAG,KAAK,CAAC,KAAK,GAAG,EAAE;AACjC;AAEA;AACA,SAAS,aAAa,CAAC,IAAY,EAAE,KAAa,EAAA;;AAEhD,IAAA,KAAK,CAAC,SAAS,GAAG,CAAC;AAEnB,IAAA,IAAI,SAAS,GAAG,EAAE;IAClB,IAAI,UAAU,GAAG,CAAC;AAClB,IAAA,IAAI,KAAK;;AAGT,IAAA,IAAI,IAAI,CAAC,MAAM,GAAG,KAAK,EAAE;;QAEvB,MAAM,SAAS,GAAG,IAAI;AACtB,QAAA,IAAI,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,GAAG,SAAS,CAAC;AAExD,QAAA,OAAO,aAAa,IAAI,CAAC,EAAE;AACzB,YAAA,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,aAAa,EAAE,aAAa,GAAG,SAAS,CAAC;AACtE,YAAA,KAAK,CAAC,SAAS,GAAG,CAAC;AAEnB,YAAA,IAAI,cAAc,GAAG,EAAE;YACvB,IAAI,eAAe,GAAG,CAAC;AAEvB,YAAA,OAAO,CAAC,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,EAAE;AAC3C,gBAAA,cAAc,GAAG,KAAK,CAAC,KAAK;AAC5B,gBAAA,eAAe,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM;;AAGnC,YAAA,IAAI,cAAc,KAAK,EAAE,EAAE;AACzB,gBAAA,OAAO,aAAa,GAAG,cAAc,GAAG,eAAe;;;AAIzD,YAAA,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,aAAa,GAAG,SAAS,GAAG,GAAG,CAAC,GAAG,CAAC;YAChE,IAAI,aAAa,IAAI,CAAC;gBAAE;;QAE1B,OAAO,EAAE;;;AAIX,IAAA,OAAO,CAAC,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE;AAC1C,QAAA,SAAS,GAAG,KAAK,CAAC,KAAK;AACvB,QAAA,UAAU,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM;;AAG9B,IAAA,OAAO,SAAS,KAAK,EAAE,GAAG,EAAE,GAAG,SAAS,GAAG,UAAU;AACvD;AAEA;AACA,SAAS,gBAAgB,CAAC,IAAY,EAAE,SAAS,GAAG,UAAU,EAAA;AAC5D,IAAA,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;AAAE,QAAA,OAAO,CAAC;;AAGxC,IAAA,KAAK,MAAM,YAAY,IAAI,gBAAgB,EAAE;AAC3C,QAAA,KAAK,MAAM,OAAO,IAAI,YAAY,EAAE;AAClC,YAAA,MAAM,QAAQ,GACZ,SAAS,KAAK;kBACV,aAAa,CAAC,IAAI,EAAE,OAAO,CAAC,KAAK;kBACjC,cAAc,CAAC,IAAI,EAAE,OAAO,CAAC,KAAK,CAAC;AAEzC,YAAA,IAAI,QAAQ,KAAK,EAAE,EAAE;AACnB,gBAAA,OAAO,QAAQ;;;;;AAMrB,IAAA,OAAO,SAAS,KAAK,UAAU,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC;AACnD;AAEA;;;;;;;;;;AAUG;AACG,SAAU,gBAAgB,CAC9B,aAAiC,EACjC,YAAY,GAAG,GAAG,EAClB,iBAAiB,GAAG,GAAG,EAAA;;AAGvB,IAAA,MAAM,UAAU,GAAG,EAAE,GAAG,aAAa,EAAE;;AAGvC,IAAA,IAAI,UAAU,CAAC,OAAO,EAAE;QACtB,UAAU,CAAC,OAAO,GAAG,CAAC,GAAG,UAAU,CAAC,OAAO,CAAC;;AAE9C,IAAA,IAAI,UAAU,CAAC,UAAU,EAAE;QACzB,UAAU,CAAC,UAAU,GAAG,CAAC,GAAG,UAAU,CAAC,UAAU,CAAC;;;AAIpD,IAAA,MAAM,kBAAkB,GAAG,CAAC,SAAS,EAAE,YAAY,CAAU;AAE7D,IAAA,KAAK,MAAM,UAAU,IAAI,kBAAkB,EAAE;AAC3C,QAAA,IAAI,CAAC,UAAU,CAAC,UAAsC,CAAC;YAAE;;AAGzD,QAAA,UAAU,CAAC,UAAU,CAAC,GAAG,UAAU,CAAC,UAAU,CAAC,EAAE,GAAG,CAAC,CAAC,MAAM,KAAI;AAC9D,YAAA,IACE,MAAM,CAAC,OAAO,IAAI,IAAI;gBACtB,MAAM,CAAC,OAAO,KAAK,EAAE;gBACrB,CAAC,MAAM,CAAC,UAAU;AAClB,gBAAA,MAAM,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC,EAC9B;gBACA,OAAO,MAAM,CAAC;;;AAIhB,YAAA,MAAM,UAAU,GAAG,EAAE,GAAG,MAAM,EAAE;AAChC,YAAA,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO;YAC9B,MAAM,UAAU,GAAG,EAAE;;AAGrB,YAAA,KAAK,MAAM,SAAS,IAAI,MAAM,CAAC,UAAU,EAAE;AACzC,gBAAA,MAAM,aAAa,GAAG,SAAS,CAAC,IAAI;gBAEpC,IAAI,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,aAAa,CAAC;AAC7C,gBAAA,IAAI,YAAY,GAAG,aAAa,CAAC,MAAM;AAEvC,gBAAA,IAAI,QAAQ,KAAK,EAAE,EAAE;;AAEnB,oBAAA,MAAM,iBAAiB,GAAG,aAAa,CAAC,IAAI,EAAE;AAC9C,oBAAA,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,iBAAiB,CAAC;AAE7C,oBAAA,IAAI,QAAQ,KAAK,EAAE,EAAE;wBACnB,UAAU,CAAC,IAAI,CAAC;4BACd,IAAI,EAAE,SAAS,CAAC,IAAI;4BACpB,KAAK,EAAE,SAAS,CAAC,KAAK;AACvB,yBAAA,CAAC;wBACF;;AAEF,oBAAA,YAAY,GAAG,iBAAiB,CAAC,MAAM;;;AAIzC,gBAAA,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,GAAG,YAAY,CAAC;AACtD,gBAAA,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CACtB,OAAO,CAAC,MAAM,EACd,QAAQ,GAAG,YAAY,GAAG,YAAY,CACvC;AAED,gBAAA,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,GAAG,iBAAiB,CAAC;AACjE,gBAAA,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAC3B,OAAO,CAAC,MAAM,EACd,OAAO,GAAG,iBAAiB,CAC5B;;gBAGD,MAAM,QAAQ,GAAG,OAAO,CAAC,SAAS,CAAC,cAAc,EAAE,SAAS,CAAC;gBAC7D,MAAM,QAAQ,GAAG,OAAO,CAAC,SAAS,CAAC,OAAO,EAAE,YAAY,CAAC;;gBAGzD,MAAM,gBAAgB,GAAG,gBAAgB,CAAC,QAAQ,EAAE,UAAU,CAAC;gBAC/D,MAAM,gBAAgB,GAAG,gBAAgB,CAAC,QAAQ,EAAE,SAAS,CAAC;;AAG9D,gBAAA,MAAM,UAAU,GAAG,cAAc,GAAG,gBAAgB;AACpD,gBAAA,MAAM,QAAQ,GAAG,OAAO,GAAG,gBAAgB;;gBAG3C,MAAM,qBAAqB,GAAG;AAC3B,qBAAA,SAAS,CAAC,UAAU,EAAE,QAAQ;AAC9B,qBAAA,IAAI,EAAE;gBACT,UAAU,CAAC,IAAI,CAAC;AACd,oBAAA,IAAI,EAAE,qBAAqB;oBAC3B,KAAK,EAAE,SAAS,CAAC,KAAK;AACvB,iBAAA,CAAC;;YAGJ,OAAO,UAAU,CAAC,OAAO;AACzB,YAAA,UAAU,CAAC,UAAU,GAAG,UAAU;AAClC,YAAA,OAAO,UAAU;AACnB,SAAC,CAAC;;AAGJ,IAAA,OAAO,UAAU;AACnB;;;;"}

package/dist/esm/tools/search/tool.mjs

@@ -68,7 +68,7 @@ const createSearchTool = (config = {}) => {
                 : undefined,
         });
         const output = formatResultsForLLM(searchResult);
-        return [output, searchResult];
+        return [output, { [Constants.WEB_SEARCH]: { ...searchResult } }];
     }, {
         name: Constants.WEB_SEARCH,
         description: `

package/dist/esm/tools/search/tool.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"tool.mjs","sources":["../../../../src/tools/search/tool.ts"],"sourcesContent":["/* eslint-disable no-console */\nimport { z } from 'zod';\nimport { tool, DynamicStructuredTool } from '@langchain/core/tools';\nimport type * as t from './types';\nimport { createSearchAPI, createSourceProcessor } from './search';\nimport { createFirecrawlScraper } from './firecrawl';\nimport { expandHighlights } from './highlights';\nimport { formatResultsForLLM } from './format';\nimport { createReranker } from './rerankers';\nimport { Constants } from '@/common';\n\nconst SearchToolSchema = z.object({\n  query: z\n    .string()\n    .describe(\n      'The search query string that specifies what should be searched for.'\n    ),\n});\n\nexport const createSearchTool = (\n  config: t.SearchToolConfig = {}\n): DynamicStructuredTool<typeof SearchToolSchema> => {\n  const {\n    searchProvider = 'serper',\n    serperApiKey,\n    searxngInstanceUrl,\n    searxngApiKey,\n    rerankerType = 'cohere',\n    topResults = 5,\n    strategies = ['no_extraction'],\n    filterContent = true,\n    firecrawlApiKey,\n    firecrawlApiUrl,\n    firecrawlFormats = ['markdown', 'html'],\n    jinaApiKey,\n    cohereApiKey,\n    onSearchResults: _onSearchResults,\n  } = config;\n\n  const searchAPI = createSearchAPI({\n    searchProvider,\n    serperApiKey,\n    searxngInstanceUrl,\n    searxngApiKey,\n  });\n\n  const firecrawlScraper = createFirecrawlScraper({\n    apiKey: firecrawlApiKey ?? process.env.FIRECRAWL_API_KEY,\n    apiUrl: firecrawlApiUrl,\n    formats: firecrawlFormats,\n  });\n\n  const selectedReranker = createReranker({\n    rerankerType,\n    jinaApiKey,\n    cohereApiKey,\n  });\n\n  if (!selectedReranker) {\n    console.warn('No reranker selected. Using default ranking.');\n  }\n\n  const sourceProcessor = createSourceProcessor(\n    {\n      reranker: selectedReranker,\n      topResults,\n      strategies,\n      filterContent,\n    },\n    firecrawlScraper\n  );\n\n  const search = async ({\n    query,\n    proMode = true,\n    maxSources = 5,\n    onSearchResults,\n  }: {\n    query: string;\n    proMode?: boolean;\n    maxSources?: number;\n    onSearchResults?: (sources: t.SearchResult) => void;\n  }): Promise<t.SearchResultData> => {\n    try {\n      const sources = await searchAPI.getSources(query);\n      onSearchResults?.(sources);\n\n      if (!sources.success) {\n        throw new Error(sources.error ?? 'Search failed');\n      }\n\n      const processedSources = await sourceProcessor.processSources(\n        sources,\n        maxSources,\n        query,\n        proMode\n      );\n      return expandHighlights(processedSources);\n    } catch (error) {\n      console.error('Error in search:', error);\n      return {\n        organic: [],\n        topStories: [],\n        images: [],\n        relatedSearches: [],\n        error: error instanceof Error ? error.message : String(error),\n      };\n    }\n  };\n\n  return tool<typeof SearchToolSchema>(\n    async ({ query }, runnableConfig) => {\n      const searchResult = await search({\n        query,\n        onSearchResults: _onSearchResults\n          ? (result): void => {\n            _onSearchResults(result, runnableConfig);\n          }\n          : undefined,\n      });\n      const output = formatResultsForLLM(searchResult);\n      return [output, searchResult];\n    },\n    {\n      name: Constants.WEB_SEARCH,\n      description: `\nReal-time search. Results have required unique citation anchors.\n\nAnchors:\n- \\\\ue202turn0searchN (web), \\\\ue202turn0newsN (news), \\\\ue202turn0imageN (image)\n\nSpecial Markers:\n- \\\\ue203...\\\\ue204 — mark start/end of cited span\n- \\\\ue200...\\\\ue201 — composite/group block (e.g. \\\\ue200cite\\\\ue202turn0search1\\\\ue202turn0news2\\\\ue201)\n- \\\\ue206 — marks grouped/summary citation areas\n\n**CITE EVERY NON-OBVIOUS FACT/QUOTE:**\nInsert the anchor marker(s) immediately after the statement:\n- \"Pure functions produce same output \\\\ue202turn0search0.\"\n- Multiple: \"Benefits \\\\ue202turn0search0\\\\ue202turn0news0.\"\n- Span: \\\\ue203Key: first-class functions\\\\ue204\\\\ue202turn0news1\n- Group: \"Functional languages.\"\\\\ue206 or \\\\ue200cite\\\\ue202turn0search0\\\\ue202turn0news1\\\\ue201\n- Image: \"See photo \\\\ue202turn0image0.\"\n\n**NEVER use markdown links, [1], or footnotes. CITE ONLY with anchors provided.**\n`.trim(),\n      schema: SearchToolSchema,\n      responseFormat: Constants.CONTENT_AND_ARTIFACT,\n    }\n  );\n};\n"],"names":[],"mappings":";;;;;;;;;AAAA;AAWA,MAAM,gBAAgB,GAAG,CAAC,CAAC,MAAM,CAAC;AAChC,IAAA,KAAK,EAAE;AACJ,SAAA,MAAM;SACN,QAAQ,CACP,qEAAqE,CACtE;AACJ,CAAA,CAAC;MAEW,gBAAgB,GAAG,CAC9B,MAA6B,GAAA,EAAE,KACmB;IAClD,MAAM,EACJ,cAAc,GAAG,QAAQ,EACzB,YAAY,EACZ,kBAAkB,EAClB,aAAa,EACb,YAAY,GAAG,QAAQ,EACvB,UAAU,GAAG,CAAC,EACd,UAAU,GAAG,CAAC,eAAe,CAAC,EAC9B,aAAa,GAAG,IAAI,EACpB,eAAe,EACf,eAAe,EACf,gBAAgB,GAAG,CAAC,UAAU,EAAE,MAAM,CAAC,EACvC,UAAU,EACV,YAAY,EACZ,eAAe,EAAE,gBAAgB,GAClC,GAAG,MAAM;IAEV,MAAM,SAAS,GAAG,eAAe,CAAC;QAChC,cAAc;QACd,YAAY;QACZ,kBAAkB;QAClB,aAAa;AACd,KAAA,CAAC;IAEF,MAAM,gBAAgB,GAAG,sBAAsB,CAAC;AAC9C,QAAA,MAAM,EAAE,eAAe,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB;AACxD,QAAA,MAAM,EAAE,eAAe;AACvB,QAAA,OAAO,EAAE,gBAAgB;AAC1B,KAAA,CAAC;IAEF,MAAM,gBAAgB,GAAG,cAAc,CAAC;QACtC,YAAY;QACZ,UAAU;QACV,YAAY;AACb,KAAA,CAAC;IAEF,IAAI,CAAC,gBAAgB,EAAE;AACrB,QAAA,OAAO,CAAC,IAAI,CAAC,8CAA8C,CAAC;;IAG9D,MAAM,eAAe,GAAG,qBAAqB,CAC3C;AACE,QAAA,QAAQ,EAAE,gBAAgB;QAC1B,WAGD,EACD,gBAAgB,CACjB;AAED,IAAA,MAAM,MAAM,GAAG,OAAO,EACpB,KAAK,EACL,OAAO,GAAG,IAAI,EACd,UAAU,GAAG,CAAC,EACd,eAAe,GAMhB,KAAiC;AAChC,QAAA,IAAI;YACF,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,UAAU,CAAC,KAAK,CAAC;AACjD,YAAA,eAAe,GAAG,OAAO,CAAC;AAE1B,YAAA,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE;gBACpB,MAAM,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,IAAI,eAAe,CAAC;;AAGnD,YAAA,MAAM,gBAAgB,GAAG,MAAM,eAAe,CAAC,cAAc,CAC3D,OAAO,EACP,UAAU,EACV,KAAK,EACL,OAAO,CACR;AACD,YAAA,OAAO,gBAAgB,CAAC,gBAAgB,CAAC;;QACzC,OAAO,KAAK,EAAE;AACd,YAAA,OAAO,CAAC,KAAK,CAAC,kBAAkB,EAAE,KAAK,CAAC;YACxC,OAAO;AACL,gBAAA,OAAO,EAAE,EAAE;AACX,gBAAA,UAAU,EAAE,EAAE;AACd,gBAAA,MAAM,EAAE,EAAE;AACV,gBAAA,eAAe,EAAE,EAAE;AACnB,gBAAA,KAAK,EAAE,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;aAC9D;;AAEL,KAAC;IAED,OAAO,IAAI,CACT,OAAO,EAAE,KAAK,EAAE,EAAE,cAAc,KAAI;AAClC,QAAA,MAAM,YAAY,GAAG,MAAM,MAAM,CAAC;YAChC,KAAK;AACL,YAAA,eAAe,EAAE;AACf,kBAAE,CAAC,MAAM,KAAU;AACjB,oBAAA,gBAAgB,CAAC,MAAM,EAAE,cAAc,CAAC;;AAE1C,kBAAE,SAAS;AACd,SAAA,CAAC;AACF,QAAA,MAAM,MAAM,GAAG,mBAAmB,CAAC,YAAY,CAAC;AAChD,QAAA,OAAO,CAAC,MAAM,EAAE,YAAY,CAAC;AAC/B,KAAC,EACD;QACE,IAAI,EAAE,SAAS,CAAC,UAAU;AAC1B,QAAA,WAAW,EAAE;;;;;;;;;;;;;;;;;;;;AAoBlB,CAAA,CAAC,IAAI,EAAE;AACF,QAAA,MAAM,EAAE,gBAAgB;QACxB,cAAc,EAAE,SAAS,CAAC,oBAAoB;AAC/C,KAAA,CACF;AACH;;;;"}
+{"version":3,"file":"tool.mjs","sources":["../../../../src/tools/search/tool.ts"],"sourcesContent":["/* eslint-disable no-console */\nimport { z } from 'zod';\nimport { tool, DynamicStructuredTool } from '@langchain/core/tools';\nimport type * as t from './types';\nimport { createSearchAPI, createSourceProcessor } from './search';\nimport { createFirecrawlScraper } from './firecrawl';\nimport { expandHighlights } from './highlights';\nimport { formatResultsForLLM } from './format';\nimport { createReranker } from './rerankers';\nimport { Constants } from '@/common';\n\nconst SearchToolSchema = z.object({\n  query: z\n    .string()\n    .describe(\n      'The search query string that specifies what should be searched for.'\n    ),\n});\n\nexport const createSearchTool = (\n  config: t.SearchToolConfig = {}\n): DynamicStructuredTool<typeof SearchToolSchema> => {\n  const {\n    searchProvider = 'serper',\n    serperApiKey,\n    searxngInstanceUrl,\n    searxngApiKey,\n    rerankerType = 'cohere',\n    topResults = 5,\n    strategies = ['no_extraction'],\n    filterContent = true,\n    firecrawlApiKey,\n    firecrawlApiUrl,\n    firecrawlFormats = ['markdown', 'html'],\n    jinaApiKey,\n    cohereApiKey,\n    onSearchResults: _onSearchResults,\n  } = config;\n\n  const searchAPI = createSearchAPI({\n    searchProvider,\n    serperApiKey,\n    searxngInstanceUrl,\n    searxngApiKey,\n  });\n\n  const firecrawlScraper = createFirecrawlScraper({\n    apiKey: firecrawlApiKey ?? process.env.FIRECRAWL_API_KEY,\n    apiUrl: firecrawlApiUrl,\n    formats: firecrawlFormats,\n  });\n\n  const selectedReranker = createReranker({\n    rerankerType,\n    jinaApiKey,\n    cohereApiKey,\n  });\n\n  if (!selectedReranker) {\n    console.warn('No reranker selected. Using default ranking.');\n  }\n\n  const sourceProcessor = createSourceProcessor(\n    {\n      reranker: selectedReranker,\n      topResults,\n      strategies,\n      filterContent,\n    },\n    firecrawlScraper\n  );\n\n  const search = async ({\n    query,\n    proMode = true,\n    maxSources = 5,\n    onSearchResults,\n  }: {\n    query: string;\n    proMode?: boolean;\n    maxSources?: number;\n    onSearchResults?: (sources: t.SearchResult) => void;\n  }): Promise<t.SearchResultData> => {\n    try {\n      const sources = await searchAPI.getSources(query);\n      onSearchResults?.(sources);\n\n      if (!sources.success) {\n        throw new Error(sources.error ?? 'Search failed');\n      }\n\n      const processedSources = await sourceProcessor.processSources(\n        sources,\n        maxSources,\n        query,\n        proMode\n      );\n      return expandHighlights(processedSources);\n    } catch (error) {\n      console.error('Error in search:', error);\n      return {\n        organic: [],\n        topStories: [],\n        images: [],\n        relatedSearches: [],\n        error: error instanceof Error ? error.message : String(error),\n      };\n    }\n  };\n\n  return tool<typeof SearchToolSchema>(\n    async ({ query }, runnableConfig) => {\n      const searchResult = await search({\n        query,\n        onSearchResults: _onSearchResults\n          ? (result): void => {\n            _onSearchResults(result, runnableConfig);\n          }\n          : undefined,\n      });\n      const output = formatResultsForLLM(searchResult);\n      return [output, { [Constants.WEB_SEARCH]: { ...searchResult } }];\n    },\n    {\n      name: Constants.WEB_SEARCH,\n      description: `\nReal-time search. Results have required unique citation anchors.\n\nAnchors:\n- \\\\ue202turn0searchN (web), \\\\ue202turn0newsN (news), \\\\ue202turn0imageN (image)\n\nSpecial Markers:\n- \\\\ue203...\\\\ue204 — mark start/end of cited span\n- \\\\ue200...\\\\ue201 — composite/group block (e.g. \\\\ue200cite\\\\ue202turn0search1\\\\ue202turn0news2\\\\ue201)\n- \\\\ue206 — marks grouped/summary citation areas\n\n**CITE EVERY NON-OBVIOUS FACT/QUOTE:**\nInsert the anchor marker(s) immediately after the statement:\n- \"Pure functions produce same output \\\\ue202turn0search0.\"\n- Multiple: \"Benefits \\\\ue202turn0search0\\\\ue202turn0news0.\"\n- Span: \\\\ue203Key: first-class functions\\\\ue204\\\\ue202turn0news1\n- Group: \"Functional languages.\"\\\\ue206 or \\\\ue200cite\\\\ue202turn0search0\\\\ue202turn0news1\\\\ue201\n- Image: \"See photo \\\\ue202turn0image0.\"\n\n**NEVER use markdown links, [1], or footnotes. CITE ONLY with anchors provided.**\n`.trim(),\n      schema: SearchToolSchema,\n      responseFormat: Constants.CONTENT_AND_ARTIFACT,\n    }\n  );\n};\n"],"names":[],"mappings":";;;;;;;;;AAAA;AAWA,MAAM,gBAAgB,GAAG,CAAC,CAAC,MAAM,CAAC;AAChC,IAAA,KAAK,EAAE;AACJ,SAAA,MAAM;SACN,QAAQ,CACP,qEAAqE,CACtE;AACJ,CAAA,CAAC;MAEW,gBAAgB,GAAG,CAC9B,MAA6B,GAAA,EAAE,KACmB;IAClD,MAAM,EACJ,cAAc,GAAG,QAAQ,EACzB,YAAY,EACZ,kBAAkB,EAClB,aAAa,EACb,YAAY,GAAG,QAAQ,EACvB,UAAU,GAAG,CAAC,EACd,UAAU,GAAG,CAAC,eAAe,CAAC,EAC9B,aAAa,GAAG,IAAI,EACpB,eAAe,EACf,eAAe,EACf,gBAAgB,GAAG,CAAC,UAAU,EAAE,MAAM,CAAC,EACvC,UAAU,EACV,YAAY,EACZ,eAAe,EAAE,gBAAgB,GAClC,GAAG,MAAM;IAEV,MAAM,SAAS,GAAG,eAAe,CAAC;QAChC,cAAc;QACd,YAAY;QACZ,kBAAkB;QAClB,aAAa;AACd,KAAA,CAAC;IAEF,MAAM,gBAAgB,GAAG,sBAAsB,CAAC;AAC9C,QAAA,MAAM,EAAE,eAAe,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB;AACxD,QAAA,MAAM,EAAE,eAAe;AACvB,QAAA,OAAO,EAAE,gBAAgB;AAC1B,KAAA,CAAC;IAEF,MAAM,gBAAgB,GAAG,cAAc,CAAC;QACtC,YAAY;QACZ,UAAU;QACV,YAAY;AACb,KAAA,CAAC;IAEF,IAAI,CAAC,gBAAgB,EAAE;AACrB,QAAA,OAAO,CAAC,IAAI,CAAC,8CAA8C,CAAC;;IAG9D,MAAM,eAAe,GAAG,qBAAqB,CAC3C;AACE,QAAA,QAAQ,EAAE,gBAAgB;QAC1B,WAGD,EACD,gBAAgB,CACjB;AAED,IAAA,MAAM,MAAM,GAAG,OAAO,EACpB,KAAK,EACL,OAAO,GAAG,IAAI,EACd,UAAU,GAAG,CAAC,EACd,eAAe,GAMhB,KAAiC;AAChC,QAAA,IAAI;YACF,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,UAAU,CAAC,KAAK,CAAC;AACjD,YAAA,eAAe,GAAG,OAAO,CAAC;AAE1B,YAAA,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE;gBACpB,MAAM,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,IAAI,eAAe,CAAC;;AAGnD,YAAA,MAAM,gBAAgB,GAAG,MAAM,eAAe,CAAC,cAAc,CAC3D,OAAO,EACP,UAAU,EACV,KAAK,EACL,OAAO,CACR;AACD,YAAA,OAAO,gBAAgB,CAAC,gBAAgB,CAAC;;QACzC,OAAO,KAAK,EAAE;AACd,YAAA,OAAO,CAAC,KAAK,CAAC,kBAAkB,EAAE,KAAK,CAAC;YACxC,OAAO;AACL,gBAAA,OAAO,EAAE,EAAE;AACX,gBAAA,UAAU,EAAE,EAAE;AACd,gBAAA,MAAM,EAAE,EAAE;AACV,gBAAA,eAAe,EAAE,EAAE;AACnB,gBAAA,KAAK,EAAE,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;aAC9D;;AAEL,KAAC;IAED,OAAO,IAAI,CACT,OAAO,EAAE,KAAK,EAAE,EAAE,cAAc,KAAI;AAClC,QAAA,MAAM,YAAY,GAAG,MAAM,MAAM,CAAC;YAChC,KAAK;AACL,YAAA,eAAe,EAAE;AACf,kBAAE,CAAC,MAAM,KAAU;AACjB,oBAAA,gBAAgB,CAAC,MAAM,EAAE,cAAc,CAAC;;AAE1C,kBAAE,SAAS;AACd,SAAA,CAAC;AACF,QAAA,MAAM,MAAM,GAAG,mBAAmB,CAAC,YAAY,CAAC;AAChD,QAAA,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,SAAS,CAAC,UAAU,GAAG,EAAE,GAAG,YAAY,EAAE,EAAE,CAAC;AAClE,KAAC,EACD;QACE,IAAI,EAAE,SAAS,CAAC,UAAU;AAC1B,QAAA,WAAW,EAAE;;;;;;;;;;;;;;;;;;;;AAoBlB,CAAA,CAAC,IAAI,EAAE;AACF,QAAA,MAAM,EAAE,gBAAgB;QACxB,cAAc,EAAE,SAAS,CAAC,oBAAoB;AAC/C,KAAA,CACF;AACH;;;;"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@librechat/agents",
-  "version": "2.4.31",
+  "version": "2.4.311",
   "main": "./dist/cjs/main.cjs",
   "module": "./dist/esm/main.mjs",
   "types": "./dist/types/index.d.ts",

package/src/tools/search/firecrawl.ts

@@ -152,7 +152,7 @@ export class FirecrawlScraper {
     this.apiUrl = `${baseUrl.replace(/\/+$/, '')}/v1/scrape`;
 
     this.defaultFormats = config.formats ?? ['markdown', 'html'];
-    this.timeout = config.timeout ?? 30000;
+    this.timeout = config.timeout ?? 15000;
 
     if (!this.apiKey) {
       console.warn('FIRECRAWL_API_KEY is not set. Scraping will not work.');

package/src/tools/search/format.ts

@@ -13,8 +13,8 @@ export function formatResultsForLLM(results: t.SearchResultData): string {
     addSection('Web Results');
     organic.forEach((r, i) => {
       output += [
-        `Source ${i + 1}: ${r.title ?? '(no title)'}`,
-        `Citation Anchor: \\ue202turn0search${i + 1}`,
+        `Source ${i}: ${r.title ?? '(no title)'}`,
+        `Citation Anchor: \\ue202turn0search${i}`,
         `URL: ${r.link}`,
         r.snippet != null ? `Summary: ${r.snippet}` : '',
         r.date != null ? `Date: ${r.date}` : '',
@@ -38,7 +38,7 @@ export function formatResultsForLLM(results: t.SearchResultData): string {
   //   addSection('News Results');
   //   topStores.forEach((r, i) => {
   //     output += [
-  //       `Anchor: \ue202turn0news${i + 1}`,
+  //       `Anchor: \ue202turn0news${i}`,
   //       `Title: ${r.title ?? '(no title)'}`,
   //       `URL: ${r.link}`,
   //       r.snippet != null ? `Snippet: ${r.snippet}` : '',
@@ -55,7 +55,7 @@ export function formatResultsForLLM(results: t.SearchResultData): string {
   //   addSection('Image Results');
   //   images.forEach((img, i) => {
   //     output += [
-  //       `Anchor: \ue202turn0image${i + 1}`,
+  //       `Anchor: \ue202turn0image${i}`,
   //       `Title: ${img.title ?? '(no title)'}`,
   //       `Image URL: ${img.imageUrl}`,
   //       ''

package/src/tools/search/highlights.ts

@@ -228,6 +228,7 @@ export function expandHighlights(
         });
       }
 
+      delete resultCopy.content;
       resultCopy.highlights = highlights;
       return resultCopy;
     });

package/src/tools/search/tool.ts

@@ -119,7 +119,7 @@ export const createSearchTool = (
           : undefined,
       });
       const output = formatResultsForLLM(searchResult);
-      return [output, searchResult];
+      return [output, { [Constants.WEB_SEARCH]: { ...searchResult } }];
     },
     {
       name: Constants.WEB_SEARCH,