@librechat/agents 2.4.31 → 2.4.33

package/dist/esm/tools/search/firecrawl.mjs

@@ -1,35 +1,7 @@
 import axios from 'axios';
+import { processContent } from './content.mjs';
+import { createDefaultLogger } from './utils.mjs';
 
-/* eslint-disable no-console */
-const getDomainName = (link, metadata) => {
-    try {
-        const url = metadata?.sourceURL ?? metadata?.url ?? (link || '');
-        const domain = new URL(url).hostname.replace(/^www\./, '');
-        if (domain) {
-            return domain;
-        }
-    }
-    catch (e) {
-        // URL parsing failed
-        console.error('Error parsing URL:', e);
-    }
-    return;
-};
-function getAttribution(link, metadata) {
-    if (!metadata)
-        return getDomainName(link, metadata);
-    const possibleAttributions = [
-        metadata.ogSiteName,
-        metadata['og:site_name'],
-        metadata.title?.split('|').pop()?.trim(),
-        metadata['twitter:site']?.replace(/^@/, ''),
-    ];
-    const attribution = possibleAttributions.find((attr) => attr != null && typeof attr === 'string' && attr.trim() !== '');
-    if (attribution != null) {
-        return attribution;
-    }
-    return getDomainName(link, metadata);
-}
 /**
  * Firecrawl scraper implementation
  * Uses the Firecrawl API to scrape web pages
@@ -39,6 +11,7 @@ class FirecrawlScraper {
     apiUrl;
     defaultFormats;
     timeout;
+    logger;
     constructor(config = {}) {
         this.apiKey = config.apiKey ?? process.env.FIRECRAWL_API_KEY ?? '';
         const baseUrl = config.apiUrl ??
@@ -46,11 +19,12 @@ class FirecrawlScraper {
             'https://api.firecrawl.dev';
         this.apiUrl = `${baseUrl.replace(/\/+$/, '')}/v1/scrape`;
         this.defaultFormats = config.formats ?? ['markdown', 'html'];
-        this.timeout = config.timeout ?? 30000;
+        this.timeout = config.timeout ?? 15000;
+        this.logger = config.logger || createDefaultLogger();
         if (!this.apiKey) {
-            console.warn('FIRECRAWL_API_KEY is not set. Scraping will not work.');
+            this.logger.warn('FIRECRAWL_API_KEY is not set. Scraping will not work.');
         }
-        console.log(`Firecrawl scraper initialized with API URL: ${this.apiUrl}`);
+        this.logger.debug(`Firecrawl scraper initialized with API URL: ${this.apiUrl}`);
     }
     /**
      * Scrape a single URL
@@ -104,21 +78,30 @@ class FirecrawlScraper {
      */
     extractContent(response) {
         if (!response.success || !response.data) {
-            return '';
+            return ['', undefined];
+        }
+        if (response.data.markdown != null && response.data.html != null) {
+            try {
+                const { markdown, ...rest } = processContent(response.data.html, response.data.markdown);
+                return [markdown, rest];
+            }
+            catch (error) {
+                this.logger.error('Error processing content:', error);
+                return [response.data.markdown, undefined];
+            }
         }
-        // Prefer markdown content if available
-        if (response.data.markdown != null) {
-            return response.data.markdown;
+        else if (response.data.markdown != null) {
+            return [response.data.markdown, undefined];
         }
         // Fall back to HTML content
         if (response.data.html != null) {
-            return response.data.html;
+            return [response.data.html, undefined];
         }
         // Fall back to raw HTML content
         if (response.data.rawHtml != null) {
-            return response.data.rawHtml;
+            return [response.data.rawHtml, undefined];
         }
-        return '';
+        return ['', undefined];
     }
     /**
      * Extract metadata from scrape response
@@ -141,5 +124,5 @@ const createFirecrawlScraper = (config = {}) => {
     return new FirecrawlScraper(config);
 };
 
-export { FirecrawlScraper, createFirecrawlScraper, getAttribution };
+export { FirecrawlScraper, createFirecrawlScraper };
 //# sourceMappingURL=firecrawl.mjs.map

package/dist/esm/tools/search/firecrawl.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"firecrawl.mjs","sources":["../../../../src/tools/search/firecrawl.ts"],"sourcesContent":["/* eslint-disable no-console */\nimport axios from 'axios';\n\nexport interface FirecrawlScrapeOptions {\n  formats?: string[];\n  includeTags?: string[];\n  excludeTags?: string[];\n  headers?: Record<string, string>;\n  waitFor?: number;\n  timeout?: number;\n}\n\ninterface ScrapeMetadata {\n  // Core source information\n  sourceURL?: string;\n  url?: string;\n  scrapeId?: string;\n  statusCode?: number;\n  // Basic metadata\n  title?: string;\n  description?: string;\n  language?: string;\n  favicon?: string;\n  viewport?: string;\n  robots?: string;\n  'theme-color'?: string;\n  // Open Graph metadata\n  'og:url'?: string;\n  'og:title'?: string;\n  'og:description'?: string;\n  'og:type'?: string;\n  'og:image'?: string;\n  'og:image:width'?: string;\n  'og:image:height'?: string;\n  'og:site_name'?: string;\n  ogUrl?: string;\n  ogTitle?: string;\n  ogDescription?: string;\n  ogImage?: string;\n  ogSiteName?: string;\n  // Article metadata\n  'article:author'?: string;\n  'article:published_time'?: string;\n  'article:modified_time'?: string;\n  'article:section'?: string;\n  'article:tag'?: string;\n  'article:publisher'?: string;\n  publishedTime?: string;\n  modifiedTime?: string;\n  // Twitter metadata\n  'twitter:site'?: string;\n  'twitter:creator'?: string;\n  'twitter:card'?: string;\n  'twitter:image'?: string;\n  'twitter:dnt'?: string;\n  'twitter:app:name:iphone'?: string;\n  'twitter:app:id:iphone'?: string;\n  'twitter:app:url:iphone'?: string;\n  'twitter:app:name:ipad'?: string;\n  'twitter:app:id:ipad'?: string;\n  'twitter:app:url:ipad'?: string;\n  'twitter:app:name:googleplay'?: string;\n  'twitter:app:id:googleplay'?: string;\n  'twitter:app:url:googleplay'?: string;\n  // Facebook metadata\n  'fb:app_id'?: string;\n  // App links\n  'al:ios:url'?: string;\n  'al:ios:app_name'?: string;\n  'al:ios:app_store_id'?: string;\n  // Allow for additional properties that might be present\n  [key: string]: string | number | boolean | null | undefined;\n}\n\nexport interface FirecrawlScrapeResponse {\n  success: boolean;\n  data?: {\n    markdown?: string;\n    html?: string;\n    rawHtml?: string;\n    screenshot?: string;\n    links?: string[];\n    metadata?: ScrapeMetadata;\n  };\n  error?: string;\n}\n\nexport interface FirecrawlScraperConfig {\n  apiKey?: string;\n  apiUrl?: string;\n  formats?: string[];\n  timeout?: number;\n}\nconst getDomainName = (\n  link: string,\n  metadata?: ScrapeMetadata\n): string | undefined => {\n  try {\n    const url = metadata?.sourceURL ?? metadata?.url ?? (link || '');\n    const domain = new URL(url).hostname.replace(/^www\\./, '');\n    if (domain) {\n      return domain;\n    }\n  } catch (e) {\n    // URL parsing failed\n    console.error('Error parsing URL:', e);\n  }\n\n  return;\n};\n\nexport function getAttribution(\n  link: string,\n  metadata?: ScrapeMetadata\n): string | undefined {\n  if (!metadata) return getDomainName(link, metadata);\n\n  const possibleAttributions = [\n    metadata.ogSiteName,\n    metadata['og:site_name'],\n    metadata.title?.split('|').pop()?.trim(),\n    metadata['twitter:site']?.replace(/^@/, ''),\n  ];\n\n  const attribution = possibleAttributions.find(\n    (attr) => attr != null && typeof attr === 'string' && attr.trim() !== ''\n  );\n  if (attribution != null) {\n    return attribution;\n  }\n\n  return getDomainName(link, metadata);\n}\n\n/**\n * Firecrawl scraper implementation\n * Uses the Firecrawl API to scrape web pages\n */\nexport class FirecrawlScraper {\n  private apiKey: string;\n  private apiUrl: string;\n  private defaultFormats: string[];\n  private timeout: number;\n\n  constructor(config: FirecrawlScraperConfig = {}) {\n    this.apiKey = config.apiKey ?? process.env.FIRECRAWL_API_KEY ?? '';\n\n    const baseUrl =\n      config.apiUrl ??\n      process.env.FIRECRAWL_BASE_URL ??\n      'https://api.firecrawl.dev';\n    this.apiUrl = `${baseUrl.replace(/\\/+$/, '')}/v1/scrape`;\n\n    this.defaultFormats = config.formats ?? ['markdown', 'html'];\n    this.timeout = config.timeout ?? 30000;\n\n    if (!this.apiKey) {\n      console.warn('FIRECRAWL_API_KEY is not set. Scraping will not work.');\n    }\n\n    console.log(`Firecrawl scraper initialized with API URL: ${this.apiUrl}`);\n  }\n\n  /**\n   * Scrape a single URL\n   * @param url URL to scrape\n   * @param options Scrape options\n   * @returns Scrape response\n   */\n  async scrapeUrl(\n    url: string,\n    options: FirecrawlScrapeOptions = {}\n  ): Promise<[string, FirecrawlScrapeResponse]> {\n    if (!this.apiKey) {\n      return [\n        url,\n        {\n          success: false,\n          error: 'FIRECRAWL_API_KEY is not set',\n        },\n      ];\n    }\n\n    try {\n      const response = await axios.post(\n        this.apiUrl,\n        {\n          url,\n          formats: options.formats || this.defaultFormats,\n          includeTags: options.includeTags,\n          excludeTags: options.excludeTags,\n          headers: options.headers,\n          waitFor: options.waitFor,\n          timeout: options.timeout ?? this.timeout,\n        },\n        {\n          headers: {\n            'Content-Type': 'application/json',\n            Authorization: `Bearer ${this.apiKey}`,\n          },\n          timeout: this.timeout,\n        }\n      );\n\n      return [url, response.data];\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      return [\n        url,\n        {\n          success: false,\n          error: `Firecrawl API request failed: ${errorMessage}`,\n        },\n      ];\n    }\n  }\n\n  /**\n   * Extract content from scrape response\n   * @param response Scrape response\n   * @returns Extracted content or empty string if not available\n   */\n  extractContent(response: FirecrawlScrapeResponse): string {\n    if (!response.success || !response.data) {\n      return '';\n    }\n\n    // Prefer markdown content if available\n    if (response.data.markdown != null) {\n      return response.data.markdown;\n    }\n\n    // Fall back to HTML content\n    if (response.data.html != null) {\n      return response.data.html;\n    }\n\n    // Fall back to raw HTML content\n    if (response.data.rawHtml != null) {\n      return response.data.rawHtml;\n    }\n\n    return '';\n  }\n\n  /**\n   * Extract metadata from scrape response\n   * @param response Scrape response\n   * @returns Metadata object\n   */\n  extractMetadata(response: FirecrawlScrapeResponse): ScrapeMetadata {\n    if (!response.success || !response.data || !response.data.metadata) {\n      return {};\n    }\n\n    return response.data.metadata;\n  }\n}\n\n/**\n * Create a Firecrawl scraper instance\n * @param config Scraper configuration\n * @returns Firecrawl scraper instance\n */\nexport const createFirecrawlScraper = (\n  config: FirecrawlScraperConfig = {}\n): FirecrawlScraper => {\n  return new FirecrawlScraper(config);\n};\n"],"names":[],"mappings":";;AAAA;AA6FA,MAAM,aAAa,GAAG,CACpB,IAAY,EACZ,QAAyB,KACH;AACtB,IAAA,IAAI;AACF,QAAA,MAAM,GAAG,GAAG,QAAQ,EAAE,SAAS,IAAI,QAAQ,EAAE,GAAG,KAAK,IAAI,IAAI,EAAE,CAAC;AAChE,QAAA,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC;QAC1D,IAAI,MAAM,EAAE;AACV,YAAA,OAAO,MAAM;;;IAEf,OAAO,CAAC,EAAE;;AAEV,QAAA,OAAO,CAAC,KAAK,CAAC,oBAAoB,EAAE,CAAC,CAAC;;IAGxC;AACF,CAAC;AAEe,SAAA,cAAc,CAC5B,IAAY,EACZ,QAAyB,EAAA;AAEzB,IAAA,IAAI,CAAC,QAAQ;AAAE,QAAA,OAAO,aAAa,CAAC,IAAI,EAAE,QAAQ,CAAC;AAEnD,IAAA,MAAM,oBAAoB,GAAG;AAC3B,QAAA,QAAQ,CAAC,UAAU;QACnB,QAAQ,CAAC,cAAc,CAAC;AACxB,QAAA,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE;QACxC,QAAQ,CAAC,cAAc,CAAC,EAAE,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;KAC5C;IAED,MAAM,WAAW,GAAG,oBAAoB,CAAC,IAAI,CAC3C,CAAC,IAAI,KAAK,IAAI,IAAI,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CACzE;AACD,IAAA,IAAI,WAAW,IAAI,IAAI,EAAE;AACvB,QAAA,OAAO,WAAW;;AAGpB,IAAA,OAAO,aAAa,CAAC,IAAI,EAAE,QAAQ,CAAC;AACtC;AAEA;;;AAGG;MACU,gBAAgB,CAAA;AACnB,IAAA,MAAM;AACN,IAAA,MAAM;AACN,IAAA,cAAc;AACd,IAAA,OAAO;AAEf,IAAA,WAAA,CAAY,SAAiC,EAAE,EAAA;AAC7C,QAAA,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,EAAE;AAElE,QAAA,MAAM,OAAO,GACX,MAAM,CAAC,MAAM;YACb,OAAO,CAAC,GAAG,CAAC,kBAAkB;AAC9B,YAAA,2BAA2B;AAC7B,QAAA,IAAI,CAAC,MAAM,GAAG,CAAA,EAAG,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,YAAY;AAExD,QAAA,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,OAAO,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC;QAC5D,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,KAAK;AAEtC,QAAA,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;AAChB,YAAA,OAAO,CAAC,IAAI,CAAC,uDAAuD,CAAC;;QAGvE,OAAO,CAAC,GAAG,CAAC,CAAA,4CAAA,EAA+C,IAAI,CAAC,MAAM,CAAE,CAAA,CAAC;;AAG3E;;;;;AAKG;AACH,IAAA,MAAM,SAAS,CACb,GAAW,EACX,UAAkC,EAAE,EAAA;AAEpC,QAAA,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YAChB,OAAO;gBACL,GAAG;AACH,gBAAA;AACE,oBAAA,OAAO,EAAE,KAAK;AACd,oBAAA,KAAK,EAAE,8BAA8B;AACtC,iBAAA;aACF;;AAGH,QAAA,IAAI;YACF,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAC/B,IAAI,CAAC,MAAM,EACX;gBACE,GAAG;AACH,gBAAA,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,cAAc;gBAC/C,WAAW,EAAE,OAAO,CAAC,WAAW;gBAChC,WAAW,EAAE,OAAO,CAAC,WAAW;gBAChC,OAAO,EAAE,OAAO,CAAC,OAAO;gBACxB,OAAO,EAAE,OAAO,CAAC,OAAO;AACxB,gBAAA,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO;aACzC,EACD;AACE,gBAAA,OAAO,EAAE;AACP,oBAAA,cAAc,EAAE,kBAAkB;AAClC,oBAAA,aAAa,EAAE,CAAA,OAAA,EAAU,IAAI,CAAC,MAAM,CAAE,CAAA;AACvC,iBAAA;gBACD,OAAO,EAAE,IAAI,CAAC,OAAO;AACtB,aAAA,CACF;AAED,YAAA,OAAO,CAAC,GAAG,EAAE,QAAQ,CAAC,IAAI,CAAC;;QAC3B,OAAO,KAAK,EAAE;AACd,YAAA,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;YACxD,OAAO;gBACL,GAAG;AACH,gBAAA;AACE,oBAAA,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,CAAiC,8BAAA,EAAA,YAAY,CAAE,CAAA;AACvD,iBAAA;aACF;;;AAIL;;;;AAIG;AACH,IAAA,cAAc,CAAC,QAAiC,EAAA;QAC9C,IAAI,CAAC,QAAQ,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE;AACvC,YAAA,OAAO,EAAE;;;QAIX,IAAI,QAAQ,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,EAAE;AAClC,YAAA,OAAO,QAAQ,CAAC,IAAI,CAAC,QAAQ;;;QAI/B,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,EAAE;AAC9B,YAAA,OAAO,QAAQ,CAAC,IAAI,CAAC,IAAI;;;QAI3B,IAAI,QAAQ,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,EAAE;AACjC,YAAA,OAAO,QAAQ,CAAC,IAAI,CAAC,OAAO;;AAG9B,QAAA,OAAO,EAAE;;AAGX;;;;AAIG;AACH,IAAA,eAAe,CAAC,QAAiC,EAAA;AAC/C,QAAA,IAAI,CAAC,QAAQ,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE;AAClE,YAAA,OAAO,EAAE;;AAGX,QAAA,OAAO,QAAQ,CAAC,IAAI,CAAC,QAAQ;;AAEhC;AAED;;;;AAIG;MACU,sBAAsB,GAAG,CACpC,MAAiC,GAAA,EAAE,KACf;AACpB,IAAA,OAAO,IAAI,gBAAgB,CAAC,MAAM,CAAC;AACrC;;;;"}
+{"version":3,"file":"firecrawl.mjs","sources":["../../../../src/tools/search/firecrawl.ts"],"sourcesContent":["import axios from 'axios';\nimport { processContent } from './content';\nimport type * as t from './types';\nimport { createDefaultLogger } from './utils';\n\n/**\n * Firecrawl scraper implementation\n * Uses the Firecrawl API to scrape web pages\n */\nexport class FirecrawlScraper {\n  private apiKey: string;\n  private apiUrl: string;\n  private defaultFormats: string[];\n  private timeout: number;\n  private logger: t.Logger;\n\n  constructor(config: t.FirecrawlScraperConfig = {}) {\n    this.apiKey = config.apiKey ?? process.env.FIRECRAWL_API_KEY ?? '';\n\n    const baseUrl =\n      config.apiUrl ??\n      process.env.FIRECRAWL_BASE_URL ??\n      'https://api.firecrawl.dev';\n    this.apiUrl = `${baseUrl.replace(/\\/+$/, '')}/v1/scrape`;\n\n    this.defaultFormats = config.formats ?? ['markdown', 'html'];\n    this.timeout = config.timeout ?? 15000;\n\n    this.logger = config.logger || createDefaultLogger();\n\n    if (!this.apiKey) {\n      this.logger.warn('FIRECRAWL_API_KEY is not set. Scraping will not work.');\n    }\n\n    this.logger.debug(\n      `Firecrawl scraper initialized with API URL: ${this.apiUrl}`\n    );\n  }\n\n  /**\n   * Scrape a single URL\n   * @param url URL to scrape\n   * @param options Scrape options\n   * @returns Scrape response\n   */\n  async scrapeUrl(\n    url: string,\n    options: t.FirecrawlScrapeOptions = {}\n  ): Promise<[string, t.FirecrawlScrapeResponse]> {\n    if (!this.apiKey) {\n      return [\n        url,\n        {\n          success: false,\n          error: 'FIRECRAWL_API_KEY is not set',\n        },\n      ];\n    }\n\n    try {\n      const response = await axios.post(\n        this.apiUrl,\n        {\n          url,\n          formats: options.formats || this.defaultFormats,\n          includeTags: options.includeTags,\n          excludeTags: options.excludeTags,\n          headers: options.headers,\n          waitFor: options.waitFor,\n          timeout: options.timeout ?? this.timeout,\n        },\n        {\n          headers: {\n            'Content-Type': 'application/json',\n            Authorization: `Bearer ${this.apiKey}`,\n          },\n          timeout: this.timeout,\n        }\n      );\n\n      return [url, response.data];\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      return [\n        url,\n        {\n          success: false,\n          error: `Firecrawl API request failed: ${errorMessage}`,\n        },\n      ];\n    }\n  }\n\n  /**\n   * Extract content from scrape response\n   * @param response Scrape response\n   * @returns Extracted content or empty string if not available\n   */\n  extractContent(\n    response: t.FirecrawlScrapeResponse\n  ): [string, undefined | t.References] {\n    if (!response.success || !response.data) {\n      return ['', undefined];\n    }\n\n    if (response.data.markdown != null && response.data.html != null) {\n      try {\n        const { markdown, ...rest } = processContent(\n          response.data.html,\n          response.data.markdown\n        );\n        return [markdown, rest];\n      } catch (error) {\n        this.logger.error('Error processing content:', error);\n        return [response.data.markdown, undefined];\n      }\n    } else if (response.data.markdown != null) {\n      return [response.data.markdown, undefined];\n    }\n\n    // Fall back to HTML content\n    if (response.data.html != null) {\n      return [response.data.html, undefined];\n    }\n\n    // Fall back to raw HTML content\n    if (response.data.rawHtml != null) {\n      return [response.data.rawHtml, undefined];\n    }\n\n    return ['', undefined];\n  }\n\n  /**\n   * Extract metadata from scrape response\n   * @param response Scrape response\n   * @returns Metadata object\n   */\n  extractMetadata(response: t.FirecrawlScrapeResponse): t.ScrapeMetadata {\n    if (!response.success || !response.data || !response.data.metadata) {\n      return {};\n    }\n\n    return response.data.metadata;\n  }\n}\n\n/**\n * Create a Firecrawl scraper instance\n * @param config Scraper configuration\n * @returns Firecrawl scraper instance\n */\nexport const createFirecrawlScraper = (\n  config: t.FirecrawlScraperConfig = {}\n): FirecrawlScraper => {\n  return new FirecrawlScraper(config);\n};\n"],"names":[],"mappings":";;;;AAKA;;;AAGG;MACU,gBAAgB,CAAA;AACnB,IAAA,MAAM;AACN,IAAA,MAAM;AACN,IAAA,cAAc;AACd,IAAA,OAAO;AACP,IAAA,MAAM;AAEd,IAAA,WAAA,CAAY,SAAmC,EAAE,EAAA;AAC/C,QAAA,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,EAAE;AAElE,QAAA,MAAM,OAAO,GACX,MAAM,CAAC,MAAM;YACb,OAAO,CAAC,GAAG,CAAC,kBAAkB;AAC9B,YAAA,2BAA2B;AAC7B,QAAA,IAAI,CAAC,MAAM,GAAG,CAAA,EAAG,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,YAAY;AAExD,QAAA,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,OAAO,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC;QAC5D,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,KAAK;QAEtC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,mBAAmB,EAAE;AAEpD,QAAA,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;AAChB,YAAA,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,uDAAuD,CAAC;;QAG3E,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,CAA+C,4CAAA,EAAA,IAAI,CAAC,MAAM,CAAE,CAAA,CAC7D;;AAGH;;;;;AAKG;AACH,IAAA,MAAM,SAAS,CACb,GAAW,EACX,UAAoC,EAAE,EAAA;AAEtC,QAAA,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YAChB,OAAO;gBACL,GAAG;AACH,gBAAA;AACE,oBAAA,OAAO,EAAE,KAAK;AACd,oBAAA,KAAK,EAAE,8BAA8B;AACtC,iBAAA;aACF;;AAGH,QAAA,IAAI;YACF,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAC/B,IAAI,CAAC,MAAM,EACX;gBACE,GAAG;AACH,gBAAA,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,cAAc;gBAC/C,WAAW,EAAE,OAAO,CAAC,WAAW;gBAChC,WAAW,EAAE,OAAO,CAAC,WAAW;gBAChC,OAAO,EAAE,OAAO,CAAC,OAAO;gBACxB,OAAO,EAAE,OAAO,CAAC,OAAO;AACxB,gBAAA,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO;aACzC,EACD;AACE,gBAAA,OAAO,EAAE;AACP,oBAAA,cAAc,EAAE,kBAAkB;AAClC,oBAAA,aAAa,EAAE,CAAA,OAAA,EAAU,IAAI,CAAC,MAAM,CAAE,CAAA;AACvC,iBAAA;gBACD,OAAO,EAAE,IAAI,CAAC,OAAO;AACtB,aAAA,CACF;AAED,YAAA,OAAO,CAAC,GAAG,EAAE,QAAQ,CAAC,IAAI,CAAC;;QAC3B,OAAO,KAAK,EAAE;AACd,YAAA,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;YACxD,OAAO;gBACL,GAAG;AACH,gBAAA;AACE,oBAAA,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,CAAiC,8BAAA,EAAA,YAAY,CAAE,CAAA;AACvD,iBAAA;aACF;;;AAIL;;;;AAIG;AACH,IAAA,cAAc,CACZ,QAAmC,EAAA;QAEnC,IAAI,CAAC,QAAQ,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE;AACvC,YAAA,OAAO,CAAC,EAAE,EAAE,SAAS,CAAC;;AAGxB,QAAA,IAAI,QAAQ,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,EAAE;AAChE,YAAA,IAAI;gBACF,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAI,EAAE,GAAG,cAAc,CAC1C,QAAQ,CAAC,IAAI,CAAC,IAAI,EAClB,QAAQ,CAAC,IAAI,CAAC,QAAQ,CACvB;AACD,gBAAA,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC;;YACvB,OAAO,KAAK,EAAE;gBACd,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,2BAA2B,EAAE,KAAK,CAAC;gBACrD,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC;;;aAEvC,IAAI,QAAQ,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,EAAE;YACzC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC;;;QAI5C,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,EAAE;YAC9B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,SAAS,CAAC;;;QAIxC,IAAI,QAAQ,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,EAAE;YACjC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC;;AAG3C,QAAA,OAAO,CAAC,EAAE,EAAE,SAAS,CAAC;;AAGxB;;;;AAIG;AACH,IAAA,eAAe,CAAC,QAAmC,EAAA;AACjD,QAAA,IAAI,CAAC,QAAQ,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE;AAClE,YAAA,OAAO,EAAE;;AAGX,QAAA,OAAO,QAAQ,CAAC,IAAI,CAAC,QAAQ;;AAEhC;AAED;;;;AAIG;MACU,sBAAsB,GAAG,CACpC,MAAmC,GAAA,EAAE,KACjB;AACpB,IAAA,OAAO,IAAI,gBAAgB,CAAC,MAAM,CAAC;AACrC;;;;"}

package/dist/esm/tools/search/format.mjs

@@ -1,113 +1,200 @@
-function formatResultsForLLM(results) {
-    let output = '';
+import { getDomainName, fileExtRegex } from './utils.mjs';
+
+function addHighlightSection() {
+    return ['\n## Highlights', ''];
+}
+// Helper function to format a source (organic or top story)
+function formatSource(source, index, turn, sourceType, references) {
+    /** Array of all lines to include in the output */
+    const outputLines = [];
+    // Add the title
+    outputLines.push(`# ${sourceType.charAt(0).toUpperCase() + sourceType.slice(1)} ${index}: ${source.title != null && source.title ? `"${source.title}"` : '(no title)'}`);
+    outputLines.push(`\nAnchor: \\ue202turn${turn}${sourceType}${index}`);
+    outputLines.push(`URL: ${source.link}`);
+    // Add optional fields
+    if ('snippet' in source && source.snippet != null) {
+        outputLines.push(`Summary: ${source.snippet}`);
+    }
+    if (source.date != null) {
+        outputLines.push(`Date: ${source.date}`);
+    }
+    if (source.attribution != null) {
+        outputLines.push(`Source: ${source.attribution}`);
+    }
+    // Add highlight section or empty line
+    if ((source.highlights?.length ?? 0) > 0) {
+        outputLines.push(...addHighlightSection());
+    }
+    else {
+        outputLines.push('');
+    }
+    // Process highlights if they exist
+    (source.highlights ?? [])
+        .filter((h) => h.text.trim().length > 0)
+        .forEach((h, hIndex) => {
+        outputLines.push(`### Highlight ${hIndex + 1} [Relevance: ${h.score.toFixed(2)}]`);
+        outputLines.push('');
+        outputLines.push('```text');
+        outputLines.push(h.text.trim());
+        outputLines.push('```');
+        outputLines.push('');
+        if (h.references != null && h.references.length) {
+            let hasHeader = false;
+            const refLines = [];
+            for (let j = 0; j < h.references.length; j++) {
+                const ref = h.references[j];
+                if (ref.reference.originalUrl.includes('mailto:')) {
+                    continue;
+                }
+                references.push({
+                    type: ref.type,
+                    link: ref.reference.originalUrl,
+                    attribution: getDomainName(ref.reference.originalUrl),
+                    title: (((ref.reference.title ?? '') || ref.reference.text) ??
+                        '').split('\n')[0],
+                });
+                if (ref.type !== 'link') {
+                    continue;
+                }
+                if (fileExtRegex.test(ref.reference.originalUrl)) {
+                    continue;
+                }
+                if (!hasHeader) {
+                    refLines.push('Core References:');
+                    hasHeader = true;
+                }
+                refLines.push(`- ${ref.type}#${ref.originalIndex + 1}: ${ref.reference.originalUrl}`);
+                refLines.push(`\t- Anchor: \\ue202turn${turn}ref${references.length - 1}`);
+            }
+            if (hasHeader) {
+                outputLines.push(...refLines);
+                outputLines.push('');
+            }
+        }
+        if (hIndex < (source.highlights?.length ?? 0) - 1) {
+            outputLines.push('---');
+            outputLines.push('');
+        }
+    });
+    outputLines.push('');
+    return outputLines.join('\n');
+}
+function formatResultsForLLM(turn, results) {
+    /** Array to collect all output lines */
+    const outputLines = [];
     const addSection = (title) => {
-        output += `\n=== ${title} ===\n`;
+        outputLines.push('');
+        outputLines.push(`=== ${title} ===`);
+        outputLines.push('');
     };
+    const references = [];
     // Organic (web) results
-    const organic = results.organic ?? [];
-    if (organic.length) {
-        addSection('Web Results');
-        organic.forEach((r, i) => {
-            output += [
-                `Source ${i + 1}: ${r.title ?? '(no title)'}`,
-                `Citation Anchor: \\ue202turn0search${i + 1}`,
-                `URL: ${r.link}`,
-                r.snippet != null ? `Summary: ${r.snippet}` : '',
-                r.date != null ? `Date: ${r.date}` : '',
-                r.attribution != null ? `Source: ${r.attribution}` : '',
-                '',
-                '--- Content Highlights ---',
-                ...(r.highlights ?? [])
-                    .filter((h) => h.text.trim().length > 0)
-                    .map((h) => `[Relevance: ${h.score.toFixed(2)}]\n${h.text.trim()}`),
-                '',
-            ]
-                .filter(Boolean)
-                .join('\n');
-        });
+    if (results.organic?.length != null && results.organic.length > 0) {
+        addSection(`Web Results, Turn ${turn}`);
+        for (let i = 0; i < results.organic.length; i++) {
+            const r = results.organic[i];
+            outputLines.push(formatSource(r, i, turn, 'search', references));
+            delete results.organic[i].highlights;
+        }
+    }
+    // Top stories (news)
+    const topStories = results.topStories ?? [];
+    if (topStories.length) {
+        addSection('News Results');
+        for (let i = 0; i < topStories.length; i++) {
+            const r = topStories[i];
+            outputLines.push(formatSource(r, i, turn, 'news', references));
+            if (results.topStories?.[i]?.highlights) {
+                delete results.topStories[i].highlights;
+            }
+        }
     }
-    // Ignoring these sections for now
-    // // Top stories (news)
-    // const topStores = results.topStories ?? [];
-    // if (topStores.length) {
-    //   addSection('News Results');
-    //   topStores.forEach((r, i) => {
-    //     output += [
-    //       `Anchor: \ue202turn0news${i + 1}`,
-    //       `Title: ${r.title ?? '(no title)'}`,
-    //       `URL: ${r.link}`,
-    //       r.snippet != null ? `Snippet: ${r.snippet}` : '',
-    //       r.date != null ? `Date: ${r.date}` : '',
-    //       r.attribution != null ? `Source: ${r.attribution}` : '',
-    //       ''
-    //     ].filter(Boolean).join('\n');
-    //   });
-    // }
     // // Images
     // const images = results.images ?? [];
     // if (images.length) {
     //   addSection('Image Results');
-    //   images.forEach((img, i) => {
-    //     output += [
-    //       `Anchor: \ue202turn0image${i + 1}`,
-    //       `Title: ${img.title ?? '(no title)'}`,
-    //       `Image URL: ${img.imageUrl}`,
-    //       ''
-    //     ].join('\n');
-    //   });
+    //   const imageLines = images.map((img, i) => [
+    //     `Anchor: \ue202turn0image${i}`,
+    //     `Title: ${img.title ?? '(no title)'}`,
+    //     `Image URL: ${img.imageUrl}`,
+    //     ''
+    //   ].join('\n'));
+    //   outputLines.push(imageLines.join('\n'));
     // }
     // Knowledge Graph
     if (results.knowledgeGraph != null) {
         addSection('Knowledge Graph');
-        output += [
-            `Title: ${results.knowledgeGraph.title ?? '(no title)'}`,
+        const kgLines = [
+            `**Title:** ${results.knowledgeGraph.title ?? '(no title)'}`,
+            results.knowledgeGraph.type != null
+                ? `**Type:** ${results.knowledgeGraph.type}`
+                : '',
             results.knowledgeGraph.description != null
-                ? `Description: ${results.knowledgeGraph.description}`
+                ? `**Description:** ${results.knowledgeGraph.description}`
                 : '',
-            results.knowledgeGraph.type != null
-                ? `Type: ${results.knowledgeGraph.type}`
+            results.knowledgeGraph.descriptionSource != null
+                ? `**Description Source:** ${results.knowledgeGraph.descriptionSource}`
+                : '',
+            results.knowledgeGraph.descriptionLink != null
+                ? `**Description Link:** ${results.knowledgeGraph.descriptionLink}`
                 : '',
             results.knowledgeGraph.imageUrl != null
-                ? `Image URL: ${results.knowledgeGraph.imageUrl}`
+                ? `**Image URL:** ${results.knowledgeGraph.imageUrl}`
+                : '',
+            results.knowledgeGraph.website != null
+                ? `**Website:** ${results.knowledgeGraph.website}`
                 : '',
             results.knowledgeGraph.attributes != null
-                ? `Attributes: ${JSON.stringify(results.knowledgeGraph.attributes, null, 2)}`
+                ? `**Attributes:**\n\`\`\`json\n${JSON.stringify(results.knowledgeGraph.attributes, null, 2)}\n\`\`\``
                 : '',
             '',
-        ]
-            .filter(Boolean)
-            .join('\n');
+        ].filter(Boolean);
+        outputLines.push(kgLines.join('\n\n'));
     }
     // Answer Box
     if (results.answerBox != null) {
         addSection('Answer Box');
-        output += [
+        const abLines = [
             results.answerBox.title != null
-                ? `Title: ${results.answerBox.title}`
-                : '',
-            results.answerBox.answer != null
-                ? `Answer: ${results.answerBox.answer}`
+                ? `**Title:** ${results.answerBox.title}`
                 : '',
             results.answerBox.snippet != null
-                ? `Snippet: ${results.answerBox.snippet}`
+                ? `**Snippet:** ${results.answerBox.snippet}`
+                : '',
+            results.answerBox.snippetHighlighted != null
+                ? `**Snippet Highlighted:** ${results.answerBox.snippetHighlighted
+                    .map((s) => `\`${s}\``)
+                    .join(' ')}`
+                : '',
+            results.answerBox.link != null
+                ? `**Link:** ${results.answerBox.link}`
                 : '',
-            results.answerBox.date != null ? `Date: ${results.answerBox.date}` : '',
             '',
-        ]
-            .filter(Boolean)
-            .join('\n');
+        ].filter(Boolean);
+        outputLines.push(abLines.join('\n\n'));
     }
     // People also ask
     const peopleAlsoAsk = results.peopleAlsoAsk ?? [];
     if (peopleAlsoAsk.length) {
         addSection('People Also Ask');
-        peopleAlsoAsk.forEach((p, _i) => {
-            output += [`Q: ${p.question}`, `A: ${p.answer}`, '']
-                .filter(Boolean)
-                .join('\n');
+        const paaLines = [];
+        peopleAlsoAsk.forEach((p, i) => {
+            const questionLines = [
+                `### Question ${i + 1}:`,
+                `"${p.question}"`,
+                `${p.snippet != null && p.snippet ? `Snippet: ${p.snippet}` : ''}`,
+                `${p.title != null && p.title ? `Title: ${p.title}` : ''}`,
+                `${p.link != null && p.link ? `Link: ${p.link}` : ''}`,
+                '',
+            ].filter(Boolean);
+            paaLines.push(questionLines.join('\n\n'));
         });
+        outputLines.push(paaLines.join(''));
     }
-    return output.trim();
+    return {
+        output: outputLines.join('\n').trim(),
+        references,
+    };
 }
 
 export { formatResultsForLLM };

package/dist/esm/tools/search/format.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"format.mjs","sources":["../../../../src/tools/search/format.ts"],"sourcesContent":["import type * as t from './types';\n\nexport function formatResultsForLLM(results: t.SearchResultData): string {\n  let output = '';\n\n  const addSection = (title: string): void => {\n    output += `\\n=== ${title} ===\\n`;\n  };\n\n  // Organic (web) results\n  const organic = results.organic ?? [];\n  if (organic.length) {\n    addSection('Web Results');\n    organic.forEach((r, i) => {\n      output += [\n        `Source ${i + 1}: ${r.title ?? '(no title)'}`,\n        `Citation Anchor: \\\\ue202turn0search${i + 1}`,\n        `URL: ${r.link}`,\n        r.snippet != null ? `Summary: ${r.snippet}` : '',\n        r.date != null ? `Date: ${r.date}` : '',\n        r.attribution != null ? `Source: ${r.attribution}` : '',\n        '',\n        '--- Content Highlights ---',\n        ...(r.highlights ?? [])\n          .filter((h) => h.text.trim().length > 0)\n          .map((h) => `[Relevance: ${h.score.toFixed(2)}]\\n${h.text.trim()}`),\n        '',\n      ]\n        .filter(Boolean)\n        .join('\\n');\n    });\n  }\n\n  // Ignoring these sections for now\n  // // Top stories (news)\n  // const topStores = results.topStories ?? [];\n  // if (topStores.length) {\n  //   addSection('News Results');\n  //   topStores.forEach((r, i) => {\n  //     output += [\n  //       `Anchor: \\ue202turn0news${i + 1}`,\n  //       `Title: ${r.title ?? '(no title)'}`,\n  //       `URL: ${r.link}`,\n  //       r.snippet != null ? `Snippet: ${r.snippet}` : '',\n  //       r.date != null ? `Date: ${r.date}` : '',\n  //       r.attribution != null ? `Source: ${r.attribution}` : '',\n  //       ''\n  //     ].filter(Boolean).join('\\n');\n  //   });\n  // }\n\n  // // Images\n  // const images = results.images ?? [];\n  // if (images.length) {\n  //   addSection('Image Results');\n  //   images.forEach((img, i) => {\n  //     output += [\n  //       `Anchor: \\ue202turn0image${i + 1}`,\n  //       `Title: ${img.title ?? '(no title)'}`,\n  //       `Image URL: ${img.imageUrl}`,\n  //       ''\n  //     ].join('\\n');\n  //   });\n  // }\n\n  // Knowledge Graph\n  if (results.knowledgeGraph != null) {\n    addSection('Knowledge Graph');\n    output += [\n      `Title: ${results.knowledgeGraph.title ?? '(no title)'}`,\n      results.knowledgeGraph.description != null\n        ? `Description: ${results.knowledgeGraph.description}`\n        : '',\n      results.knowledgeGraph.type != null\n        ? `Type: ${results.knowledgeGraph.type}`\n        : '',\n      results.knowledgeGraph.imageUrl != null\n        ? `Image URL: ${results.knowledgeGraph.imageUrl}`\n        : '',\n      results.knowledgeGraph.attributes != null\n        ? `Attributes: ${JSON.stringify(results.knowledgeGraph.attributes, null, 2)}`\n        : '',\n      '',\n    ]\n      .filter(Boolean)\n      .join('\\n');\n  }\n\n  // Answer Box\n  if (results.answerBox != null) {\n    addSection('Answer Box');\n    output += [\n      results.answerBox.title != null\n        ? `Title: ${results.answerBox.title}`\n        : '',\n      results.answerBox.answer != null\n        ? `Answer: ${results.answerBox.answer}`\n        : '',\n      results.answerBox.snippet != null\n        ? `Snippet: ${results.answerBox.snippet}`\n        : '',\n      results.answerBox.date != null ? `Date: ${results.answerBox.date}` : '',\n      '',\n    ]\n      .filter(Boolean)\n      .join('\\n');\n  }\n\n  // People also ask\n  const peopleAlsoAsk = results.peopleAlsoAsk ?? [];\n  if (peopleAlsoAsk.length) {\n    addSection('People Also Ask');\n    peopleAlsoAsk.forEach((p, _i) => {\n      output += [`Q: ${p.question}`, `A: ${p.answer}`, '']\n        .filter(Boolean)\n        .join('\\n');\n    });\n  }\n\n  return output.trim();\n}\n"],"names":[],"mappings":"AAEM,SAAU,mBAAmB,CAAC,OAA2B,EAAA;IAC7D,IAAI,MAAM,GAAG,EAAE;AAEf,IAAA,MAAM,UAAU,GAAG,CAAC,KAAa,KAAU;AACzC,QAAA,MAAM,IAAI,CAAA,MAAA,EAAS,KAAK,CAAA,MAAA,CAAQ;AAClC,KAAC;;AAGD,IAAA,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,EAAE;AACrC,IAAA,IAAI,OAAO,CAAC,MAAM,EAAE;QAClB,UAAU,CAAC,aAAa,CAAC;QACzB,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,KAAI;AACvB,YAAA,MAAM,IAAI;gBACR,CAAU,OAAA,EAAA,CAAC,GAAG,CAAC,CAAA,EAAA,EAAK,CAAC,CAAC,KAAK,IAAI,YAAY,CAAE,CAAA;gBAC7C,CAAsC,mCAAA,EAAA,CAAC,GAAG,CAAC,CAAE,CAAA;gBAC7C,CAAQ,KAAA,EAAA,CAAC,CAAC,IAAI,CAAE,CAAA;AAChB,gBAAA,CAAC,CAAC,OAAO,IAAI,IAAI,GAAG,CAAY,SAAA,EAAA,CAAC,CAAC,OAAO,CAAA,CAAE,GAAG,EAAE;AAChD,gBAAA,CAAC,CAAC,IAAI,IAAI,IAAI,GAAG,CAAS,MAAA,EAAA,CAAC,CAAC,IAAI,CAAA,CAAE,GAAG,EAAE;AACvC,gBAAA,CAAC,CAAC,WAAW,IAAI,IAAI,GAAG,CAAW,QAAA,EAAA,CAAC,CAAC,WAAW,CAAA,CAAE,GAAG,EAAE;gBACvD,EAAE;gBACF,4BAA4B;AAC5B,gBAAA,GAAG,CAAC,CAAC,CAAC,UAAU,IAAI,EAAE;AACnB,qBAAA,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC;qBACtC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAA,YAAA,EAAe,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA,GAAA,EAAM,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAA,CAAE,CAAC;gBACrE,EAAE;AACH;iBACE,MAAM,CAAC,OAAO;iBACd,IAAI,CAAC,IAAI,CAAC;AACf,SAAC,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAoCJ,IAAA,IAAI,OAAO,CAAC,cAAc,IAAI,IAAI,EAAE;QAClC,UAAU,CAAC,iBAAiB,CAAC;AAC7B,QAAA,MAAM,IAAI;AACR,YAAA,CAAA,OAAA,EAAU,OAAO,CAAC,cAAc,CAAC,KAAK,IAAI,YAAY,CAAE,CAAA;AACxD,YAAA,OAAO,CAAC,cAAc,CAAC,WAAW,IAAI;AACpC,kBAAE,CAAgB,aAAA,EAAA,OAAO,CAAC,cAAc,CAAC,WAAW,CAAE;AACtD,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,cAAc,CAAC,IAAI,IAAI;AAC7B,kBAAE,CAAS,MAAA,EAAA,OAAO,CAAC,cAAc,CAAC,IAAI,CAAE;AACxC,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,cAAc,CAAC,QAAQ,IAAI;AACjC,kBAAE,CAAc,WAAA,EAAA,OAAO,CAAC,cAAc,CAAC,QAAQ,CAAE;AACjD,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,cAAc,CAAC,UAAU,IAAI;AACnC,kBAAE,CAAe,YAAA,EAAA,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,cAAc,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,CAAE;AAC7E,kBAAE,EAAE;YACN,EAAE;AACH;aACE,MAAM,CAAC,OAAO;aACd,IAAI,CAAC,IAAI,CAAC;;;AAIf,IAAA,IAAI,OAAO,CAAC,SAAS,IAAI,IAAI,EAAE;QAC7B,UAAU,CAAC,YAAY,CAAC;AACxB,QAAA,MAAM,IAAI;AACR,YAAA,OAAO,CAAC,SAAS,CAAC,KAAK,IAAI;AACzB,kBAAE,CAAU,OAAA,EAAA,OAAO,CAAC,SAAS,CAAC,KAAK,CAAE;AACrC,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,SAAS,CAAC,MAAM,IAAI;AAC1B,kBAAE,CAAW,QAAA,EAAA,OAAO,CAAC,SAAS,CAAC,MAAM,CAAE;AACvC,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,SAAS,CAAC,OAAO,IAAI;AAC3B,kBAAE,CAAY,SAAA,EAAA,OAAO,CAAC,SAAS,CAAC,OAAO,CAAE;AACzC,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,SAAS,CAAC,IAAI,IAAI,IAAI,GAAG,SAAS,OAAO,CAAC,SAAS,CAAC,IAAI,EAAE,GAAG,EAAE;YACvE,EAAE;AACH;aACE,MAAM,CAAC,OAAO;aACd,IAAI,CAAC,IAAI,CAAC;;;AAIf,IAAA,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,EAAE;AACjD,IAAA,IAAI,aAAa,CAAC,MAAM,EAAE;QACxB,UAAU,CAAC,iBAAiB,CAAC;QAC7B,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,KAAI;AAC9B,YAAA,MAAM,IAAI,CAAC,CAAM,GAAA,EAAA,CAAC,CAAC,QAAQ,CAAA,CAAE,EAAE,CAAA,GAAA,EAAM,CAAC,CAAC,MAAM,CAAE,CAAA,EAAE,EAAE;iBAChD,MAAM,CAAC,OAAO;iBACd,IAAI,CAAC,IAAI,CAAC;AACf,SAAC,CAAC;;AAGJ,IAAA,OAAO,MAAM,CAAC,IAAI,EAAE;AACtB;;;;"}
+{"version":3,"file":"format.mjs","sources":["../../../../src/tools/search/format.ts"],"sourcesContent":["import type * as t from './types';\nimport { getDomainName, fileExtRegex } from './utils';\n\nfunction addHighlightSection(): string[] {\n  return ['\\n## Highlights', ''];\n}\n\n// Helper function to format a source (organic or top story)\nfunction formatSource(\n  source: t.ValidSource,\n  index: number,\n  turn: number,\n  sourceType: 'search' | 'news',\n  references: t.ResultReference[]\n): string {\n  /** Array of all lines to include in the output */\n  const outputLines: string[] = [];\n\n  // Add the title\n  outputLines.push(\n    `# ${sourceType.charAt(0).toUpperCase() + sourceType.slice(1)} ${index}: ${source.title != null && source.title ? `\"${source.title}\"` : '(no title)'}`\n  );\n  outputLines.push(`\\nAnchor: \\\\ue202turn${turn}${sourceType}${index}`);\n  outputLines.push(`URL: ${source.link}`);\n\n  // Add optional fields\n  if ('snippet' in source && source.snippet != null) {\n    outputLines.push(`Summary: ${source.snippet}`);\n  }\n\n  if (source.date != null) {\n    outputLines.push(`Date: ${source.date}`);\n  }\n\n  if (source.attribution != null) {\n    outputLines.push(`Source: ${source.attribution}`);\n  }\n\n  // Add highlight section or empty line\n  if ((source.highlights?.length ?? 0) > 0) {\n    outputLines.push(...addHighlightSection());\n  } else {\n    outputLines.push('');\n  }\n\n  // Process highlights if they exist\n  (source.highlights ?? [])\n    .filter((h) => h.text.trim().length > 0)\n    .forEach((h, hIndex) => {\n      outputLines.push(\n        `### Highlight ${hIndex + 1} [Relevance: ${h.score.toFixed(2)}]`\n      );\n      outputLines.push('');\n      outputLines.push('```text');\n      outputLines.push(h.text.trim());\n      outputLines.push('```');\n      outputLines.push('');\n\n      if (h.references != null && h.references.length) {\n        let hasHeader = false;\n        const refLines: string[] = [];\n\n        for (let j = 0; j < h.references.length; j++) {\n          const ref = h.references[j];\n          if (ref.reference.originalUrl.includes('mailto:')) {\n            continue;\n          }\n          references.push({\n            type: ref.type,\n            link: ref.reference.originalUrl,\n            attribution: getDomainName(ref.reference.originalUrl),\n            title: (\n              ((ref.reference.title ?? '') || ref.reference.text) ??\n              ''\n            ).split('\\n')[0],\n          });\n\n          if (ref.type !== 'link') {\n            continue;\n          }\n\n          if (fileExtRegex.test(ref.reference.originalUrl)) {\n            continue;\n          }\n\n          if (!hasHeader) {\n            refLines.push('Core References:');\n            hasHeader = true;\n          }\n\n          refLines.push(\n            `- ${ref.type}#${ref.originalIndex + 1}: ${ref.reference.originalUrl}`\n          );\n          refLines.push(\n            `\\t- Anchor: \\\\ue202turn${turn}ref${references.length - 1}`\n          );\n        }\n\n        if (hasHeader) {\n          outputLines.push(...refLines);\n          outputLines.push('');\n        }\n      }\n\n      if (hIndex < (source.highlights?.length ?? 0) - 1) {\n        outputLines.push('---');\n        outputLines.push('');\n      }\n    });\n\n  outputLines.push('');\n  return outputLines.join('\\n');\n}\n\nexport function formatResultsForLLM(\n  turn: number,\n  results: t.SearchResultData\n): { output: string; references: t.ResultReference[] } {\n  /** Array to collect all output lines */\n  const outputLines: string[] = [];\n\n  const addSection = (title: string): void => {\n    outputLines.push('');\n    outputLines.push(`=== ${title} ===`);\n    outputLines.push('');\n  };\n\n  const references: t.ResultReference[] = [];\n\n  // Organic (web) results\n  if (results.organic?.length != null && results.organic.length > 0) {\n    addSection(`Web Results, Turn ${turn}`);\n    for (let i = 0; i < results.organic.length; i++) {\n      const r = results.organic[i];\n      outputLines.push(formatSource(r, i, turn, 'search', references));\n      delete results.organic[i].highlights;\n    }\n  }\n\n  // Top stories (news)\n  const topStories = results.topStories ?? [];\n  if (topStories.length) {\n    addSection('News Results');\n    for (let i = 0; i < topStories.length; i++) {\n      const r = topStories[i];\n      outputLines.push(formatSource(r, i, turn, 'news', references));\n      if (results.topStories?.[i]?.highlights) {\n        delete results.topStories[i].highlights;\n      }\n    }\n  }\n\n  // // Images\n  // const images = results.images ?? [];\n  // if (images.length) {\n  //   addSection('Image Results');\n  //   const imageLines = images.map((img, i) => [\n  //     `Anchor: \\ue202turn0image${i}`,\n  //     `Title: ${img.title ?? '(no title)'}`,\n  //     `Image URL: ${img.imageUrl}`,\n  //     ''\n  //   ].join('\\n'));\n  //   outputLines.push(imageLines.join('\\n'));\n  // }\n\n  // Knowledge Graph\n  if (results.knowledgeGraph != null) {\n    addSection('Knowledge Graph');\n    const kgLines = [\n      `**Title:** ${results.knowledgeGraph.title ?? '(no title)'}`,\n      results.knowledgeGraph.type != null\n        ? `**Type:** ${results.knowledgeGraph.type}`\n        : '',\n      results.knowledgeGraph.description != null\n        ? `**Description:** ${results.knowledgeGraph.description}`\n        : '',\n      results.knowledgeGraph.descriptionSource != null\n        ? `**Description Source:** ${results.knowledgeGraph.descriptionSource}`\n        : '',\n      results.knowledgeGraph.descriptionLink != null\n        ? `**Description Link:** ${results.knowledgeGraph.descriptionLink}`\n        : '',\n      results.knowledgeGraph.imageUrl != null\n        ? `**Image URL:** ${results.knowledgeGraph.imageUrl}`\n        : '',\n      results.knowledgeGraph.website != null\n        ? `**Website:** ${results.knowledgeGraph.website}`\n        : '',\n      results.knowledgeGraph.attributes != null\n        ? `**Attributes:**\\n\\`\\`\\`json\\n${JSON.stringify(\n          results.knowledgeGraph.attributes,\n          null,\n          2\n        )}\\n\\`\\`\\``\n        : '',\n      '',\n    ].filter(Boolean);\n\n    outputLines.push(kgLines.join('\\n\\n'));\n  }\n\n  // Answer Box\n  if (results.answerBox != null) {\n    addSection('Answer Box');\n    const abLines = [\n      results.answerBox.title != null\n        ? `**Title:** ${results.answerBox.title}`\n        : '',\n      results.answerBox.snippet != null\n        ? `**Snippet:** ${results.answerBox.snippet}`\n        : '',\n      results.answerBox.snippetHighlighted != null\n        ? `**Snippet Highlighted:** ${results.answerBox.snippetHighlighted\n          .map((s) => `\\`${s}\\``)\n          .join(' ')}`\n        : '',\n      results.answerBox.link != null\n        ? `**Link:** ${results.answerBox.link}`\n        : '',\n      '',\n    ].filter(Boolean);\n\n    outputLines.push(abLines.join('\\n\\n'));\n  }\n\n  // People also ask\n  const peopleAlsoAsk = results.peopleAlsoAsk ?? [];\n  if (peopleAlsoAsk.length) {\n    addSection('People Also Ask');\n\n    const paaLines: string[] = [];\n    peopleAlsoAsk.forEach((p, i) => {\n      const questionLines = [\n        `### Question ${i + 1}:`,\n        `\"${p.question}\"`,\n        `${p.snippet != null && p.snippet ? `Snippet: ${p.snippet}` : ''}`,\n        `${p.title != null && p.title ? `Title: ${p.title}` : ''}`,\n        `${p.link != null && p.link ? `Link: ${p.link}` : ''}`,\n        '',\n      ].filter(Boolean);\n\n      paaLines.push(questionLines.join('\\n\\n'));\n    });\n\n    outputLines.push(paaLines.join(''));\n  }\n\n  return {\n    output: outputLines.join('\\n').trim(),\n    references,\n  };\n}\n"],"names":[],"mappings":";;AAGA,SAAS,mBAAmB,GAAA;AAC1B,IAAA,OAAO,CAAC,iBAAiB,EAAE,EAAE,CAAC;AAChC;AAEA;AACA,SAAS,YAAY,CACnB,MAAqB,EACrB,KAAa,EACb,IAAY,EACZ,UAA6B,EAC7B,UAA+B,EAAA;;IAG/B,MAAM,WAAW,GAAa,EAAE;;IAGhC,WAAW,CAAC,IAAI,CACd,CAAA,EAAA,EAAK,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA,CAAA,EAAI,KAAK,CAAA,EAAA,EAAK,MAAM,CAAC,KAAK,IAAI,IAAI,IAAI,MAAM,CAAC,KAAK,GAAG,IAAI,MAAM,CAAC,KAAK,CAAG,CAAA,CAAA,GAAG,YAAY,CAAE,CAAA,CACvJ;IACD,WAAW,CAAC,IAAI,CAAC,CAAwB,qBAAA,EAAA,IAAI,CAAG,EAAA,UAAU,CAAG,EAAA,KAAK,CAAE,CAAA,CAAC;IACrE,WAAW,CAAC,IAAI,CAAC,CAAA,KAAA,EAAQ,MAAM,CAAC,IAAI,CAAE,CAAA,CAAC;;IAGvC,IAAI,SAAS,IAAI,MAAM,IAAI,MAAM,CAAC,OAAO,IAAI,IAAI,EAAE;QACjD,WAAW,CAAC,IAAI,CAAC,CAAA,SAAA,EAAY,MAAM,CAAC,OAAO,CAAE,CAAA,CAAC;;AAGhD,IAAA,IAAI,MAAM,CAAC,IAAI,IAAI,IAAI,EAAE;QACvB,WAAW,CAAC,IAAI,CAAC,CAAA,MAAA,EAAS,MAAM,CAAC,IAAI,CAAE,CAAA,CAAC;;AAG1C,IAAA,IAAI,MAAM,CAAC,WAAW,IAAI,IAAI,EAAE;QAC9B,WAAW,CAAC,IAAI,CAAC,CAAA,QAAA,EAAW,MAAM,CAAC,WAAW,CAAE,CAAA,CAAC;;;AAInD,IAAA,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,MAAM,IAAI,CAAC,IAAI,CAAC,EAAE;AACxC,QAAA,WAAW,CAAC,IAAI,CAAC,GAAG,mBAAmB,EAAE,CAAC;;SACrC;AACL,QAAA,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;;;AAItB,IAAA,CAAC,MAAM,CAAC,UAAU,IAAI,EAAE;AACrB,SAAA,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC;AACtC,SAAA,OAAO,CAAC,CAAC,CAAC,EAAE,MAAM,KAAI;AACrB,QAAA,WAAW,CAAC,IAAI,CACd,iBAAiB,MAAM,GAAG,CAAC,CAAgB,aAAA,EAAA,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA,CAAA,CAAG,CACjE;AACD,QAAA,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;AACpB,QAAA,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC;QAC3B,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;AAC/B,QAAA,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC;AACvB,QAAA,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;AAEpB,QAAA,IAAI,CAAC,CAAC,UAAU,IAAI,IAAI,IAAI,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE;YAC/C,IAAI,SAAS,GAAG,KAAK;YACrB,MAAM,QAAQ,GAAa,EAAE;AAE7B,YAAA,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;gBAC5C,MAAM,GAAG,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC;gBAC3B,IAAI,GAAG,CAAC,SAAS,CAAC,WAAW,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;oBACjD;;gBAEF,UAAU,CAAC,IAAI,CAAC;oBACd,IAAI,EAAE,GAAG,CAAC,IAAI;AACd,oBAAA,IAAI,EAAE,GAAG,CAAC,SAAS,CAAC,WAAW;oBAC/B,WAAW,EAAE,aAAa,CAAC,GAAG,CAAC,SAAS,CAAC,WAAW,CAAC;AACrD,oBAAA,KAAK,EAAE,CACL,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,IAAI,EAAE,KAAK,GAAG,CAAC,SAAS,CAAC,IAAI;wBAClD,EAAE,EACF,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACjB,iBAAA,CAAC;AAEF,gBAAA,IAAI,GAAG,CAAC,IAAI,KAAK,MAAM,EAAE;oBACvB;;gBAGF,IAAI,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,WAAW,CAAC,EAAE;oBAChD;;gBAGF,IAAI,CAAC,SAAS,EAAE;AACd,oBAAA,QAAQ,CAAC,IAAI,CAAC,kBAAkB,CAAC;oBACjC,SAAS,GAAG,IAAI;;gBAGlB,QAAQ,CAAC,IAAI,CACX,CAAA,EAAA,EAAK,GAAG,CAAC,IAAI,IAAI,GAAG,CAAC,aAAa,GAAG,CAAC,KAAK,GAAG,CAAC,SAAS,CAAC,WAAW,CAAE,CAAA,CACvE;AACD,gBAAA,QAAQ,CAAC,IAAI,CACX,CAAA,uBAAA,EAA0B,IAAI,CAAA,GAAA,EAAM,UAAU,CAAC,MAAM,GAAG,CAAC,CAAA,CAAE,CAC5D;;YAGH,IAAI,SAAS,EAAE;AACb,gBAAA,WAAW,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC;AAC7B,gBAAA,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;;;AAIxB,QAAA,IAAI,MAAM,GAAG,CAAC,MAAM,CAAC,UAAU,EAAE,MAAM,IAAI,CAAC,IAAI,CAAC,EAAE;AACjD,YAAA,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC;AACvB,YAAA,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;;AAExB,KAAC,CAAC;AAEJ,IAAA,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;AACpB,IAAA,OAAO,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC;AAC/B;AAEgB,SAAA,mBAAmB,CACjC,IAAY,EACZ,OAA2B,EAAA;;IAG3B,MAAM,WAAW,GAAa,EAAE;AAEhC,IAAA,MAAM,UAAU,GAAG,CAAC,KAAa,KAAU;AACzC,QAAA,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;AACpB,QAAA,WAAW,CAAC,IAAI,CAAC,OAAO,KAAK,CAAA,IAAA,CAAM,CAAC;AACpC,QAAA,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;AACtB,KAAC;IAED,MAAM,UAAU,GAAwB,EAAE;;AAG1C,IAAA,IAAI,OAAO,CAAC,OAAO,EAAE,MAAM,IAAI,IAAI,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE;AACjE,QAAA,UAAU,CAAC,CAAA,kBAAA,EAAqB,IAAI,CAAA,CAAE,CAAC;AACvC,QAAA,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YAC/C,MAAM,CAAC,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;AAC5B,YAAA,WAAW,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,EAAE,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC;YAChE,OAAO,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,UAAU;;;;AAKxC,IAAA,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,EAAE;AAC3C,IAAA,IAAI,UAAU,CAAC,MAAM,EAAE;QACrB,UAAU,CAAC,cAAc,CAAC;AAC1B,QAAA,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;AAC1C,YAAA,MAAM,CAAC,GAAG,UAAU,CAAC,CAAC,CAAC;AACvB,YAAA,WAAW,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;YAC9D,IAAI,OAAO,CAAC,UAAU,GAAG,CAAC,CAAC,EAAE,UAAU,EAAE;gBACvC,OAAO,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,UAAU;;;;;;;;;;;;;;;;;AAmB7C,IAAA,IAAI,OAAO,CAAC,cAAc,IAAI,IAAI,EAAE;QAClC,UAAU,CAAC,iBAAiB,CAAC;AAC7B,QAAA,MAAM,OAAO,GAAG;AACd,YAAA,CAAA,WAAA,EAAc,OAAO,CAAC,cAAc,CAAC,KAAK,IAAI,YAAY,CAAE,CAAA;AAC5D,YAAA,OAAO,CAAC,cAAc,CAAC,IAAI,IAAI;AAC7B,kBAAE,CAAa,UAAA,EAAA,OAAO,CAAC,cAAc,CAAC,IAAI,CAAE;AAC5C,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,cAAc,CAAC,WAAW,IAAI;AACpC,kBAAE,CAAoB,iBAAA,EAAA,OAAO,CAAC,cAAc,CAAC,WAAW,CAAE;AAC1D,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,cAAc,CAAC,iBAAiB,IAAI;AAC1C,kBAAE,CAA2B,wBAAA,EAAA,OAAO,CAAC,cAAc,CAAC,iBAAiB,CAAE;AACvE,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,cAAc,CAAC,eAAe,IAAI;AACxC,kBAAE,CAAyB,sBAAA,EAAA,OAAO,CAAC,cAAc,CAAC,eAAe,CAAE;AACnE,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,cAAc,CAAC,QAAQ,IAAI;AACjC,kBAAE,CAAkB,eAAA,EAAA,OAAO,CAAC,cAAc,CAAC,QAAQ,CAAE;AACrD,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,cAAc,CAAC,OAAO,IAAI;AAChC,kBAAE,CAAgB,aAAA,EAAA,OAAO,CAAC,cAAc,CAAC,OAAO,CAAE;AAClD,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,cAAc,CAAC,UAAU,IAAI;AACnC,kBAAE,CAAgC,6BAAA,EAAA,IAAI,CAAC,SAAS,CAC9C,OAAO,CAAC,cAAc,CAAC,UAAU,EACjC,IAAI,EACJ,CAAC,CACF,CAAU,QAAA;AACX,kBAAE,EAAE;YACN,EAAE;AACH,SAAA,CAAC,MAAM,CAAC,OAAO,CAAC;QAEjB,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;;;AAIxC,IAAA,IAAI,OAAO,CAAC,SAAS,IAAI,IAAI,EAAE;QAC7B,UAAU,CAAC,YAAY,CAAC;AACxB,QAAA,MAAM,OAAO,GAAG;AACd,YAAA,OAAO,CAAC,SAAS,CAAC,KAAK,IAAI;AACzB,kBAAE,CAAc,WAAA,EAAA,OAAO,CAAC,SAAS,CAAC,KAAK,CAAE;AACzC,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,SAAS,CAAC,OAAO,IAAI;AAC3B,kBAAE,CAAgB,aAAA,EAAA,OAAO,CAAC,SAAS,CAAC,OAAO,CAAE;AAC7C,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,SAAS,CAAC,kBAAkB,IAAI;AACtC,kBAAE,CAA4B,yBAAA,EAAA,OAAO,CAAC,SAAS,CAAC;qBAC7C,GAAG,CAAC,CAAC,CAAC,KAAK,CAAA,EAAA,EAAK,CAAC,CAAA,EAAA,CAAI;qBACrB,IAAI,CAAC,GAAG,CAAC,CAAE;AACd,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,SAAS,CAAC,IAAI,IAAI;AACxB,kBAAE,CAAa,UAAA,EAAA,OAAO,CAAC,SAAS,CAAC,IAAI,CAAE;AACvC,kBAAE,EAAE;YACN,EAAE;AACH,SAAA,CAAC,MAAM,CAAC,OAAO,CAAC;QAEjB,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;;;AAIxC,IAAA,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,EAAE;AACjD,IAAA,IAAI,aAAa,CAAC,MAAM,EAAE;QACxB,UAAU,CAAC,iBAAiB,CAAC;QAE7B,MAAM,QAAQ,GAAa,EAAE;QAC7B,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,KAAI;AAC7B,YAAA,MAAM,aAAa,GAAG;gBACpB,CAAgB,aAAA,EAAA,CAAC,GAAG,CAAC,CAAG,CAAA,CAAA;gBACxB,CAAI,CAAA,EAAA,CAAC,CAAC,QAAQ,CAAG,CAAA,CAAA;gBACjB,CAAG,EAAA,CAAC,CAAC,OAAO,IAAI,IAAI,IAAI,CAAC,CAAC,OAAO,GAAG,CAAA,SAAA,EAAY,CAAC,CAAC,OAAO,EAAE,GAAG,EAAE,CAAE,CAAA;gBAClE,CAAG,EAAA,CAAC,CAAC,KAAK,IAAI,IAAI,IAAI,CAAC,CAAC,KAAK,GAAG,CAAA,OAAA,EAAU,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,CAAE,CAAA;gBAC1D,CAAG,EAAA,CAAC,CAAC,IAAI,IAAI,IAAI,IAAI,CAAC,CAAC,IAAI,GAAG,CAAA,MAAA,EAAS,CAAC,CAAC,IAAI,EAAE,GAAG,EAAE,CAAE,CAAA;gBACtD,EAAE;AACH,aAAA,CAAC,MAAM,CAAC,OAAO,CAAC;YAEjB,QAAQ,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AAC3C,SAAC,CAAC;QAEF,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;;IAGrC,OAAO;QACL,MAAM,EAAE,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE;QACrC,UAAU;KACX;AACH;;;;"}

package/dist/esm/tools/search/highlights.mjs

@@ -101,6 +101,57 @@ function findBestBoundary(text, direction = 'backward') {
     // No match found, use character boundary
     return direction === 'backward' ? text.length : 0;
 }
+/**
+ * Tracks references used in a highlight without changing their numbers
+ */
+function trackReferencesInHighlight(text, sourceResult // Source containing the original references
+) {
+    // Track used references
+    const references = [];
+    if (!text || text.length === 0 || !text.includes('#')) {
+        return { references }; // Early return
+    }
+    // Quick check for reference markers
+    if (!text.includes('link#') &&
+        !text.includes('image#') &&
+        !text.includes('video#')) {
+        return { references };
+    }
+    // Get references from the source if available
+    const sourceRefs = sourceResult.references || {
+        links: [],
+        images: [],
+        videos: [],
+    };
+    // Find references but don't modify text
+    const refRegex = /\((link|image|video)#(\d+)(?:\s+"([^"]*)")?\)/g;
+    let match;
+    while ((match = refRegex.exec(text)) !== null) {
+        const [, type, indexStr] = match;
+        const originalIndex = parseInt(indexStr, 10) - 1; // Convert to 0-based
+        // Get the source array for this type
+        const refType = type;
+        const sourceArray = sourceRefs[`${refType}s`];
+        // Skip if invalid reference
+        if (!sourceArray ||
+            originalIndex < 0 ||
+            originalIndex >= sourceArray.length) {
+            continue; // Skip invalid references
+        }
+        // Get original reference
+        const reference = sourceArray[originalIndex];
+        // Track if not already tracked
+        const alreadyTracked = references.some((ref) => ref.type === refType && ref.originalIndex === originalIndex);
+        if (!alreadyTracked) {
+            references.push({
+                type: refType,
+                originalIndex,
+                reference,
+            });
+        }
+    }
+    return { references };
+}
 /**
  * Expand highlights in search results using smart boundary detection.
  *
@@ -110,19 +161,16 @@ function findBestBoundary(text, direction = 'backward') {
  * @param searchResults - Search results object
  * @param mainExpandBy - Primary expansion size on each side (default: 300)
  * @param separatorExpandBy - Additional range to look for separators (default: 150)
- * @returns Copy of search results with expanded highlights
+ * @returns Copy of search results with expanded highlights and tracked references
  */
 function expandHighlights(searchResults, mainExpandBy = 300, separatorExpandBy = 150) {
-    // 1. Avoid full deep copy - only copy what we modify
+    // Avoid deep copy - only copy what we modify
     const resultCopy = { ...searchResults };
-    // Only deep copy the relevant arrays
-    if (resultCopy.organic) {
+    if (resultCopy.organic)
         resultCopy.organic = [...resultCopy.organic];
-    }
-    if (resultCopy.topStories) {
+    if (resultCopy.topStories)
         resultCopy.topStories = [...resultCopy.topStories];
-    }
-    // 5. Process the results efficiently
+    // Process the results efficiently
     const processResultTypes = ['organic', 'topStories'];
     for (const resultType of processResultTypes) {
         if (!resultCopy[resultType])
@@ -141,17 +189,18 @@ function expandHighlights(searchResults, mainExpandBy = 300, separatorExpandBy =
             const highlights = [];
             // Process each highlight
             for (const highlight of result.highlights) {
-                const highlightText = highlight.text;
-                let startPos = content.indexOf(highlightText);
-                let highlightLen = highlightText.length;
+                const { references } = trackReferencesInHighlight(highlight.text, result);
+                let startPos = content.indexOf(highlight.text);
+                let highlightLen = highlight.text.length;
                 if (startPos === -1) {
                     // Try with stripped whitespace
-                    const strippedHighlight = highlightText.trim();
+                    const strippedHighlight = highlight.text.trim();
                     startPos = content.indexOf(strippedHighlight);
                     if (startPos === -1) {
                         highlights.push({
                             text: highlight.text,
                             score: highlight.score,
+                            references,
                         });
                         continue;
                     }
@@ -178,9 +227,12 @@ function expandHighlights(searchResults, mainExpandBy = 300, separatorExpandBy =
                 highlights.push({
                     text: expandedHighlightText,
                     score: highlight.score,
+                    references,
                 });
             }
             resultCopy.highlights = highlights;
+            delete resultCopy.content;
+            delete resultCopy.references;
             return resultCopy;
         });
     }