@librechat/agents 2.4.316 → 2.4.318

package/dist/cjs/tools/search/content.cjs

@@ -0,0 +1,140 @@
+'use strict';
+
+var cheerio = require('cheerio');
+
+function _interopNamespaceDefault(e) {
+    var n = Object.create(null);
+    if (e) {
+        Object.keys(e).forEach(function (k) {
+            if (k !== 'default') {
+                var d = Object.getOwnPropertyDescriptor(e, k);
+                Object.defineProperty(n, k, d.get ? d : {
+                    enumerable: true,
+                    get: function () { return e[k]; }
+                });
+            }
+        });
+    }
+    n.default = e;
+    return Object.freeze(n);
+}
+
+var cheerio__namespace = /*#__PURE__*/_interopNamespaceDefault(cheerio);
+
+function processContent(html, markdown) {
+    const linkMap = new Map();
+    const imageMap = new Map();
+    const videoMap = new Map();
+    const iframeMap = new Map();
+    const $ = cheerio__namespace.load(html, {
+        xmlMode: false,
+    });
+    // Extract all media references
+    $('a[href]').each((_, el) => {
+        const href = $(el).attr('href');
+        if (href != null && href) {
+            linkMap.set(href, {
+                originalUrl: href,
+                title: $(el).attr('title'),
+                text: $(el).text().trim(),
+            });
+        }
+    });
+    $('img[src]').each((_, el) => {
+        const src = $(el).attr('src');
+        if (src != null && src) {
+            imageMap.set(src, {
+                originalUrl: src,
+                title: $(el).attr('alt') ?? $(el).attr('title'),
+            });
+        }
+    });
+    // Handle videos (dedicated video elements and video platforms in iframes)
+    $('video[src], iframe[src*="youtube"], iframe[src*="vimeo"]').each((_, el) => {
+        const src = $(el).attr('src');
+        if (src != null && src) {
+            videoMap.set(src, {
+                originalUrl: src,
+                title: $(el).attr('title'),
+            });
+        }
+    });
+    // Handle all other generic iframes that aren't already captured as videos
+    $('iframe').each((_, el) => {
+        const src = $(el).attr('src');
+        if (src != null &&
+            src &&
+            !src.includes('youtube') &&
+            !src.includes('vimeo')) {
+            iframeMap.set(src, {
+                originalUrl: src,
+                title: $(el).attr('title'),
+            });
+        }
+    });
+    // Create lookup maps with indices
+    const linkIndexMap = new Map();
+    const imageIndexMap = new Map();
+    const videoIndexMap = new Map();
+    const iframeIndexMap = new Map();
+    Array.from(linkMap.keys()).forEach((url, i) => linkIndexMap.set(url, i + 1));
+    Array.from(imageMap.keys()).forEach((url, i) => imageIndexMap.set(url, i + 1));
+    Array.from(videoMap.keys()).forEach((url, i) => videoIndexMap.set(url, i + 1));
+    Array.from(iframeMap.keys()).forEach((url, i) => iframeIndexMap.set(url, i + 1));
+    // Process the markdown
+    let result = markdown;
+    // Replace each URL one by one, starting with the longest URLs first to avoid partial matches
+    const allUrls = [
+        ...Array.from(imageMap.keys()).map((url) => ({
+            url,
+            type: 'image',
+            idx: imageIndexMap.get(url),
+        })),
+        ...Array.from(videoMap.keys()).map((url) => ({
+            url,
+            type: 'video',
+            idx: videoIndexMap.get(url),
+        })),
+        ...Array.from(iframeMap.keys()).map((url) => ({
+            url,
+            type: 'iframe',
+            idx: iframeIndexMap.get(url),
+        })),
+        ...Array.from(linkMap.keys()).map((url) => ({
+            url,
+            type: 'link',
+            idx: linkIndexMap.get(url),
+        })),
+    ].sort((a, b) => b.url.length - a.url.length);
+    // Create a function to escape special characters in URLs for regex
+    function escapeRegex(string) {
+        return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+    }
+    // Replace each URL in the markdown
+    for (const { url, type, idx } of allUrls) {
+        // Create a regex that captures URLs in markdown links
+        const regex = new RegExp(`\\(${escapeRegex(url)}(?:\\s+"[^"]*")?\\)`, 'g');
+        result = result.replace(regex, (match) => {
+            // Keep any title attribute that might exist
+            const titleMatch = match.match(/\s+"([^"]*)"/);
+            const titlePart = titleMatch ? ` "${titleMatch[1]}"` : '';
+            return `(${type}#${idx}${titlePart})`;
+        });
+    }
+    iframeMap.clear();
+    const links = Array.from(linkMap.values());
+    linkMap.clear();
+    const images = Array.from(imageMap.values());
+    imageMap.clear();
+    const videos = Array.from(videoMap.values());
+    videoMap.clear();
+    return {
+        markdown: result,
+        links,
+        images,
+        videos,
+    };
+}
+
+exports.processContent = processContent;
+//# sourceMappingURL=content.cjs.map

package/dist/cjs/tools/search/content.cjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"content.cjs","sources":["../../../../src/tools/search/content.ts"],"sourcesContent":["import * as cheerio from 'cheerio';\nimport type { References, MediaReference } from './types';\n\nexport function processContent(\n  html: string,\n  markdown: string\n): {\n  markdown: string;\n} & References {\n  const linkMap = new Map<string, MediaReference>();\n  const imageMap = new Map<string, MediaReference>();\n  const videoMap = new Map<string, MediaReference>();\n  const iframeMap = new Map<string, MediaReference>();\n\n  const $ = cheerio.load(html, {\n    xmlMode: false,\n  });\n\n  // Extract all media references\n  $('a[href]').each((_, el) => {\n    const href = $(el).attr('href');\n    if (href != null && href) {\n      linkMap.set(href, {\n        originalUrl: href,\n        title: $(el).attr('title'),\n        text: $(el).text().trim(),\n      });\n    }\n  });\n\n  $('img[src]').each((_, el) => {\n    const src = $(el).attr('src');\n    if (src != null && src) {\n      imageMap.set(src, {\n        originalUrl: src,\n        title: $(el).attr('alt') ?? $(el).attr('title'),\n      });\n    }\n  });\n\n  // Handle videos (dedicated video elements and video platforms in iframes)\n  $('video[src], iframe[src*=\"youtube\"], iframe[src*=\"vimeo\"]').each(\n    (_, el) => {\n      const src = $(el).attr('src');\n      if (src != null && src) {\n        videoMap.set(src, {\n          originalUrl: src,\n          title: $(el).attr('title'),\n        });\n      }\n    }\n  );\n\n  // Handle all other generic iframes that aren't already captured as videos\n  $('iframe').each((_, el) => {\n    const src = $(el).attr('src');\n    if (\n      src != null &&\n      src &&\n      !src.includes('youtube') &&\n      !src.includes('vimeo')\n    ) {\n      iframeMap.set(src, {\n        originalUrl: src,\n        title: $(el).attr('title'),\n      });\n    }\n  });\n\n  // Create lookup maps with indices\n  const linkIndexMap = new Map<string, number>();\n  const imageIndexMap = new Map<string, number>();\n  const videoIndexMap = new Map<string, number>();\n  const iframeIndexMap = new Map<string, number>();\n\n  Array.from(linkMap.keys()).forEach((url, i) => linkIndexMap.set(url, i + 1));\n  Array.from(imageMap.keys()).forEach((url, i) =>\n    imageIndexMap.set(url, i + 1)\n  );\n  Array.from(videoMap.keys()).forEach((url, i) =>\n    videoIndexMap.set(url, i + 1)\n  );\n  Array.from(iframeMap.keys()).forEach((url, i) =>\n    iframeIndexMap.set(url, i + 1)\n  );\n\n  // Process the markdown\n  let result = markdown;\n\n  // Replace each URL one by one, starting with the longest URLs first to avoid partial matches\n  const allUrls = [\n    ...Array.from(imageMap.keys()).map((url) => ({\n      url,\n      type: 'image',\n      idx: imageIndexMap.get(url),\n    })),\n    ...Array.from(videoMap.keys()).map((url) => ({\n      url,\n      type: 'video',\n      idx: videoIndexMap.get(url),\n    })),\n    ...Array.from(iframeMap.keys()).map((url) => ({\n      url,\n      type: 'iframe',\n      idx: iframeIndexMap.get(url),\n    })),\n    ...Array.from(linkMap.keys()).map((url) => ({\n      url,\n      type: 'link',\n      idx: linkIndexMap.get(url),\n    })),\n  ].sort((a, b) => b.url.length - a.url.length);\n\n  // Create a function to escape special characters in URLs for regex\n  function escapeRegex(string: string): string {\n    return string.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\$&');\n  }\n\n  // Replace each URL in the markdown\n  for (const { url, type, idx } of allUrls) {\n    // Create a regex that captures URLs in markdown links\n    const regex = new RegExp(`\\\\(${escapeRegex(url)}(?:\\\\s+\"[^\"]*\")?\\\\)`, 'g');\n\n    result = result.replace(regex, (match) => {\n      // Keep any title attribute that might exist\n      const titleMatch = match.match(/\\s+\"([^\"]*)\"/);\n      const titlePart = titleMatch ? ` \"${titleMatch[1]}\"` : '';\n\n      return `(${type}#${idx}${titlePart})`;\n    });\n  }\n\n  iframeMap.clear();\n  const links = Array.from(linkMap.values());\n  linkMap.clear();\n  const images = Array.from(imageMap.values());\n  imageMap.clear();\n  const videos = Array.from(videoMap.values());\n  videoMap.clear();\n\n  return {\n    markdown: result,\n    links,\n    images,\n    videos,\n  };\n}\n"],"names":["cheerio"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;AAGgB,SAAA,cAAc,CAC5B,IAAY,EACZ,QAAgB,EAAA;AAIhB,IAAA,MAAM,OAAO,GAAG,IAAI,GAAG,EAA0B;AACjD,IAAA,MAAM,QAAQ,GAAG,IAAI,GAAG,EAA0B;AAClD,IAAA,MAAM,QAAQ,GAAG,IAAI,GAAG,EAA0B;AAClD,IAAA,MAAM,SAAS,GAAG,IAAI,GAAG,EAA0B;AAEnD,IAAA,MAAM,CAAC,GAAGA,kBAAO,CAAC,IAAI,CAAC,IAAI,EAAE;AAC3B,QAAA,OAAO,EAAE,KAAK;AACf,KAAA,CAAC;;IAGF,CAAC,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,KAAI;QAC1B,MAAM,IAAI,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;AAC/B,QAAA,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,EAAE;AACxB,YAAA,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE;AAChB,gBAAA,WAAW,EAAE,IAAI;gBACjB,KAAK,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC;gBAC1B,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE;AAC1B,aAAA,CAAC;;AAEN,KAAC,CAAC;IAEF,CAAC,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,KAAI;QAC3B,MAAM,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC;AAC7B,QAAA,IAAI,GAAG,IAAI,IAAI,IAAI,GAAG,EAAE;AACtB,YAAA,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE;AAChB,gBAAA,WAAW,EAAE,GAAG;AAChB,gBAAA,KAAK,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC;AAChD,aAAA,CAAC;;AAEN,KAAC,CAAC;;IAGF,CAAC,CAAC,0DAA0D,CAAC,CAAC,IAAI,CAChE,CAAC,CAAC,EAAE,EAAE,KAAI;QACR,MAAM,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC;AAC7B,QAAA,IAAI,GAAG,IAAI,IAAI,IAAI,GAAG,EAAE;AACtB,YAAA,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE;AAChB,gBAAA,WAAW,EAAE,GAAG;gBAChB,KAAK,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC;AAC3B,aAAA,CAAC;;AAEN,KAAC,CACF;;IAGD,CAAC,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,KAAI;QACzB,MAAM,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC;QAC7B,IACE,GAAG,IAAI,IAAI;YACX,GAAG;AACH,YAAA,CAAC,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC;AACxB,YAAA,CAAC,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,EACtB;AACA,YAAA,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE;AACjB,gBAAA,WAAW,EAAE,GAAG;gBAChB,KAAK,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC;AAC3B,aAAA,CAAC;;AAEN,KAAC,CAAC;;AAGF,IAAA,MAAM,YAAY,GAAG,IAAI,GAAG,EAAkB;AAC9C,IAAA,MAAM,aAAa,GAAG,IAAI,GAAG,EAAkB;AAC/C,IAAA,MAAM,aAAa,GAAG,IAAI,GAAG,EAAkB;AAC/C,IAAA,MAAM,cAAc,GAAG,IAAI,GAAG,EAAkB;AAEhD,IAAA,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,CAAC,KAAK,YAAY,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;AAC5E,IAAA,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,CAAC,KACzC,aAAa,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAC9B;AACD,IAAA,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,CAAC,KACzC,aAAa,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAC9B;AACD,IAAA,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,CAAC,KAC1C,cAAc,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAC/B;;IAGD,IAAI,MAAM,GAAG,QAAQ;;AAGrB,IAAA,MAAM,OAAO,GAAG;AACd,QAAA,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM;YAC3C,GAAG;AACH,YAAA,IAAI,EAAE,OAAO;AACb,YAAA,GAAG,EAAE,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC;AAC5B,SAAA,CAAC,CAAC;AACH,QAAA,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM;YAC3C,GAAG;AACH,YAAA,IAAI,EAAE,OAAO;AACb,YAAA,GAAG,EAAE,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC;AAC5B,SAAA,CAAC,CAAC;AACH,QAAA,GAAG,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM;YAC5C,GAAG;AACH,YAAA,IAAI,EAAE,QAAQ;AACd,YAAA,GAAG,EAAE,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC;AAC7B,SAAA,CAAC,CAAC;AACH,QAAA,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM;YAC1C,GAAG;AACH,YAAA,IAAI,EAAE,MAAM;AACZ,YAAA,GAAG,EAAE,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC;AAC3B,SAAA,CAAC,CAAC;KACJ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC;;IAG7C,SAAS,WAAW,CAAC,MAAc,EAAA;QACjC,OAAO,MAAM,CAAC,OAAO,CAAC,qBAAqB,EAAE,MAAM,CAAC;;;IAItD,KAAK,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,OAAO,EAAE;;AAExC,QAAA,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,CAAM,GAAA,EAAA,WAAW,CAAC,GAAG,CAAC,CAAA,mBAAA,CAAqB,EAAE,GAAG,CAAC;QAE1E,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,KAAK,KAAI;;YAEvC,MAAM,UAAU,GAAG,KAAK,CAAC,KAAK,CAAC,cAAc,CAAC;AAC9C,YAAA,MAAM,SAAS,GAAG,UAAU,GAAG,CAAK,EAAA,EAAA,UAAU,CAAC,CAAC,CAAC,CAAG,CAAA,CAAA,GAAG,EAAE;AAEzD,YAAA,OAAO,IAAI,IAAI,CAAA,CAAA,EAAI,GAAG,CAAG,EAAA,SAAS,GAAG;AACvC,SAAC,CAAC;;IAGJ,SAAS,CAAC,KAAK,EAAE;IACjB,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;IAC1C,OAAO,CAAC,KAAK,EAAE;IACf,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;IAC5C,QAAQ,CAAC,KAAK,EAAE;IAChB,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;IAC5C,QAAQ,CAAC,KAAK,EAAE;IAEhB,OAAO;AACL,QAAA,QAAQ,EAAE,MAAM;QAChB,KAAK;QACL,MAAM;QACN,MAAM;KACP;AACH;;;;"}

package/dist/cjs/tools/search/firecrawl.cjs

@@ -1,37 +1,9 @@
 'use strict';
 
 var axios = require('axios');
+var content = require('./content.cjs');
 
 /* eslint-disable no-console */
-const getDomainName = (link, metadata) => {
-    try {
-        const url = metadata?.sourceURL ?? metadata?.url ?? (link || '');
-        const domain = new URL(url).hostname.replace(/^www\./, '');
-        if (domain) {
-            return domain;
-        }
-    }
-    catch (e) {
-        // URL parsing failed
-        console.error('Error parsing URL:', e);
-    }
-    return;
-};
-function getAttribution(link, metadata) {
-    if (!metadata)
-        return getDomainName(link, metadata);
-    const possibleAttributions = [
-        metadata.ogSiteName,
-        metadata['og:site_name'],
-        metadata.title?.split('|').pop()?.trim(),
-        metadata['twitter:site']?.replace(/^@/, ''),
-    ];
-    const attribution = possibleAttributions.find((attr) => attr != null && typeof attr === 'string' && attr.trim() !== '');
-    if (attribution != null) {
-        return attribution;
-    }
-    return getDomainName(link, metadata);
-}
 /**
  * Firecrawl scraper implementation
  * Uses the Firecrawl API to scrape web pages
@@ -106,21 +78,30 @@ class FirecrawlScraper {
      */
     extractContent(response) {
         if (!response.success || !response.data) {
-            return '';
+            return ['', undefined];
+        }
+        if (response.data.markdown != null && response.data.html != null) {
+            try {
+                const { markdown, ...rest } = content.processContent(response.data.html, response.data.markdown);
+                return [markdown, rest];
+            }
+            catch (error) {
+                console.error('Error processing content:', error);
+                return [response.data.markdown, undefined];
+            }
         }
-        // Prefer markdown content if available
-        if (response.data.markdown != null) {
-            return response.data.markdown;
+        else if (response.data.markdown != null) {
+            return [response.data.markdown, undefined];
         }
         // Fall back to HTML content
         if (response.data.html != null) {
-            return response.data.html;
+            return [response.data.html, undefined];
         }
         // Fall back to raw HTML content
         if (response.data.rawHtml != null) {
-            return response.data.rawHtml;
+            return [response.data.rawHtml, undefined];
         }
-        return '';
+        return ['', undefined];
     }
     /**
      * Extract metadata from scrape response
@@ -145,5 +126,4 @@ const createFirecrawlScraper = (config = {}) => {
 
 exports.FirecrawlScraper = FirecrawlScraper;
 exports.createFirecrawlScraper = createFirecrawlScraper;
-exports.getAttribution = getAttribution;
 //# sourceMappingURL=firecrawl.cjs.map

package/dist/cjs/tools/search/firecrawl.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"firecrawl.cjs","sources":["../../../../src/tools/search/firecrawl.ts"],"sourcesContent":["/* eslint-disable no-console */\nimport axios from 'axios';\n\nexport interface FirecrawlScrapeOptions {\n  formats?: string[];\n  includeTags?: string[];\n  excludeTags?: string[];\n  headers?: Record<string, string>;\n  waitFor?: number;\n  timeout?: number;\n}\n\ninterface ScrapeMetadata {\n  // Core source information\n  sourceURL?: string;\n  url?: string;\n  scrapeId?: string;\n  statusCode?: number;\n  // Basic metadata\n  title?: string;\n  description?: string;\n  language?: string;\n  favicon?: string;\n  viewport?: string;\n  robots?: string;\n  'theme-color'?: string;\n  // Open Graph metadata\n  'og:url'?: string;\n  'og:title'?: string;\n  'og:description'?: string;\n  'og:type'?: string;\n  'og:image'?: string;\n  'og:image:width'?: string;\n  'og:image:height'?: string;\n  'og:site_name'?: string;\n  ogUrl?: string;\n  ogTitle?: string;\n  ogDescription?: string;\n  ogImage?: string;\n  ogSiteName?: string;\n  // Article metadata\n  'article:author'?: string;\n  'article:published_time'?: string;\n  'article:modified_time'?: string;\n  'article:section'?: string;\n  'article:tag'?: string;\n  'article:publisher'?: string;\n  publishedTime?: string;\n  modifiedTime?: string;\n  // Twitter metadata\n  'twitter:site'?: string;\n  'twitter:creator'?: string;\n  'twitter:card'?: string;\n  'twitter:image'?: string;\n  'twitter:dnt'?: string;\n  'twitter:app:name:iphone'?: string;\n  'twitter:app:id:iphone'?: string;\n  'twitter:app:url:iphone'?: string;\n  'twitter:app:name:ipad'?: string;\n  'twitter:app:id:ipad'?: string;\n  'twitter:app:url:ipad'?: string;\n  'twitter:app:name:googleplay'?: string;\n  'twitter:app:id:googleplay'?: string;\n  'twitter:app:url:googleplay'?: string;\n  // Facebook metadata\n  'fb:app_id'?: string;\n  // App links\n  'al:ios:url'?: string;\n  'al:ios:app_name'?: string;\n  'al:ios:app_store_id'?: string;\n  // Allow for additional properties that might be present\n  [key: string]: string | number | boolean | null | undefined;\n}\n\nexport interface FirecrawlScrapeResponse {\n  success: boolean;\n  data?: {\n    markdown?: string;\n    html?: string;\n    rawHtml?: string;\n    screenshot?: string;\n    links?: string[];\n    metadata?: ScrapeMetadata;\n  };\n  error?: string;\n}\n\nexport interface FirecrawlScraperConfig {\n  apiKey?: string;\n  apiUrl?: string;\n  formats?: string[];\n  timeout?: number;\n}\nconst getDomainName = (\n  link: string,\n  metadata?: ScrapeMetadata\n): string | undefined => {\n  try {\n    const url = metadata?.sourceURL ?? metadata?.url ?? (link || '');\n    const domain = new URL(url).hostname.replace(/^www\\./, '');\n    if (domain) {\n      return domain;\n    }\n  } catch (e) {\n    // URL parsing failed\n    console.error('Error parsing URL:', e);\n  }\n\n  return;\n};\n\nexport function getAttribution(\n  link: string,\n  metadata?: ScrapeMetadata\n): string | undefined {\n  if (!metadata) return getDomainName(link, metadata);\n\n  const possibleAttributions = [\n    metadata.ogSiteName,\n    metadata['og:site_name'],\n    metadata.title?.split('|').pop()?.trim(),\n    metadata['twitter:site']?.replace(/^@/, ''),\n  ];\n\n  const attribution = possibleAttributions.find(\n    (attr) => attr != null && typeof attr === 'string' && attr.trim() !== ''\n  );\n  if (attribution != null) {\n    return attribution;\n  }\n\n  return getDomainName(link, metadata);\n}\n\n/**\n * Firecrawl scraper implementation\n * Uses the Firecrawl API to scrape web pages\n */\nexport class FirecrawlScraper {\n  private apiKey: string;\n  private apiUrl: string;\n  private defaultFormats: string[];\n  private timeout: number;\n\n  constructor(config: FirecrawlScraperConfig = {}) {\n    this.apiKey = config.apiKey ?? process.env.FIRECRAWL_API_KEY ?? '';\n\n    const baseUrl =\n      config.apiUrl ??\n      process.env.FIRECRAWL_BASE_URL ??\n      'https://api.firecrawl.dev';\n    this.apiUrl = `${baseUrl.replace(/\\/+$/, '')}/v1/scrape`;\n\n    this.defaultFormats = config.formats ?? ['markdown', 'html'];\n    this.timeout = config.timeout ?? 15000;\n\n    if (!this.apiKey) {\n      console.warn('FIRECRAWL_API_KEY is not set. Scraping will not work.');\n    }\n\n    console.log(`Firecrawl scraper initialized with API URL: ${this.apiUrl}`);\n  }\n\n  /**\n   * Scrape a single URL\n   * @param url URL to scrape\n   * @param options Scrape options\n   * @returns Scrape response\n   */\n  async scrapeUrl(\n    url: string,\n    options: FirecrawlScrapeOptions = {}\n  ): Promise<[string, FirecrawlScrapeResponse]> {\n    if (!this.apiKey) {\n      return [\n        url,\n        {\n          success: false,\n          error: 'FIRECRAWL_API_KEY is not set',\n        },\n      ];\n    }\n\n    try {\n      const response = await axios.post(\n        this.apiUrl,\n        {\n          url,\n          formats: options.formats || this.defaultFormats,\n          includeTags: options.includeTags,\n          excludeTags: options.excludeTags,\n          headers: options.headers,\n          waitFor: options.waitFor,\n          timeout: options.timeout ?? this.timeout,\n        },\n        {\n          headers: {\n            'Content-Type': 'application/json',\n            Authorization: `Bearer ${this.apiKey}`,\n          },\n          timeout: this.timeout,\n        }\n      );\n\n      return [url, response.data];\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      return [\n        url,\n        {\n          success: false,\n          error: `Firecrawl API request failed: ${errorMessage}`,\n        },\n      ];\n    }\n  }\n\n  /**\n   * Extract content from scrape response\n   * @param response Scrape response\n   * @returns Extracted content or empty string if not available\n   */\n  extractContent(response: FirecrawlScrapeResponse): string {\n    if (!response.success || !response.data) {\n      return '';\n    }\n\n    // Prefer markdown content if available\n    if (response.data.markdown != null) {\n      return response.data.markdown;\n    }\n\n    // Fall back to HTML content\n    if (response.data.html != null) {\n      return response.data.html;\n    }\n\n    // Fall back to raw HTML content\n    if (response.data.rawHtml != null) {\n      return response.data.rawHtml;\n    }\n\n    return '';\n  }\n\n  /**\n   * Extract metadata from scrape response\n   * @param response Scrape response\n   * @returns Metadata object\n   */\n  extractMetadata(response: FirecrawlScrapeResponse): ScrapeMetadata {\n    if (!response.success || !response.data || !response.data.metadata) {\n      return {};\n    }\n\n    return response.data.metadata;\n  }\n}\n\n/**\n * Create a Firecrawl scraper instance\n * @param config Scraper configuration\n * @returns Firecrawl scraper instance\n */\nexport const createFirecrawlScraper = (\n  config: FirecrawlScraperConfig = {}\n): FirecrawlScraper => {\n  return new FirecrawlScraper(config);\n};\n"],"names":[],"mappings":";;;;AAAA;AA6FA,MAAM,aAAa,GAAG,CACpB,IAAY,EACZ,QAAyB,KACH;AACtB,IAAA,IAAI;AACF,QAAA,MAAM,GAAG,GAAG,QAAQ,EAAE,SAAS,IAAI,QAAQ,EAAE,GAAG,KAAK,IAAI,IAAI,EAAE,CAAC;AAChE,QAAA,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC;QAC1D,IAAI,MAAM,EAAE;AACV,YAAA,OAAO,MAAM;;;IAEf,OAAO,CAAC,EAAE;;AAEV,QAAA,OAAO,CAAC,KAAK,CAAC,oBAAoB,EAAE,CAAC,CAAC;;IAGxC;AACF,CAAC;AAEe,SAAA,cAAc,CAC5B,IAAY,EACZ,QAAyB,EAAA;AAEzB,IAAA,IAAI,CAAC,QAAQ;AAAE,QAAA,OAAO,aAAa,CAAC,IAAI,EAAE,QAAQ,CAAC;AAEnD,IAAA,MAAM,oBAAoB,GAAG;AAC3B,QAAA,QAAQ,CAAC,UAAU;QACnB,QAAQ,CAAC,cAAc,CAAC;AACxB,QAAA,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE;QACxC,QAAQ,CAAC,cAAc,CAAC,EAAE,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;KAC5C;IAED,MAAM,WAAW,GAAG,oBAAoB,CAAC,IAAI,CAC3C,CAAC,IAAI,KAAK,IAAI,IAAI,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CACzE;AACD,IAAA,IAAI,WAAW,IAAI,IAAI,EAAE;AACvB,QAAA,OAAO,WAAW;;AAGpB,IAAA,OAAO,aAAa,CAAC,IAAI,EAAE,QAAQ,CAAC;AACtC;AAEA;;;AAGG;MACU,gBAAgB,CAAA;AACnB,IAAA,MAAM;AACN,IAAA,MAAM;AACN,IAAA,cAAc;AACd,IAAA,OAAO;AAEf,IAAA,WAAA,CAAY,SAAiC,EAAE,EAAA;AAC7C,QAAA,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,EAAE;AAElE,QAAA,MAAM,OAAO,GACX,MAAM,CAAC,MAAM;YACb,OAAO,CAAC,GAAG,CAAC,kBAAkB;AAC9B,YAAA,2BAA2B;AAC7B,QAAA,IAAI,CAAC,MAAM,GAAG,CAAA,EAAG,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,YAAY;AAExD,QAAA,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,OAAO,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC;QAC5D,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,KAAK;AAEtC,QAAA,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;AAChB,YAAA,OAAO,CAAC,IAAI,CAAC,uDAAuD,CAAC;;QAGvE,OAAO,CAAC,GAAG,CAAC,CAAA,4CAAA,EAA+C,IAAI,CAAC,MAAM,CAAE,CAAA,CAAC;;AAG3E;;;;;AAKG;AACH,IAAA,MAAM,SAAS,CACb,GAAW,EACX,UAAkC,EAAE,EAAA;AAEpC,QAAA,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YAChB,OAAO;gBACL,GAAG;AACH,gBAAA;AACE,oBAAA,OAAO,EAAE,KAAK;AACd,oBAAA,KAAK,EAAE,8BAA8B;AACtC,iBAAA;aACF;;AAGH,QAAA,IAAI;YACF,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAC/B,IAAI,CAAC,MAAM,EACX;gBACE,GAAG;AACH,gBAAA,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,cAAc;gBAC/C,WAAW,EAAE,OAAO,CAAC,WAAW;gBAChC,WAAW,EAAE,OAAO,CAAC,WAAW;gBAChC,OAAO,EAAE,OAAO,CAAC,OAAO;gBACxB,OAAO,EAAE,OAAO,CAAC,OAAO;AACxB,gBAAA,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO;aACzC,EACD;AACE,gBAAA,OAAO,EAAE;AACP,oBAAA,cAAc,EAAE,kBAAkB;AAClC,oBAAA,aAAa,EAAE,CAAA,OAAA,EAAU,IAAI,CAAC,MAAM,CAAE,CAAA;AACvC,iBAAA;gBACD,OAAO,EAAE,IAAI,CAAC,OAAO;AACtB,aAAA,CACF;AAED,YAAA,OAAO,CAAC,GAAG,EAAE,QAAQ,CAAC,IAAI,CAAC;;QAC3B,OAAO,KAAK,EAAE;AACd,YAAA,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;YACxD,OAAO;gBACL,GAAG;AACH,gBAAA;AACE,oBAAA,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,CAAiC,8BAAA,EAAA,YAAY,CAAE,CAAA;AACvD,iBAAA;aACF;;;AAIL;;;;AAIG;AACH,IAAA,cAAc,CAAC,QAAiC,EAAA;QAC9C,IAAI,CAAC,QAAQ,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE;AACvC,YAAA,OAAO,EAAE;;;QAIX,IAAI,QAAQ,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,EAAE;AAClC,YAAA,OAAO,QAAQ,CAAC,IAAI,CAAC,QAAQ;;;QAI/B,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,EAAE;AAC9B,YAAA,OAAO,QAAQ,CAAC,IAAI,CAAC,IAAI;;;QAI3B,IAAI,QAAQ,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,EAAE;AACjC,YAAA,OAAO,QAAQ,CAAC,IAAI,CAAC,OAAO;;AAG9B,QAAA,OAAO,EAAE;;AAGX;;;;AAIG;AACH,IAAA,eAAe,CAAC,QAAiC,EAAA;AAC/C,QAAA,IAAI,CAAC,QAAQ,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE;AAClE,YAAA,OAAO,EAAE;;AAGX,QAAA,OAAO,QAAQ,CAAC,IAAI,CAAC,QAAQ;;AAEhC;AAED;;;;AAIG;MACU,sBAAsB,GAAG,CACpC,MAAiC,GAAA,EAAE,KACf;AACpB,IAAA,OAAO,IAAI,gBAAgB,CAAC,MAAM,CAAC;AACrC;;;;;;"}
+{"version":3,"file":"firecrawl.cjs","sources":["../../../../src/tools/search/firecrawl.ts"],"sourcesContent":["/* eslint-disable no-console */\nimport axios from 'axios';\nimport { processContent } from './content';\nimport type * as t from './types';\n\n/**\n * Firecrawl scraper implementation\n * Uses the Firecrawl API to scrape web pages\n */\nexport class FirecrawlScraper {\n  private apiKey: string;\n  private apiUrl: string;\n  private defaultFormats: string[];\n  private timeout: number;\n\n  constructor(config: t.FirecrawlScraperConfig = {}) {\n    this.apiKey = config.apiKey ?? process.env.FIRECRAWL_API_KEY ?? '';\n\n    const baseUrl =\n      config.apiUrl ??\n      process.env.FIRECRAWL_BASE_URL ??\n      'https://api.firecrawl.dev';\n    this.apiUrl = `${baseUrl.replace(/\\/+$/, '')}/v1/scrape`;\n\n    this.defaultFormats = config.formats ?? ['markdown', 'html'];\n    this.timeout = config.timeout ?? 15000;\n\n    if (!this.apiKey) {\n      console.warn('FIRECRAWL_API_KEY is not set. Scraping will not work.');\n    }\n\n    console.log(`Firecrawl scraper initialized with API URL: ${this.apiUrl}`);\n  }\n\n  /**\n   * Scrape a single URL\n   * @param url URL to scrape\n   * @param options Scrape options\n   * @returns Scrape response\n   */\n  async scrapeUrl(\n    url: string,\n    options: t.FirecrawlScrapeOptions = {}\n  ): Promise<[string, t.FirecrawlScrapeResponse]> {\n    if (!this.apiKey) {\n      return [\n        url,\n        {\n          success: false,\n          error: 'FIRECRAWL_API_KEY is not set',\n        },\n      ];\n    }\n\n    try {\n      const response = await axios.post(\n        this.apiUrl,\n        {\n          url,\n          formats: options.formats || this.defaultFormats,\n          includeTags: options.includeTags,\n          excludeTags: options.excludeTags,\n          headers: options.headers,\n          waitFor: options.waitFor,\n          timeout: options.timeout ?? this.timeout,\n        },\n        {\n          headers: {\n            'Content-Type': 'application/json',\n            Authorization: `Bearer ${this.apiKey}`,\n          },\n          timeout: this.timeout,\n        }\n      );\n\n      return [url, response.data];\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      return [\n        url,\n        {\n          success: false,\n          error: `Firecrawl API request failed: ${errorMessage}`,\n        },\n      ];\n    }\n  }\n\n  /**\n   * Extract content from scrape response\n   * @param response Scrape response\n   * @returns Extracted content or empty string if not available\n   */\n  extractContent(\n    response: t.FirecrawlScrapeResponse\n  ): [string, undefined | t.References] {\n    if (!response.success || !response.data) {\n      return ['', undefined];\n    }\n\n    if (response.data.markdown != null && response.data.html != null) {\n      try {\n        const { markdown, ...rest } = processContent(\n          response.data.html,\n          response.data.markdown\n        );\n        return [markdown, rest];\n      } catch (error) {\n        console.error('Error processing content:', error);\n        return [response.data.markdown, undefined];\n      }\n    } else if (response.data.markdown != null) {\n      return [response.data.markdown, undefined];\n    }\n\n    // Fall back to HTML content\n    if (response.data.html != null) {\n      return [response.data.html, undefined];\n    }\n\n    // Fall back to raw HTML content\n    if (response.data.rawHtml != null) {\n      return [response.data.rawHtml, undefined];\n    }\n\n    return ['', undefined];\n  }\n\n  /**\n   * Extract metadata from scrape response\n   * @param response Scrape response\n   * @returns Metadata object\n   */\n  extractMetadata(response: t.FirecrawlScrapeResponse): t.ScrapeMetadata {\n    if (!response.success || !response.data || !response.data.metadata) {\n      return {};\n    }\n\n    return response.data.metadata;\n  }\n}\n\n/**\n * Create a Firecrawl scraper instance\n * @param config Scraper configuration\n * @returns Firecrawl scraper instance\n */\nexport const createFirecrawlScraper = (\n  config: t.FirecrawlScraperConfig = {}\n): FirecrawlScraper => {\n  return new FirecrawlScraper(config);\n};\n"],"names":["processContent"],"mappings":";;;;;AAAA;AAKA;;;AAGG;MACU,gBAAgB,CAAA;AACnB,IAAA,MAAM;AACN,IAAA,MAAM;AACN,IAAA,cAAc;AACd,IAAA,OAAO;AAEf,IAAA,WAAA,CAAY,SAAmC,EAAE,EAAA;AAC/C,QAAA,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,EAAE;AAElE,QAAA,MAAM,OAAO,GACX,MAAM,CAAC,MAAM;YACb,OAAO,CAAC,GAAG,CAAC,kBAAkB;AAC9B,YAAA,2BAA2B;AAC7B,QAAA,IAAI,CAAC,MAAM,GAAG,CAAA,EAAG,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,YAAY;AAExD,QAAA,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,OAAO,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC;QAC5D,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,KAAK;AAEtC,QAAA,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;AAChB,YAAA,OAAO,CAAC,IAAI,CAAC,uDAAuD,CAAC;;QAGvE,OAAO,CAAC,GAAG,CAAC,CAAA,4CAAA,EAA+C,IAAI,CAAC,MAAM,CAAE,CAAA,CAAC;;AAG3E;;;;;AAKG;AACH,IAAA,MAAM,SAAS,CACb,GAAW,EACX,UAAoC,EAAE,EAAA;AAEtC,QAAA,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YAChB,OAAO;gBACL,GAAG;AACH,gBAAA;AACE,oBAAA,OAAO,EAAE,KAAK;AACd,oBAAA,KAAK,EAAE,8BAA8B;AACtC,iBAAA;aACF;;AAGH,QAAA,IAAI;YACF,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAC/B,IAAI,CAAC,MAAM,EACX;gBACE,GAAG;AACH,gBAAA,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,cAAc;gBAC/C,WAAW,EAAE,OAAO,CAAC,WAAW;gBAChC,WAAW,EAAE,OAAO,CAAC,WAAW;gBAChC,OAAO,EAAE,OAAO,CAAC,OAAO;gBACxB,OAAO,EAAE,OAAO,CAAC,OAAO;AACxB,gBAAA,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO;aACzC,EACD;AACE,gBAAA,OAAO,EAAE;AACP,oBAAA,cAAc,EAAE,kBAAkB;AAClC,oBAAA,aAAa,EAAE,CAAA,OAAA,EAAU,IAAI,CAAC,MAAM,CAAE,CAAA;AACvC,iBAAA;gBACD,OAAO,EAAE,IAAI,CAAC,OAAO;AACtB,aAAA,CACF;AAED,YAAA,OAAO,CAAC,GAAG,EAAE,QAAQ,CAAC,IAAI,CAAC;;QAC3B,OAAO,KAAK,EAAE;AACd,YAAA,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;YACxD,OAAO;gBACL,GAAG;AACH,gBAAA;AACE,oBAAA,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,CAAiC,8BAAA,EAAA,YAAY,CAAE,CAAA;AACvD,iBAAA;aACF;;;AAIL;;;;AAIG;AACH,IAAA,cAAc,CACZ,QAAmC,EAAA;QAEnC,IAAI,CAAC,QAAQ,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE;AACvC,YAAA,OAAO,CAAC,EAAE,EAAE,SAAS,CAAC;;AAGxB,QAAA,IAAI,QAAQ,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,EAAE;AAChE,YAAA,IAAI;gBACF,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAI,EAAE,GAAGA,sBAAc,CAC1C,QAAQ,CAAC,IAAI,CAAC,IAAI,EAClB,QAAQ,CAAC,IAAI,CAAC,QAAQ,CACvB;AACD,gBAAA,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC;;YACvB,OAAO,KAAK,EAAE;AACd,gBAAA,OAAO,CAAC,KAAK,CAAC,2BAA2B,EAAE,KAAK,CAAC;gBACjD,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC;;;aAEvC,IAAI,QAAQ,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,EAAE;YACzC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC;;;QAI5C,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,EAAE;YAC9B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,SAAS,CAAC;;;QAIxC,IAAI,QAAQ,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,EAAE;YACjC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC;;AAG3C,QAAA,OAAO,CAAC,EAAE,EAAE,SAAS,CAAC;;AAGxB;;;;AAIG;AACH,IAAA,eAAe,CAAC,QAAmC,EAAA;AACjD,QAAA,IAAI,CAAC,QAAQ,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE;AAClE,YAAA,OAAO,EAAE;;AAGX,QAAA,OAAO,QAAQ,CAAC,IAAI,CAAC,QAAQ;;AAEhC;AAED;;;;AAIG;MACU,sBAAsB,GAAG,CACpC,MAAmC,GAAA,EAAE,KACjB;AACpB,IAAA,OAAO,IAAI,gBAAgB,CAAC,MAAM,CAAC;AACrC;;;;;"}

package/dist/cjs/tools/search/format.cjs

@@ -1,32 +1,59 @@
 'use strict';
 
+var utils = require('./utils.cjs');
+
 function formatResultsForLLM(turn, results) {
     let output = '';
     const addSection = (title) => {
         output += `\n=== ${title} ===\n`;
     };
+    const references = [];
     // Organic (web) results
-    const organic = results.organic ?? [];
-    if (organic.length) {
-        addSection('Web Results, Turn ' + turn);
-        organic.forEach((r, i) => {
+    if (results.organic?.length != null && results.organic.length > 0) {
+        addSection(`Web Results, Turn ${turn}`);
+        for (let i = 0; i < results.organic.length; i++) {
+            const r = results.organic[i];
             output += [
-                `Source ${i}: ${r.title ?? '(no title)'}`,
-                `Citation Anchor: \\ue202turn${turn}search${i}`,
+                `# Source ${i}: "${r.title ?? '(no title)'}"`,
+                `Anchor: \\ue202turn${turn}search${i}`,
                 `URL: ${r.link}`,
                 r.snippet != null ? `Summary: ${r.snippet}` : '',
                 r.date != null ? `Date: ${r.date}` : '',
                 r.attribution != null ? `Source: ${r.attribution}` : '',
                 '',
-                '--- Content Highlights ---',
-                ...(r.highlights ?? [])
-                    .filter((h) => h.text.trim().length > 0)
-                    .map((h) => `[Relevance: ${h.score.toFixed(2)}]\n${h.text.trim()}`),
+                '\n## Highlights\n\n',
+                '',
                 '',
             ]
                 .filter(Boolean)
                 .join('\n');
-        });
+            (r.highlights ?? [])
+                .filter((h) => h.text.trim().length > 0)
+                .forEach((h, hIndex) => {
+                output += `### Highlight ${hIndex + 1} [Relevance: ${h.score.toFixed(2)}]\n\n`;
+                output += '```text\n' + h.text.trim() + '\n```\n\n';
+                if (h.references != null && h.references.length) {
+                    output += 'Core References:\n';
+                    output += h.references
+                        .map((ref) => {
+                        references.push({
+                            link: ref.reference.originalUrl,
+                            attribution: utils.getDomainName(ref.reference.originalUrl),
+                            title: (((ref.reference.title ?? '') || ref.reference.text) ??
+                                '').split('\n')[0],
+                        });
+                        return `- ${ref.type}#${ref.originalIndex + 1}: ${ref.reference.originalUrl}\n\t- Anchor: \\ue202turn${turn}ref${references.length - 1}`;
+                    })
+                        .join('\n');
+                    output += '\n\n';
+                }
+                if (hIndex < (r.highlights?.length ?? 0) - 1) {
+                    output += '---\n\n';
+                }
+            });
+            delete results.organic[i].highlights;
+            output += '\n';
+        }
     }
     // Ignoring these sections for now
     // // Top stories (news)
@@ -62,54 +89,77 @@ function formatResultsForLLM(turn, results) {
     if (results.knowledgeGraph != null) {
         addSection('Knowledge Graph');
         output += [
-            `Title: ${results.knowledgeGraph.title ?? '(no title)'}`,
+            `**Title:** ${results.knowledgeGraph.title ?? '(no title)'}`,
+            results.knowledgeGraph.type != null
+                ? `**Type:** ${results.knowledgeGraph.type}`
+                : '',
             results.knowledgeGraph.description != null
-                ? `Description: ${results.knowledgeGraph.description}`
+                ? `**Description:** ${results.knowledgeGraph.description}`
                 : '',
-            results.knowledgeGraph.type != null
-                ? `Type: ${results.knowledgeGraph.type}`
+            results.knowledgeGraph.descriptionSource != null
+                ? `**Description Source:** ${results.knowledgeGraph.descriptionSource}`
+                : '',
+            results.knowledgeGraph.descriptionLink != null
+                ? `**Description Link:** ${results.knowledgeGraph.descriptionLink}`
                 : '',
             results.knowledgeGraph.imageUrl != null
-                ? `Image URL: ${results.knowledgeGraph.imageUrl}`
+                ? `**Image URL:** ${results.knowledgeGraph.imageUrl}`
+                : '',
+            results.knowledgeGraph.website != null
+                ? `**Website:** ${results.knowledgeGraph.website}`
                 : '',
             results.knowledgeGraph.attributes != null
-                ? `Attributes: ${JSON.stringify(results.knowledgeGraph.attributes, null, 2)}`
+                ? `**Attributes:**\n\`\`\`json\n${JSON.stringify(results.knowledgeGraph.attributes, null, 2)}\n\`\`\``
                 : '',
             '',
         ]
             .filter(Boolean)
-            .join('\n');
+            .join('\n\n');
     }
     // Answer Box
     if (results.answerBox != null) {
         addSection('Answer Box');
         output += [
             results.answerBox.title != null
-                ? `Title: ${results.answerBox.title}`
-                : '',
-            results.answerBox.answer != null
-                ? `Answer: ${results.answerBox.answer}`
+                ? `**Title:** ${results.answerBox.title}`
                 : '',
             results.answerBox.snippet != null
-                ? `Snippet: ${results.answerBox.snippet}`
+                ? `**Snippet:** ${results.answerBox.snippet}`
+                : '',
+            results.answerBox.snippetHighlighted != null
+                ? `**Snippet Highlighted:** ${results.answerBox.snippetHighlighted
+                    .map((s) => `\`${s}\``)
+                    .join(' ')}`
+                : '',
+            results.answerBox.link != null
+                ? `**Link:** ${results.answerBox.link}`
                 : '',
-            results.answerBox.date != null ? `Date: ${results.answerBox.date}` : '',
             '',
         ]
             .filter(Boolean)
-            .join('\n');
+            .join('\n\n');
     }
     // People also ask
     const peopleAlsoAsk = results.peopleAlsoAsk ?? [];
     if (peopleAlsoAsk.length) {
         addSection('People Also Ask');
-        peopleAlsoAsk.forEach((p, _i) => {
-            output += [`Q: ${p.question}`, `A: ${p.answer}`, '']
+        peopleAlsoAsk.forEach((p, i) => {
+            output += [
+                `### Question ${i + 1}:`,
+                `"${p.question}"`,
+                `${p.snippet != null && p.snippet ? `Snippet: ${p.snippet}}` : ''}`,
+                `${p.title != null && p.title ? `Title: ${p.title}` : ''}`,
+                `${p.link != null && p.link ? `Link: ${p.link}` : ''}`,
+                '',
+            ]
                 .filter(Boolean)
-                .join('\n');
+                .join('\n\n');
         });
     }
-    return output.trim();
+    return {
+        output: output.trim(),
+        references,
+    };
 }
 
 exports.formatResultsForLLM = formatResultsForLLM;

package/dist/cjs/tools/search/format.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"format.cjs","sources":["../../../../src/tools/search/format.ts"],"sourcesContent":["import type * as t from './types';\n\nexport function formatResultsForLLM(\n  turn: number,\n  results: t.SearchResultData\n): string {\n  let output = '';\n\n  const addSection = (title: string): void => {\n    output += `\\n=== ${title} ===\\n`;\n  };\n\n  // Organic (web) results\n  const organic = results.organic ?? [];\n  if (organic.length) {\n    addSection('Web Results, Turn ' + turn);\n    organic.forEach((r, i) => {\n      output += [\n        `Source ${i}: ${r.title ?? '(no title)'}`,\n        `Citation Anchor: \\\\ue202turn${turn}search${i}`,\n        `URL: ${r.link}`,\n        r.snippet != null ? `Summary: ${r.snippet}` : '',\n        r.date != null ? `Date: ${r.date}` : '',\n        r.attribution != null ? `Source: ${r.attribution}` : '',\n        '',\n        '--- Content Highlights ---',\n        ...(r.highlights ?? [])\n          .filter((h) => h.text.trim().length > 0)\n          .map((h) => `[Relevance: ${h.score.toFixed(2)}]\\n${h.text.trim()}`),\n        '',\n      ]\n        .filter(Boolean)\n        .join('\\n');\n    });\n  }\n\n  // Ignoring these sections for now\n  // // Top stories (news)\n  // const topStores = results.topStories ?? [];\n  // if (topStores.length) {\n  //   addSection('News Results');\n  //   topStores.forEach((r, i) => {\n  //     output += [\n  //       `Anchor: \\ue202turn0news${i}`,\n  //       `Title: ${r.title ?? '(no title)'}`,\n  //       `URL: ${r.link}`,\n  //       r.snippet != null ? `Snippet: ${r.snippet}` : '',\n  //       r.date != null ? `Date: ${r.date}` : '',\n  //       r.attribution != null ? `Source: ${r.attribution}` : '',\n  //       ''\n  //     ].filter(Boolean).join('\\n');\n  //   });\n  // }\n\n  // // Images\n  // const images = results.images ?? [];\n  // if (images.length) {\n  //   addSection('Image Results');\n  //   images.forEach((img, i) => {\n  //     output += [\n  //       `Anchor: \\ue202turn0image${i}`,\n  //       `Title: ${img.title ?? '(no title)'}`,\n  //       `Image URL: ${img.imageUrl}`,\n  //       ''\n  //     ].join('\\n');\n  //   });\n  // }\n\n  // Knowledge Graph\n  if (results.knowledgeGraph != null) {\n    addSection('Knowledge Graph');\n    output += [\n      `Title: ${results.knowledgeGraph.title ?? '(no title)'}`,\n      results.knowledgeGraph.description != null\n        ? `Description: ${results.knowledgeGraph.description}`\n        : '',\n      results.knowledgeGraph.type != null\n        ? `Type: ${results.knowledgeGraph.type}`\n        : '',\n      results.knowledgeGraph.imageUrl != null\n        ? `Image URL: ${results.knowledgeGraph.imageUrl}`\n        : '',\n      results.knowledgeGraph.attributes != null\n        ? `Attributes: ${JSON.stringify(results.knowledgeGraph.attributes, null, 2)}`\n        : '',\n      '',\n    ]\n      .filter(Boolean)\n      .join('\\n');\n  }\n\n  // Answer Box\n  if (results.answerBox != null) {\n    addSection('Answer Box');\n    output += [\n      results.answerBox.title != null\n        ? `Title: ${results.answerBox.title}`\n        : '',\n      results.answerBox.answer != null\n        ? `Answer: ${results.answerBox.answer}`\n        : '',\n      results.answerBox.snippet != null\n        ? `Snippet: ${results.answerBox.snippet}`\n        : '',\n      results.answerBox.date != null ? `Date: ${results.answerBox.date}` : '',\n      '',\n    ]\n      .filter(Boolean)\n      .join('\\n');\n  }\n\n  // People also ask\n  const peopleAlsoAsk = results.peopleAlsoAsk ?? [];\n  if (peopleAlsoAsk.length) {\n    addSection('People Also Ask');\n    peopleAlsoAsk.forEach((p, _i) => {\n      output += [`Q: ${p.question}`, `A: ${p.answer}`, '']\n        .filter(Boolean)\n        .join('\\n');\n    });\n  }\n\n  return output.trim();\n}\n"],"names":[],"mappings":";;AAEgB,SAAA,mBAAmB,CACjC,IAAY,EACZ,OAA2B,EAAA;IAE3B,IAAI,MAAM,GAAG,EAAE;AAEf,IAAA,MAAM,UAAU,GAAG,CAAC,KAAa,KAAU;AACzC,QAAA,MAAM,IAAI,CAAA,MAAA,EAAS,KAAK,CAAA,MAAA,CAAQ;AAClC,KAAC;;AAGD,IAAA,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,EAAE;AACrC,IAAA,IAAI,OAAO,CAAC,MAAM,EAAE;AAClB,QAAA,UAAU,CAAC,oBAAoB,GAAG,IAAI,CAAC;QACvC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,KAAI;AACvB,YAAA,MAAM,IAAI;AACR,gBAAA,CAAA,OAAA,EAAU,CAAC,CAAK,EAAA,EAAA,CAAC,CAAC,KAAK,IAAI,YAAY,CAAE,CAAA;gBACzC,CAA+B,4BAAA,EAAA,IAAI,CAAS,MAAA,EAAA,CAAC,CAAE,CAAA;gBAC/C,CAAQ,KAAA,EAAA,CAAC,CAAC,IAAI,CAAE,CAAA;AAChB,gBAAA,CAAC,CAAC,OAAO,IAAI,IAAI,GAAG,CAAY,SAAA,EAAA,CAAC,CAAC,OAAO,CAAA,CAAE,GAAG,EAAE;AAChD,gBAAA,CAAC,CAAC,IAAI,IAAI,IAAI,GAAG,CAAS,MAAA,EAAA,CAAC,CAAC,IAAI,CAAA,CAAE,GAAG,EAAE;AACvC,gBAAA,CAAC,CAAC,WAAW,IAAI,IAAI,GAAG,CAAW,QAAA,EAAA,CAAC,CAAC,WAAW,CAAA,CAAE,GAAG,EAAE;gBACvD,EAAE;gBACF,4BAA4B;AAC5B,gBAAA,GAAG,CAAC,CAAC,CAAC,UAAU,IAAI,EAAE;AACnB,qBAAA,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC;qBACtC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAA,YAAA,EAAe,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA,GAAA,EAAM,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAA,CAAE,CAAC;gBACrE,EAAE;AACH;iBACE,MAAM,CAAC,OAAO;iBACd,IAAI,CAAC,IAAI,CAAC;AACf,SAAC,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAoCJ,IAAA,IAAI,OAAO,CAAC,cAAc,IAAI,IAAI,EAAE;QAClC,UAAU,CAAC,iBAAiB,CAAC;AAC7B,QAAA,MAAM,IAAI;AACR,YAAA,CAAA,OAAA,EAAU,OAAO,CAAC,cAAc,CAAC,KAAK,IAAI,YAAY,CAAE,CAAA;AACxD,YAAA,OAAO,CAAC,cAAc,CAAC,WAAW,IAAI;AACpC,kBAAE,CAAgB,aAAA,EAAA,OAAO,CAAC,cAAc,CAAC,WAAW,CAAE;AACtD,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,cAAc,CAAC,IAAI,IAAI;AAC7B,kBAAE,CAAS,MAAA,EAAA,OAAO,CAAC,cAAc,CAAC,IAAI,CAAE;AACxC,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,cAAc,CAAC,QAAQ,IAAI;AACjC,kBAAE,CAAc,WAAA,EAAA,OAAO,CAAC,cAAc,CAAC,QAAQ,CAAE;AACjD,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,cAAc,CAAC,UAAU,IAAI;AACnC,kBAAE,CAAe,YAAA,EAAA,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,cAAc,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,CAAE;AAC7E,kBAAE,EAAE;YACN,EAAE;AACH;aACE,MAAM,CAAC,OAAO;aACd,IAAI,CAAC,IAAI,CAAC;;;AAIf,IAAA,IAAI,OAAO,CAAC,SAAS,IAAI,IAAI,EAAE;QAC7B,UAAU,CAAC,YAAY,CAAC;AACxB,QAAA,MAAM,IAAI;AACR,YAAA,OAAO,CAAC,SAAS,CAAC,KAAK,IAAI;AACzB,kBAAE,CAAU,OAAA,EAAA,OAAO,CAAC,SAAS,CAAC,KAAK,CAAE;AACrC,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,SAAS,CAAC,MAAM,IAAI;AAC1B,kBAAE,CAAW,QAAA,EAAA,OAAO,CAAC,SAAS,CAAC,MAAM,CAAE;AACvC,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,SAAS,CAAC,OAAO,IAAI;AAC3B,kBAAE,CAAY,SAAA,EAAA,OAAO,CAAC,SAAS,CAAC,OAAO,CAAE;AACzC,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,SAAS,CAAC,IAAI,IAAI,IAAI,GAAG,SAAS,OAAO,CAAC,SAAS,CAAC,IAAI,EAAE,GAAG,EAAE;YACvE,EAAE;AACH;aACE,MAAM,CAAC,OAAO;aACd,IAAI,CAAC,IAAI,CAAC;;;AAIf,IAAA,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,EAAE;AACjD,IAAA,IAAI,aAAa,CAAC,MAAM,EAAE;QACxB,UAAU,CAAC,iBAAiB,CAAC;QAC7B,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,KAAI;AAC9B,YAAA,MAAM,IAAI,CAAC,CAAM,GAAA,EAAA,CAAC,CAAC,QAAQ,CAAA,CAAE,EAAE,CAAA,GAAA,EAAM,CAAC,CAAC,MAAM,CAAE,CAAA,EAAE,EAAE;iBAChD,MAAM,CAAC,OAAO;iBACd,IAAI,CAAC,IAAI,CAAC;AACf,SAAC,CAAC;;AAGJ,IAAA,OAAO,MAAM,CAAC,IAAI,EAAE;AACtB;;;;"}
+{"version":3,"file":"format.cjs","sources":["../../../../src/tools/search/format.ts"],"sourcesContent":["import type * as t from './types';\nimport { getDomainName } from './utils';\n\nexport function formatResultsForLLM(\n  turn: number,\n  results: t.SearchResultData\n): { output: string; references: t.ResultReference[] } {\n  let output = '';\n\n  const addSection = (title: string): void => {\n    output += `\\n=== ${title} ===\\n`;\n  };\n\n  const references: t.ResultReference[] = [];\n  // Organic (web) results\n  if (results.organic?.length != null && results.organic.length > 0) {\n    addSection(`Web Results, Turn ${turn}`);\n    for (let i = 0; i < results.organic.length; i++) {\n      const r = results.organic[i];\n      output += [\n        `# Source ${i}: \"${r.title ?? '(no title)'}\"`,\n        `Anchor: \\\\ue202turn${turn}search${i}`,\n        `URL: ${r.link}`,\n        r.snippet != null ? `Summary: ${r.snippet}` : '',\n        r.date != null ? `Date: ${r.date}` : '',\n        r.attribution != null ? `Source: ${r.attribution}` : '',\n        '',\n        '\\n## Highlights\\n\\n',\n        '',\n        '',\n      ]\n        .filter(Boolean)\n        .join('\\n');\n\n      (r.highlights ?? [])\n        .filter((h) => h.text.trim().length > 0)\n        .forEach((h, hIndex) => {\n          output += `### Highlight ${hIndex + 1} [Relevance: ${h.score.toFixed(2)}]\\n\\n`;\n          output += '```text\\n' + h.text.trim() + '\\n```\\n\\n';\n\n          if (h.references != null && h.references.length) {\n            output += 'Core References:\\n';\n            output += h.references\n              .map((ref) => {\n                references.push({\n                  link: ref.reference.originalUrl,\n                  attribution: getDomainName(ref.reference.originalUrl),\n                  title: (\n                    ((ref.reference.title ?? '') || ref.reference.text) ??\n                    ''\n                  ).split('\\n')[0],\n                });\n                return `- ${ref.type}#${ref.originalIndex + 1}: ${ref.reference.originalUrl}\\n\\t- Anchor: \\\\ue202turn${turn}ref${references.length - 1}`;\n              })\n              .join('\\n');\n            output += '\\n\\n';\n          }\n\n          if (hIndex < (r.highlights?.length ?? 0) - 1) {\n            output += '---\\n\\n';\n          }\n        });\n\n      delete results.organic[i].highlights;\n      output += '\\n';\n    }\n  }\n\n  // Ignoring these sections for now\n  // // Top stories (news)\n  // const topStores = results.topStories ?? [];\n  // if (topStores.length) {\n  //   addSection('News Results');\n  //   topStores.forEach((r, i) => {\n  //     output += [\n  //       `Anchor: \\ue202turn0news${i}`,\n  //       `Title: ${r.title ?? '(no title)'}`,\n  //       `URL: ${r.link}`,\n  //       r.snippet != null ? `Snippet: ${r.snippet}` : '',\n  //       r.date != null ? `Date: ${r.date}` : '',\n  //       r.attribution != null ? `Source: ${r.attribution}` : '',\n  //       ''\n  //     ].filter(Boolean).join('\\n');\n  //   });\n  // }\n\n  // // Images\n  // const images = results.images ?? [];\n  // if (images.length) {\n  //   addSection('Image Results');\n  //   images.forEach((img, i) => {\n  //     output += [\n  //       `Anchor: \\ue202turn0image${i}`,\n  //       `Title: ${img.title ?? '(no title)'}`,\n  //       `Image URL: ${img.imageUrl}`,\n  //       ''\n  //     ].join('\\n');\n  //   });\n  // }\n\n  // Knowledge Graph\n  if (results.knowledgeGraph != null) {\n    addSection('Knowledge Graph');\n    output += [\n      `**Title:** ${results.knowledgeGraph.title ?? '(no title)'}`,\n      results.knowledgeGraph.type != null\n        ? `**Type:** ${results.knowledgeGraph.type}`\n        : '',\n      results.knowledgeGraph.description != null\n        ? `**Description:** ${results.knowledgeGraph.description}`\n        : '',\n      results.knowledgeGraph.descriptionSource != null\n        ? `**Description Source:** ${results.knowledgeGraph.descriptionSource}`\n        : '',\n      results.knowledgeGraph.descriptionLink != null\n        ? `**Description Link:** ${results.knowledgeGraph.descriptionLink}`\n        : '',\n      results.knowledgeGraph.imageUrl != null\n        ? `**Image URL:** ${results.knowledgeGraph.imageUrl}`\n        : '',\n      results.knowledgeGraph.website != null\n        ? `**Website:** ${results.knowledgeGraph.website}`\n        : '',\n      results.knowledgeGraph.attributes != null\n        ? `**Attributes:**\\n\\`\\`\\`json\\n${JSON.stringify(\n          results.knowledgeGraph.attributes,\n          null,\n          2\n        )}\\n\\`\\`\\``\n        : '',\n      '',\n    ]\n      .filter(Boolean)\n      .join('\\n\\n');\n  }\n\n  // Answer Box\n  if (results.answerBox != null) {\n    addSection('Answer Box');\n    output += [\n      results.answerBox.title != null\n        ? `**Title:** ${results.answerBox.title}`\n        : '',\n      results.answerBox.snippet != null\n        ? `**Snippet:** ${results.answerBox.snippet}`\n        : '',\n      results.answerBox.snippetHighlighted != null\n        ? `**Snippet Highlighted:** ${results.answerBox.snippetHighlighted\n          .map((s) => `\\`${s}\\``)\n          .join(' ')}`\n        : '',\n      results.answerBox.link != null\n        ? `**Link:** ${results.answerBox.link}`\n        : '',\n      '',\n    ]\n      .filter(Boolean)\n      .join('\\n\\n');\n  }\n\n  // People also ask\n  const peopleAlsoAsk = results.peopleAlsoAsk ?? [];\n  if (peopleAlsoAsk.length) {\n    addSection('People Also Ask');\n    peopleAlsoAsk.forEach((p, i) => {\n      output += [\n        `### Question ${i + 1}:`,\n        `\"${p.question}\"`,\n        `${p.snippet != null && p.snippet ? `Snippet: ${p.snippet}}` : ''}`,\n        `${p.title != null && p.title ? `Title: ${p.title}` : ''}`,\n        `${p.link != null && p.link ? `Link: ${p.link}` : ''}`,\n        '',\n      ]\n        .filter(Boolean)\n        .join('\\n\\n');\n    });\n  }\n  return {\n    output: output.trim(),\n    references,\n  };\n}\n"],"names":["getDomainName"],"mappings":";;;;AAGgB,SAAA,mBAAmB,CACjC,IAAY,EACZ,OAA2B,EAAA;IAE3B,IAAI,MAAM,GAAG,EAAE;AAEf,IAAA,MAAM,UAAU,GAAG,CAAC,KAAa,KAAU;AACzC,QAAA,MAAM,IAAI,CAAA,MAAA,EAAS,KAAK,CAAA,MAAA,CAAQ;AAClC,KAAC;IAED,MAAM,UAAU,GAAwB,EAAE;;AAE1C,IAAA,IAAI,OAAO,CAAC,OAAO,EAAE,MAAM,IAAI,IAAI,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE;AACjE,QAAA,UAAU,CAAC,CAAA,kBAAA,EAAqB,IAAI,CAAA,CAAE,CAAC;AACvC,QAAA,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YAC/C,MAAM,CAAC,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;AAC5B,YAAA,MAAM,IAAI;AACR,gBAAA,CAAA,SAAA,EAAY,CAAC,CAAM,GAAA,EAAA,CAAC,CAAC,KAAK,IAAI,YAAY,CAAG,CAAA,CAAA;gBAC7C,CAAsB,mBAAA,EAAA,IAAI,CAAS,MAAA,EAAA,CAAC,CAAE,CAAA;gBACtC,CAAQ,KAAA,EAAA,CAAC,CAAC,IAAI,CAAE,CAAA;AAChB,gBAAA,CAAC,CAAC,OAAO,IAAI,IAAI,GAAG,CAAY,SAAA,EAAA,CAAC,CAAC,OAAO,CAAA,CAAE,GAAG,EAAE;AAChD,gBAAA,CAAC,CAAC,IAAI,IAAI,IAAI,GAAG,CAAS,MAAA,EAAA,CAAC,CAAC,IAAI,CAAA,CAAE,GAAG,EAAE;AACvC,gBAAA,CAAC,CAAC,WAAW,IAAI,IAAI,GAAG,CAAW,QAAA,EAAA,CAAC,CAAC,WAAW,CAAA,CAAE,GAAG,EAAE;gBACvD,EAAE;gBACF,qBAAqB;gBACrB,EAAE;gBACF,EAAE;AACH;iBACE,MAAM,CAAC,OAAO;iBACd,IAAI,CAAC,IAAI,CAAC;AAEb,YAAA,CAAC,CAAC,CAAC,UAAU,IAAI,EAAE;AAChB,iBAAA,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC;AACtC,iBAAA,OAAO,CAAC,CAAC,CAAC,EAAE,MAAM,KAAI;AACrB,gBAAA,MAAM,IAAI,CAAA,cAAA,EAAiB,MAAM,GAAG,CAAC,CAAgB,aAAA,EAAA,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO;gBAC9E,MAAM,IAAI,WAAW,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,WAAW;AAEnD,gBAAA,IAAI,CAAC,CAAC,UAAU,IAAI,IAAI,IAAI,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE;oBAC/C,MAAM,IAAI,oBAAoB;oBAC9B,MAAM,IAAI,CAAC,CAAC;AACT,yBAAA,GAAG,CAAC,CAAC,GAAG,KAAI;wBACX,UAAU,CAAC,IAAI,CAAC;AACd,4BAAA,IAAI,EAAE,GAAG,CAAC,SAAS,CAAC,WAAW;4BAC/B,WAAW,EAAEA,mBAAa,CAAC,GAAG,CAAC,SAAS,CAAC,WAAW,CAAC;AACrD,4BAAA,KAAK,EAAE,CACL,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,IAAI,EAAE,KAAK,GAAG,CAAC,SAAS,CAAC,IAAI;gCAClD,EAAE,EACF,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACjB,yBAAA,CAAC;wBACF,OAAO,CAAA,EAAA,EAAK,GAAG,CAAC,IAAI,CAAA,CAAA,EAAI,GAAG,CAAC,aAAa,GAAG,CAAC,CAAK,EAAA,EAAA,GAAG,CAAC,SAAS,CAAC,WAAW,CAAA,yBAAA,EAA4B,IAAI,CAAA,GAAA,EAAM,UAAU,CAAC,MAAM,GAAG,CAAC,CAAA,CAAE;AAC1I,qBAAC;yBACA,IAAI,CAAC,IAAI,CAAC;oBACb,MAAM,IAAI,MAAM;;AAGlB,gBAAA,IAAI,MAAM,GAAG,CAAC,CAAC,CAAC,UAAU,EAAE,MAAM,IAAI,CAAC,IAAI,CAAC,EAAE;oBAC5C,MAAM,IAAI,SAAS;;AAEvB,aAAC,CAAC;YAEJ,OAAO,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,UAAU;YACpC,MAAM,IAAI,IAAI;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAqClB,IAAA,IAAI,OAAO,CAAC,cAAc,IAAI,IAAI,EAAE;QAClC,UAAU,CAAC,iBAAiB,CAAC;AAC7B,QAAA,MAAM,IAAI;AACR,YAAA,CAAA,WAAA,EAAc,OAAO,CAAC,cAAc,CAAC,KAAK,IAAI,YAAY,CAAE,CAAA;AAC5D,YAAA,OAAO,CAAC,cAAc,CAAC,IAAI,IAAI;AAC7B,kBAAE,CAAa,UAAA,EAAA,OAAO,CAAC,cAAc,CAAC,IAAI,CAAE;AAC5C,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,cAAc,CAAC,WAAW,IAAI;AACpC,kBAAE,CAAoB,iBAAA,EAAA,OAAO,CAAC,cAAc,CAAC,WAAW,CAAE;AAC1D,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,cAAc,CAAC,iBAAiB,IAAI;AAC1C,kBAAE,CAA2B,wBAAA,EAAA,OAAO,CAAC,cAAc,CAAC,iBAAiB,CAAE;AACvE,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,cAAc,CAAC,eAAe,IAAI;AACxC,kBAAE,CAAyB,sBAAA,EAAA,OAAO,CAAC,cAAc,CAAC,eAAe,CAAE;AACnE,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,cAAc,CAAC,QAAQ,IAAI;AACjC,kBAAE,CAAkB,eAAA,EAAA,OAAO,CAAC,cAAc,CAAC,QAAQ,CAAE;AACrD,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,cAAc,CAAC,OAAO,IAAI;AAChC,kBAAE,CAAgB,aAAA,EAAA,OAAO,CAAC,cAAc,CAAC,OAAO,CAAE;AAClD,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,cAAc,CAAC,UAAU,IAAI;AACnC,kBAAE,CAAgC,6BAAA,EAAA,IAAI,CAAC,SAAS,CAC9C,OAAO,CAAC,cAAc,CAAC,UAAU,EACjC,IAAI,EACJ,CAAC,CACF,CAAU,QAAA;AACX,kBAAE,EAAE;YACN,EAAE;AACH;aACE,MAAM,CAAC,OAAO;aACd,IAAI,CAAC,MAAM,CAAC;;;AAIjB,IAAA,IAAI,OAAO,CAAC,SAAS,IAAI,IAAI,EAAE;QAC7B,UAAU,CAAC,YAAY,CAAC;AACxB,QAAA,MAAM,IAAI;AACR,YAAA,OAAO,CAAC,SAAS,CAAC,KAAK,IAAI;AACzB,kBAAE,CAAc,WAAA,EAAA,OAAO,CAAC,SAAS,CAAC,KAAK,CAAE;AACzC,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,SAAS,CAAC,OAAO,IAAI;AAC3B,kBAAE,CAAgB,aAAA,EAAA,OAAO,CAAC,SAAS,CAAC,OAAO,CAAE;AAC7C,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,SAAS,CAAC,kBAAkB,IAAI;AACtC,kBAAE,CAA4B,yBAAA,EAAA,OAAO,CAAC,SAAS,CAAC;qBAC7C,GAAG,CAAC,CAAC,CAAC,KAAK,CAAA,EAAA,EAAK,CAAC,CAAA,EAAA,CAAI;qBACrB,IAAI,CAAC,GAAG,CAAC,CAAE;AACd,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,SAAS,CAAC,IAAI,IAAI;AACxB,kBAAE,CAAa,UAAA,EAAA,OAAO,CAAC,SAAS,CAAC,IAAI,CAAE;AACvC,kBAAE,EAAE;YACN,EAAE;AACH;aACE,MAAM,CAAC,OAAO;aACd,IAAI,CAAC,MAAM,CAAC;;;AAIjB,IAAA,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,EAAE;AACjD,IAAA,IAAI,aAAa,CAAC,MAAM,EAAE;QACxB,UAAU,CAAC,iBAAiB,CAAC;QAC7B,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,KAAI;AAC7B,YAAA,MAAM,IAAI;gBACR,CAAgB,aAAA,EAAA,CAAC,GAAG,CAAC,CAAG,CAAA,CAAA;gBACxB,CAAI,CAAA,EAAA,CAAC,CAAC,QAAQ,CAAG,CAAA,CAAA;gBACjB,CAAG,EAAA,CAAC,CAAC,OAAO,IAAI,IAAI,IAAI,CAAC,CAAC,OAAO,GAAG,CAAA,SAAA,EAAY,CAAC,CAAC,OAAO,GAAG,GAAG,EAAE,CAAE,CAAA;gBACnE,CAAG,EAAA,CAAC,CAAC,KAAK,IAAI,IAAI,IAAI,CAAC,CAAC,KAAK,GAAG,CAAA,OAAA,EAAU,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,CAAE,CAAA;gBAC1D,CAAG,EAAA,CAAC,CAAC,IAAI,IAAI,IAAI,IAAI,CAAC,CAAC,IAAI,GAAG,CAAA,MAAA,EAAS,CAAC,CAAC,IAAI,EAAE,GAAG,EAAE,CAAE,CAAA;gBACtD,EAAE;AACH;iBACE,MAAM,CAAC,OAAO;iBACd,IAAI,CAAC,MAAM,CAAC;AACjB,SAAC,CAAC;;IAEJ,OAAO;AACL,QAAA,MAAM,EAAE,MAAM,CAAC,IAAI,EAAE;QACrB,UAAU;KACX;AACH;;;;"}

package/dist/cjs/tools/search/highlights.cjs

@@ -103,6 +103,57 @@ function findBestBoundary(text, direction = 'backward') {
     // No match found, use character boundary
     return direction === 'backward' ? text.length : 0;
 }
+/**
+ * Tracks references used in a highlight without changing their numbers
+ */
+function trackReferencesInHighlight(text, sourceResult // Source containing the original references
+) {
+    // Track used references
+    const references = [];
+    if (!text || text.length === 0 || !text.includes('#')) {
+        return { references }; // Early return
+    }
+    // Quick check for reference markers
+    if (!text.includes('link#') &&
+        !text.includes('image#') &&
+        !text.includes('video#')) {
+        return { references };
+    }
+    // Get references from the source if available
+    const sourceRefs = sourceResult.references || {
+        links: [],
+        images: [],
+        videos: [],
+    };
+    // Find references but don't modify text
+    const refRegex = /\((link|image|video)#(\d+)(?:\s+"([^"]*)")?\)/g;
+    let match;
+    while ((match = refRegex.exec(text)) !== null) {
+        const [, type, indexStr] = match;
+        const originalIndex = parseInt(indexStr, 10) - 1; // Convert to 0-based
+        // Get the source array for this type
+        const refType = type;
+        const sourceArray = sourceRefs[`${refType}s`];
+        // Skip if invalid reference
+        if (!sourceArray ||
+            originalIndex < 0 ||
+            originalIndex >= sourceArray.length) {
+            continue; // Skip invalid references
+        }
+        // Get original reference
+        const reference = sourceArray[originalIndex];
+        // Track if not already tracked
+        const alreadyTracked = references.some((ref) => ref.type === refType && ref.originalIndex === originalIndex);
+        if (!alreadyTracked) {
+            references.push({
+                type: refType,
+                originalIndex,
+                reference,
+            });
+        }
+    }
+    return { references };
+}
 /**
  * Expand highlights in search results using smart boundary detection.
  *
@@ -112,19 +163,16 @@ function findBestBoundary(text, direction = 'backward') {
  * @param searchResults - Search results object
  * @param mainExpandBy - Primary expansion size on each side (default: 300)
  * @param separatorExpandBy - Additional range to look for separators (default: 150)
- * @returns Copy of search results with expanded highlights
+ * @returns Copy of search results with expanded highlights and tracked references
  */
 function expandHighlights(searchResults, mainExpandBy = 300, separatorExpandBy = 150) {
-    // 1. Avoid full deep copy - only copy what we modify
+    // Avoid deep copy - only copy what we modify
     const resultCopy = { ...searchResults };
-    // Only deep copy the relevant arrays
-    if (resultCopy.organic) {
+    if (resultCopy.organic)
         resultCopy.organic = [...resultCopy.organic];
-    }
-    if (resultCopy.topStories) {
+    if (resultCopy.topStories)
         resultCopy.topStories = [...resultCopy.topStories];
-    }
-    // 5. Process the results efficiently
+    // Process the results efficiently
     const processResultTypes = ['organic', 'topStories'];
     for (const resultType of processResultTypes) {
         if (!resultCopy[resultType])
@@ -143,17 +191,18 @@ function expandHighlights(searchResults, mainExpandBy = 300, separatorExpandBy =
             const highlights = [];
             // Process each highlight
             for (const highlight of result.highlights) {
-                const highlightText = highlight.text;
-                let startPos = content.indexOf(highlightText);
-                let highlightLen = highlightText.length;
+                const { references } = trackReferencesInHighlight(highlight.text, result);
+                let startPos = content.indexOf(highlight.text);
+                let highlightLen = highlight.text.length;
                 if (startPos === -1) {
                     // Try with stripped whitespace
-                    const strippedHighlight = highlightText.trim();
+                    const strippedHighlight = highlight.text.trim();
                     startPos = content.indexOf(strippedHighlight);
                     if (startPos === -1) {
                         highlights.push({
                             text: highlight.text,
                             score: highlight.score,
+                            references,
                         });
                         continue;
                     }
@@ -180,10 +229,12 @@ function expandHighlights(searchResults, mainExpandBy = 300, separatorExpandBy =
                 highlights.push({
                     text: expandedHighlightText,
                     score: highlight.score,
+                    references,
                 });
             }
-            delete resultCopy.content;
             resultCopy.highlights = highlights;
+            delete resultCopy.content;
+            delete resultCopy.references;
             return resultCopy;
         });
     }