@librechat/agents 2.4.319 → 2.4.321

package/dist/cjs/tools/search/format.cjs

@@ -2,10 +2,86 @@
 
 var utils = require('./utils.cjs');
 
+function addHighlightSection() {
+    return ['\n## Highlights', ''];
+}
+// Helper function to format a source (organic or top story)
+function formatSource(source, index, turn, sourceType, references) {
+    /** Array of all lines to include in the output */
+    const outputLines = [];
+    // Add the title
+    outputLines.push(`# ${sourceType.charAt(0).toUpperCase() + sourceType.slice(1)} ${index}: ${source.title != null && source.title ? `"${source.title}"` : '(no title)'}`);
+    outputLines.push(`\nAnchor: \\ue202turn${turn}${sourceType}${index}`);
+    outputLines.push(`URL: ${source.link}`);
+    // Add optional fields
+    if ('snippet' in source && source.snippet != null) {
+        outputLines.push(`Summary: ${source.snippet}`);
+    }
+    if (source.date != null) {
+        outputLines.push(`Date: ${source.date}`);
+    }
+    if (source.attribution != null) {
+        outputLines.push(`Source: ${source.attribution}`);
+    }
+    // Add highlight section or empty line
+    if ((source.highlights?.length ?? 0) > 0) {
+        outputLines.push(...addHighlightSection());
+    }
+    else {
+        outputLines.push('');
+    }
+    // Process highlights if they exist
+    (source.highlights ?? [])
+        .filter((h) => h.text.trim().length > 0)
+        .forEach((h, hIndex) => {
+        outputLines.push(`### Highlight ${hIndex + 1} [Relevance: ${h.score.toFixed(2)}]`);
+        outputLines.push('');
+        outputLines.push('```text');
+        outputLines.push(h.text.trim());
+        outputLines.push('```');
+        outputLines.push('');
+        if (h.references != null && h.references.length) {
+            let hasHeader = false;
+            const refLines = [];
+            for (let j = 0; j < h.references.length; j++) {
+                const ref = h.references[j];
+                references.push({
+                    type: ref.type,
+                    link: ref.reference.originalUrl,
+                    attribution: utils.getDomainName(ref.reference.originalUrl),
+                    title: (((ref.reference.title ?? '') || ref.reference.text) ??
+                        '').split('\n')[0],
+                });
+                if (ref.type !== 'link') {
+                    continue;
+                }
+                if (!hasHeader) {
+                    refLines.push('Core References:');
+                    hasHeader = true;
+                }
+                refLines.push(`- ${ref.type}#${ref.originalIndex + 1}: ${ref.reference.originalUrl}`);
+                refLines.push(`\t- Anchor: \\ue202turn${turn}ref${references.length - 1}`);
+            }
+            if (hasHeader) {
+                outputLines.push(...refLines);
+                outputLines.push('');
+            }
+        }
+        if (hIndex < (source.highlights?.length ?? 0) - 1) {
+            outputLines.push('---');
+            outputLines.push('');
+        }
+    });
+    outputLines.push('');
+    return outputLines.join('\n');
+}
 function formatResultsForLLM(turn, results) {
-    let output = '';
+    /** Array to collect all output lines */
+    const outputLines = [];
     const addSection = (title) => {
-        output += `\n=== ${title} ===\n`;
+        outputLines.push('');
+        outputLines.push(`=== ${title} ===`);
+        outputLines.push('');
     };
     const references = [];
     // Organic (web) results
@@ -13,82 +89,38 @@ function formatResultsForLLM(turn, results) {
         addSection(`Web Results, Turn ${turn}`);
         for (let i = 0; i < results.organic.length; i++) {
             const r = results.organic[i];
-            output += [
-                `# Source ${i}: "${r.title ?? '(no title)'}"`,
-                `Anchor: \\ue202turn${turn}search${i}`,
-                `URL: ${r.link}`,
-                r.snippet != null ? `Summary: ${r.snippet}` : '',
-                r.date != null ? `Date: ${r.date}` : '',
-                r.attribution != null ? `Source: ${r.attribution}` : '',
-                '',
-                '\n## Highlights\n\n',
-                '',
-                '',
-            ]
-                .filter(Boolean)
-                .join('\n');
-            (r.highlights ?? [])
-                .filter((h) => h.text.trim().length > 0)
-                .forEach((h, hIndex) => {
-                output += `### Highlight ${hIndex + 1} [Relevance: ${h.score.toFixed(2)}]\n\n`;
-                output += '```text\n' + h.text.trim() + '\n```\n\n';
-                if (h.references != null && h.references.length) {
-                    output += 'Core References:\n';
-                    output += h.references
-                        .map((ref) => {
-                        references.push({
-                            link: ref.reference.originalUrl,
-                            attribution: utils.getDomainName(ref.reference.originalUrl),
-                            title: (((ref.reference.title ?? '') || ref.reference.text) ??
-                                '').split('\n')[0],
-                        });
-                        return `- ${ref.type}#${ref.originalIndex + 1}: ${ref.reference.originalUrl}\n\t- Anchor: \\ue202turn${turn}ref${references.length - 1}`;
-                    })
-                        .join('\n');
-                    output += '\n\n';
-                }
-                if (hIndex < (r.highlights?.length ?? 0) - 1) {
-                    output += '---\n\n';
-                }
-            });
+            outputLines.push(formatSource(r, i, turn, 'search', references));
             delete results.organic[i].highlights;
-            output += '\n';
         }
     }
-    // Ignoring these sections for now
-    // // Top stories (news)
-    // const topStores = results.topStories ?? [];
-    // if (topStores.length) {
-    //   addSection('News Results');
-    //   topStores.forEach((r, i) => {
-    //     output += [
-    //       `Anchor: \ue202turn0news${i}`,
-    //       `Title: ${r.title ?? '(no title)'}`,
-    //       `URL: ${r.link}`,
-    //       r.snippet != null ? `Snippet: ${r.snippet}` : '',
-    //       r.date != null ? `Date: ${r.date}` : '',
-    //       r.attribution != null ? `Source: ${r.attribution}` : '',
-    //       ''
-    //     ].filter(Boolean).join('\n');
-    //   });
-    // }
+    // Top stories (news)
+    const topStories = results.topStories ?? [];
+    if (topStories.length) {
+        addSection('News Results');
+        for (let i = 0; i < topStories.length; i++) {
+            const r = topStories[i];
+            outputLines.push(formatSource(r, i, turn, 'news', references));
+            if (results.topStories?.[i]?.highlights) {
+                delete results.topStories[i].highlights;
+            }
+        }
+    }
     // // Images
     // const images = results.images ?? [];
     // if (images.length) {
     //   addSection('Image Results');
-    //   images.forEach((img, i) => {
-    //     output += [
-    //       `Anchor: \ue202turn0image${i}`,
-    //       `Title: ${img.title ?? '(no title)'}`,
-    //       `Image URL: ${img.imageUrl}`,
-    //       ''
-    //     ].join('\n');
-    //   });
+    //   const imageLines = images.map((img, i) => [
+    //     `Anchor: \ue202turn0image${i}`,
+    //     `Title: ${img.title ?? '(no title)'}`,
+    //     `Image URL: ${img.imageUrl}`,
+    //     ''
+    //   ].join('\n'));
+    //   outputLines.push(imageLines.join('\n'));
     // }
     // Knowledge Graph
     if (results.knowledgeGraph != null) {
         addSection('Knowledge Graph');
-        output += [
+        const kgLines = [
             `**Title:** ${results.knowledgeGraph.title ?? '(no title)'}`,
             results.knowledgeGraph.type != null
                 ? `**Type:** ${results.knowledgeGraph.type}`
@@ -112,14 +144,13 @@ function formatResultsForLLM(turn, results) {
                 ? `**Attributes:**\n\`\`\`json\n${JSON.stringify(results.knowledgeGraph.attributes, null, 2)}\n\`\`\``
                 : '',
             '',
-        ]
-            .filter(Boolean)
-            .join('\n\n');
+        ].filter(Boolean);
+        outputLines.push(kgLines.join('\n\n'));
     }
     // Answer Box
     if (results.answerBox != null) {
         addSection('Answer Box');
-        output += [
+        const abLines = [
             results.answerBox.title != null
                 ? `**Title:** ${results.answerBox.title}`
                 : '',
@@ -135,29 +166,29 @@ function formatResultsForLLM(turn, results) {
                 ? `**Link:** ${results.answerBox.link}`
                 : '',
             '',
-        ]
-            .filter(Boolean)
-            .join('\n\n');
+        ].filter(Boolean);
+        outputLines.push(abLines.join('\n\n'));
     }
     // People also ask
     const peopleAlsoAsk = results.peopleAlsoAsk ?? [];
     if (peopleAlsoAsk.length) {
         addSection('People Also Ask');
+        const paaLines = [];
         peopleAlsoAsk.forEach((p, i) => {
-            output += [
+            const questionLines = [
                 `### Question ${i + 1}:`,
                 `"${p.question}"`,
-                `${p.snippet != null && p.snippet ? `Snippet: ${p.snippet}}` : ''}`,
+                `${p.snippet != null && p.snippet ? `Snippet: ${p.snippet}` : ''}`,
                 `${p.title != null && p.title ? `Title: ${p.title}` : ''}`,
                 `${p.link != null && p.link ? `Link: ${p.link}` : ''}`,
                 '',
-            ]
-                .filter(Boolean)
-                .join('\n\n');
+            ].filter(Boolean);
+            paaLines.push(questionLines.join('\n\n'));
         });
+        outputLines.push(paaLines.join(''));
     }
     return {
-        output: output.trim(),
+        output: outputLines.join('\n').trim(),
         references,
     };
 }

package/dist/cjs/tools/search/format.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"format.cjs","sources":["../../../../src/tools/search/format.ts"],"sourcesContent":["import type * as t from './types';\nimport { getDomainName } from './utils';\n\nexport function formatResultsForLLM(\n  turn: number,\n  results: t.SearchResultData\n): { output: string; references: t.ResultReference[] } {\n  let output = '';\n\n  const addSection = (title: string): void => {\n    output += `\\n=== ${title} ===\\n`;\n  };\n\n  const references: t.ResultReference[] = [];\n  // Organic (web) results\n  if (results.organic?.length != null && results.organic.length > 0) {\n    addSection(`Web Results, Turn ${turn}`);\n    for (let i = 0; i < results.organic.length; i++) {\n      const r = results.organic[i];\n      output += [\n        `# Source ${i}: \"${r.title ?? '(no title)'}\"`,\n        `Anchor: \\\\ue202turn${turn}search${i}`,\n        `URL: ${r.link}`,\n        r.snippet != null ? `Summary: ${r.snippet}` : '',\n        r.date != null ? `Date: ${r.date}` : '',\n        r.attribution != null ? `Source: ${r.attribution}` : '',\n        '',\n        '\\n## Highlights\\n\\n',\n        '',\n        '',\n      ]\n        .filter(Boolean)\n        .join('\\n');\n\n      (r.highlights ?? [])\n        .filter((h) => h.text.trim().length > 0)\n        .forEach((h, hIndex) => {\n          output += `### Highlight ${hIndex + 1} [Relevance: ${h.score.toFixed(2)}]\\n\\n`;\n          output += '```text\\n' + h.text.trim() + '\\n```\\n\\n';\n\n          if (h.references != null && h.references.length) {\n            output += 'Core References:\\n';\n            output += h.references\n              .map((ref) => {\n                references.push({\n                  link: ref.reference.originalUrl,\n                  attribution: getDomainName(ref.reference.originalUrl),\n                  title: (\n                    ((ref.reference.title ?? '') || ref.reference.text) ??\n                    ''\n                  ).split('\\n')[0],\n                });\n                return `- ${ref.type}#${ref.originalIndex + 1}: ${ref.reference.originalUrl}\\n\\t- Anchor: \\\\ue202turn${turn}ref${references.length - 1}`;\n              })\n              .join('\\n');\n            output += '\\n\\n';\n          }\n\n          if (hIndex < (r.highlights?.length ?? 0) - 1) {\n            output += '---\\n\\n';\n          }\n        });\n\n      delete results.organic[i].highlights;\n      output += '\\n';\n    }\n  }\n\n  // Ignoring these sections for now\n  // // Top stories (news)\n  // const topStores = results.topStories ?? [];\n  // if (topStores.length) {\n  //   addSection('News Results');\n  //   topStores.forEach((r, i) => {\n  //     output += [\n  //       `Anchor: \\ue202turn0news${i}`,\n  //       `Title: ${r.title ?? '(no title)'}`,\n  //       `URL: ${r.link}`,\n  //       r.snippet != null ? `Snippet: ${r.snippet}` : '',\n  //       r.date != null ? `Date: ${r.date}` : '',\n  //       r.attribution != null ? `Source: ${r.attribution}` : '',\n  //       ''\n  //     ].filter(Boolean).join('\\n');\n  //   });\n  // }\n\n  // // Images\n  // const images = results.images ?? [];\n  // if (images.length) {\n  //   addSection('Image Results');\n  //   images.forEach((img, i) => {\n  //     output += [\n  //       `Anchor: \\ue202turn0image${i}`,\n  //       `Title: ${img.title ?? '(no title)'}`,\n  //       `Image URL: ${img.imageUrl}`,\n  //       ''\n  //     ].join('\\n');\n  //   });\n  // }\n\n  // Knowledge Graph\n  if (results.knowledgeGraph != null) {\n    addSection('Knowledge Graph');\n    output += [\n      `**Title:** ${results.knowledgeGraph.title ?? '(no title)'}`,\n      results.knowledgeGraph.type != null\n        ? `**Type:** ${results.knowledgeGraph.type}`\n        : '',\n      results.knowledgeGraph.description != null\n        ? `**Description:** ${results.knowledgeGraph.description}`\n        : '',\n      results.knowledgeGraph.descriptionSource != null\n        ? `**Description Source:** ${results.knowledgeGraph.descriptionSource}`\n        : '',\n      results.knowledgeGraph.descriptionLink != null\n        ? `**Description Link:** ${results.knowledgeGraph.descriptionLink}`\n        : '',\n      results.knowledgeGraph.imageUrl != null\n        ? `**Image URL:** ${results.knowledgeGraph.imageUrl}`\n        : '',\n      results.knowledgeGraph.website != null\n        ? `**Website:** ${results.knowledgeGraph.website}`\n        : '',\n      results.knowledgeGraph.attributes != null\n        ? `**Attributes:**\\n\\`\\`\\`json\\n${JSON.stringify(\n          results.knowledgeGraph.attributes,\n          null,\n          2\n        )}\\n\\`\\`\\``\n        : '',\n      '',\n    ]\n      .filter(Boolean)\n      .join('\\n\\n');\n  }\n\n  // Answer Box\n  if (results.answerBox != null) {\n    addSection('Answer Box');\n    output += [\n      results.answerBox.title != null\n        ? `**Title:** ${results.answerBox.title}`\n        : '',\n      results.answerBox.snippet != null\n        ? `**Snippet:** ${results.answerBox.snippet}`\n        : '',\n      results.answerBox.snippetHighlighted != null\n        ? `**Snippet Highlighted:** ${results.answerBox.snippetHighlighted\n          .map((s) => `\\`${s}\\``)\n          .join(' ')}`\n        : '',\n      results.answerBox.link != null\n        ? `**Link:** ${results.answerBox.link}`\n        : '',\n      '',\n    ]\n      .filter(Boolean)\n      .join('\\n\\n');\n  }\n\n  // People also ask\n  const peopleAlsoAsk = results.peopleAlsoAsk ?? [];\n  if (peopleAlsoAsk.length) {\n    addSection('People Also Ask');\n    peopleAlsoAsk.forEach((p, i) => {\n      output += [\n        `### Question ${i + 1}:`,\n        `\"${p.question}\"`,\n        `${p.snippet != null && p.snippet ? `Snippet: ${p.snippet}}` : ''}`,\n        `${p.title != null && p.title ? `Title: ${p.title}` : ''}`,\n        `${p.link != null && p.link ? `Link: ${p.link}` : ''}`,\n        '',\n      ]\n        .filter(Boolean)\n        .join('\\n\\n');\n    });\n  }\n  return {\n    output: output.trim(),\n    references,\n  };\n}\n"],"names":["getDomainName"],"mappings":";;;;AAGgB,SAAA,mBAAmB,CACjC,IAAY,EACZ,OAA2B,EAAA;IAE3B,IAAI,MAAM,GAAG,EAAE;AAEf,IAAA,MAAM,UAAU,GAAG,CAAC,KAAa,KAAU;AACzC,QAAA,MAAM,IAAI,CAAA,MAAA,EAAS,KAAK,CAAA,MAAA,CAAQ;AAClC,KAAC;IAED,MAAM,UAAU,GAAwB,EAAE;;AAE1C,IAAA,IAAI,OAAO,CAAC,OAAO,EAAE,MAAM,IAAI,IAAI,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE;AACjE,QAAA,UAAU,CAAC,CAAA,kBAAA,EAAqB,IAAI,CAAA,CAAE,CAAC;AACvC,QAAA,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YAC/C,MAAM,CAAC,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;AAC5B,YAAA,MAAM,IAAI;AACR,gBAAA,CAAA,SAAA,EAAY,CAAC,CAAM,GAAA,EAAA,CAAC,CAAC,KAAK,IAAI,YAAY,CAAG,CAAA,CAAA;gBAC7C,CAAsB,mBAAA,EAAA,IAAI,CAAS,MAAA,EAAA,CAAC,CAAE,CAAA;gBACtC,CAAQ,KAAA,EAAA,CAAC,CAAC,IAAI,CAAE,CAAA;AAChB,gBAAA,CAAC,CAAC,OAAO,IAAI,IAAI,GAAG,CAAY,SAAA,EAAA,CAAC,CAAC,OAAO,CAAA,CAAE,GAAG,EAAE;AAChD,gBAAA,CAAC,CAAC,IAAI,IAAI,IAAI,GAAG,CAAS,MAAA,EAAA,CAAC,CAAC,IAAI,CAAA,CAAE,GAAG,EAAE;AACvC,gBAAA,CAAC,CAAC,WAAW,IAAI,IAAI,GAAG,CAAW,QAAA,EAAA,CAAC,CAAC,WAAW,CAAA,CAAE,GAAG,EAAE;gBACvD,EAAE;gBACF,qBAAqB;gBACrB,EAAE;gBACF,EAAE;AACH;iBACE,MAAM,CAAC,OAAO;iBACd,IAAI,CAAC,IAAI,CAAC;AAEb,YAAA,CAAC,CAAC,CAAC,UAAU,IAAI,EAAE;AAChB,iBAAA,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC;AACtC,iBAAA,OAAO,CAAC,CAAC,CAAC,EAAE,MAAM,KAAI;AACrB,gBAAA,MAAM,IAAI,CAAA,cAAA,EAAiB,MAAM,GAAG,CAAC,CAAgB,aAAA,EAAA,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO;gBAC9E,MAAM,IAAI,WAAW,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,WAAW;AAEnD,gBAAA,IAAI,CAAC,CAAC,UAAU,IAAI,IAAI,IAAI,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE;oBAC/C,MAAM,IAAI,oBAAoB;oBAC9B,MAAM,IAAI,CAAC,CAAC;AACT,yBAAA,GAAG,CAAC,CAAC,GAAG,KAAI;wBACX,UAAU,CAAC,IAAI,CAAC;AACd,4BAAA,IAAI,EAAE,GAAG,CAAC,SAAS,CAAC,WAAW;4BAC/B,WAAW,EAAEA,mBAAa,CAAC,GAAG,CAAC,SAAS,CAAC,WAAW,CAAC;AACrD,4BAAA,KAAK,EAAE,CACL,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,IAAI,EAAE,KAAK,GAAG,CAAC,SAAS,CAAC,IAAI;gCAClD,EAAE,EACF,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACjB,yBAAA,CAAC;wBACF,OAAO,CAAA,EAAA,EAAK,GAAG,CAAC,IAAI,CAAA,CAAA,EAAI,GAAG,CAAC,aAAa,GAAG,CAAC,CAAK,EAAA,EAAA,GAAG,CAAC,SAAS,CAAC,WAAW,CAAA,yBAAA,EAA4B,IAAI,CAAA,GAAA,EAAM,UAAU,CAAC,MAAM,GAAG,CAAC,CAAA,CAAE;AAC1I,qBAAC;yBACA,IAAI,CAAC,IAAI,CAAC;oBACb,MAAM,IAAI,MAAM;;AAGlB,gBAAA,IAAI,MAAM,GAAG,CAAC,CAAC,CAAC,UAAU,EAAE,MAAM,IAAI,CAAC,IAAI,CAAC,EAAE;oBAC5C,MAAM,IAAI,SAAS;;AAEvB,aAAC,CAAC;YAEJ,OAAO,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,UAAU;YACpC,MAAM,IAAI,IAAI;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAqClB,IAAA,IAAI,OAAO,CAAC,cAAc,IAAI,IAAI,EAAE;QAClC,UAAU,CAAC,iBAAiB,CAAC;AAC7B,QAAA,MAAM,IAAI;AACR,YAAA,CAAA,WAAA,EAAc,OAAO,CAAC,cAAc,CAAC,KAAK,IAAI,YAAY,CAAE,CAAA;AAC5D,YAAA,OAAO,CAAC,cAAc,CAAC,IAAI,IAAI;AAC7B,kBAAE,CAAa,UAAA,EAAA,OAAO,CAAC,cAAc,CAAC,IAAI,CAAE;AAC5C,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,cAAc,CAAC,WAAW,IAAI;AACpC,kBAAE,CAAoB,iBAAA,EAAA,OAAO,CAAC,cAAc,CAAC,WAAW,CAAE;AAC1D,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,cAAc,CAAC,iBAAiB,IAAI;AAC1C,kBAAE,CAA2B,wBAAA,EAAA,OAAO,CAAC,cAAc,CAAC,iBAAiB,CAAE;AACvE,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,cAAc,CAAC,eAAe,IAAI;AACxC,kBAAE,CAAyB,sBAAA,EAAA,OAAO,CAAC,cAAc,CAAC,eAAe,CAAE;AACnE,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,cAAc,CAAC,QAAQ,IAAI;AACjC,kBAAE,CAAkB,eAAA,EAAA,OAAO,CAAC,cAAc,CAAC,QAAQ,CAAE;AACrD,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,cAAc,CAAC,OAAO,IAAI;AAChC,kBAAE,CAAgB,aAAA,EAAA,OAAO,CAAC,cAAc,CAAC,OAAO,CAAE;AAClD,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,cAAc,CAAC,UAAU,IAAI;AACnC,kBAAE,CAAgC,6BAAA,EAAA,IAAI,CAAC,SAAS,CAC9C,OAAO,CAAC,cAAc,CAAC,UAAU,EACjC,IAAI,EACJ,CAAC,CACF,CAAU,QAAA;AACX,kBAAE,EAAE;YACN,EAAE;AACH;aACE,MAAM,CAAC,OAAO;aACd,IAAI,CAAC,MAAM,CAAC;;;AAIjB,IAAA,IAAI,OAAO,CAAC,SAAS,IAAI,IAAI,EAAE;QAC7B,UAAU,CAAC,YAAY,CAAC;AACxB,QAAA,MAAM,IAAI;AACR,YAAA,OAAO,CAAC,SAAS,CAAC,KAAK,IAAI;AACzB,kBAAE,CAAc,WAAA,EAAA,OAAO,CAAC,SAAS,CAAC,KAAK,CAAE;AACzC,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,SAAS,CAAC,OAAO,IAAI;AAC3B,kBAAE,CAAgB,aAAA,EAAA,OAAO,CAAC,SAAS,CAAC,OAAO,CAAE;AAC7C,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,SAAS,CAAC,kBAAkB,IAAI;AACtC,kBAAE,CAA4B,yBAAA,EAAA,OAAO,CAAC,SAAS,CAAC;qBAC7C,GAAG,CAAC,CAAC,CAAC,KAAK,CAAA,EAAA,EAAK,CAAC,CAAA,EAAA,CAAI;qBACrB,IAAI,CAAC,GAAG,CAAC,CAAE;AACd,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,SAAS,CAAC,IAAI,IAAI;AACxB,kBAAE,CAAa,UAAA,EAAA,OAAO,CAAC,SAAS,CAAC,IAAI,CAAE;AACvC,kBAAE,EAAE;YACN,EAAE;AACH;aACE,MAAM,CAAC,OAAO;aACd,IAAI,CAAC,MAAM,CAAC;;;AAIjB,IAAA,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,EAAE;AACjD,IAAA,IAAI,aAAa,CAAC,MAAM,EAAE;QACxB,UAAU,CAAC,iBAAiB,CAAC;QAC7B,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,KAAI;AAC7B,YAAA,MAAM,IAAI;gBACR,CAAgB,aAAA,EAAA,CAAC,GAAG,CAAC,CAAG,CAAA,CAAA;gBACxB,CAAI,CAAA,EAAA,CAAC,CAAC,QAAQ,CAAG,CAAA,CAAA;gBACjB,CAAG,EAAA,CAAC,CAAC,OAAO,IAAI,IAAI,IAAI,CAAC,CAAC,OAAO,GAAG,CAAA,SAAA,EAAY,CAAC,CAAC,OAAO,GAAG,GAAG,EAAE,CAAE,CAAA;gBACnE,CAAG,EAAA,CAAC,CAAC,KAAK,IAAI,IAAI,IAAI,CAAC,CAAC,KAAK,GAAG,CAAA,OAAA,EAAU,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,CAAE,CAAA;gBAC1D,CAAG,EAAA,CAAC,CAAC,IAAI,IAAI,IAAI,IAAI,CAAC,CAAC,IAAI,GAAG,CAAA,MAAA,EAAS,CAAC,CAAC,IAAI,EAAE,GAAG,EAAE,CAAE,CAAA;gBACtD,EAAE;AACH;iBACE,MAAM,CAAC,OAAO;iBACd,IAAI,CAAC,MAAM,CAAC;AACjB,SAAC,CAAC;;IAEJ,OAAO;AACL,QAAA,MAAM,EAAE,MAAM,CAAC,IAAI,EAAE;QACrB,UAAU;KACX;AACH;;;;"}
+{"version":3,"file":"format.cjs","sources":["../../../../src/tools/search/format.ts"],"sourcesContent":["import type * as t from './types';\nimport { getDomainName } from './utils';\n\nfunction addHighlightSection(): string[] {\n  return ['\\n## Highlights', ''];\n}\n\n// Helper function to format a source (organic or top story)\nfunction formatSource(\n  source: t.ValidSource,\n  index: number,\n  turn: number,\n  sourceType: 'search' | 'news',\n  references: t.ResultReference[]\n): string {\n  /** Array of all lines to include in the output */\n  const outputLines: string[] = [];\n\n  // Add the title\n  outputLines.push(\n    `# ${sourceType.charAt(0).toUpperCase() + sourceType.slice(1)} ${index}: ${source.title != null && source.title ? `\"${source.title}\"` : '(no title)'}`\n  );\n  outputLines.push(`\\nAnchor: \\\\ue202turn${turn}${sourceType}${index}`);\n  outputLines.push(`URL: ${source.link}`);\n\n  // Add optional fields\n  if ('snippet' in source && source.snippet != null) {\n    outputLines.push(`Summary: ${source.snippet}`);\n  }\n\n  if (source.date != null) {\n    outputLines.push(`Date: ${source.date}`);\n  }\n\n  if (source.attribution != null) {\n    outputLines.push(`Source: ${source.attribution}`);\n  }\n\n  // Add highlight section or empty line\n  if ((source.highlights?.length ?? 0) > 0) {\n    outputLines.push(...addHighlightSection());\n  } else {\n    outputLines.push('');\n  }\n\n  // Process highlights if they exist\n  (source.highlights ?? [])\n    .filter((h) => h.text.trim().length > 0)\n    .forEach((h, hIndex) => {\n      outputLines.push(\n        `### Highlight ${hIndex + 1} [Relevance: ${h.score.toFixed(2)}]`\n      );\n      outputLines.push('');\n      outputLines.push('```text');\n      outputLines.push(h.text.trim());\n      outputLines.push('```');\n      outputLines.push('');\n\n      if (h.references != null && h.references.length) {\n        let hasHeader = false;\n        const refLines: string[] = [];\n\n        for (let j = 0; j < h.references.length; j++) {\n          const ref = h.references[j];\n          references.push({\n            type: ref.type,\n            link: ref.reference.originalUrl,\n            attribution: getDomainName(ref.reference.originalUrl),\n            title: (\n              ((ref.reference.title ?? '') || ref.reference.text) ??\n              ''\n            ).split('\\n')[0],\n          });\n\n          if (ref.type !== 'link') {\n            continue;\n          }\n\n          if (!hasHeader) {\n            refLines.push('Core References:');\n            hasHeader = true;\n          }\n\n          refLines.push(\n            `- ${ref.type}#${ref.originalIndex + 1}: ${ref.reference.originalUrl}`\n          );\n          refLines.push(\n            `\\t- Anchor: \\\\ue202turn${turn}ref${references.length - 1}`\n          );\n        }\n\n        if (hasHeader) {\n          outputLines.push(...refLines);\n          outputLines.push('');\n        }\n      }\n\n      if (hIndex < (source.highlights?.length ?? 0) - 1) {\n        outputLines.push('---');\n        outputLines.push('');\n      }\n    });\n\n  outputLines.push('');\n  return outputLines.join('\\n');\n}\n\nexport function formatResultsForLLM(\n  turn: number,\n  results: t.SearchResultData\n): { output: string; references: t.ResultReference[] } {\n  /** Array to collect all output lines */\n  const outputLines: string[] = [];\n\n  const addSection = (title: string): void => {\n    outputLines.push('');\n    outputLines.push(`=== ${title} ===`);\n    outputLines.push('');\n  };\n\n  const references: t.ResultReference[] = [];\n\n  // Organic (web) results\n  if (results.organic?.length != null && results.organic.length > 0) {\n    addSection(`Web Results, Turn ${turn}`);\n    for (let i = 0; i < results.organic.length; i++) {\n      const r = results.organic[i];\n      outputLines.push(formatSource(r, i, turn, 'search', references));\n      delete results.organic[i].highlights;\n    }\n  }\n\n  // Top stories (news)\n  const topStories = results.topStories ?? [];\n  if (topStories.length) {\n    addSection('News Results');\n    for (let i = 0; i < topStories.length; i++) {\n      const r = topStories[i];\n      outputLines.push(formatSource(r, i, turn, 'news', references));\n      if (results.topStories?.[i]?.highlights) {\n        delete results.topStories[i].highlights;\n      }\n    }\n  }\n\n  // // Images\n  // const images = results.images ?? [];\n  // if (images.length) {\n  //   addSection('Image Results');\n  //   const imageLines = images.map((img, i) => [\n  //     `Anchor: \\ue202turn0image${i}`,\n  //     `Title: ${img.title ?? '(no title)'}`,\n  //     `Image URL: ${img.imageUrl}`,\n  //     ''\n  //   ].join('\\n'));\n  //   outputLines.push(imageLines.join('\\n'));\n  // }\n\n  // Knowledge Graph\n  if (results.knowledgeGraph != null) {\n    addSection('Knowledge Graph');\n    const kgLines = [\n      `**Title:** ${results.knowledgeGraph.title ?? '(no title)'}`,\n      results.knowledgeGraph.type != null\n        ? `**Type:** ${results.knowledgeGraph.type}`\n        : '',\n      results.knowledgeGraph.description != null\n        ? `**Description:** ${results.knowledgeGraph.description}`\n        : '',\n      results.knowledgeGraph.descriptionSource != null\n        ? `**Description Source:** ${results.knowledgeGraph.descriptionSource}`\n        : '',\n      results.knowledgeGraph.descriptionLink != null\n        ? `**Description Link:** ${results.knowledgeGraph.descriptionLink}`\n        : '',\n      results.knowledgeGraph.imageUrl != null\n        ? `**Image URL:** ${results.knowledgeGraph.imageUrl}`\n        : '',\n      results.knowledgeGraph.website != null\n        ? `**Website:** ${results.knowledgeGraph.website}`\n        : '',\n      results.knowledgeGraph.attributes != null\n        ? `**Attributes:**\\n\\`\\`\\`json\\n${JSON.stringify(\n          results.knowledgeGraph.attributes,\n          null,\n          2\n        )}\\n\\`\\`\\``\n        : '',\n      '',\n    ].filter(Boolean);\n\n    outputLines.push(kgLines.join('\\n\\n'));\n  }\n\n  // Answer Box\n  if (results.answerBox != null) {\n    addSection('Answer Box');\n    const abLines = [\n      results.answerBox.title != null\n        ? `**Title:** ${results.answerBox.title}`\n        : '',\n      results.answerBox.snippet != null\n        ? `**Snippet:** ${results.answerBox.snippet}`\n        : '',\n      results.answerBox.snippetHighlighted != null\n        ? `**Snippet Highlighted:** ${results.answerBox.snippetHighlighted\n          .map((s) => `\\`${s}\\``)\n          .join(' ')}`\n        : '',\n      results.answerBox.link != null\n        ? `**Link:** ${results.answerBox.link}`\n        : '',\n      '',\n    ].filter(Boolean);\n\n    outputLines.push(abLines.join('\\n\\n'));\n  }\n\n  // People also ask\n  const peopleAlsoAsk = results.peopleAlsoAsk ?? [];\n  if (peopleAlsoAsk.length) {\n    addSection('People Also Ask');\n\n    const paaLines: string[] = [];\n    peopleAlsoAsk.forEach((p, i) => {\n      const questionLines = [\n        `### Question ${i + 1}:`,\n        `\"${p.question}\"`,\n        `${p.snippet != null && p.snippet ? `Snippet: ${p.snippet}` : ''}`,\n        `${p.title != null && p.title ? `Title: ${p.title}` : ''}`,\n        `${p.link != null && p.link ? `Link: ${p.link}` : ''}`,\n        '',\n      ].filter(Boolean);\n\n      paaLines.push(questionLines.join('\\n\\n'));\n    });\n\n    outputLines.push(paaLines.join(''));\n  }\n\n  return {\n    output: outputLines.join('\\n').trim(),\n    references,\n  };\n}\n"],"names":["getDomainName"],"mappings":";;;;AAGA,SAAS,mBAAmB,GAAA;AAC1B,IAAA,OAAO,CAAC,iBAAiB,EAAE,EAAE,CAAC;AAChC;AAEA;AACA,SAAS,YAAY,CACnB,MAAqB,EACrB,KAAa,EACb,IAAY,EACZ,UAA6B,EAC7B,UAA+B,EAAA;;IAG/B,MAAM,WAAW,GAAa,EAAE;;IAGhC,WAAW,CAAC,IAAI,CACd,CAAA,EAAA,EAAK,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA,CAAA,EAAI,KAAK,CAAA,EAAA,EAAK,MAAM,CAAC,KAAK,IAAI,IAAI,IAAI,MAAM,CAAC,KAAK,GAAG,IAAI,MAAM,CAAC,KAAK,CAAG,CAAA,CAAA,GAAG,YAAY,CAAE,CAAA,CACvJ;IACD,WAAW,CAAC,IAAI,CAAC,CAAwB,qBAAA,EAAA,IAAI,CAAG,EAAA,UAAU,CAAG,EAAA,KAAK,CAAE,CAAA,CAAC;IACrE,WAAW,CAAC,IAAI,CAAC,CAAA,KAAA,EAAQ,MAAM,CAAC,IAAI,CAAE,CAAA,CAAC;;IAGvC,IAAI,SAAS,IAAI,MAAM,IAAI,MAAM,CAAC,OAAO,IAAI,IAAI,EAAE;QACjD,WAAW,CAAC,IAAI,CAAC,CAAA,SAAA,EAAY,MAAM,CAAC,OAAO,CAAE,CAAA,CAAC;;AAGhD,IAAA,IAAI,MAAM,CAAC,IAAI,IAAI,IAAI,EAAE;QACvB,WAAW,CAAC,IAAI,CAAC,CAAA,MAAA,EAAS,MAAM,CAAC,IAAI,CAAE,CAAA,CAAC;;AAG1C,IAAA,IAAI,MAAM,CAAC,WAAW,IAAI,IAAI,EAAE;QAC9B,WAAW,CAAC,IAAI,CAAC,CAAA,QAAA,EAAW,MAAM,CAAC,WAAW,CAAE,CAAA,CAAC;;;AAInD,IAAA,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,MAAM,IAAI,CAAC,IAAI,CAAC,EAAE;AACxC,QAAA,WAAW,CAAC,IAAI,CAAC,GAAG,mBAAmB,EAAE,CAAC;;SACrC;AACL,QAAA,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;;;AAItB,IAAA,CAAC,MAAM,CAAC,UAAU,IAAI,EAAE;AACrB,SAAA,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC;AACtC,SAAA,OAAO,CAAC,CAAC,CAAC,EAAE,MAAM,KAAI;AACrB,QAAA,WAAW,CAAC,IAAI,CACd,iBAAiB,MAAM,GAAG,CAAC,CAAgB,aAAA,EAAA,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA,CAAA,CAAG,CACjE;AACD,QAAA,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;AACpB,QAAA,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC;QAC3B,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;AAC/B,QAAA,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC;AACvB,QAAA,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;AAEpB,QAAA,IAAI,CAAC,CAAC,UAAU,IAAI,IAAI,IAAI,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE;YAC/C,IAAI,SAAS,GAAG,KAAK;YACrB,MAAM,QAAQ,GAAa,EAAE;AAE7B,YAAA,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;gBAC5C,MAAM,GAAG,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC;gBAC3B,UAAU,CAAC,IAAI,CAAC;oBACd,IAAI,EAAE,GAAG,CAAC,IAAI;AACd,oBAAA,IAAI,EAAE,GAAG,CAAC,SAAS,CAAC,WAAW;oBAC/B,WAAW,EAAEA,mBAAa,CAAC,GAAG,CAAC,SAAS,CAAC,WAAW,CAAC;AACrD,oBAAA,KAAK,EAAE,CACL,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,IAAI,EAAE,KAAK,GAAG,CAAC,SAAS,CAAC,IAAI;wBAClD,EAAE,EACF,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACjB,iBAAA,CAAC;AAEF,gBAAA,IAAI,GAAG,CAAC,IAAI,KAAK,MAAM,EAAE;oBACvB;;gBAGF,IAAI,CAAC,SAAS,EAAE;AACd,oBAAA,QAAQ,CAAC,IAAI,CAAC,kBAAkB,CAAC;oBACjC,SAAS,GAAG,IAAI;;gBAGlB,QAAQ,CAAC,IAAI,CACX,CAAA,EAAA,EAAK,GAAG,CAAC,IAAI,IAAI,GAAG,CAAC,aAAa,GAAG,CAAC,KAAK,GAAG,CAAC,SAAS,CAAC,WAAW,CAAE,CAAA,CACvE;AACD,gBAAA,QAAQ,CAAC,IAAI,CACX,CAAA,uBAAA,EAA0B,IAAI,CAAA,GAAA,EAAM,UAAU,CAAC,MAAM,GAAG,CAAC,CAAA,CAAE,CAC5D;;YAGH,IAAI,SAAS,EAAE;AACb,gBAAA,WAAW,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC;AAC7B,gBAAA,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;;;AAIxB,QAAA,IAAI,MAAM,GAAG,CAAC,MAAM,CAAC,UAAU,EAAE,MAAM,IAAI,CAAC,IAAI,CAAC,EAAE;AACjD,YAAA,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC;AACvB,YAAA,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;;AAExB,KAAC,CAAC;AAEJ,IAAA,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;AACpB,IAAA,OAAO,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC;AAC/B;AAEgB,SAAA,mBAAmB,CACjC,IAAY,EACZ,OAA2B,EAAA;;IAG3B,MAAM,WAAW,GAAa,EAAE;AAEhC,IAAA,MAAM,UAAU,GAAG,CAAC,KAAa,KAAU;AACzC,QAAA,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;AACpB,QAAA,WAAW,CAAC,IAAI,CAAC,OAAO,KAAK,CAAA,IAAA,CAAM,CAAC;AACpC,QAAA,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;AACtB,KAAC;IAED,MAAM,UAAU,GAAwB,EAAE;;AAG1C,IAAA,IAAI,OAAO,CAAC,OAAO,EAAE,MAAM,IAAI,IAAI,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE;AACjE,QAAA,UAAU,CAAC,CAAA,kBAAA,EAAqB,IAAI,CAAA,CAAE,CAAC;AACvC,QAAA,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YAC/C,MAAM,CAAC,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;AAC5B,YAAA,WAAW,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,EAAE,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC;YAChE,OAAO,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,UAAU;;;;AAKxC,IAAA,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,EAAE;AAC3C,IAAA,IAAI,UAAU,CAAC,MAAM,EAAE;QACrB,UAAU,CAAC,cAAc,CAAC;AAC1B,QAAA,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;AAC1C,YAAA,MAAM,CAAC,GAAG,UAAU,CAAC,CAAC,CAAC;AACvB,YAAA,WAAW,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;YAC9D,IAAI,OAAO,CAAC,UAAU,GAAG,CAAC,CAAC,EAAE,UAAU,EAAE;gBACvC,OAAO,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,UAAU;;;;;;;;;;;;;;;;;AAmB7C,IAAA,IAAI,OAAO,CAAC,cAAc,IAAI,IAAI,EAAE;QAClC,UAAU,CAAC,iBAAiB,CAAC;AAC7B,QAAA,MAAM,OAAO,GAAG;AACd,YAAA,CAAA,WAAA,EAAc,OAAO,CAAC,cAAc,CAAC,KAAK,IAAI,YAAY,CAAE,CAAA;AAC5D,YAAA,OAAO,CAAC,cAAc,CAAC,IAAI,IAAI;AAC7B,kBAAE,CAAa,UAAA,EAAA,OAAO,CAAC,cAAc,CAAC,IAAI,CAAE;AAC5C,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,cAAc,CAAC,WAAW,IAAI;AACpC,kBAAE,CAAoB,iBAAA,EAAA,OAAO,CAAC,cAAc,CAAC,WAAW,CAAE;AAC1D,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,cAAc,CAAC,iBAAiB,IAAI;AAC1C,kBAAE,CAA2B,wBAAA,EAAA,OAAO,CAAC,cAAc,CAAC,iBAAiB,CAAE;AACvE,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,cAAc,CAAC,eAAe,IAAI;AACxC,kBAAE,CAAyB,sBAAA,EAAA,OAAO,CAAC,cAAc,CAAC,eAAe,CAAE;AACnE,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,cAAc,CAAC,QAAQ,IAAI;AACjC,kBAAE,CAAkB,eAAA,EAAA,OAAO,CAAC,cAAc,CAAC,QAAQ,CAAE;AACrD,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,cAAc,CAAC,OAAO,IAAI;AAChC,kBAAE,CAAgB,aAAA,EAAA,OAAO,CAAC,cAAc,CAAC,OAAO,CAAE;AAClD,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,cAAc,CAAC,UAAU,IAAI;AACnC,kBAAE,CAAgC,6BAAA,EAAA,IAAI,CAAC,SAAS,CAC9C,OAAO,CAAC,cAAc,CAAC,UAAU,EACjC,IAAI,EACJ,CAAC,CACF,CAAU,QAAA;AACX,kBAAE,EAAE;YACN,EAAE;AACH,SAAA,CAAC,MAAM,CAAC,OAAO,CAAC;QAEjB,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;;;AAIxC,IAAA,IAAI,OAAO,CAAC,SAAS,IAAI,IAAI,EAAE;QAC7B,UAAU,CAAC,YAAY,CAAC;AACxB,QAAA,MAAM,OAAO,GAAG;AACd,YAAA,OAAO,CAAC,SAAS,CAAC,KAAK,IAAI;AACzB,kBAAE,CAAc,WAAA,EAAA,OAAO,CAAC,SAAS,CAAC,KAAK,CAAE;AACzC,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,SAAS,CAAC,OAAO,IAAI;AAC3B,kBAAE,CAAgB,aAAA,EAAA,OAAO,CAAC,SAAS,CAAC,OAAO,CAAE;AAC7C,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,SAAS,CAAC,kBAAkB,IAAI;AACtC,kBAAE,CAA4B,yBAAA,EAAA,OAAO,CAAC,SAAS,CAAC;qBAC7C,GAAG,CAAC,CAAC,CAAC,KAAK,CAAA,EAAA,EAAK,CAAC,CAAA,EAAA,CAAI;qBACrB,IAAI,CAAC,GAAG,CAAC,CAAE;AACd,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,SAAS,CAAC,IAAI,IAAI;AACxB,kBAAE,CAAa,UAAA,EAAA,OAAO,CAAC,SAAS,CAAC,IAAI,CAAE;AACvC,kBAAE,EAAE;YACN,EAAE;AACH,SAAA,CAAC,MAAM,CAAC,OAAO,CAAC;QAEjB,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;;;AAIxC,IAAA,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,EAAE;AACjD,IAAA,IAAI,aAAa,CAAC,MAAM,EAAE;QACxB,UAAU,CAAC,iBAAiB,CAAC;QAE7B,MAAM,QAAQ,GAAa,EAAE;QAC7B,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,KAAI;AAC7B,YAAA,MAAM,aAAa,GAAG;gBACpB,CAAgB,aAAA,EAAA,CAAC,GAAG,CAAC,CAAG,CAAA,CAAA;gBACxB,CAAI,CAAA,EAAA,CAAC,CAAC,QAAQ,CAAG,CAAA,CAAA;gBACjB,CAAG,EAAA,CAAC,CAAC,OAAO,IAAI,IAAI,IAAI,CAAC,CAAC,OAAO,GAAG,CAAA,SAAA,EAAY,CAAC,CAAC,OAAO,EAAE,GAAG,EAAE,CAAE,CAAA;gBAClE,CAAG,EAAA,CAAC,CAAC,KAAK,IAAI,IAAI,IAAI,CAAC,CAAC,KAAK,GAAG,CAAA,OAAA,EAAU,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,CAAE,CAAA;gBAC1D,CAAG,EAAA,CAAC,CAAC,IAAI,IAAI,IAAI,IAAI,CAAC,CAAC,IAAI,GAAG,CAAA,MAAA,EAAS,CAAC,CAAC,IAAI,EAAE,GAAG,EAAE,CAAE,CAAA;gBACtD,EAAE;AACH,aAAA,CAAC,MAAM,CAAC,OAAO,CAAC;YAEjB,QAAQ,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AAC3C,SAAC,CAAC;QAEF,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;;IAGrC,OAAO;QACL,MAAM,EAAE,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE;QACrC,UAAU;KACX;AACH;;;;"}

package/dist/cjs/tools/search/search.cjs

@@ -40,7 +40,7 @@ const chunker = {
         return Promise.all(promises);
     },
 };
-const createSourceUpdateCallback = (sourceMap) => {
+function createSourceUpdateCallback(sourceMap) {
     return (link, update) => {
         const source = sourceMap.get(link);
         if (source) {
@@ -50,7 +50,7 @@ const createSourceUpdateCallback = (sourceMap) => {
             });
         }
     };
-};
+}
 const getHighlights = async ({ query, content, reranker, topResults = 5, }) => {
     if (!content) {
         console.warn('No content provided for highlights');
@@ -225,11 +225,12 @@ const createSourceProcessor = (config = {}, scraperInstance) => {
     reranker, } = config;
     const firecrawlScraper = scraperInstance;
     const webScraper = {
-        scrapeMany: async ({ query, links, }) => {
+        scrapeMany: async ({ query, links, onGetHighlights, }) => {
             console.log(`Scraping ${links.length} links with Firecrawl`);
             const promises = [];
             try {
-                for (const currentLink of links) {
+                for (let i = 0; i < links.length; i++) {
+                    const currentLink = links[i];
                     const promise = firecrawlScraper
                         .scrapeUrl(currentLink, {})
                         .then(([url, response]) => {
@@ -247,7 +248,7 @@ const createSourceProcessor = (config = {}, scraperInstance) => {
                             url,
                             attribution,
                             error: true,
-                            content: `Failed to scrape ${url}: ${response.error ?? 'Unknown error'}`,
+                            content: '',
                         };
                     })
                         .then(async (result) => {
@@ -263,6 +264,9 @@ const createSourceProcessor = (config = {}, scraperInstance) => {
                                 reranker,
                                 content: result.content,
                             });
+                            if (onGetHighlights) {
+                                onGetHighlights(result.url);
+                            }
                             return {
                                 ...result,
                                 highlights,
@@ -280,7 +284,7 @@ const createSourceProcessor = (config = {}, scraperInstance) => {
                         return {
                             url: currentLink,
                             error: true,
-                            content: `Failed to scrape ${currentLink}: ${error.message ?? 'Unknown error'}`,
+                            content: '',
                         };
                     });
                     promises.push(promise);
@@ -293,10 +297,14 @@ const createSourceProcessor = (config = {}, scraperInstance) => {
             }
         },
     };
-    const fetchContents = async ({ links, query, target, onContentScraped, }) => {
+    const fetchContents = async ({ links, query, target, onGetHighlights, onContentScraped, }) => {
         const initialLinks = links.slice(0, target);
         // const remainingLinks = links.slice(target).reverse();
-        const results = await webScraper.scrapeMany({ query, links: initialLinks });
+        const results = await webScraper.scrapeMany({
+            query,
+            links: initialLinks,
+            onGetHighlights,
+        });
         for (const result of results) {
             if (result.error === true) {
                 continue;
@@ -310,7 +318,7 @@ const createSourceProcessor = (config = {}, scraperInstance) => {
             });
         }
     };
-    const processSources = async (result, numElements, query, proMode = false) => {
+    const processSources = async ({ result, numElements, query, proMode = true, onGetHighlights, }) => {
         try {
             if (!result.data) {
                 return {
@@ -334,6 +342,7 @@ const createSourceProcessor = (config = {}, scraperInstance) => {
                 await fetchContents({
                     query,
                     target: 1,
+                    onGetHighlights,
                     onContentScraped,
                     links: [wikiSources[0].link],
                 });
@@ -350,38 +359,44 @@ const createSourceProcessor = (config = {}, scraperInstance) => {
                 return result.data;
             }
             const sourceMap = new Map();
-            const allLinks = [];
-            for (const source of result.data.organic) {
-                if (source.link) {
-                    allLinks.push(source.link);
-                    sourceMap.set(source.link, source);
-                }
-            }
-            if (allLinks.length === 0) {
+            const organicLinksSet = new Set();
+            // Collect organic links
+            const organicLinks = collectLinks(result.data.organic, sourceMap, organicLinksSet);
+            // Collect top story links, excluding any that are already in organic links
+            const topStories = result.data.topStories ?? [];
+            const topStoryLinks = collectLinks(topStories, sourceMap, organicLinksSet);
+            if (organicLinks.length === 0 && topStoryLinks.length === 0) {
                 return result.data;
             }
             const onContentScraped = createSourceUpdateCallback(sourceMap);
-            await fetchContents({
-                links: allLinks,
-                query,
-                onContentScraped,
-                target: numElements,
-            });
-            for (let i = 0; i < result.data.organic.length; i++) {
-                const source = result.data.organic[i];
-                const updatedSource = sourceMap.get(source.link);
-                if (updatedSource) {
-                    result.data.organic[i] = {
-                        ...source,
-                        ...updatedSource,
-                    };
-                }
+            const promises = [];
+            // Process organic links
+            if (organicLinks.length > 0) {
+                promises.push(fetchContents({
+                    query,
+                    onGetHighlights,
+                    onContentScraped,
+                    links: organicLinks,
+                    target: numElements,
+                }));
             }
-            const successfulSources = result.data.organic
-                .filter((source) => source.content != null && !source.content.startsWith('Failed'))
-                .slice(0, numElements);
-            if (successfulSources.length > 0) {
-                result.data.organic = successfulSources;
+            // Process top story links
+            if (topStoryLinks.length > 0) {
+                promises.push(fetchContents({
+                    query,
+                    onGetHighlights,
+                    onContentScraped,
+                    links: topStoryLinks,
+                    target: numElements,
+                }));
+            }
+            await Promise.all(promises);
+            // Update sources with scraped content
+            if (result.data.organic.length > 0) {
+                updateSourcesWithContent(result.data.organic, sourceMap);
+            }
+            if (topStories.length > 0) {
+                updateSourcesWithContent(topStories, sourceMap);
             }
             return result.data;
         }
@@ -402,6 +417,37 @@ const createSourceProcessor = (config = {}, scraperInstance) => {
         topResults,
     };
 };
+/** Helper function to collect links and update sourceMap */
+function collectLinks(sources, sourceMap, existingLinksSet) {
+    const links = [];
+    for (const source of sources) {
+        if (source.link) {
+            // For topStories, only add if not already in organic links
+            if (existingLinksSet && existingLinksSet.has(source.link)) {
+                continue;
+            }
+            links.push(source.link);
+            if (existingLinksSet) {
+                existingLinksSet.add(source.link);
+            }
+            sourceMap.set(source.link, source);
+        }
+    }
+    return links;
+}
+/** Helper function to update sources with scraped content */
+function updateSourcesWithContent(sources, sourceMap) {
+    for (let i = 0; i < sources.length; i++) {
+        const source = sources[i];
+        const updatedSource = sourceMap.get(source.link);
+        if (updatedSource) {
+            sources[i] = {
+                ...source,
+                ...updatedSource,
+            };
+        }
+    }
+}
 
 exports.createSearchAPI = createSearchAPI;
 exports.createSourceProcessor = createSourceProcessor;