@librechat/agents 3.2.31 → 3.2.33

package/src/tools/search/source-processing.test.ts

@@ -0,0 +1,373 @@
+import type * as t from './types';
+import { executeParallelSearches } from './tool';
+import { createSourceProcessor } from './search';
+import { expandHighlights } from './highlights';
+import { BaseReranker } from './rerankers';
+
+const noopLog = (..._args: unknown[]): void => {};
+const silentLogger = {
+  error: noopLog,
+  warn: noopLog,
+  info: noopLog,
+  debug: noopLog,
+} as t.Logger;
+
+class RecordingReranker extends BaseReranker {
+  public rerankCalls: string[][] = [];
+
+  constructor() {
+    super(silentLogger);
+  }
+
+  async rerank(
+    _query: string,
+    documents: string[],
+    topK: number = 5
+  ): Promise<t.Highlight[]> {
+    this.rerankCalls.push(documents);
+    return this.getDefaultRanking(documents, topK);
+  }
+}
+
+const createFakeScraper = (
+  contentByUrl: Record<string, string>,
+  scrapedUrls: string[] = []
+): t.BaseScraper => ({
+  scrapeUrl: async (url: string): Promise<[string, t.AnyScraperResponse]> => {
+    scrapedUrls.push(url);
+    return [
+      url,
+      { success: true, data: { markdown: contentByUrl[url] ?? '' } },
+    ];
+  },
+  extractContent: (
+    response: t.AnyScraperResponse
+  ): [string, undefined | t.References] => [
+    (response as t.FirecrawlScrapeResponse).data?.markdown ?? '',
+    undefined,
+  ],
+  extractMetadata: (): t.GenericScrapeMetadata => ({}),
+});
+
+const makeOrganic = (link: string): t.ProcessedOrganic => ({
+  link,
+  title: `Title for ${link}`,
+  snippet: `Snippet for ${link}`,
+});
+
+const makeStory = (link: string): t.ProcessedTopStory => ({
+  link,
+  title: `Story for ${link}`,
+});
+
+const makeLongContent = (chars: number): string => {
+  const line =
+    'lorem ipsum dolor sit amet consectetur adipiscing elit sed do eiusmod\n';
+  return line.repeat(Math.ceil(chars / line.length)).slice(0, chars);
+};
+
+describe('expandHighlights content stripping', () => {
+  test('strips content and references from sources without highlights', () => {
+    const references: t.References = { links: [], images: [], videos: [] };
+    const data: t.SearchResultData = {
+      organic: [
+        {
+          ...makeOrganic('https://a.com'),
+          content: 'X'.repeat(100000),
+          references,
+        },
+      ],
+      topStories: [
+        { ...makeStory('https://news.com'), content: 'Y'.repeat(100000) },
+      ],
+    };
+
+    const result = expandHighlights(data);
+
+    expect(result.organic?.[0].content).toBeUndefined();
+    expect(result.organic?.[0].references).toBeUndefined();
+    expect(result.organic?.[0].title).toBe('Title for https://a.com');
+    expect(result.organic?.[0].snippet).toBe('Snippet for https://a.com');
+    expect(result.topStories?.[0].content).toBeUndefined();
+    expect(result.topStories?.[0].title).toBe('Story for https://news.com');
+    expect(data.organic?.[0].content).toBeDefined();
+  });
+
+  test('strips content when highlights array is empty', () => {
+    const data: t.SearchResultData = {
+      organic: [
+        {
+          ...makeOrganic('https://a.com'),
+          content: 'X'.repeat(5000),
+          highlights: [],
+        },
+      ],
+    };
+
+    const result = expandHighlights(data);
+
+    expect(result.organic?.[0].content).toBeUndefined();
+    expect(result.organic?.[0].highlights).toEqual([]);
+  });
+
+  test('returns sources without content or references unchanged', () => {
+    const source = makeOrganic('https://a.com');
+    const result = expandHighlights({ organic: [source] });
+
+    expect(result.organic?.[0]).toBe(source);
+  });
+
+  test('still expands highlights and strips content on the normal path', () => {
+    const highlightText = 'THE KEY FACT OF THIS PAGE IS RIGHT HERE.';
+    const content = `${makeLongContent(2000)} ${highlightText} ${makeLongContent(2000)}`;
+    const data: t.SearchResultData = {
+      organic: [
+        {
+          ...makeOrganic('https://a.com'),
+          content,
+          highlights: [{ text: highlightText, score: 0.9 }],
+        },
+      ],
+    };
+
+    const result = expandHighlights(data);
+    const highlight = result.organic?.[0].highlights?.[0];
+
+    expect(highlight?.text).toContain(highlightText);
+    expect(highlight?.text.length).toBeGreaterThan(highlightText.length);
+    expect(highlight?.score).toBe(0.9);
+    expect(result.organic?.[0].content).toBeUndefined();
+    expect(result.organic?.[0].references).toBeUndefined();
+  });
+});
+
+describe('createSourceProcessor content capping', () => {
+  const link = 'https://a.com';
+  const baseFields = {
+    query: 'test query',
+    proMode: true,
+    onGetHighlights: undefined,
+  };
+
+  test('caps stored content and reranker input at maxContentLength', async () => {
+    const reranker = new RecordingReranker();
+    const scraper = createFakeScraper({ [link]: makeLongContent(200000) });
+    const processor = createSourceProcessor(
+      { reranker, maxContentLength: 1000, logger: silentLogger },
+      scraper
+    );
+
+    const data = await processor.processSources({
+      ...baseFields,
+      news: false,
+      numElements: 5,
+      result: { success: true, data: { organic: [makeOrganic(link)] } },
+    });
+
+    expect(data.organic?.[0].content?.length).toBe(1000);
+    const rerankedChars = reranker.rerankCalls
+      .flat()
+      .reduce((sum, doc) => sum + doc.length, 0);
+    expect(rerankedChars).toBeGreaterThan(0);
+    expect(rerankedChars).toBeLessThan(2000);
+  });
+
+  test('respects SEARCH_MAX_CONTENT_LENGTH env var when config is not set', async () => {
+    process.env.SEARCH_MAX_CONTENT_LENGTH = '500';
+    try {
+      const reranker = new RecordingReranker();
+      const scraper = createFakeScraper({ [link]: makeLongContent(10000) });
+      const processor = createSourceProcessor(
+        { reranker, logger: silentLogger },
+        scraper
+      );
+
+      const data = await processor.processSources({
+        ...baseFields,
+        news: false,
+        numElements: 5,
+        result: { success: true, data: { organic: [makeOrganic(link)] } },
+      });
+
+      expect(data.organic?.[0].content?.length).toBe(500);
+    } finally {
+      delete process.env.SEARCH_MAX_CONTENT_LENGTH;
+    }
+  });
+
+  test('caps content at 50,000 chars by default', async () => {
+    const reranker = new RecordingReranker();
+    const scraper = createFakeScraper({ [link]: makeLongContent(60000) });
+    const processor = createSourceProcessor(
+      { reranker, logger: silentLogger },
+      scraper
+    );
+
+    const data = await processor.processSources({
+      ...baseFields,
+      news: false,
+      numElements: 5,
+      result: { success: true, data: { organic: [makeOrganic(link)] } },
+    });
+
+    expect(data.organic?.[0].content?.length).toBe(50000);
+  });
+});
+
+describe('createSourceProcessor topStories capping', () => {
+  const storyLinks = Array.from(
+    { length: 8 },
+    (_, i) => `https://news${i}.com`
+  );
+
+  const createProcessorWithStories = (
+    scrapedUrls: string[]
+  ): ReturnType<typeof createSourceProcessor> => {
+    const contentByUrl = Object.fromEntries(
+      storyLinks.map((storyLink) => [storyLink, makeLongContent(500)])
+    );
+    contentByUrl['https://a.com'] = makeLongContent(500);
+    const scraper = createFakeScraper(contentByUrl, scrapedUrls);
+    return createSourceProcessor(
+      { reranker: new RecordingReranker(), logger: silentLogger },
+      scraper
+    );
+  };
+
+  test('caps topStories to numElements when news is enabled', async () => {
+    const scrapedUrls: string[] = [];
+    const processor = createProcessorWithStories(scrapedUrls);
+
+    const data = await processor.processSources({
+      query: 'test query',
+      proMode: true,
+      onGetHighlights: undefined,
+      news: true,
+      numElements: 3,
+      result: {
+        success: true,
+        data: { organic: [], topStories: storyLinks.map(makeStory) },
+      },
+    });
+
+    expect(data.topStories?.length).toBe(3);
+    expect(data.topStories?.map((story) => story.link)).toEqual(
+      storyLinks.slice(0, 3)
+    );
+    expect(scrapedUrls).toEqual(storyLinks.slice(0, 3));
+    expect(data.topStories?.[0].content).toBeDefined();
+  });
+
+  test('caps topStories when organic results are missing', async () => {
+    const scrapedUrls: string[] = [];
+    const processor = createProcessorWithStories(scrapedUrls);
+
+    const data = await processor.processSources({
+      query: 'test query',
+      proMode: true,
+      onGetHighlights: undefined,
+      news: true,
+      numElements: 3,
+      result: {
+        success: true,
+        data: { topStories: storyLinks.map(makeStory) },
+      },
+    });
+
+    expect(data.topStories?.length).toBe(3);
+    expect(scrapedUrls).toEqual([]);
+  });
+
+  test('caps topStories on the empty-links early return with news disabled', async () => {
+    const scrapedUrls: string[] = [];
+    const processor = createProcessorWithStories(scrapedUrls);
+
+    const data = await processor.processSources({
+      query: 'test query',
+      proMode: true,
+      onGetHighlights: undefined,
+      news: false,
+      numElements: 3,
+      result: {
+        success: true,
+        data: { organic: [], topStories: storyLinks.map(makeStory) },
+      },
+    });
+
+    expect(data.topStories?.length).toBe(3);
+    expect(scrapedUrls).toEqual([]);
+  });
+
+  test('caps topStories to numElements even when news is disabled', async () => {
+    const scrapedUrls: string[] = [];
+    const processor = createProcessorWithStories(scrapedUrls);
+
+    const data = await processor.processSources({
+      query: 'test query',
+      proMode: true,
+      onGetHighlights: undefined,
+      news: false,
+      numElements: 3,
+      result: {
+        success: true,
+        data: {
+          organic: [makeOrganic('https://a.com')],
+          topStories: storyLinks.map(makeStory),
+        },
+      },
+    });
+
+    expect(data.topStories?.length).toBe(3);
+    expect(scrapedUrls).toEqual(['https://a.com']);
+  });
+});
+
+describe('executeParallelSearches topStories dedupe', () => {
+  test('dedupes topStories by link across main and news searches', async () => {
+    const mainResult: t.SearchResult = {
+      success: true,
+      data: {
+        organic: [makeOrganic('https://a.com')],
+        topStories: [makeStory('https://n1.com')],
+        news: [
+          { title: 'N1 from main news', link: 'https://n1.com' },
+          { title: 'N2 from main news', link: 'https://n2.com' },
+        ],
+      },
+    };
+    const newsResult: t.SearchResult = {
+      success: true,
+      data: {
+        news: [
+          { title: 'N2 from news search', link: 'https://n2.com' },
+          { title: 'N3 from news search', link: 'https://n3.com' },
+          { title: 'No link' },
+        ],
+      },
+    };
+    const searchAPI = {
+      getSources: async (
+        params: t.GetSourcesParams
+      ): Promise<t.SearchResult> =>
+        params.type === 'news' ? newsResult : mainResult,
+    };
+
+    const merged = await executeParallelSearches({
+      searchAPI,
+      query: 'test query',
+      safeSearch: 1,
+      images: false,
+      videos: false,
+      news: true,
+      logger: silentLogger,
+    });
+
+    expect(merged.data?.topStories?.map((story) => story.link)).toEqual([
+      'https://n1.com',
+      'https://n2.com',
+      'https://n3.com',
+    ]);
+    expect(merged.data?.topStories?.[0].title).toBe('Story for https://n1.com');
+    expect(merged.data?.news).toBeUndefined();
+  });
+});

package/src/tools/search/tool.ts

@@ -23,9 +23,10 @@ import { createReranker } from './rerankers';
 import { Constants } from '@/common';
 
 /**
- * Executes parallel searches and merges the results
+ * Executes parallel searches and merges the results,
+ * deduplicating top stories by link
  */
-async function executeParallelSearches({
+export async function executeParallelSearches({
   searchAPI,
   query,
   date,
@@ -173,6 +174,23 @@ async function executeParallelSearches({
     }
   });
 
+  if (
+    mergedResults.topStories !== undefined &&
+    mergedResults.topStories.length > 1
+  ) {
+    /** The main search's own news results and the parallel news sub-search
+     * frequently return the same stories — keep the first occurrence of each
+     * link so duplicates aren't scraped, reranked, and formatted repeatedly */
+    const seenLinks = new Set<string>();
+    mergedResults.topStories = mergedResults.topStories.filter((story) => {
+      if (!story.link || seenLinks.has(story.link)) {
+        return false;
+      }
+      seenLinks.add(story.link);
+      return true;
+    });
+  }
+
   return { success: true, data: mergedResults };
 }
 
@@ -340,6 +358,7 @@ export const createSearchTool = (
     tavilySearchOptions,
     rerankerType = 'cohere',
     topResults = 5,
+    maxContentLength,
     strategies = ['no_extraction'],
     filterContent = true,
     safeSearch = 1,
@@ -444,6 +463,7 @@ export const createSearchTool = (
     {
       reranker: selectedReranker,
       topResults,
+      maxContentLength,
       strategies,
       filterContent,
       logger,

package/src/tools/search/types.ts

@@ -133,6 +133,10 @@ export interface ScrapeResult {
 
 export interface ProcessSourcesConfig {
   topResults?: number;
+  /** Max chars of scraped content stored per source and passed to the
+   * chunker/reranker. Defaults to 50,000; also configurable via the
+   * `SEARCH_MAX_CONTENT_LENGTH` env var. */
+  maxContentLength?: number;
   strategies?: string[];
   filterContent?: boolean;
   reranker?: BaseReranker;

package/dist/cjs/langchain/google-common.cjs.map

@@ -1 +0,0 @@
-{"version":3,"file":"google-common.cjs","sources":[],"sourcesContent":[],"names":[],"mappings":";;"}

package/dist/cjs/langchain/index.cjs.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.cjs","sources":[],"sourcesContent":[],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;"}

package/dist/cjs/langchain/language_models/chat_models.cjs.map

@@ -1 +0,0 @@
-{"version":3,"file":"chat_models.cjs","sources":[],"sourcesContent":[],"names":[],"mappings":";;"}

package/dist/cjs/langchain/messages/tool.cjs.map

@@ -1 +0,0 @@
-{"version":3,"file":"tool.cjs","sources":[],"sourcesContent":[],"names":[],"mappings":";;"}

package/dist/cjs/langchain/messages.cjs.map

@@ -1 +0,0 @@
-{"version":3,"file":"messages.cjs","sources":[],"sourcesContent":[],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;"}

package/dist/cjs/langchain/openai.cjs.map

@@ -1 +0,0 @@
-{"version":3,"file":"openai.cjs","sources":[],"sourcesContent":[],"names":[],"mappings":";;"}

package/dist/cjs/langchain/prompts.cjs.map

@@ -1 +0,0 @@
-{"version":3,"file":"prompts.cjs","sources":[],"sourcesContent":[],"names":[],"mappings":";;;;;;;;;"}

package/dist/cjs/langchain/runnables.cjs.map

@@ -1 +0,0 @@
-{"version":3,"file":"runnables.cjs","sources":[],"sourcesContent":[],"names":[],"mappings":";;;;;;;;;;;;;;;;;"}

package/dist/cjs/langchain/tools.cjs.map

@@ -1 +0,0 @@
-{"version":3,"file":"tools.cjs","sources":[],"sourcesContent":[],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;"}

package/dist/cjs/langchain/utils/env.cjs.map

@@ -1 +0,0 @@
-{"version":3,"file":"env.cjs","sources":[],"sourcesContent":[],"names":[],"mappings":";;;;;;;;;"}

package/dist/cjs/main.cjs.map

@@ -1 +0,0 @@
-{"version":3,"file":"main.cjs","sources":[],"sourcesContent":[],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;"}

package/dist/esm/langchain/google-common.mjs.map

@@ -1 +0,0 @@
-{"version":3,"file":"google-common.mjs","sources":[],"sourcesContent":[],"names":[],"mappings":""}

package/dist/esm/langchain/index.mjs.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.mjs","sources":[],"sourcesContent":[],"names":[],"mappings":";;;"}

package/dist/esm/langchain/language_models/chat_models.mjs.map

@@ -1 +0,0 @@
-{"version":3,"file":"chat_models.mjs","sources":[],"sourcesContent":[],"names":[],"mappings":""}

package/dist/esm/langchain/messages/tool.mjs.map

@@ -1 +0,0 @@
-{"version":3,"file":"tool.mjs","sources":[],"sourcesContent":[],"names":[],"mappings":""}

package/dist/esm/langchain/messages.mjs.map

@@ -1 +0,0 @@
-{"version":3,"file":"messages.mjs","sources":[],"sourcesContent":[],"names":[],"mappings":""}

package/dist/esm/langchain/openai.mjs.map

@@ -1 +0,0 @@
-{"version":3,"file":"openai.mjs","sources":[],"sourcesContent":[],"names":[],"mappings":""}

package/dist/esm/langchain/prompts.mjs.map

@@ -1 +0,0 @@
-{"version":3,"file":"prompts.mjs","sources":[],"sourcesContent":[],"names":[],"mappings":""}

package/dist/esm/langchain/runnables.mjs.map

@@ -1 +0,0 @@
-{"version":3,"file":"runnables.mjs","sources":[],"sourcesContent":[],"names":[],"mappings":""}

package/dist/esm/langchain/tools.mjs.map

@@ -1 +0,0 @@
-{"version":3,"file":"tools.mjs","sources":[],"sourcesContent":[],"names":[],"mappings":""}

package/dist/esm/langchain/utils/env.mjs.map

@@ -1 +0,0 @@
-{"version":3,"file":"env.mjs","sources":[],"sourcesContent":[],"names":[],"mappings":""}

package/dist/esm/main.mjs.map

@@ -1 +0,0 @@
-{"version":3,"file":"main.mjs","sources":[],"sourcesContent":[],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;"}