@librechat/agents 3.1.75 → 3.1.76

package/src/specs/summarization.test.ts

@@ -138,6 +138,7 @@ async function createSummarizationRun(opts: {
   tools?: t.GraphTools;
   indexTokenCountMap?: Record<string, number>;
   llmConfigOverride?: Record<string, unknown>;
+  maxSummaryTokens?: number;
 }): Promise<Run<t.IState>> {
   const llmConfig = {
     ...getLLMConfig(opts.agentProvider),
@@ -157,6 +158,7 @@ async function createSummarizationRun(opts: {
       summarizationConfig: {
         provider: opts.summarizationProvider,
         model: opts.summarizationModel,
+        maxSummaryTokens: opts.maxSummaryTokens,
       },
     },
     returnContent: true,
@@ -244,6 +246,33 @@ function buildIndexTokenCountMap(
   return map;
 }
 
+function sumTokenCountMap(map: Record<string, number | undefined>): number {
+  let total = 0;
+  for (const key in map) {
+    total += map[key] ?? 0;
+  }
+  return total;
+}
+
+function createSeededTokenAuditHistory(): BaseMessage[] {
+  const details =
+    'Token audit context preserves index token counts, summary replacement, calibration data, and post-summary continuity. ' +
+    'Important retained values: alpha=1024, beta=2048, gamma=4096, checksum TOKEN-AUDIT-7F3. ' +
+    'The repeated detail intentionally exceeds a compact context budget. ';
+  const padding = details.repeat(8);
+
+  return [
+    new HumanMessage(
+      `Audit turn 1: establish the accounting scenario. ${padding}`
+    ),
+    new AIMessage(`Recorded turn 1 accounting notes. ${padding}`),
+    new HumanMessage(`Audit turn 2: add more retained details. ${padding}`),
+    new AIMessage(`Recorded turn 2 accounting notes. ${padding}`),
+    new HumanMessage(`Audit turn 3: preserve final identifiers. ${padding}`),
+    new AIMessage(`Recorded turn 3 accounting notes. ${padding}`),
+  ];
+}
+
 function logTurn(
   label: string,
   conversationHistory: BaseMessage[],
@@ -2417,10 +2446,10 @@ const hasAnyApiKey =
   test('token count map is accurate after summarization cycle', async () => {
     const spies = createSpies();
     let collectedUsage: UsageMetadata[] = [];
-    const conversationHistory: BaseMessage[] = [];
+    const conversationHistory = createSeededTokenAuditHistory();
     const tokenCounter = await createTokenCounter();
 
-    const createRun = async (maxTokens = 4000): Promise<Run<t.IState>> => {
+    const createRun = async (maxTokens = 1200): Promise<Run<t.IState>> => {
       collectedUsage = [];
       const { aggregateContent } = createContentAggregator();
       const indexTokenCountMap = buildIndexTokenCountMap(
@@ -2432,80 +2461,44 @@ const hasAnyApiKey =
         summarizationProvider,
         summarizationModel,
         maxContextTokens: maxTokens,
-        instructions: INSTRUCTIONS,
+        instructions:
+          'You are a concise assistant. Preserve checkpoint context and answer in one short sentence.',
         collectedUsage,
         aggregateContent,
         spies,
         tokenCounter,
         indexTokenCountMap,
+        maxSummaryTokens: 300,
+        tools: [],
+        llmConfigOverride: {
+          maxTokens: 128,
+        },
       });
     };
 
-    // Accumulate messages over 6 turns at generous budget
-    let run = await createRun();
-    await runTurn(
-      { run, conversationHistory },
-      'What is 42 * 58? Calculator.',
-      streamConfig
-    );
-
-    run = await createRun();
-    await runTurn(
-      { run, conversationHistory },
-      'Now compute 2436 + 1000. Calculator.',
-      streamConfig
-    );
-
-    run = await createRun();
-    await runTurn(
-      { run, conversationHistory },
-      'What is 3436 / 4? Calculator.',
-      streamConfig
-    );
-
-    run = await createRun();
-    await runTurn(
-      { run, conversationHistory },
-      'Compute 999 * 2. Calculator.',
-      streamConfig
-    );
-
-    run = await createRun();
-    await runTurn(
-      { run, conversationHistory },
-      'What is 2^10? Calculator. Also list everything.',
-      streamConfig
+    const originalMap = buildIndexTokenCountMap(
+      conversationHistory,
+      tokenCounter
     );
+    const originalTokenTotal = sumTokenCountMap(originalMap);
+    expect(originalTokenTotal).toBeGreaterThan(1200);
 
-    run = await createRun();
+    const run = await createRun();
     await runTurn(
       { run, conversationHistory },
-      'Calculate 355 / 113. Calculator.',
+      'Acknowledge the preserved token audit context in one short sentence.',
       streamConfig
     );
 
-    // Squeeze progressively to force summarization
-    for (const squeeze of [3500, 3200, 3100, 3000, 2800, 2500, 2000]) {
-      if (spies.onSummarizeStartSpy.mock.calls.length > 0) {
-        break;
-      }
-      run = await createRun(squeeze);
-      await runTurn(
-        { run, conversationHistory },
-        `What is ${squeeze} - 1000? Calculator.`,
-        streamConfig
-      );
-    }
-
-    // Verify summarization fired
     expect(spies.onSummarizeCompleteSpy).toHaveBeenCalled();
 
     const completePayload = spies.onSummarizeCompleteSpy.mock
       .calls[0][0] as t.SummarizeCompleteEvent;
-    expect(completePayload.summary!.tokenCount).toBeGreaterThan(10);
-    expect(completePayload.summary!.tokenCount).toBeLessThan(1500);
+    const summaryTokenCount = completePayload.summary!.tokenCount ?? 0;
+    expect(summaryTokenCount).toBeGreaterThan(10);
+    expect(summaryTokenCount).toBeLessThan(1500);
+    expect(summaryTokenCount).toBeLessThan(originalTokenTotal);
 
-    // Token accounting: collectedUsage should have valid entries
     const validUsage = collectedUsage.filter(
       (u: Partial<UsageMetadata>) =>
         u.input_tokens != null && u.input_tokens > 0
@@ -2513,8 +2506,8 @@ const hasAnyApiKey =
     expect(validUsage.length).toBeGreaterThan(0);
 
     console.log(
-      `  Token audit: summary=${completePayload.summary!.tokenCount} tokens, ` +
-        `usageEntries=${validUsage.length}`
+      `  Token audit: summary=${summaryTokenCount} tokens, ` +
+        `preTotal=${originalTokenTotal}, usageEntries=${validUsage.length}`
     );
   }, 180_000);
 

package/src/tools/ToolNode.ts

@@ -33,6 +33,7 @@ import {
 } from '@/utils/truncation';
 import { safeDispatchCustomEvent } from '@/utils/events';
 import { executeHooks } from '@/hooks';
+import { toLangChainContent } from '@/messages/langchain';
 import { Constants, GraphEvents, CODE_EXECUTION_TOOLS } from '@/common';
 import {
   buildReferenceKey,
@@ -1282,7 +1283,10 @@ export class ToolNode<T = any> extends RunnableCallable<T, T> {
       if (msg.skillName != null) additional_kwargs.skillName = msg.skillName;
 
       converted.push(
-        new HumanMessage({ content: msg.content, additional_kwargs })
+        new HumanMessage({
+          content: toLangChainContent(msg.content),
+          additional_kwargs,
+        })
       );
     }
     return converted;

package/src/tools/search/search.ts

@@ -2,6 +2,7 @@ import axios from 'axios';
 import { RecursiveCharacterTextSplitter } from '@langchain/textsplitters';
 import type * as t from './types';
 import { getAttribution, createDefaultLogger } from './utils';
+import { createTavilyAPI } from './tavily-search';
 import { BaseReranker } from './rerankers';
 
 const chunker = {
@@ -418,15 +419,20 @@ export const createSearchAPI = (
     serperApiKey,
     searxngInstanceUrl,
     searxngApiKey,
+    tavilyApiKey,
+    tavilySearchUrl,
+    tavilySearchOptions,
   } = config;
 
   if (searchProvider.toLowerCase() === 'serper') {
     return createSerperAPI(serperApiKey);
   } else if (searchProvider.toLowerCase() === 'searxng') {
     return createSearXNGAPI(searxngInstanceUrl, searxngApiKey);
+  } else if (searchProvider.toLowerCase() === 'tavily') {
+    return createTavilyAPI(tavilyApiKey, tavilySearchUrl, tavilySearchOptions);
   } else {
     throw new Error(
-      `Invalid search provider: ${searchProvider}. Must be 'serper' or 'searxng'`
+      `Invalid search provider: ${searchProvider}. Must be 'serper', 'searxng', or 'tavily'`
     );
   }
 };
@@ -454,6 +460,56 @@ export const createSourceProcessor = (
   const logger_ = logger || createDefaultLogger();
   const scraper = scraperInstance;
 
+  const processResponse = (
+    url: string,
+    response: t.AnyScraperResponse
+  ): t.ScrapeResult => {
+    const rawMetadata = scraper.extractMetadata(response);
+    const metadata =
+      Object.keys(rawMetadata).length > 0 ? rawMetadata : undefined;
+    const attribution = getAttribution(url, metadata, logger_);
+
+    if (response.success && response.data) {
+      const [content, references] = scraper.extractContent(response);
+      return {
+        url,
+        references,
+        attribution,
+        content: chunker.cleanText(content),
+      };
+    }
+
+    logger_.error(
+      `Error scraping ${url}: ${response.error ?? 'Unknown error'}`
+    );
+    return { url, attribution, error: true, content: '' };
+  };
+
+  const addHighlights = async (
+    result: t.ScrapeResult,
+    query: string,
+    onGetHighlights: t.SearchToolConfig['onGetHighlights']
+  ): Promise<t.ScrapeResult> => {
+    if (result.error != null) {
+      return result;
+    }
+    try {
+      const highlights = await getHighlights({
+        query,
+        reranker,
+        content: result.content,
+        logger: logger_,
+      });
+      if (onGetHighlights) {
+        onGetHighlights(result.url);
+      }
+      return { ...result, highlights };
+    } catch (error) {
+      logger_.error('Error processing scraped content:', error);
+      return result;
+    }
+  };
+
   const webScraper = {
     scrapeMany: async ({
       query,
@@ -465,80 +521,34 @@ export const createSourceProcessor = (
       onGetHighlights: t.SearchToolConfig['onGetHighlights'];
     }): Promise<Array<t.ScrapeResult>> => {
       logger_.debug(`Scraping ${links.length} links`);
-      const promises: Array<Promise<t.ScrapeResult>> = [];
       try {
-        for (let i = 0; i < links.length; i++) {
-          const currentLink = links[i];
-          const promise: Promise<t.ScrapeResult> = scraper
-            .scrapeUrl(currentLink, {})
-            .then(([url, response]) => {
-              const attribution = getAttribution(
-                url,
-                response.data?.metadata,
-                logger_
-              );
-              if (response.success && response.data) {
-                const [content, references] = scraper.extractContent(response);
-                return {
-                  url,
-                  references,
-                  attribution,
-                  content: chunker.cleanText(content),
-                } as t.ScrapeResult;
-              } else {
-                logger_.error(
-                  `Error scraping ${url}: ${response.error ?? 'Unknown error'}`
-                );
-              }
-
-              return {
-                url,
-                attribution,
-                error: true,
-                content: '',
-              } as t.ScrapeResult;
-            })
-            .then(async (result) => {
-              try {
-                if (result.error != null) {
-                  logger_.error(
-                    `Error scraping ${result.url}: ${result.content}`
-                  );
-                  return {
-                    ...result,
-                  };
-                }
-                const highlights = await getHighlights({
-                  query,
-                  reranker,
-                  content: result.content,
-                  logger: logger_,
-                });
-                if (onGetHighlights) {
-                  onGetHighlights(result.url);
-                }
-                return {
-                  ...result,
-                  highlights,
-                };
-              } catch (error) {
-                logger_.error('Error processing scraped content:', error);
-                return {
-                  ...result,
-                };
-              }
-            })
-            .catch((error) => {
-              logger_.error(`Error scraping ${currentLink}:`, error);
-              return {
-                url: currentLink,
-                error: true,
-                content: '',
-              };
-            });
-          promises.push(promise);
+        let responses: Array<[string, t.AnyScraperResponse]>;
+
+        if (scraper.scrapeUrls) {
+          responses = await scraper.scrapeUrls(links);
+        } else {
+          responses = await Promise.all(
+            links.map((link) =>
+              scraper
+                .scrapeUrl(link, {})
+                .catch((error): [string, t.AnyScraperResponse] => {
+                  logger_.error(`Error scraping ${link}:`, error);
+                  return [link, { success: false, error: String(error) }];
+                })
+            )
+          );
         }
-        return await Promise.all(promises);
+
+        const withHighlights = await Promise.all(
+          responses.map(([url, response]) =>
+            addHighlights(
+              processResponse(url, response),
+              query,
+              onGetHighlights
+            )
+          )
+        );
+        return withHighlights;
       } catch (error) {
         logger_.error('Error in scrapeMany:', error);
         return [];

package/src/tools/search/tavily-scraper.ts

@@ -0,0 +1,235 @@
+import axios from 'axios';
+import type * as t from './types';
+import { createDefaultLogger } from './utils';
+
+const DEFAULT_BASIC_TIMEOUT = 15000;
+const DEFAULT_ADVANCED_TIMEOUT = 30000;
+const MAX_BATCH_SIZE = 20;
+
+const getDefaultTimeout = (extractDepth: 'basic' | 'advanced'): number =>
+  extractDepth === 'advanced'
+    ? DEFAULT_ADVANCED_TIMEOUT
+    : DEFAULT_BASIC_TIMEOUT;
+
+const normalizeUrlKey = (url: string): string => {
+  try {
+    const parsedUrl = new URL(url);
+    parsedUrl.hash = '';
+    if (parsedUrl.pathname.length > 1) {
+      parsedUrl.pathname = parsedUrl.pathname.replace(/\/+$/, '');
+    }
+    return parsedUrl.toString();
+  } catch {
+    return url;
+  }
+};
+
+const setUrlResult = (
+  map: Map<string, t.TavilyExtractResult>,
+  result: t.TavilyExtractResult
+): void => {
+  map.set(result.url, result);
+  const normalizedUrl = normalizeUrlKey(result.url);
+  if (!map.has(normalizedUrl)) {
+    map.set(normalizedUrl, result);
+  }
+};
+
+export class TavilyScraper implements t.BaseScraper {
+  private apiKey: string;
+  private apiUrl: string;
+  private timeout: number;
+  private payloadTimeout: number | undefined;
+  private logger: t.Logger;
+  private extractDepth: 'basic' | 'advanced';
+  private includeImages: boolean;
+  private includeFavicon: boolean;
+  private format: 'markdown' | 'text' | undefined;
+
+  constructor(config: t.TavilyScraperConfig = {}) {
+    this.apiKey = config.apiKey ?? process.env.TAVILY_API_KEY ?? '';
+    this.apiUrl =
+      config.apiUrl ??
+      process.env.TAVILY_EXTRACT_URL ??
+      'https://api.tavily.com/extract';
+    this.payloadTimeout = config.timeout;
+    this.extractDepth = config.extractDepth ?? 'basic';
+    this.timeout = config.timeout ?? getDefaultTimeout(this.extractDepth);
+    this.includeImages = config.includeImages ?? false;
+    this.includeFavicon = config.includeFavicon ?? false;
+    this.format = config.format;
+    this.logger = config.logger || createDefaultLogger();
+
+    if (!this.apiKey) {
+      this.logger.warn('TAVILY_API_KEY is not set. Scraping will not work.');
+    }
+  }
+
+  async scrapeUrl(
+    url: string,
+    options: t.TavilyScrapeOptions = {}
+  ): Promise<[string, t.TavilyScrapeResponse]> {
+    const results = await this.scrapeUrls([url], options);
+    return results[0];
+  }
+
+  async scrapeUrls(
+    urls: string[],
+    options: t.TavilyScrapeOptions = {}
+  ): Promise<Array<[string, t.TavilyScrapeResponse]>> {
+    if (!this.apiKey) {
+      return urls.map((url) => [
+        url,
+        { success: false, error: 'TAVILY_API_KEY is not set' },
+      ]);
+    }
+
+    const batches: string[][] = [];
+    for (let i = 0; i < urls.length; i += MAX_BATCH_SIZE) {
+      batches.push(urls.slice(i, i + MAX_BATCH_SIZE));
+    }
+
+    const allResults: Array<[string, t.TavilyScrapeResponse]> = [];
+
+    for (const batch of batches) {
+      const batchResults = await this.extractBatch(batch, options);
+      allResults.push(...batchResults);
+    }
+
+    return allResults;
+  }
+
+  private async extractBatch(
+    urls: string[],
+    options: t.TavilyScrapeOptions = {}
+  ): Promise<Array<[string, t.TavilyScrapeResponse]>> {
+    try {
+      const includeFavicon = options.includeFavicon ?? this.includeFavicon;
+      const format = options.format ?? this.format;
+      const extractDepth = options.extractDepth ?? this.extractDepth;
+      const payload: t.TavilyExtractPayload = {
+        urls,
+        extract_depth: extractDepth,
+        include_images: options.includeImages ?? this.includeImages,
+      };
+
+      if (includeFavicon) {
+        payload.include_favicon = true;
+      }
+      if (format != null) {
+        payload.format = format;
+      }
+
+      const effectiveTimeout =
+        options.timeout ??
+        this.payloadTimeout ??
+        (options.extractDepth != null
+          ? getDefaultTimeout(extractDepth)
+          : this.timeout);
+      const payloadTimeout = options.timeout ?? this.payloadTimeout;
+      if (payloadTimeout != null) {
+        payload.timeout = Math.min(Math.max(payloadTimeout / 1000, 1), 60);
+      }
+
+      const response = await axios.post<{
+        results?: t.TavilyExtractResult[];
+        failed_results?: t.TavilyExtractResult[];
+      }>(this.apiUrl, payload, {
+        headers: {
+          Authorization: `Bearer ${this.apiKey}`,
+          'Content-Type': 'application/json',
+        },
+        timeout: effectiveTimeout,
+      });
+
+      const data = response.data;
+      const successMap = new Map<string, t.TavilyExtractResult>();
+      const failedMap = new Map<string, t.TavilyExtractResult>();
+
+      for (const result of data.results ?? []) {
+        setUrlResult(successMap, result);
+      }
+      for (const result of data.failed_results ?? []) {
+        setUrlResult(failedMap, result);
+      }
+
+      return urls.map((url): [string, t.TavilyScrapeResponse] => {
+        const success =
+          successMap.get(url) ?? successMap.get(normalizeUrlKey(url));
+        if (success && success.error == null) {
+          return [
+            url,
+            {
+              success: true,
+              data: {
+                rawContent: success.raw_content ?? '',
+                images: success.images ?? [],
+                favicon: success.favicon,
+              },
+            },
+          ];
+        }
+
+        const failed =
+          failedMap.get(url) ?? failedMap.get(normalizeUrlKey(url));
+        const error =
+          success?.error ??
+          failed?.error ??
+          'URL not found in Tavily Extract response';
+        return [url, { success: false, error }];
+      });
+    } catch (error) {
+      const errorMessage =
+        error instanceof Error ? error.message : String(error);
+      return urls.map((url) => [
+        url,
+        {
+          success: false,
+          error: `Tavily Extract API request failed: ${errorMessage}`,
+        },
+      ]);
+    }
+  }
+
+  extractContent(
+    response: t.TavilyScrapeResponse
+  ): [string, undefined | t.References] {
+    if (!response.success || !response.data) {
+      return ['', undefined];
+    }
+
+    const content = response.data.rawContent ?? '';
+    const images = response.data.images ?? [];
+
+    const references: t.References | undefined =
+      images.length > 0
+        ? {
+            links: [],
+            images: images.map((imageUrl) => ({ originalUrl: imageUrl })),
+            videos: [],
+          }
+        : undefined;
+
+    return [content, references];
+  }
+
+  extractMetadata(response: t.TavilyScrapeResponse): t.GenericScrapeMetadata {
+    if (!response.success || !response.data) {
+      return {};
+    }
+
+    const metadata: t.GenericScrapeMetadata = {
+      images_count: response.data.images?.length ?? 0,
+    };
+    if (response.data.favicon != null) {
+      metadata.favicon = response.data.favicon;
+    }
+    return metadata;
+  }
+}
+
+export const createTavilyScraper = (
+  config: t.TavilyScraperConfig = {}
+): TavilyScraper => {
+  return new TavilyScraper(config);
+};