@librechat/agents 3.7.7 → 3.7.9

package/src/summarization/index.ts

@@ -1,5 +1,16 @@
 import type { SummarizationTrigger } from '@/types';
 
+/**
+ * The summary-boundary seam a caller compacting outside a run needs. The two
+ * default prompts stay out of it: LibreChat's manual flow deliberately words
+ * its own, so exporting these would publish an API with no consumer.
+ */
+export {
+  buildSummarizationInstruction,
+  buildSummaryCarrierText,
+  separateSummarizationParameters,
+} from './shared';
+
 const VALID_TRIGGER_TYPES = [
   'token_ratio',
   'remaining_tokens',

package/src/summarization/node.ts

@@ -1,16 +1,27 @@
-import {
-  AIMessage,
-  ToolMessage,
-  HumanMessage,
-  SystemMessage,
-} from '@langchain/core/messages';
+import { AIMessage, ToolMessage, HumanMessage } from '@langchain/core/messages';
 import type { UsageMetadata, BaseMessage } from '@langchain/core/messages';
 import type { RunnableConfig } from '@langchain/core/runnables';
+import type { RenderedCompactionSemanticIndex } from '@/summarization/semanticIndex';
 import type { StreamLimitState } from '@/llm/streamLimits';
 import type { AgentContext } from '@/agents/AgentContext';
+import type { EncodingName } from '@/utils/tokens';
 import type { HookRegistry } from '@/hooks';
 import type { OnChunk } from '@/llm/invoke';
 import type * as t from '@/types';
+import {
+  DEFAULT_SUMMARIZATION_PROMPT,
+  DEFAULT_UPDATE_SUMMARIZATION_PROMPT,
+  buildSummaryCarrierText,
+  separateSummarizationParameters,
+  buildSummarizationInstruction,
+} from './shared';
+import {
+  addTailCacheControl,
+  addBedrockTailCacheControl,
+  resolvePromptCacheTtl,
+  resolveBedrockPromptCacheTtl,
+  type PromptCacheTtl,
+} from '@/messages/cache';
 import {
   cloneToolMessageWithContent,
   compactToolContent,
@@ -22,18 +33,16 @@ import {
   StreamLimitExceededError,
   STREAM_LIMIT_EPOCH_KEY,
 } from '@/llm/streamLimits';
-import {
-  addTailCacheControl,
-  addBedrockTailCacheControl,
-  resolvePromptCacheTtl,
-  resolveBedrockPromptCacheTtl,
-  type PromptCacheTtl,
-} from '@/messages/cache';
 import {
   DEFAULT_RETAIN_RECENT_TURNS,
   resolveIntraTurnRetainTokens,
   splitAtRecencyBoundary,
 } from '@/messages/recency';
+import {
+  createTokenCounter,
+  encodingForModel,
+  encodingOfTokenCounter,
+} from '@/utils/tokens';
 import {
   Constants,
   ContentTypes,
@@ -41,19 +50,22 @@ import {
   StepTypes,
   Providers,
 } from '@/common';
+import {
+  getMaxOutputTokensKey,
+  resolveClientOptionsModel,
+} from '@/llm/request';
+import { renderCompactionSemanticIndex } from '@/summarization/semanticIndex';
 import { safeDispatchCustomEvent, emitAgentLog } from '@/utils/events';
-import { attemptInvoke, tryFallbackProviders } from '@/llm/invoke';
 import { prepareToolsForPromptCache } from '@/llm/promptCacheTools';
+import { attemptInvoke, tryFallbackProviders } from '@/llm/invoke';
 import { calculateMaxToolResultChars } from '@/utils/truncation';
 import { makeIsDeferred } from '@/messages/anthropicToolCache';
 import { createRemoveAllMessage } from '@/messages/reducer';
-import { getMaxOutputTokensKey } from '@/llm/request';
+import { getProviderFamily } from '@/llm/providerRegistry';
 import { initializeModel } from '@/llm/init';
 import { getChunkContent } from '@/stream';
 import { executeHooks } from '@/hooks';
 
-const SUMMARIZATION_PARAM_KEYS = new Set(['maxSummaryTokens']);
-
 /**
  * Default number of recent user-led turns preserved verbatim during
  * compaction.  A turn begins at a HumanMessage and includes every
@@ -64,89 +76,6 @@ const SUMMARIZATION_PARAM_KEYS = new Set(['maxSummaryTokens']);
  * `retainRecent.turns` to `0` reverts to the legacy "summarize every
  * message" behavior.
  */
-/**
- * Token overhead of the XML wrapper + instruction text added around the
- * summary at injection time in AgentContext.buildSystemRunnable:
- * `<summary>\n${text}\n</summary>\n\nYour context window was compacted...`
- * ~33 tokens on Anthropic, ~24-27 on OpenAI.  Using 33 as a safe ceiling.
- */
-const SUMMARY_WRAPPER_OVERHEAD_TOKENS = 33;
-
-/** Structured checkpoint prompt for fresh summarization (no prior summary). */
-export const DEFAULT_SUMMARIZATION_PROMPT = `Hold on, before you continue I need you to write me a checkpoint of everything so far. Your context window is filling up and this checkpoint replaces the messages above, so capture everything you need to pick right back up.
-
-Don't second-guess or fact-check anything you did, your tool results reflect exactly what happened. If a tool result appears truncated, that's just a display artifact from context management: the tool executed fully. Just record what you did and what you observed. Only the checkpoint, don't respond to me or continue the conversation.
-
-## Checkpoint
-
-## Goal
-What I asked you to do and any sub-goals you identified.
-
-## Constraints & Preferences
-Any rules, preferences, or configuration I established.
-
-## Progress
-### Done
-- What you completed and the outcomes
-
-### In Progress
-- What you're currently working on
-
-## Key Decisions
-Decisions you made and why.
-
-## Next Steps
-Concrete task actions remaining, in priority order.
-
-## Critical Context
-Exact identifiers, names, error messages, URLs, and details you need to preserve verbatim.
-
-Rules:
-- Record what you did and observed, don't judge or re-evaluate it
-- For each tool call: the tool name, key inputs, and the outcome
-- Preserve exact identifiers, names, errors, and references verbatim
-- Short declarative sentences
-- Skip empty sections`;
-
-/** Prompt for re-compaction when a prior summary exists. */
-export const DEFAULT_UPDATE_SUMMARIZATION_PROMPT = `Hold on again, update your checkpoint. Merge the new messages into your existing checkpoint and give me a single consolidated replacement.
-
-Keep it roughly the same length as your last checkpoint. Compress older details to make room for what's new, don't just append. Give recent actions more detail, compress older items to one-liners.
-
-Don't fact-check or second-guess anything, your tool results are ground truth. If a tool result appears truncated, that's just a display artifact: the tool executed fully. Only the checkpoint, don't respond to me or continue the conversation.
-
-Rules:
-- Merge new progress into existing sections, don't duplicate headers
-- Compress older completed items into one-line entries
-- Move items from "In Progress" to "Done" when you completed them
-- Update "Next Steps" to reflect current task priorities.
-- For each new tool call: the tool name, key inputs, and the outcome
-- Preserve exact identifiers, names, errors, and references verbatim
-- Skip empty sections`;
-
-function separateParameters(parameters: Record<string, unknown>): {
-  llmParams: Record<string, unknown>;
-  maxSummaryTokens?: number;
-} {
-  const llmParams: Record<string, unknown> = {};
-  let maxSummaryTokens: number | undefined;
-
-  for (const [key, value] of Object.entries(parameters)) {
-    if (SUMMARIZATION_PARAM_KEYS.has(key)) {
-      if (
-        key === 'maxSummaryTokens' &&
-        typeof value === 'number' &&
-        value > 0
-      ) {
-        maxSummaryTokens = value;
-      }
-    } else {
-      llmParams[key] = value;
-    }
-  }
-
-  return { llmParams, maxSummaryTokens };
-}
 
 /**
  * Generates a structural metadata summary without making an LLM call.
@@ -338,7 +267,7 @@ function buildSummarizationClientConfig(
     summarizationConfig?.updatePrompt ?? DEFAULT_UPDATE_SUMMARIZATION_PROMPT;
 
   const { llmParams, maxSummaryTokens: paramMaxSummaryTokens } =
-    separateParameters(parameters);
+    separateSummarizationParameters(parameters);
 
   const isSelfSummarize = provider === (agentContext.provider as string);
   const baseOptions =
@@ -373,23 +302,104 @@ function buildSummarizationClientConfig(
   };
 }
 
-/** Computes the token count for a summary, preferring provider output tokens when available. */
-function computeSummaryTokenCount(
+/**
+ * Sizes a summary from the text that will actually be re-injected, carrier
+ * included, so the stored count is a measurement rather than a body count plus
+ * a remembered constant.
+ *
+ * The provider's `output_tokens` is deliberately not consulted. On a reasoning
+ * summarizer the two diverge by the hidden thinking, which is billed but never
+ * written into the checkpoint, so using it here would make every later context
+ * calculation reserve room for tokens that are never sent. Provider usage stays
+ * exclusively a billing input.
+ *
+ * A missing host counter is not hypothetical: `shouldSummarizeOverflow` fires
+ * precisely when there is nothing to count with, so that branch is the
+ * overflow-recovery summary, and its count is persisted and then reserved by
+ * `AgentContext.instructionTokens` on the retry. Undercounting it is what makes
+ * the retry overflow again, which rules out a character heuristic: measured
+ * against `o200k_base` and Anthropic's tokenizer, four-characters-per-token
+ * understates base64 by 1.5x and Korean by 4.6x, and coefficients large enough
+ * to cover those overestimate English prose by roughly 4x. So this falls back
+ * to the tokenizer this package already bundles rather than to an estimate.
+ *
+ * A host counter that is present is not automatically the right one either. In
+ * a heterogeneous multi-agent run `Run.create` derives a single counter from
+ * `agents[0]` and `StandardGraph` hands that same counter to every
+ * `AgentContext`, so a Claude agent behind a GPT first agent would otherwise
+ * measure its carrier in `o200k_base` and under-reserve by up to 1.9x on CJK.
+ * When the counter came from `createTokenCounter` its encoding is known, so a
+ * disagreement with the receiving agent's encoding takes the bundled path
+ * instead. A counter the host built itself is unstamped and stays authoritative:
+ * its units are the ones the rest of that host's accounting is denominated in.
+ */
+async function computeSummaryTokenCount(
   summaryText: string,
-  summaryUsage: Partial<UsageMetadata> | undefined,
-  tokenCounter?: (message: BaseMessage) => number
-): number {
-  const providerOutputTokens = Number(summaryUsage?.output_tokens) || 0;
-  if (providerOutputTokens > 0) {
-    return providerOutputTokens + SUMMARY_WRAPPER_OVERHEAD_TOKENS;
+  agentContext: AgentContext
+): Promise<number> {
+  const carrier = new HumanMessage(buildSummaryCarrierText(summaryText));
+  const encoding = encodingForReceivingAgent(agentContext);
+  const hostCounter = agentContext.tokenCounter;
+  const hostEncoding =
+    hostCounter != null ? encodingOfTokenCounter(hostCounter) : undefined;
+  if (
+    hostCounter != null &&
+    (hostEncoding == null || hostEncoding === encoding)
+  ) {
+    return hostCounter(carrier);
   }
-  if (tokenCounter) {
-    return (
-      tokenCounter(new SystemMessage(summaryText)) +
-      SUMMARY_WRAPPER_OVERHEAD_TOKENS
-    );
+  const bundledCounter = await createTokenCounter(encoding);
+  return bundledCounter(carrier);
+}
+
+/**
+ * Encoding of the model that will *receive* the carrier.
+ *
+ * That is the agent's own model, never the summarizer's. The two are the same
+ * only by default: `summarizationConfig.model` is undefined for ordinary
+ * self-summarization and can name a different provider entirely when a cheap
+ * dedicated summarizer is configured. The count produced here is spent by
+ * `AgentContext.instructionTokens` against the agent's context window, so it
+ * has to be denominated in the agent's tokenizer, and Anthropic's counts run
+ * well above `o200k_base` on the same text: measuring CJK with the wrong one
+ * understates it by up to 1.9x.
+ *
+ * Both signals are read, because only one of them can ever be positive.
+ * `encodingForModel` matches the substring `claude`, so a name it accepts is
+ * proof, which is what keeps a Claude model reached through Bedrock or
+ * OpenRouter resolving to `claude`. A name it rejects is only the absence of
+ * proof: `production` is an opaque deployment alias, not a statement that the
+ * model behind it is something other than Claude. Both option keys are
+ * consulted for that name, since `modelName` is LangChain's alias for `model`
+ * and hosts configure agents through either, so reading one key alone would
+ * report an unconfigured model and quietly hand a Claude agent the
+ * `o200k_base` tokenizer.
+ *
+ * The provider therefore decides every case the name leaves open, not just the
+ * case where no model was recorded at all, and it is read as a family rather
+ * than as one enum value: a host can register its own provider with
+ * `family: 'anthropic'` (the trait `isThinkingEnabled` already reads the same
+ * way), and such a provider serves Claude behind whatever name and deployment
+ * alias the host chose. `BEDROCK` is left out by the same rule, since its
+ * family is `bedrock` and it also serves Llama, Titan and Mistral. The enum
+ * check stays ahead of the family lookup because the registry is populated by
+ * importing `@/llm/providers`, which a root-barrel consumer defers.
+ *
+ * On an opaque alias this deliberately parts ways with `Run.create`, which
+ * infers from the model name alone and stamps its counter `o200k_base`. That
+ * stamp records which tokenizer the counter is, not which one the agent needs,
+ * so the disagreement routes the carrier to the bundled Claude tokenizer by
+ * the same rule that rejects another agent's counter.
+ */
+function encodingForReceivingAgent(agentContext: AgentContext): EncodingName {
+  const model = resolveClientOptionsModel(agentContext.clientOptions);
+  if (model != null && encodingForModel(model) === 'claude') {
+    return 'claude';
   }
-  return 0;
+  return agentContext.provider === Providers.ANTHROPIC ||
+    getProviderFamily(agentContext.provider) === 'anthropic'
+    ? 'claude'
+    : 'o200k_base';
 }
 
 /**
@@ -625,6 +635,7 @@ async function executeSummarizationWithFallback(params: {
   summarizeConfig?: RunnableConfig;
   stepId: string;
   usePromptCache: boolean;
+  semanticIndex: RenderedCompactionSemanticIndex;
   log: LogFn;
   /** Carries the run's stream limits so the event cap covers summary streams. */
   graph?: StreamLimitState & {
@@ -651,6 +662,7 @@ async function executeSummarizationWithFallback(params: {
     summarizeConfig,
     stepId,
     usePromptCache,
+    semanticIndex,
     log,
     graph,
   } = params;
@@ -693,6 +705,7 @@ async function executeSummarizationWithFallback(params: {
       promptText: clientConfig.promptText,
       updatePromptText: clientConfig.updatePromptText,
       priorSummaryText,
+      semanticIndexAppendix: semanticIndex.appendix,
       config: summarizeConfig,
       stepId,
       provider: clientConfig.provider,
@@ -704,20 +717,20 @@ async function executeSummarizationWithFallback(params: {
         clientConfig.provider === Providers.OPENROUTER ||
         clientConfig.provider === Providers.BEDROCK
           ? (
-                clientConfig.clientOptions as {
-                  promptCacheTtl?: PromptCacheTtl;
-                }
+              clientConfig.clientOptions as {
+                promptCacheTtl?: PromptCacheTtl;
+              }
           ).promptCacheTtl
           : undefined,
       bedrockModelId:
         clientConfig.provider === Providers.BEDROCK
           ? resolveBedrockCompactionCacheModel(
-            clientConfig.clientOptions as
-              | {
-                applicationInferenceProfile?: string;
-                model?: string;
-              }
-              | undefined
+              clientConfig.clientOptions as
+                | {
+                    applicationInferenceProfile?: string;
+                    model?: string;
+                  }
+                | undefined
           )
           : undefined,
       log,
@@ -785,7 +798,8 @@ async function executeSummarizationWithFallback(params: {
               buildSummarizationInstruction(
                 clientConfig.promptText,
                 clientConfig.updatePromptText,
-                priorSummaryText
+                priorSummaryText,
+                semanticIndex.appendix
               )
             ),
           ],
@@ -1092,6 +1106,29 @@ export function createSummarizeNode({
       return { summarizationRequest: undefined };
     }
 
+    /**
+     * A summarizer that has already returned nothing several times in a row
+     * keeps returning nothing, and every empty result leaves the message set
+     * exactly as it was — so the next prune cycle re-triggers on identical
+     * state. Stopping here bounds that loop instead of letting the run spend
+     * its recursion budget dispatching empty summary steps.
+     */
+    if (agentContext.summarizationExhausted) {
+      emitAgentLog(
+        config,
+        'warn',
+        'summarize',
+        'Summarization skipped — consecutive attempts produced no usable summary',
+        {
+          failures: agentContext.summarizationFailures,
+          reason: request.reason ?? 'trigger',
+        },
+        { runId: graph.runId, agentId: request.agentId }
+      );
+      agentContext.markSummarizationTriggered(state.messages.length);
+      return { summarizationRequest: undefined };
+    }
+
     const maxCtx = agentContext.maxContextTokens ?? 0;
     if (maxCtx > 0 && agentContext.instructionTokens >= maxCtx) {
       emitAgentLog(
@@ -1188,6 +1225,10 @@ export function createSummarizeNode({
       agentContext,
       agentContext.summarizationConfig
     );
+    const semanticIndex = renderCompactionSemanticIndex(
+      agentContext.compactionSemanticIndex,
+      messagesToRefine
+    );
 
     const stepKey = `summarize-${request.agentId}`;
     const [stepId, stepIndex] = generateStepId(stepKey);
@@ -1229,6 +1270,8 @@ export function createSummarizeNode({
           model: clientConfig.modelName,
           messagesToRefineCount: messagesToRefine.length,
           summaryVersion: agentContext.summaryVersion + 1,
+          semanticIndexEntryCount: semanticIndex.entryCount,
+          semanticIndexCharCount: semanticIndex.charCount,
         } satisfies t.SummarizeStartEvent,
         runnableConfig
       );
@@ -1276,6 +1319,9 @@ export function createSummarizeNode({
       isSelfSummarize: isSelfSummarizeModel,
       hasPromptCache,
       provider: clientConfig.provider,
+      semanticIndexEntryCount: semanticIndex.entryCount,
+      semanticIndexCharCount: semanticIndex.charCount,
+      semanticIndexOmittedEntryCount: semanticIndex.omittedEntryCount,
     });
 
     const summarizeConfig: RunnableConfig | undefined = config
@@ -1291,6 +1337,10 @@ export function createSummarizeNode({
           agentId: request.agentId,
           summarization_provider: clientConfig.provider,
           summarization_model: clientConfig.modelName,
+          compaction_semantic_index_entries: semanticIndex.entryCount,
+          compaction_semantic_index_chars: semanticIndex.charCount,
+          compaction_semantic_index_omitted_entries:
+            semanticIndex.omittedEntryCount,
           /**
              * Per-call model attribution for usage consumers (the subagent
              * usage-capture handler): the summarizer's model can differ from
@@ -1340,6 +1390,7 @@ export function createSummarizeNode({
       summarizeConfig,
       stepId,
       usePromptCache: isSelfSummarizeModel && hasPromptCache,
+      semanticIndex,
       log,
       graph,
     });
@@ -1365,6 +1416,7 @@ export function createSummarizeNode({
         `Summarization failed during ${preservationReason}; keeping history rather than replacing it with a metadata stub`
       );
       agentContext.markSummarizationTriggered(state.messages.length);
+      agentContext.recordSummarizationFailure();
       /**
        * The run step was already dispatched, so it has to be resolved here or
        * consumers tracking step lifecycle keep an unfinished placeholder for
@@ -1393,7 +1445,22 @@ export function createSummarizeNode({
     }
 
     if (!rawText) {
-      agentContext.markSummarizationTriggered(0);
+      /**
+       * An empty summary compacts nothing, so the state the pruner sees next
+       * is byte-identical to the one that just triggered. Resetting the guard
+       * to `0` here made `shouldSkipSummarization` answer `false` forever,
+       * and the agent node re-triggered on that unchanged state until the
+       * graph hit its recursion cap — a loop of empty summary steps, each one
+       * a billed model call. Recording the current count keeps the guard
+       * honest; the failure tally bounds the retries once new messages do
+       * arrive and legitimately lift it.
+       */
+      agentContext.markSummarizationTriggered(state.messages.length);
+      agentContext.recordSummarizationFailure();
+      log('warn', 'Summarization produced empty output', {
+        failures: agentContext.summarizationFailures,
+        messagesToRefineCount: messagesToRefine.length,
+      });
       if (runnableConfig) {
         await safeDispatchCustomEvent(
           GraphEvents.ON_SUMMARIZE_COMPLETE,
@@ -1411,10 +1478,9 @@ export function createSummarizeNode({
 
     const summaryText = enrichSummary(rawText, messagesToRefine);
 
-    const tokenCount = computeSummaryTokenCount(
+    const tokenCount = await computeSummaryTokenCount(
       summaryText,
-      summaryUsage,
-      agentContext.tokenCounter
+      agentContext
     );
 
     if (usedIntraTurnFallback) {
@@ -1541,23 +1607,6 @@ function extractResponseText(response: { content: string | object }): string {
   return parts.join('').trim();
 }
 
-function buildSummarizationInstruction(
-  promptText: string,
-  updatePromptText: string | undefined,
-  priorSummaryText: string
-): string {
-  const effectivePrompt = priorSummaryText
-    ? (updatePromptText ?? promptText)
-    : promptText;
-  const parts = [effectivePrompt];
-  if (priorSummaryText) {
-    parts.push(
-      `\n\n<previous-summary>\n${priorSummaryText}\n</previous-summary>`
-    );
-  }
-  return parts.join('');
-}
-
 /** Creates an `onChunk` callback that dispatches `ON_SUMMARIZE_DELTA` events for streaming. */
 export function createSummarizationChunkHandler({
   stepId,
@@ -1666,10 +1715,7 @@ export function applySummarizationHistoryCache(params: {
   if (params.provider === Providers.BEDROCK) {
     return addBedrockTailCacheControl(
       [...params.messages],
-      resolveBedrockPromptCacheTtl(
-        params.promptCacheTtl,
-        params.bedrockModelId
-      )
+      resolveBedrockPromptCacheTtl(params.promptCacheTtl, params.bedrockModelId)
     );
   }
   if (
@@ -1685,9 +1731,7 @@ export function applySummarizationHistoryCache(params: {
 }
 
 export function resolveBedrockCompactionCacheModel(
-  options:
-    | { applicationInferenceProfile?: string; model?: string }
-    | undefined
+  options: { applicationInferenceProfile?: string; model?: string } | undefined
 ): string | undefined {
   return options?.model;
 }
@@ -1705,6 +1749,7 @@ async function summarizeWithCacheHit({
   promptText,
   updatePromptText,
   priorSummaryText,
+  semanticIndexAppendix,
   config,
   stepId,
   provider,
@@ -1720,6 +1765,7 @@ async function summarizeWithCacheHit({
   promptText: string;
   updatePromptText?: string;
   priorSummaryText: string;
+  semanticIndexAppendix?: string;
   config?: RunnableConfig;
   stepId?: string;
   provider: t.ProviderName;
@@ -1733,7 +1779,8 @@ async function summarizeWithCacheHit({
   const instruction = buildSummarizationInstruction(
     promptText,
     updatePromptText,
-    priorSummaryText
+    priorSummaryText,
+    semanticIndexAppendix
   );
 
   const cachedHistory = applySummarizationHistoryCache({
@@ -1743,10 +1790,7 @@ async function summarizeWithCacheHit({
     promptCacheTtl,
     bedrockModelId,
   });
-  const invokeMessages = [
-    ...cachedHistory,
-    new HumanMessage(instruction),
-  ];
+  const invokeMessages = [...cachedHistory, new HumanMessage(instruction)];
 
   const result = await attemptInvoke(
     {