@librechat/agents 3.0.17 → 3.0.19

package/src/scripts/test-multi-agent-list-handoff.ts

@@ -4,11 +4,12 @@ import { config } from 'dotenv';
 config();
 
 import { HumanMessage, BaseMessage } from '@langchain/core/messages';
-import { Run } from '@/run';
+import type * as t from '@/types';
+import { labelContentByAgent, formatAgentMessages } from '@/messages/format';
 import { ChatModelStreamHandler, createContentAggregator } from '@/stream';
-import { Providers, GraphEvents, Constants } from '@/common';
+import { Providers, GraphEvents, Constants, StepTypes } from '@/common';
 import { ToolEndHandler, ModelEndHandler } from '@/events';
-import type * as t from '@/types';
+import { Run } from '@/run';
 
 const conversationHistory: BaseMessage[] = [];
 
@@ -188,10 +189,6 @@ async function testSupervisorListHandoff() {
   try {
     // Test with different queries
     const testQueries = [
-      // 'How can we analyze user engagement metrics to improve our product?',
-      // 'What security measures should we implement for our new API?',
-      // 'Can you help design a better onboarding flow for our mobile app?',
-      // 'We need to set up a CI/CD pipeline for our microservices.',
       'What are the legal implications of using GPL-licensed code in our product?',
     ];
 
@@ -204,9 +201,9 @@ async function testSupervisorListHandoff() {
     };
 
     for (const query of testQueries) {
-      console.log(`\n${'='.repeat(60)}`);
-      console.log(`USER QUERY: "${query}"`);
-      console.log('='.repeat(60));
+      console.log(`\n${'='.repeat(80)}`);
+      console.log(`FIRST RUN - USER QUERY: "${query}"`);
+      console.log('='.repeat(80));
 
       // Reset conversation
       conversationHistory.length = 0;
@@ -216,7 +213,7 @@ async function testSupervisorListHandoff() {
       const runConfig = createSupervisorGraphWithListEdge();
       const run = await Run.create(runConfig);
 
-      console.log('Processing request...');
+      console.log('Processing first request...');
 
       // Process with streaming
       const inputs = {
@@ -230,9 +227,59 @@ async function testSupervisorListHandoff() {
         conversationHistory.push(...finalMessages);
       }
 
-      // Show summary
+      // Demo: Map contentParts to agentIds
+      console.log(`\n${'─'.repeat(60)}`);
+      console.log('CONTENT PARTS TO AGENT MAPPING:');
+      console.log('─'.repeat(60));
+
+      if (run.Graph) {
+        // Get the mapping of contentPart index to agentId
+        const contentPartAgentMap = run.Graph.getContentPartAgentMap();
+
+        console.log(`\nTotal content parts: ${contentParts.length}`);
+        console.log(`\nContent Part → Agent Mapping:`);
+
+        contentPartAgentMap.forEach((agentId, index) => {
+          const contentPart = contentParts[index];
+          const contentType = contentPart?.type || 'unknown';
+          const preview =
+            contentType === 'text'
+              ? (contentPart as any).text?.slice(0, 50) || ''
+              : contentType === 'tool_call'
+                ? `Tool: ${(contentPart as any).tool_call?.name || 'unknown'}`
+                : contentType;
+
+          console.log(
+            `  [${index}] ${agentId} → ${contentType}: ${preview}${preview.length >= 50 ? '...' : ''}`
+          );
+        });
+
+        // Show agent participation summary
+        console.log(`\n${'─'.repeat(60)}`);
+        console.log('AGENT PARTICIPATION SUMMARY:');
+        console.log('─'.repeat(60));
+
+        const activeAgents = run.Graph.getActiveAgentIds();
+        console.log(`\nActive agents (${activeAgents.length}):`, activeAgents);
+
+        const stepsByAgent = run.Graph.getRunStepsByAgent();
+        stepsByAgent.forEach((steps, agentId) => {
+          const toolCallSteps = steps.filter(
+            (s) => s.type === StepTypes.TOOL_CALLS
+          ).length;
+          const messageSteps = steps.filter(
+            (s) => s.type === StepTypes.MESSAGE_CREATION
+          ).length;
+          console.log(`\n  ${agentId}:`);
+          console.log(`    - Total steps: ${steps.length}`);
+          console.log(`    - Message steps: ${messageSteps}`);
+          console.log(`    - Tool call steps: ${toolCallSteps}`);
+        });
+      }
+
+      // Show graph structure summary
       console.log(`\n${'─'.repeat(60)}`);
-      console.log(`Graph structure:`);
+      console.log(`GRAPH STRUCTURE:`);
       console.log(`- Agents: 6 total (supervisor + 5 specialists)`);
       console.log(`- Edges: 1 edge with multiple destinations`);
       console.log(
@@ -242,6 +289,115 @@ async function testSupervisorListHandoff() {
         `- Result: Supervisor has 5 handoff tools from a single edge`
       );
       console.log('─'.repeat(60));
+
+      // =============================================================
+      // SECOND RUN: Demonstrate agent-labeled history
+      // =============================================================
+      console.log(`\n${'='.repeat(80)}`);
+      console.log(`SECOND RUN - Simulating DB Load with Agent-Labeled History`);
+      console.log('='.repeat(80));
+
+      // Simulate what happens in the main app:
+      // 1. Store contentParts + agentIdMap to "DB" (in-memory here)
+      const dbStoredContentParts = [...contentParts];
+      const dbStoredAgentIdMap = Object.fromEntries(
+        run.Graph!.getContentPartAgentMap()
+      );
+
+      console.log('\n📦 Simulating DB storage:');
+      console.log(`  - Stored ${dbStoredContentParts.length} content parts`);
+      console.log(
+        `  - Stored agent mappings for ${Object.keys(dbStoredAgentIdMap).length} parts`
+      );
+
+      // 2. On next run, load from "DB" and label by agent
+      console.log('\n📥 Loading from DB and labeling by agent...');
+
+      const agentNames = {
+        supervisor: 'Supervisor',
+        legal_advisor: 'Legal Advisor',
+        data_analyst: 'Data Analyst',
+        security_expert: 'Security Expert',
+        product_designer: 'Product Designer',
+        devops_engineer: 'DevOps Engineer',
+      };
+
+      const labeledContentParts = labelContentByAgent(
+        dbStoredContentParts.filter(
+          (p): p is t.MessageContentComplex => p != null
+        ),
+        dbStoredAgentIdMap,
+        agentNames
+      );
+
+      console.log(
+        `  - Labeled ${labeledContentParts.length} content parts by agent`
+      );
+
+      // 3. Convert labeled content parts to payload format
+      const payload: t.TPayload = [
+        {
+          role: 'user',
+          content: query,
+        },
+        {
+          role: 'assistant',
+          content: labeledContentParts,
+        },
+      ];
+
+      // 4. Format using formatAgentMessages (simulates what main app does)
+      console.log('\n🔧 Calling formatAgentMessages...');
+      const { messages: formattedMessages } = formatAgentMessages(payload);
+
+      console.log(
+        `  - Formatted into ${formattedMessages.length} BaseMessages`
+      );
+
+      // Show a preview of what the supervisor will see
+      console.log('\n👁️  Preview of formatted history for supervisor:');
+      console.log('─'.repeat(80));
+      for (let i = 0; i < formattedMessages.length; i++) {
+        const msg = formattedMessages[i];
+        const role = msg._getType();
+        const preview =
+          typeof msg.content === 'string'
+            ? msg.content.slice(0, 200)
+            : JSON.stringify(msg.content).slice(0, 200);
+        console.log(
+          `[${i}] ${role}: ${preview}${preview.length >= 200 ? '...' : ''}`
+        );
+      }
+      console.log('─'.repeat(80));
+
+      // 5. Create a new run with the formatted history + a followup question
+      console.log(
+        '\n🚀 Starting second run with agent-labeled history + followup question...'
+      );
+      const followupQuery =
+        'Can you summarize the key legal points from your previous response?';
+      console.log(`   Followup: "${followupQuery}"`);
+
+      // Reset for second run
+      const secondRunHistory: BaseMessage[] = [
+        ...formattedMessages,
+        new HumanMessage(followupQuery),
+      ];
+
+      const runConfig2 = createSupervisorGraphWithListEdge();
+      const run2 = await Run.create(runConfig2);
+
+      const inputs2 = {
+        messages: secondRunHistory,
+      };
+
+      await run2.processStream(inputs2, config);
+
+      console.log('\n✅ Second run completed successfully!');
+      console.log(
+        '   The supervisor correctly understood that the legal_advisor handled'
+      );
+      console.log('   the previous query, avoiding identity confusion.');
     }
 
     // Final summary

package/src/scripts/test-parallel-agent-labeling.ts

@@ -0,0 +1,325 @@
+#!/usr/bin/env bun
+
+import { config } from 'dotenv';
+config();
+
+import { HumanMessage, BaseMessage } from '@langchain/core/messages';
+import type * as t from '@/types';
+import { labelContentByAgent, formatAgentMessages } from '@/messages/format';
+import { ChatModelStreamHandler, createContentAggregator } from '@/stream';
+import { Providers, GraphEvents, StepTypes } from '@/common';
+import { ToolEndHandler, ModelEndHandler } from '@/events';
+import { Run } from '@/run';
+
+const conversationHistory: BaseMessage[] = [];
+
+/**
+ * Test parallel multi-agent system with agent labeling on subsequent runs
+ *
+ * Graph structure:
+ * START -> researcher
+ * researcher -> [analyst1, analyst2, analyst3] (fan-out)
+ * [analyst1, analyst2, analyst3] -> summarizer (fan-in)
+ * summarizer -> END
+ */
+async function testParallelWithAgentLabeling() {
+  console.log('Testing Parallel Multi-Agent with Agent Labeling...\n');
+
+  // Set up content aggregator
+  const { contentParts, aggregateContent } = createContentAggregator();
+
+  // Define specialized agents
+  const agents: t.AgentInputs[] = [
+    {
+      agentId: 'researcher',
+      provider: Providers.ANTHROPIC,
+      clientOptions: {
+        modelName: 'claude-haiku-4-5',
+        apiKey: process.env.ANTHROPIC_API_KEY,
+      },
+      instructions: `You are a research coordinator. Analyze the request and provide 2-3 sentence coordination brief.`,
+    },
+    {
+      agentId: 'analyst1',
+      provider: Providers.ANTHROPIC,
+      clientOptions: {
+        modelName: 'claude-haiku-4-5',
+        apiKey: process.env.ANTHROPIC_API_KEY,
+      },
+      instructions: `You are FINANCIAL ANALYST. Provide 2-3 sentence financial analysis. Start with "FINANCIAL ANALYSIS:"`,
+    },
+    {
+      agentId: 'analyst2',
+      provider: Providers.ANTHROPIC,
+      clientOptions: {
+        modelName: 'claude-haiku-4-5',
+        apiKey: process.env.ANTHROPIC_API_KEY,
+      },
+      instructions: `You are TECHNICAL ANALYST. Provide 2-3 sentence technical analysis. Start with "TECHNICAL ANALYSIS:"`,
+    },
+    {
+      agentId: 'analyst3',
+      provider: Providers.ANTHROPIC,
+      clientOptions: {
+        modelName: 'claude-haiku-4-5',
+        apiKey: process.env.ANTHROPIC_API_KEY,
+      },
+      instructions: `You are MARKET ANALYST. Provide 2-3 sentence market analysis. Start with "MARKET ANALYSIS:"`,
+    },
+    {
+      agentId: 'summarizer',
+      provider: Providers.ANTHROPIC,
+      clientOptions: {
+        modelName: 'claude-haiku-4-5',
+        apiKey: process.env.ANTHROPIC_API_KEY,
+      },
+      instructions: `You are SYNTHESIS EXPERT. Review all analyses and provide 2-3 sentence integrated summary.`,
+    },
+  ];
+
+  // Define direct edges (fan-out and fan-in)
+  const edges: t.GraphEdge[] = [
+    {
+      from: 'researcher',
+      to: ['analyst1', 'analyst2', 'analyst3'],
+      description: 'Distribute research to specialist analysts',
+      edgeType: 'direct',
+    },
+    {
+      from: ['analyst1', 'analyst2', 'analyst3'],
+      to: 'summarizer',
+      description: 'Aggregate analysis results',
+      edgeType: 'direct',
+      prompt:
+        'Based on the analyses below, provide an integrated summary:\n\n{results}',
+    },
+  ];
+
+  // Create custom handlers
+  const customHandlers = {
+    [GraphEvents.TOOL_END]: new ToolEndHandler(),
+    [GraphEvents.CHAT_MODEL_END]: new ModelEndHandler(),
+    [GraphEvents.CHAT_MODEL_STREAM]: new ChatModelStreamHandler(),
+    [GraphEvents.ON_RUN_STEP_COMPLETED]: {
+      handle: (
+        event: GraphEvents.ON_RUN_STEP_COMPLETED,
+        data: t.StreamEventData
+      ): void => {
+        aggregateContent({
+          event,
+          data: data as unknown as { result: t.ToolEndEvent },
+        });
+      },
+    },
+    [GraphEvents.ON_RUN_STEP]: {
+      handle: (
+        event: GraphEvents.ON_RUN_STEP,
+        data: t.StreamEventData
+      ): void => {
+        aggregateContent({ event, data: data as t.RunStep });
+      },
+    },
+    [GraphEvents.ON_MESSAGE_DELTA]: {
+      handle: (
+        event: GraphEvents.ON_MESSAGE_DELTA,
+        data: t.StreamEventData
+      ): void => {
+        aggregateContent({ event, data: data as t.MessageDeltaEvent });
+      },
+    },
+  };
+
+  try {
+    const query = 'What are the implications of widespread AI adoption?';
+
+    console.log(`${'='.repeat(80)}`);
+    console.log(`FIRST RUN - USER QUERY: "${query}"`);
+    console.log('='.repeat(80));
+
+    // Reset conversation
+    conversationHistory.length = 0;
+    conversationHistory.push(new HumanMessage(query));
+
+    // Create graph
+    const runConfig: t.RunConfig = {
+      runId: `parallel-test-${Date.now()}`,
+      graphConfig: {
+        type: 'multi-agent',
+        agents,
+        edges,
+      },
+      customHandlers,
+      returnContent: true,
+    };
+
+    const run = await Run.create(runConfig);
+
+    console.log('\nProcessing first run with parallel agents...\n');
+
+    const config = {
+      configurable: {
+        thread_id: 'parallel-agent-labeling-1',
+      },
+      streamMode: 'values',
+      version: 'v2' as const,
+    };
+
+    const inputs = {
+      messages: conversationHistory,
+    };
+
+    await run.processStream(inputs, config);
+    const finalMessages = run.getRunMessages();
+
+    if (finalMessages) {
+      conversationHistory.push(...finalMessages);
+    }
+
+    // Show agent participation
+    console.log(`\n${'─'.repeat(80)}`);
+    console.log('FIRST RUN - AGENT PARTICIPATION:');
+    console.log('─'.repeat(80));
+
+    if (run.Graph) {
+      const activeAgents = run.Graph.getActiveAgentIds();
+      console.log(`\nActive agents (${activeAgents.length}):`, activeAgents);
+
+      const stepsByAgent = run.Graph.getRunStepsByAgent();
+      stepsByAgent.forEach((steps, agentId) => {
+        console.log(`  ${agentId}: ${steps.length} steps`);
+      });
+
+      console.log(`\nTotal content parts: ${contentParts.length}`);
+    }
+
+    // =============================================================
+    // SECOND RUN: Test with agent-labeled history
+    // =============================================================
+    console.log(`\n${'='.repeat(80)}`);
+    console.log(`SECOND RUN - Simulating DB Load with Parallel Agent Labeling`);
+    console.log('='.repeat(80));
+
+    // Simulate DB storage
+    const dbStoredContentParts = [...contentParts];
+    const dbStoredAgentIdMap = Object.fromEntries(
+      run.Graph!.getContentPartAgentMap()
+    );
+
+    console.log('\n📦 Simulating DB storage:');
+    console.log(`  - Stored ${dbStoredContentParts.length} content parts`);
+    console.log(
+      `  - Stored agent mappings for ${Object.keys(dbStoredAgentIdMap).length} parts`
+    );
+
+    // Load and label by agent with labelNonTransferContent option
+    console.log('\n📥 Loading from DB and labeling ALL agent content...');
+
+    const agentNames = {
+      researcher: 'Researcher',
+      analyst1: 'Financial Analyst',
+      analyst2: 'Technical Analyst',
+      analyst3: 'Market Analyst',
+      summarizer: 'Synthesizer',
+    };
+
+    const labeledContentParts = labelContentByAgent(
+      dbStoredContentParts.filter(
+        (p): p is t.MessageContentComplex => p != null
+      ),
+      dbStoredAgentIdMap,
+      agentNames,
+      { labelNonTransferContent: true } // NEW: Label all content
+    );
+
+    console.log(
+      `  - Labeled ${labeledContentParts.length} content groups by agent`
+    );
+
+    // Convert to payload
+    const payload: t.TPayload = [
+      {
+        role: 'user',
+        content: query,
+      },
+      {
+        role: 'assistant',
+        content: labeledContentParts,
+      },
+    ];
+
+    // Format using formatAgentMessages
+    console.log('\n🔧 Calling formatAgentMessages...');
+    const { messages: formattedMessages } = formatAgentMessages(payload);
+
+    console.log(`  - Formatted into ${formattedMessages.length} BaseMessages`);
+
+    // Show preview
+    console.log('\n👁️  Preview of formatted history:');
+    console.log('─'.repeat(80));
+    for (let i = 0; i < formattedMessages.length; i++) {
+      const msg = formattedMessages[i];
+      const role = msg._getType();
+      const preview =
+        typeof msg.content === 'string'
+          ? msg.content.slice(0, 300)
+          : JSON.stringify(msg.content).slice(0, 300);
+      console.log(
+        `[${i}] ${role}: ${preview}${preview.length >= 300 ? '...' : ''}`
+      );
+      console.log('');
+    }
+    console.log('─'.repeat(80));
+
+    // Create a second run with labeled history
+    console.log(
+      '\n🚀 Starting second run with agent-labeled parallel history...'
+    );
+    const followupQuery = 'Which analyst identified the most significant risk?';
+    console.log(`   Followup: "${followupQuery}"`);
+
+    const secondRunHistory: BaseMessage[] = [
+      ...formattedMessages,
+      new HumanMessage(followupQuery),
+    ];
+
+    const runConfig2: t.RunConfig = {
+      runId: `parallel-test-2-${Date.now()}`,
+      graphConfig: {
+        type: 'multi-agent',
+        agents,
+        edges,
+      },
+      customHandlers,
+      returnContent: true,
+    };
+
+    const run2 = await Run.create(runConfig2);
+
+    const inputs2 = {
+      messages: secondRunHistory,
+    };
+
+    await run2.processStream(inputs2, config);
+
+    console.log('\n✅ Second run completed successfully!');
+    console.log(
+      '   The researcher correctly understood that parallel analysts handled'
+    );
+    console.log('   the previous analysis, with clear attribution per agent.');
+
+    console.log(`\n${'='.repeat(80)}`);
+    console.log('TEST COMPLETE');
+    console.log('='.repeat(80));
+    console.log(
+      '\nThis demonstrates that parallel multi-agent patterns work correctly'
+    );
+    console.log(
+      'with agent labeling, preventing confusion about who said what.'
+    );
+  } catch (error) {
+    console.error('Error in parallel agent labeling test:', error);
+  }
+}
+
+// Run the test
+testParallelWithAgentLabeling();

package/src/scripts/test-thinking-handoff-bedrock.ts

@@ -0,0 +1,153 @@
+#!/usr/bin/env bun
+
+import { config } from 'dotenv';
+config();
+
+import { HumanMessage, BaseMessage } from '@langchain/core/messages';
+import { Run } from '@/run';
+import { ChatModelStreamHandler } from '@/stream';
+import { Providers, GraphEvents } from '@/common';
+import { ToolEndHandler, ModelEndHandler } from '@/events';
+import type * as t from '@/types';
+
+const conversationHistory: BaseMessage[] = [];
+
+/**
+ * Test edge case: switching from OpenAI supervisor (no thinking) to Bedrock specialist (with thinking enabled)
+ * This should not throw an error about missing thinking blocks
+ */
+async function testBedrockThinkingHandoff() {
+  console.log('Testing OpenAI → Bedrock (with thinking) handoff...\n');
+
+  // Create custom handlers
+  const customHandlers = {
+    [GraphEvents.TOOL_END]: new ToolEndHandler(),
+    [GraphEvents.CHAT_MODEL_END]: new ModelEndHandler(),
+    [GraphEvents.CHAT_MODEL_STREAM]: new ChatModelStreamHandler(),
+    [GraphEvents.TOOL_START]: {
+      handle: (_event: string, data: t.StreamEventData): void => {
+        const toolData = data as any;
+        if (toolData?.name) {
+          console.log(`\n🔧 Tool called: ${toolData.name}`);
+        }
+      },
+    },
+  };
+
+  // Create the graph configuration
+  function createGraphConfig(): t.RunConfig {
+    console.log(
+      'Creating graph with OpenAI supervisor and Bedrock specialist with thinking enabled.\n'
+    );
+
+    const agents: t.AgentInputs[] = [
+      {
+        agentId: 'supervisor',
+        provider: Providers.OPENAI,
+        clientOptions: {
+          modelName: 'gpt-4o-mini',
+          apiKey: process.env.OPENAI_API_KEY,
+        },
+        instructions: `You are a task supervisor. When the user asks about code review, use transfer_to_code_reviewer to hand off to the specialist.`,
+        maxContextTokens: 8000,
+      },
+      {
+        agentId: 'code_reviewer',
+        provider: Providers.BEDROCK,
+        clientOptions: {
+          region: process.env.BEDROCK_AWS_REGION || 'us-east-1',
+          model: 'us.anthropic.claude-3-7-sonnet-20250219-v1:0',
+          credentials: {
+            accessKeyId: process.env.BEDROCK_AWS_ACCESS_KEY_ID!,
+            secretAccessKey: process.env.BEDROCK_AWS_SECRET_ACCESS_KEY!,
+          },
+          additionalModelRequestFields: {
+            thinking: {
+              type: 'enabled',
+              budget_tokens: 2000,
+            },
+          },
+        },
+        instructions: `You are a code review specialist using Bedrock with extended thinking. Think carefully about the code quality, best practices, and potential issues. Provide thoughtful feedback.`,
+        maxContextTokens: 8000,
+      },
+    ];
+
+    const edges: t.GraphEdge[] = [
+      {
+        from: 'supervisor',
+        to: ['code_reviewer'],
+        description: 'Transfer to code review specialist',
+        edgeType: 'handoff',
+      },
+    ];
+
+    return {
+      runId: `bedrock-thinking-handoff-test-${Date.now()}`,
+      graphConfig: {
+        type: 'multi-agent',
+        agents,
+        edges,
+      },
+      customHandlers,
+      returnContent: true,
+    };
+  }
+
+  try {
+    // Test query that should trigger a handoff
+    const query =
+      'Can you review this function and tell me if there are any issues?\n\nfunction add(a, b) { return a + b; }';
+
+    console.log(`${'='.repeat(60)}`);
+    console.log(`USER QUERY: "${query}"`);
+    console.log('='.repeat(60));
+
+    // Initialize conversation
+    conversationHistory.push(new HumanMessage(query));
+
+    // Create and run the graph
+    const runConfig = createGraphConfig();
+    const run = await Run.create(runConfig);
+
+    const config = {
+      configurable: {
+        thread_id: 'bedrock-thinking-handoff-test-1',
+      },
+      streamMode: 'values',
+      version: 'v2' as const,
+    };
+
+    console.log('\nProcessing request...\n');
+
+    // Process with streaming
+    const inputs = {
+      messages: conversationHistory,
+    };
+
+    await run.processStream(inputs, config);
+    const finalMessages = run.getRunMessages();
+
+    if (finalMessages) {
+      conversationHistory.push(...finalMessages);
+    }
+
+    // Success!
+    console.log(`\n${'='.repeat(60)}`);
+    console.log('✅ TEST PASSED');
+    console.log('='.repeat(60));
+    console.log('\nSuccessfully handed off from OpenAI (no thinking) to');
+    console.log('Bedrock with thinking enabled without errors!');
+    console.log('\nThe ensureThinkingBlockInMessages() function correctly');
+    console.log('handled the transition by converting tool sequences to');
+    console.log('HumanMessages before calling the Bedrock API.');
+  } catch (error) {
+    console.error('\n❌ TEST FAILED');
+    console.error('='.repeat(60));
+    console.error('Error:', error);
+    process.exit(1);
+  }
+}
+
+// Run the test
+testBedrockThinkingHandoff();