@illuma-ai/agents 1.0.81

package/src/scripts/test-parallel-agent-labeling.ts

@@ -0,0 +1,325 @@
+#!/usr/bin/env bun
+
+import { config } from 'dotenv';
+config();
+
+import { HumanMessage, BaseMessage } from '@langchain/core/messages';
+import type * as t from '@/types';
+import { labelContentByAgent, formatAgentMessages } from '@/messages/format';
+import { ChatModelStreamHandler, createContentAggregator } from '@/stream';
+import { Providers, GraphEvents, StepTypes } from '@/common';
+import { ToolEndHandler, ModelEndHandler } from '@/events';
+import { Run } from '@/run';
+
+const conversationHistory: BaseMessage[] = [];
+
+/**
+ * Test parallel multi-agent system with agent labeling on subsequent runs
+ *
+ * Graph structure:
+ * START -> researcher
+ * researcher -> [analyst1, analyst2, analyst3] (fan-out)
+ * [analyst1, analyst2, analyst3] -> summarizer (fan-in)
+ * summarizer -> END
+ */
+async function testParallelWithAgentLabeling() {
+  console.log('Testing Parallel Multi-Agent with Agent Labeling...\n');
+
+  // Set up content aggregator
+  const { contentParts, aggregateContent } = createContentAggregator();
+
+  // Define specialized agents
+  const agents: t.AgentInputs[] = [
+    {
+      agentId: 'researcher',
+      provider: Providers.ANTHROPIC,
+      clientOptions: {
+        modelName: 'claude-haiku-4-5',
+        apiKey: process.env.ANTHROPIC_API_KEY,
+      },
+      instructions: `You are a research coordinator. Analyze the request and provide 2-3 sentence coordination brief.`,
+    },
+    {
+      agentId: 'analyst1',
+      provider: Providers.ANTHROPIC,
+      clientOptions: {
+        modelName: 'claude-haiku-4-5',
+        apiKey: process.env.ANTHROPIC_API_KEY,
+      },
+      instructions: `You are FINANCIAL ANALYST. Provide 2-3 sentence financial analysis. Start with "FINANCIAL ANALYSIS:"`,
+    },
+    {
+      agentId: 'analyst2',
+      provider: Providers.ANTHROPIC,
+      clientOptions: {
+        modelName: 'claude-haiku-4-5',
+        apiKey: process.env.ANTHROPIC_API_KEY,
+      },
+      instructions: `You are TECHNICAL ANALYST. Provide 2-3 sentence technical analysis. Start with "TECHNICAL ANALYSIS:"`,
+    },
+    {
+      agentId: 'analyst3',
+      provider: Providers.ANTHROPIC,
+      clientOptions: {
+        modelName: 'claude-haiku-4-5',
+        apiKey: process.env.ANTHROPIC_API_KEY,
+      },
+      instructions: `You are MARKET ANALYST. Provide 2-3 sentence market analysis. Start with "MARKET ANALYSIS:"`,
+    },
+    {
+      agentId: 'summarizer',
+      provider: Providers.ANTHROPIC,
+      clientOptions: {
+        modelName: 'claude-haiku-4-5',
+        apiKey: process.env.ANTHROPIC_API_KEY,
+      },
+      instructions: `You are SYNTHESIS EXPERT. Review all analyses and provide 2-3 sentence integrated summary.`,
+    },
+  ];
+
+  // Define direct edges (fan-out and fan-in)
+  const edges: t.GraphEdge[] = [
+    {
+      from: 'researcher',
+      to: ['analyst1', 'analyst2', 'analyst3'],
+      description: 'Distribute research to specialist analysts',
+      edgeType: 'direct',
+    },
+    {
+      from: ['analyst1', 'analyst2', 'analyst3'],
+      to: 'summarizer',
+      description: 'Aggregate analysis results',
+      edgeType: 'direct',
+      prompt:
+        'Based on the analyses below, provide an integrated summary:\n\n{results}',
+    },
+  ];
+
+  // Create custom handlers
+  const customHandlers = {
+    [GraphEvents.TOOL_END]: new ToolEndHandler(),
+    [GraphEvents.CHAT_MODEL_END]: new ModelEndHandler(),
+    [GraphEvents.CHAT_MODEL_STREAM]: new ChatModelStreamHandler(),
+    [GraphEvents.ON_RUN_STEP_COMPLETED]: {
+      handle: (
+        event: GraphEvents.ON_RUN_STEP_COMPLETED,
+        data: t.StreamEventData
+      ): void => {
+        aggregateContent({
+          event,
+          data: data as unknown as { result: t.ToolEndEvent },
+        });
+      },
+    },
+    [GraphEvents.ON_RUN_STEP]: {
+      handle: (
+        event: GraphEvents.ON_RUN_STEP,
+        data: t.StreamEventData
+      ): void => {
+        aggregateContent({ event, data: data as t.RunStep });
+      },
+    },
+    [GraphEvents.ON_MESSAGE_DELTA]: {
+      handle: (
+        event: GraphEvents.ON_MESSAGE_DELTA,
+        data: t.StreamEventData
+      ): void => {
+        aggregateContent({ event, data: data as t.MessageDeltaEvent });
+      },
+    },
+  };
+
+  try {
+    const query = 'What are the implications of widespread AI adoption?';
+
+    console.log(`${'='.repeat(80)}`);
+    console.log(`FIRST RUN - USER QUERY: "${query}"`);
+    console.log('='.repeat(80));
+
+    // Reset conversation
+    conversationHistory.length = 0;
+    conversationHistory.push(new HumanMessage(query));
+
+    // Create graph
+    const runConfig: t.RunConfig = {
+      runId: `parallel-test-${Date.now()}`,
+      graphConfig: {
+        type: 'multi-agent',
+        agents,
+        edges,
+      },
+      customHandlers,
+      returnContent: true,
+    };
+
+    const run = await Run.create(runConfig);
+
+    console.log('\nProcessing first run with parallel agents...\n');
+
+    const config = {
+      configurable: {
+        thread_id: 'parallel-agent-labeling-1',
+      },
+      streamMode: 'values',
+      version: 'v2' as const,
+    };
+
+    const inputs = {
+      messages: conversationHistory,
+    };
+
+    await run.processStream(inputs, config);
+    const finalMessages = run.getRunMessages();
+
+    if (finalMessages) {
+      conversationHistory.push(...finalMessages);
+    }
+
+    // Show agent participation
+    console.log(`\n${'─'.repeat(80)}`);
+    console.log('FIRST RUN - AGENT PARTICIPATION:');
+    console.log('─'.repeat(80));
+
+    if (run.Graph) {
+      const activeAgents = run.Graph.getActiveAgentIds();
+      console.log(`\nActive agents (${activeAgents.length}):`, activeAgents);
+
+      const stepsByAgent = run.Graph.getRunStepsByAgent();
+      stepsByAgent.forEach((steps, agentId) => {
+        console.log(`  ${agentId}: ${steps.length} steps`);
+      });
+
+      console.log(`\nTotal content parts: ${contentParts.length}`);
+    }
+
+    // =============================================================
+    // SECOND RUN: Test with agent-labeled history
+    // =============================================================
+    console.log(`\n${'='.repeat(80)}`);
+    console.log(`SECOND RUN - Simulating DB Load with Parallel Agent Labeling`);
+    console.log('='.repeat(80));
+
+    // Simulate DB storage
+    const dbStoredContentParts = [...contentParts];
+    const dbStoredAgentIdMap = Object.fromEntries(
+      run.Graph!.getContentPartAgentMap()
+    );
+
+    console.log('\n📦 Simulating DB storage:');
+    console.log(`  - Stored ${dbStoredContentParts.length} content parts`);
+    console.log(
+      `  - Stored agent mappings for ${Object.keys(dbStoredAgentIdMap).length} parts`
+    );
+
+    // Load and label by agent with labelNonTransferContent option
+    console.log('\n📥 Loading from DB and labeling ALL agent content...');
+
+    const agentNames = {
+      researcher: 'Researcher',
+      analyst1: 'Financial Analyst',
+      analyst2: 'Technical Analyst',
+      analyst3: 'Market Analyst',
+      summarizer: 'Synthesizer',
+    };
+
+    const labeledContentParts = labelContentByAgent(
+      dbStoredContentParts.filter(
+        (p): p is t.MessageContentComplex => p != null
+      ),
+      dbStoredAgentIdMap,
+      agentNames,
+      { labelNonTransferContent: true } // NEW: Label all content
+    );
+
+    console.log(
+      `  - Labeled ${labeledContentParts.length} content groups by agent`
+    );
+
+    // Convert to payload
+    const payload: t.TPayload = [
+      {
+        role: 'user',
+        content: query,
+      },
+      {
+        role: 'assistant',
+        content: labeledContentParts,
+      },
+    ];
+
+    // Format using formatAgentMessages
+    console.log('\n🔧 Calling formatAgentMessages...');
+    const { messages: formattedMessages } = formatAgentMessages(payload);
+
+    console.log(`  - Formatted into ${formattedMessages.length} BaseMessages`);
+
+    // Show preview
+    console.log('\n👁️  Preview of formatted history:');
+    console.log('─'.repeat(80));
+    for (let i = 0; i < formattedMessages.length; i++) {
+      const msg = formattedMessages[i];
+      const role = msg._getType();
+      const preview =
+        typeof msg.content === 'string'
+          ? msg.content.slice(0, 300)
+          : JSON.stringify(msg.content).slice(0, 300);
+      console.log(
+        `[${i}] ${role}: ${preview}${preview.length >= 300 ? '...' : ''}`
+      );
+      console.log('');
+    }
+    console.log('─'.repeat(80));
+
+    // Create a second run with labeled history
+    console.log(
+      '\n🚀 Starting second run with agent-labeled parallel history...'
+    );
+    const followupQuery = 'Which analyst identified the most significant risk?';
+    console.log(`   Followup: "${followupQuery}"`);
+
+    const secondRunHistory: BaseMessage[] = [
+      ...formattedMessages,
+      new HumanMessage(followupQuery),
+    ];
+
+    const runConfig2: t.RunConfig = {
+      runId: `parallel-test-2-${Date.now()}`,
+      graphConfig: {
+        type: 'multi-agent',
+        agents,
+        edges,
+      },
+      customHandlers,
+      returnContent: true,
+    };
+
+    const run2 = await Run.create(runConfig2);
+
+    const inputs2 = {
+      messages: secondRunHistory,
+    };
+
+    await run2.processStream(inputs2, config);
+
+    console.log('\n✅ Second run completed successfully!');
+    console.log(
+      '   The researcher correctly understood that parallel analysts handled'
+    );
+    console.log('   the previous analysis, with clear attribution per agent.');
+
+    console.log(`\n${'='.repeat(80)}`);
+    console.log('TEST COMPLETE');
+    console.log('='.repeat(80));
+    console.log(
+      '\nThis demonstrates that parallel multi-agent patterns work correctly'
+    );
+    console.log(
+      'with agent labeling, preventing confusion about who said what.'
+    );
+  } catch (error) {
+    console.error('Error in parallel agent labeling test:', error);
+  }
+}
+
+// Run the test
+testParallelWithAgentLabeling();

package/src/scripts/test-parallel-handoffs.ts

@@ -0,0 +1,291 @@
+import { config } from 'dotenv';
+config();
+
+import { HumanMessage, BaseMessage } from '@langchain/core/messages';
+import type * as t from '@/types';
+import { ChatModelStreamHandler, createContentAggregator } from '@/stream';
+import { ToolEndHandler } from '@/events';
+import { Providers, GraphEvents } from '@/common';
+import { sleep } from '@/utils/run';
+import { Run } from '@/run';
+
+const conversationHistory: BaseMessage[] = [];
+
+/**
+ * Test parallel handoffs - where an LLM calls multiple transfer tools simultaneously
+ *
+ * Graph structure:
+ * coordinator -> [researcher, writer] (via parallel handoff tools)
+ *
+ * The coordinator agent has two transfer tools:
+ * - transfer_to_researcher
+ * - transfer_to_writer
+ *
+ * When given a task that needs both, it should call both tools in parallel.
+ */
+async function testParallelHandoffs() {
+  console.log(
+    'Testing Parallel Handoffs (LLM calling multiple transfers)...\n'
+  );
+
+  const { contentParts, aggregateContent } = createContentAggregator();
+
+  const agents: t.AgentInputs[] = [
+    {
+      agentId: 'coordinator',
+      provider: Providers.OPENAI,
+      clientOptions: {
+        modelName: 'gpt-4o-mini',
+        apiKey: process.env.OPENAI_API_KEY,
+      },
+      instructions: `You are a COORDINATOR agent. Your job is to delegate tasks to specialized agents.
+
+You have access to two transfer tools:
+- transfer_to_researcher: For research and fact-finding tasks
+- transfer_to_writer: For content creation and writing tasks
+
+IMPORTANT: When a task requires BOTH research AND writing, you MUST call BOTH transfer tools SIMULTANEOUSLY in the same response. Do not call them sequentially.
+
+For example, if asked to "research and write about X", call both transfers at once to enable parallel work.
+
+When delegating, provide clear instructions to each agent about what they should do.`,
+    },
+    {
+      agentId: 'researcher',
+      provider: Providers.ANTHROPIC,
+      clientOptions: {
+        modelName: 'claude-haiku-4-5',
+        apiKey: process.env.ANTHROPIC_API_KEY,
+      },
+      instructions: `You are a RESEARCHER. When you receive a task:
+1. Provide concise research findings (100-150 words)
+2. Start your response with "📚 RESEARCH FINDINGS:"`,
+    },
+    {
+      agentId: 'writer',
+      provider: Providers.ANTHROPIC,
+      clientOptions: {
+        modelName: 'claude-haiku-4-5',
+        apiKey: process.env.ANTHROPIC_API_KEY,
+      },
+      instructions: `You are a WRITER. When you receive a task:
+1. Provide creative content (100-150 words)
+2. Start your response with "✍️ WRITTEN CONTENT:"`,
+    },
+  ];
+
+  /**
+   * Create handoff edges from coordinator to both researcher and writer.
+   * These are separate edges so the LLM sees both transfer tools.
+   */
+  const edges: t.GraphEdge[] = [
+    {
+      from: 'coordinator',
+      to: 'researcher',
+      edgeType: 'handoff',
+      description: 'Transfer to researcher for research and fact-finding tasks',
+      prompt: 'Research task instructions',
+    },
+    {
+      from: 'coordinator',
+      to: 'writer',
+      edgeType: 'handoff',
+      description: 'Transfer to writer for content creation and writing tasks',
+      prompt: 'Writing task instructions',
+    },
+  ];
+
+  /** Track which agents are active and their timing */
+  const activeAgents = new Set<string>();
+  const agentTimings: Record<string, { start?: number; end?: number }> = {};
+  const startTime = Date.now();
+
+  const customHandlers = {
+    [GraphEvents.TOOL_END]: new ToolEndHandler(),
+    [GraphEvents.CHAT_MODEL_END]: {
+      handle: (
+        _event: string,
+        _data: t.StreamEventData,
+        metadata?: Record<string, unknown>
+      ): void => {
+        const nodeName = metadata?.langgraph_node as string;
+        if (nodeName) {
+          const elapsed = Date.now() - startTime;
+          agentTimings[nodeName] = agentTimings[nodeName] || {};
+          agentTimings[nodeName].end = elapsed;
+          activeAgents.delete(nodeName);
+          console.log(`\n⏱️  [${nodeName}] COMPLETED at ${elapsed}ms`);
+        }
+      },
+    },
+    [GraphEvents.CHAT_MODEL_START]: {
+      handle: (
+        _event: string,
+        _data: t.StreamEventData,
+        metadata?: Record<string, unknown>
+      ): void => {
+        const nodeName = metadata?.langgraph_node as string;
+        if (nodeName) {
+          const elapsed = Date.now() - startTime;
+          /** Store first start time for parallel overlap calculation */
+          if (!agentTimings[nodeName]?.start) {
+            agentTimings[nodeName] = agentTimings[nodeName] || {};
+            agentTimings[nodeName].start = elapsed;
+          }
+          activeAgents.add(nodeName);
+          console.log(`\n⏱️  [${nodeName}] STARTED at ${elapsed}ms`);
+          console.log(
+            `   Active agents: ${Array.from(activeAgents).join(', ')}`
+          );
+        }
+      },
+    },
+    [GraphEvents.CHAT_MODEL_STREAM]: new ChatModelStreamHandler(),
+    [GraphEvents.ON_RUN_STEP_COMPLETED]: {
+      handle: (
+        event: GraphEvents.ON_RUN_STEP_COMPLETED,
+        data: t.StreamEventData
+      ): void => {
+        aggregateContent({
+          event,
+          data: data as unknown as { result: t.ToolEndEvent },
+        });
+      },
+    },
+    [GraphEvents.ON_RUN_STEP]: {
+      handle: (
+        event: GraphEvents.ON_RUN_STEP,
+        data: t.StreamEventData
+      ): void => {
+        aggregateContent({ event, data: data as t.RunStep });
+      },
+    },
+    [GraphEvents.ON_RUN_STEP_DELTA]: {
+      handle: (
+        event: GraphEvents.ON_RUN_STEP_DELTA,
+        data: t.StreamEventData
+      ): void => {
+        aggregateContent({ event, data: data as t.RunStepDeltaEvent });
+      },
+    },
+    [GraphEvents.ON_MESSAGE_DELTA]: {
+      handle: (
+        event: GraphEvents.ON_MESSAGE_DELTA,
+        data: t.StreamEventData
+      ): void => {
+        aggregateContent({ event, data: data as t.MessageDeltaEvent });
+      },
+    },
+  };
+
+  const runConfig: t.RunConfig = {
+    runId: `parallel-handoffs-${Date.now()}`,
+    graphConfig: {
+      type: 'multi-agent',
+      agents,
+      edges,
+    },
+    customHandlers,
+    returnContent: true,
+  };
+
+  try {
+    const run = await Run.create(runConfig);
+
+    /** Prompt designed to trigger parallel handoffs without confusing language */
+    const userMessage = `Help me with two topics:
+1. The history of the internet
+2. A short poem about technology
+
+I need information on both topics.`;
+
+    conversationHistory.push(new HumanMessage(userMessage));
+
+    console.log('User message:', userMessage);
+    console.log(
+      '\nInvoking multi-agent graph with parallel handoff request...\n'
+    );
+
+    const config = {
+      configurable: {
+        thread_id: 'parallel-handoffs-test-1',
+      },
+      streamMode: 'values',
+      version: 'v2' as const,
+    };
+
+    const inputs = {
+      messages: conversationHistory,
+    };
+
+    await run.processStream(inputs, config);
+    const finalMessages = run.getRunMessages();
+
+    if (finalMessages) {
+      conversationHistory.push(...finalMessages);
+    }
+
+    /** Analyze parallel execution */
+    console.log('\n\n========== TIMING SUMMARY ==========');
+    console.log('Available timing keys:', Object.keys(agentTimings));
+    for (const [agent, timing] of Object.entries(agentTimings)) {
+      const duration =
+        timing.end && timing.start ? timing.end - timing.start : 'N/A';
+      console.log(
+        `${agent}: started=${timing.start}ms, ended=${timing.end}ms, duration=${duration}ms`
+      );
+    }
+
+    /** Check if researcher and writer ran in parallel (handle key variations) */
+    const researcherKey = Object.keys(agentTimings).find((k) =>
+      k.includes('researcher')
+    );
+    const writerKey = Object.keys(agentTimings).find((k) =>
+      k.includes('writer')
+    );
+    const researcherTiming = researcherKey
+      ? agentTimings[researcherKey]
+      : undefined;
+    const writerTiming = writerKey ? agentTimings[writerKey] : undefined;
+
+    if (researcherTiming && writerTiming) {
+      const bothStarted = researcherTiming.start && writerTiming.start;
+      const bothEnded = researcherTiming.end && writerTiming.end;
+
+      if (bothStarted && bothEnded) {
+        const overlap =
+          Math.min(researcherTiming.end!, writerTiming.end!) -
+          Math.max(researcherTiming.start!, writerTiming.start!);
+
+        if (overlap > 0) {
+          console.log(
+            `\n✅ PARALLEL HANDOFFS SUCCESSFUL: ${overlap}ms overlap between researcher and writer`
+          );
+        } else {
+          console.log(
+            `\n⚠️  SEQUENTIAL EXECUTION: researcher and writer did not overlap`
+          );
+          console.log(
+            `   This may indicate the LLM called transfers sequentially, not in parallel`
+          );
+        }
+      }
+    } else {
+      console.log(
+        '\n⚠️  Not all agents were invoked. Check if handoffs occurred.'
+      );
+      console.log('   researcher timing:', researcherTiming);
+      console.log('   writer timing:', writerTiming);
+    }
+    console.log('====================================\n');
+
+    console.log('Final content parts:', contentParts.length, 'parts');
+    console.dir(contentParts, { depth: null });
+    await sleep(3000);
+  } catch (error) {
+    console.error('Error in parallel handoffs test:', error);
+    throw error;
+  }
+}
+
+testParallelHandoffs();