@librechat/agents 3.0.35 → 3.0.40

package/src/scripts/code_exec_ptc.ts

@@ -0,0 +1,277 @@
+// src/scripts/code_exec_ptc.ts
+/**
+ * Live LLM test for Programmatic Tool Calling (PTC).
+ * Run with: npm run code_exec_ptc
+ *
+ * Tests PTC with a real LLM in the loop, demonstrating:
+ * 1. LLM decides when to use PTC
+ * 2. LLM writes Python code that calls tools programmatically
+ * 3. ToolNode automatically injects programmatic tools
+ * 4. Tools filtered by allowed_callers
+ */
+import { config } from 'dotenv';
+config();
+
+import { randomUUID } from 'crypto';
+import { HumanMessage, BaseMessage } from '@langchain/core/messages';
+import type { RunnableConfig } from '@langchain/core/runnables';
+import type * as t from '@/types';
+import { ChatModelStreamHandler, createContentAggregator } from '@/stream';
+import {
+  ToolEndHandler,
+  ModelEndHandler,
+  createMetadataAggregator,
+} from '@/events';
+import { getLLMConfig } from '@/utils/llmConfig';
+import { getArgs } from '@/scripts/args';
+import { GraphEvents } from '@/common';
+import { Run } from '@/run';
+import { createCodeExecutionTool } from '@/tools/CodeExecutor';
+import { createProgrammaticToolCallingTool } from '@/tools/ProgrammaticToolCalling';
+import {
+  createGetTeamMembersTool,
+  createGetExpensesTool,
+  createGetWeatherTool,
+  createProgrammaticToolRegistry,
+} from '@/test/mockTools';
+
+const conversationHistory: BaseMessage[] = [];
+
+async function testProgrammaticToolCalling(): Promise<void> {
+  const { userName, location, provider, currentDate } = await getArgs();
+  const { contentParts, aggregateContent } = createContentAggregator();
+
+  const customHandlers = {
+    [GraphEvents.TOOL_END]: new ToolEndHandler(),
+    [GraphEvents.CHAT_MODEL_END]: new ModelEndHandler(),
+    [GraphEvents.CHAT_MODEL_STREAM]: new ChatModelStreamHandler(),
+    [GraphEvents.ON_RUN_STEP_COMPLETED]: {
+      handle: (
+        event: GraphEvents.ON_RUN_STEP_COMPLETED,
+        data: t.StreamEventData
+      ): void => {
+        console.log('====== ON_RUN_STEP_COMPLETED ======');
+        console.dir(data, { depth: null });
+        aggregateContent({
+          event,
+          data: data as unknown as { result: t.ToolEndEvent },
+        });
+      },
+    },
+    [GraphEvents.ON_RUN_STEP]: {
+      handle: (
+        event: GraphEvents.ON_RUN_STEP,
+        data: t.StreamEventData
+      ): void => {
+        console.log('====== ON_RUN_STEP ======');
+        console.dir(data, { depth: null });
+        aggregateContent({ event, data: data as t.RunStep });
+      },
+    },
+    [GraphEvents.ON_RUN_STEP_DELTA]: {
+      handle: (
+        event: GraphEvents.ON_RUN_STEP_DELTA,
+        data: t.StreamEventData
+      ): void => {
+        aggregateContent({ event, data: data as t.RunStepDeltaEvent });
+      },
+    },
+    [GraphEvents.ON_MESSAGE_DELTA]: {
+      handle: (
+        event: GraphEvents.ON_MESSAGE_DELTA,
+        data: t.StreamEventData
+      ): void => {
+        aggregateContent({ event, data: data as t.MessageDeltaEvent });
+      },
+    },
+    [GraphEvents.TOOL_START]: {
+      handle: (
+        _event: string,
+        data: t.StreamEventData,
+        metadata?: Record<string, unknown>
+      ): void => {
+        console.log('====== TOOL_START ======');
+        console.dir(data, { depth: null });
+      },
+    },
+  };
+
+  const llmConfig = getLLMConfig(provider);
+
+  // Create mock tool instances
+  const teamTool = createGetTeamMembersTool();
+  const expensesTool = createGetExpensesTool();
+  const weatherTool = createGetWeatherTool();
+
+  // Create special tools
+  const codeExecTool = createCodeExecutionTool();
+  const ptcTool = createProgrammaticToolCallingTool();
+
+  // Build complete tool list and map
+  const allTools = [teamTool, expensesTool, weatherTool, codeExecTool, ptcTool];
+  const toolMap = new Map(allTools.map((t) => [t.name, t]));
+
+  // Create tool registry with allowed_callers configuration
+  // Only includes business logic tools (not special tools)
+  // Special tools (execute_code, PTC) are always bound directly to LLM
+  const toolRegistry = createProgrammaticToolRegistry();
+
+  console.log('\n' + '='.repeat(70));
+  console.log('Tool Configuration Summary:');
+  console.log('='.repeat(70));
+  console.log('Total tools:', allTools.length);
+  console.log(
+    'Programmatic-allowed:',
+    Array.from(toolRegistry.values())
+      .filter((t) => t.allowed_callers?.includes('code_execution'))
+      .map((t) => t.name)
+      .join(', ')
+  );
+  console.log(
+    'Direct-callable:',
+    Array.from(toolRegistry.values())
+      .filter((t) => !t.allowed_callers || t.allowed_callers.includes('direct'))
+      .map((t) => t.name)
+      .join(', ')
+  );
+  console.log('='.repeat(70) + '\n');
+
+  const run = await Run.create<t.IState>({
+    runId: randomUUID(),
+    graphConfig: {
+      type: 'standard',
+      llmConfig,
+      agents: [
+        {
+          agentId: 'default',
+          provider: llmConfig.provider,
+          clientOptions: llmConfig,
+          tools: allTools,
+          toolMap,
+          toolRegistry,
+          instructions:
+            'You are a friendly AI assistant with advanced coding capabilities. ' +
+            'You have access to team and expense management tools, but ONLY through programmatic code execution. ' +
+            'When you need to analyze expenses or process team data, use the programmatic_code_execution tool ' +
+            'to write Python code that calls get_team_members(), get_expenses(), and get_weather() functions. ' +
+            'These functions are async - use await. Use asyncio.gather() for parallel execution.',
+          additional_instructions: `The user's name is ${userName} and they are located in ${location}. Today is ${currentDate}.`,
+        },
+      ],
+    },
+    returnContent: true,
+    customHandlers,
+  });
+
+  const config: Partial<RunnableConfig> & {
+    version: 'v1' | 'v2';
+    run_id?: string;
+    streamMode: string;
+  } = {
+    configurable: {
+      provider,
+      thread_id: 'ptc-conversation-1',
+    },
+    streamMode: 'values',
+    version: 'v2' as const,
+  };
+
+  console.log('Test 1: Team Expense Analysis with PTC');
+  console.log('='.repeat(70) + '\n');
+
+  const userMessage1 = `Hi ${userName}! I need you to analyze our team's expenses. Please:
+
+1. Get the list of all team members
+2. For each member, fetch their expense records
+3. Calculate the total expenses per member
+4. Identify anyone who spent more than $500
+5. Show me a summary report
+
+IMPORTANT: Use the programmatic_code_execution tool to do this efficiently. 
+Don't call each tool separately - write Python code that orchestrates all the calls!`;
+
+  conversationHistory.push(new HumanMessage(userMessage1));
+
+  let inputs = {
+    messages: conversationHistory,
+  };
+
+  const finalContentParts1 = await run.processStream(inputs, config);
+  const finalMessages1 = run.getRunMessages();
+  if (finalMessages1) {
+    conversationHistory.push(...finalMessages1);
+  }
+
+  console.log('\n\n====================\n\n');
+  console.log('Content Parts:');
+  console.dir(contentParts, { depth: null });
+
+  console.log('\n\n' + '='.repeat(70));
+  console.log('Test 2: Conditional Logic and Parallel Execution');
+  console.log('='.repeat(70) + '\n');
+
+  const userMessage2 = `Great job! Now let's test some advanced patterns. Please:
+
+1. Check the weather in both San Francisco and New York (in parallel!)
+2. Based on which city has better weather (warmer), fetch the team members
+3. For the Engineering team members only, calculate their travel expenses
+4. Show me the results
+
+Again, use programmatic_code_execution for maximum efficiency. Use asyncio.gather() 
+to check both cities' weather at the same time!`;
+
+  conversationHistory.push(new HumanMessage(userMessage2));
+
+  inputs = {
+    messages: conversationHistory,
+  };
+
+  const finalContentParts2 = await run.processStream(inputs, config);
+  const finalMessages2 = run.getRunMessages();
+  if (finalMessages2) {
+    conversationHistory.push(...finalMessages2);
+  }
+
+  console.log('\n\n====================\n\n');
+  console.log('Final Content Parts:');
+  console.dir(finalContentParts2, { depth: null });
+
+  console.log('\n\n' + '='.repeat(70));
+  console.log('Generating conversation title...');
+  console.log('='.repeat(70) + '\n');
+
+  const { handleLLMEnd, collected } = createMetadataAggregator();
+  const titleResult = await run.generateTitle({
+    provider,
+    inputText: userMessage1,
+    contentParts,
+    chainOptions: {
+      callbacks: [
+        {
+          handleLLMEnd,
+        },
+      ],
+    },
+  });
+
+  console.log('Generated Title:', titleResult);
+  console.log('Collected metadata:', collected);
+}
+
+process.on('unhandledRejection', (reason, promise) => {
+  console.error('Unhandled Rejection at:', promise, 'reason:', reason);
+  console.log('Conversation history:');
+  console.dir(conversationHistory, { depth: null });
+  process.exit(1);
+});
+
+process.on('uncaughtException', (err) => {
+  console.error('Uncaught Exception:', err);
+});
+
+testProgrammaticToolCalling().catch((err) => {
+  console.error(err);
+  console.log('Conversation history:');
+  console.dir(conversationHistory, { depth: null });
+  process.exit(1);
+});

package/src/scripts/programmatic_exec.ts

@@ -0,0 +1,396 @@
+// src/scripts/programmatic_exec.ts
+/**
+ * Test script for Programmatic Tool Calling (PTC).
+ * Run with: npm run programmatic_exec
+ *
+ * Demonstrates:
+ * 1. Runtime toolMap injection - the tool map is passed at invocation time
+ * 2. Tool classification with allowed_callers (inspired by Anthropic's API)
+ *    - 'direct': Tool can only be called directly by the LLM (default)
+ *    - 'code_execution': Tool can only be called from within PTC
+ *    - Both: Tool can be called either way
+ *
+ * IMPORTANT: The Python code passed to PTC should NOT define the tool functions.
+ * The Code API automatically generates async function stubs from the tool definitions.
+ * The code should just CALL the tools as if they're already available:
+ *   - result = await get_weather(city="SF")
+ *   - results = await asyncio.gather(tool1(), tool2())
+ */
+import { config } from 'dotenv';
+config();
+
+import type { StructuredToolInterface } from '@langchain/core/tools';
+import type { LCTool, ToolMap } from '@/types';
+import { createProgrammaticToolCallingTool } from '@/tools/ProgrammaticToolCalling';
+import {
+  createGetTeamMembersTool,
+  createGetExpensesTool,
+  createGetWeatherTool,
+  createCalculatorTool,
+  createProgrammaticToolRegistry,
+} from '@/test/mockTools';
+
+// ============================================================================
+// Test Runner
+// ============================================================================
+
+interface RunTestOptions {
+  toolMap: ToolMap;
+  tools?: LCTool[];
+  session_id?: string;
+  timeout?: number;
+  showArtifact?: boolean;
+}
+
+async function runTest(
+  ptcTool: ReturnType<typeof createProgrammaticToolCallingTool>,
+  testName: string,
+  code: string,
+  options: RunTestOptions
+): Promise<void> {
+  console.log(`\n${'='.repeat(70)}`);
+  console.log(`TEST: ${testName}`);
+  console.log('='.repeat(70));
+  console.log('\nCode:');
+  console.log('```python');
+  console.log(code.trim());
+  console.log('```\n');
+
+  try {
+    const startTime = Date.now();
+
+    // Manual testing: schema params + extras (LangChain moves extras to config.toolCall)
+    const result = await ptcTool.invoke({
+      code,
+      tools: options.tools,
+      session_id: options.session_id,
+      timeout: options.timeout,
+      toolMap: options.toolMap, // Non-schema field → config.toolCall.toolMap
+    });
+
+    const duration = Date.now() - startTime;
+
+    console.log(`Result (${duration}ms):`);
+    if (Array.isArray(result)) {
+      console.log(result[0]);
+      if (options.showArtifact) {
+        console.log('\n--- Artifact ---');
+        console.dir(result[1], { depth: null });
+      }
+    } else {
+      console.log(result);
+    }
+  } catch (error) {
+    console.error('Error:', error instanceof Error ? error.message : error);
+  }
+}
+
+// ============================================================================
+// Main
+// ============================================================================
+
+async function main(): Promise<void> {
+  console.log('Programmatic Tool Calling (PTC) - Test Script');
+  console.log('==============================================');
+  console.log('Demonstrating runtime toolMap injection\n');
+
+  const apiKey = process.env.LIBRECHAT_CODE_API_KEY;
+  if (!apiKey) {
+    console.error(
+      'Error: LIBRECHAT_CODE_API_KEY environment variable is not set.'
+    );
+    console.log('Please set it in your .env file or environment.');
+    process.exit(1);
+  }
+
+  console.log('Creating mock tools...');
+  const mockTools: StructuredToolInterface[] = [
+    createGetTeamMembersTool(),
+    createGetExpensesTool(),
+    createGetWeatherTool(),
+    createCalculatorTool(),
+  ];
+
+  const toolMap: ToolMap = new Map(mockTools.map((t) => [t.name, t]));
+  const toolDefinitions = Array.from(createProgrammaticToolRegistry().values());
+
+  console.log(
+    `ToolMap contains ${toolMap.size} tools: ${Array.from(toolMap.keys()).join(', ')}`
+  );
+
+  console.log('\nCreating PTC tool (without toolMap)...');
+  const ptcTool = createProgrammaticToolCallingTool({ apiKey });
+  console.log('PTC tool created successfully!');
+  console.log(
+    'Note: toolMap will be passed at runtime with each invocation.\n'
+  );
+
+  const baseOptions = { toolMap, tools: toolDefinitions };
+
+  // =========================================================================
+  // Test 1: Simple async tool call
+  // =========================================================================
+  await runTest(
+    ptcTool,
+    'Simple async tool call',
+    `
+# Tools are auto-generated as async functions - just await them
+result = await get_weather(city="San Francisco")
+print(f"Weather in SF: {result['temperature']}°F, {result['condition']}")
+    `,
+    { ...baseOptions, showArtifact: true }
+  );
+
+  // =========================================================================
+  // Test 2: Sequential loop with await
+  // =========================================================================
+  await runTest(
+    ptcTool,
+    'Sequential loop - Process team expenses',
+    `
+# Each tool call uses await
+team = await get_team_members()
+print("Team expense report:")
+print("-" * 30)
+total = 0
+for member in team:
+    expenses = await get_expenses(user_id=member['id'])
+    member_total = sum(e['amount'] for e in expenses)
+    total += member_total
+    print(f"{member['name']}: \${member_total:.2f}")
+print("-" * 30)
+print(f"Total: \${total:.2f}")
+    `,
+    baseOptions
+  );
+
+  // =========================================================================
+  // Test 3: Parallel execution with asyncio.gather
+  // =========================================================================
+  await runTest(
+    ptcTool,
+    'Parallel execution - Weather for multiple cities',
+    `
+# Use asyncio.gather for parallel tool calls - single round-trip!
+import asyncio
+
+cities = ["San Francisco", "New York", "London"]
+results = await asyncio.gather(*[
+    get_weather(city=city)
+    for city in cities
+])
+
+print("Weather report:")
+for city, weather in zip(cities, results):
+    print(f"  {city}: {weather['temperature']}°F, {weather['condition']}")
+    `,
+    baseOptions
+  );
+
+  // =========================================================================
+  // Test 4: Chained dependencies
+  // =========================================================================
+  await runTest(
+    ptcTool,
+    'Chained dependencies - Get team then process each',
+    `
+# Get team first, then fetch expenses for each
+team = await get_team_members()
+engineering = [m for m in team if m['department'] == 'Engineering']
+
+print(f"Engineering team ({len(engineering)} members):")
+for member in engineering:
+    expenses = await get_expenses(user_id=member['id'])
+    equipment = sum(e['amount'] for e in expenses if e['category'] == 'equipment')
+    print(f"  {member['name']}: \${equipment:.2f} on equipment")
+    `,
+    baseOptions
+  );
+
+  // =========================================================================
+  // Test 5: Conditional logic
+  // =========================================================================
+  await runTest(
+    ptcTool,
+    'Conditional logic - Find high spenders',
+    `
+team = await get_team_members()
+high_spenders = []
+
+for member in team:
+    expenses = await get_expenses(user_id=member['id'])
+    total = sum(e['amount'] for e in expenses)
+    if total > 300:
+        high_spenders.append((member['name'], total))
+
+if high_spenders:
+    print("High spenders (over $300):")
+    for name, amount in sorted(high_spenders, key=lambda x: x[1], reverse=True):
+        print(f"  {name}: \${amount:.2f}")
+else:
+    print("No high spenders found.")
+    `,
+    baseOptions
+  );
+
+  // =========================================================================
+  // Test 6: Mixed parallel and sequential
+  // =========================================================================
+  await runTest(
+    ptcTool,
+    'Mixed - Parallel expense fetch after sequential team fetch',
+    `
+import asyncio
+
+# Step 1: Get team (one tool call)
+team = await get_team_members()
+print(f"Fetched {len(team)} team members")
+
+# Step 2: Get all expenses in parallel (single round-trip for all!)
+all_expenses = await asyncio.gather(*[
+    get_expenses(user_id=member['id'])
+    for member in team
+])
+
+# Step 3: Process and output
+print("\\nExpense summary:")
+for member, expenses in zip(team, all_expenses):
+    total = sum(e['amount'] for e in expenses)
+    print(f"  {member['name']}: \${total:.2f} ({len(expenses)} items)")
+    `,
+    baseOptions
+  );
+
+  // =========================================================================
+  // Test 7: Calculator usage
+  // =========================================================================
+  await runTest(
+    ptcTool,
+    'Calculator tool usage',
+    `
+# All tools are async - use await
+result1 = await calculator(expression="2 + 2 * 3")
+result2 = await calculator(expression="(10 + 5) / 3")
+
+print(f"2 + 2 * 3 = {result1['result']}")
+print(f"(10 + 5) / 3 = {result2['result']:.2f}")
+    `,
+    baseOptions
+  );
+
+  // =========================================================================
+  // Test 8: Error handling in code
+  // =========================================================================
+  await runTest(
+    ptcTool,
+    'Error handling - Invalid city',
+    `
+# Tool errors become Python exceptions - handle with try/except
+cities = ["San Francisco", "InvalidCity", "New York"]
+
+for city in cities:
+    try:
+        weather = await get_weather(city=city)
+        print(f"{city}: {weather['temperature']}°F")
+    except Exception as e:
+        print(f"{city}: Error - {e}")
+    `,
+    baseOptions
+  );
+
+  // =========================================================================
+  // Test 9: Early termination
+  // =========================================================================
+  await runTest(
+    ptcTool,
+    'Early termination - Stop when condition met',
+    `
+# Stop as soon as we find what we need - no wasted tool calls
+team = await get_team_members()
+
+for member in team:
+    expenses = await get_expenses(user_id=member['id'])
+    if any(e['category'] == 'equipment' for e in expenses):
+        print(f"First team member with equipment expense: {member['name']}")
+        equipment_total = sum(e['amount'] for e in expenses if e['category'] == 'equipment')
+        print(f"Equipment total: \${equipment_total:.2f}")
+        break
+else:
+    print("No team member has equipment expenses")
+    `,
+    baseOptions
+  );
+
+  // =========================================================================
+  // Test 10: Subset of tools
+  // =========================================================================
+  await runTest(
+    ptcTool,
+    'Subset of tools - Only weather',
+    `
+# Only the weather tool is available in this execution
+import asyncio
+
+sf, nyc = await asyncio.gather(
+    get_weather(city="San Francisco"),
+    get_weather(city="New York")
+)
+print(f"SF: {sf['temperature']}°F vs NYC: {nyc['temperature']}°F")
+difference = abs(sf['temperature'] - nyc['temperature'])
+print(f"Temperature difference: {difference}°F")
+    `,
+    {
+      ...baseOptions,
+      tools: [toolDefinitions.find((t) => t.name === 'get_weather')!],
+    }
+  );
+
+  // =========================================================================
+  // Test 11: Note about ToolNode injection
+  // =========================================================================
+  console.log(`\n${'='.repeat(70)}`);
+  console.log('NOTE: ToolNode Runtime Injection');
+  console.log('='.repeat(70));
+  console.log(
+    '\nWhen PTC is invoked through ToolNode in a real agent:\n' +
+      '- ToolNode detects call.name === "programmatic_code_execution"\n' +
+      '- ToolNode injects: { ...invokeParams, toolMap, programmaticToolDefs }\n' +
+      '- PTC tool extracts these from params (not from config)\n' +
+      '- No explicit tools parameter needed in schema\n\n' +
+      'This test demonstrates param injection with explicit tools:\n'
+  );
+
+  await runTest(
+    ptcTool,
+    'Runtime injection with explicit tools',
+    `
+# ToolNode would inject toolMap+programmaticToolDefs
+# For this test, we pass tools explicitly (same effect)
+team = await get_team_members()
+print(f"Team size: {len(team)}")
+for member in team:
+    print(f"- {member['name']} ({member['department']})")
+    `,
+    baseOptions
+  );
+
+  console.log('\n' + '='.repeat(70));
+  console.log('All tests completed!');
+  console.log('='.repeat(70) + '\n');
+  console.log('Summary of allowed_callers patterns:');
+  console.log(
+    '- get_team_members, get_expenses, calculator: code_execution only'
+  );
+  console.log('- get_weather: both direct and code_execution');
+  console.log(
+    '\nIn a real agent setup, the LLM would only see tools with allowed_callers'
+  );
+  console.log(
+    'including "direct", while PTC can call any tool with "code_execution".\n'
+  );
+}
+
+main().catch((err) => {
+  console.error('Fatal error:', err);
+  process.exit(1);
+});