illuma-agents 1.0.10 → 1.0.12

package/src/common/enum.ts

@@ -159,6 +159,8 @@ export enum Callback {
 export enum Constants {
   OFFICIAL_CODE_BASEURL = 'https://api.illuma.ai/v1',
   EXECUTE_CODE = 'execute_code',
+  TOOL_SEARCH_REGEX = 'tool_search_regex',
+  PROGRAMMATIC_TOOL_CALLING = 'run_tools_with_code',
   WEB_SEARCH = 'web_search',
   CONTENT_AND_ARTIFACT = 'content_and_artifact',
   LC_TRANSFER_TO_ = 'lc_transfer_to_',

package/src/events.ts

@@ -9,7 +9,7 @@ import type { MultiAgentGraph, StandardGraph } from '@/graphs';
 import type { Logger } from 'winston';
 import type * as t from '@/types';
 import { handleToolCalls } from '@/tools/handlers';
-import { Providers } from '@/common';
+import { Constants, Providers } from '@/common';
 
 export class HandlerRegistry {
   private handlers: Map<string, t.EventHandler> = new Map();
@@ -92,17 +92,6 @@ export class ToolEndHandler implements t.EventHandler {
     metadata?: Record<string, unknown>,
     graph?: StandardGraph | MultiAgentGraph
   ): Promise<void> {
-    const toolData = data as t.ToolEndData;
-    const output = toolData?.output as ToolMessage | undefined;
-    console.log('[ToolEndHandler] handle called', { 
-      event, 
-      hasData: !!data,
-      hasGraph: !!graph,
-      hasMetadata: !!metadata,
-      hasCallback: !!this.callback,
-      outputName: output?.name,
-      hasArtifact: !!output?.artifact,
-    });
     try {
       if (!graph || !metadata) {
         if (this.logger) {
@@ -123,6 +112,10 @@ export class ToolEndHandler implements t.EventHandler {
         return;
       }
 
+      if (metadata[Constants.PROGRAMMATIC_TOOL_CALLING] === true) {
+        return;
+      }
+
       if (this.callback) {
         await this.callback(toolEndData, metadata);
       }

package/src/graphs/Graph.ts

@@ -21,6 +21,7 @@ import {
   ToolMessage,
   SystemMessage,
   AIMessageChunk,
+  HumanMessage,
 } from '@langchain/core/messages';
 import type {
   BaseMessageFields,
@@ -46,6 +47,7 @@ import {
   formatContentStrings,
   createPruneMessages,
   addCacheControl,
+  extractToolDiscoveries,
 } from '@/messages';
 import {
   resetIfNotEmpty,
@@ -443,9 +445,11 @@ export class StandardGraph extends Graph<t.BaseGraphState, t.GraphNode> {
   initializeTools({
     currentTools,
     currentToolMap,
+    agentContext,
   }: {
     currentTools?: t.GraphTools;
     currentToolMap?: t.ToolMap;
+    agentContext?: AgentContext;
   }): CustomToolNode<t.BaseGraphState> | ToolNode<t.BaseGraphState> {
     return new CustomToolNode<t.BaseGraphState>({
       tools: (currentTools as t.GenericTool[] | undefined) ?? [],
@@ -453,6 +457,7 @@ export class StandardGraph extends Graph<t.BaseGraphState, t.GraphNode> {
       toolCallStepIds: this.toolCallStepIds,
       errorHandler: (data, metadata) =>
         StandardGraph.handleToolCallErrorStatic(this, data, metadata),
+      toolRegistry: agentContext?.toolRegistry,
     });
   }
 
@@ -603,7 +608,7 @@ export class StandardGraph extends Graph<t.BaseGraphState, t.GraphNode> {
     client.abortHandler = undefined;
   }
 
-  createCallModel(agentId = 'default', currentModel?: t.ChatModel) {
+  createCallModel(agentId = 'default') {
     return async (
       state: t.BaseGraphState,
       config?: RunnableConfig
@@ -616,15 +621,54 @@ export class StandardGraph extends Graph<t.BaseGraphState, t.GraphNode> {
         throw new Error(`Agent context not found for agentId: ${agentId}`);
       }
 
-      const model = this.overrideModel ?? currentModel;
-      if (!model) {
-        throw new Error('No Graph model found');
-      }
       if (!config) {
         throw new Error('No config provided');
       }
 
-      // Ensure token calculations are complete before proceeding
+      let { messages } = state;
+
+      // CACHE OPTIMIZATION: Inject dynamicContext as a HumanMessage at the start of conversation
+      // This keeps the system message static (cacheable) while providing dynamic context
+      // (timestamps, user info, tool context) as conversation content instead.
+      // Only inject on the first turn when messages don't already have the context marker.
+      if (
+        agentContext.dynamicContext &&
+        messages.length > 0 &&
+        !messages.some(
+          (m) =>
+            m instanceof HumanMessage &&
+            typeof m.content === 'string' &&
+            m.content.startsWith('[SESSION_CONTEXT]')
+        )
+      ) {
+        const dynamicContextMessage = new HumanMessage({
+          content: `[SESSION_CONTEXT]\n${agentContext.dynamicContext}`,
+        });
+        const ackMessage = new AIMessageChunk({
+          content: 'Understood. I have noted the session context including the current date/time (CST) and will apply it appropriately.',
+        });
+        messages = [dynamicContextMessage, ackMessage, ...messages];
+      }
+
+      // Extract tool discoveries from current turn only (similar to formatArtifactPayload pattern)
+      const discoveredNames = extractToolDiscoveries(messages);
+      if (discoveredNames.length > 0) {
+        agentContext.markToolsAsDiscovered(discoveredNames);
+      }
+
+      const toolsForBinding = agentContext.getToolsForBinding();
+      let model =
+        this.overrideModel ??
+        this.initializeModel({
+          tools: toolsForBinding,
+          provider: agentContext.provider,
+          clientOptions: agentContext.clientOptions,
+        });
+
+      if (agentContext.systemRunnable) {
+        model = agentContext.systemRunnable.pipe(model as Runnable);
+      }
+
       if (agentContext.tokenCalculationPromise) {
         await agentContext.tokenCalculationPromise;
       }
@@ -632,7 +676,6 @@ export class StandardGraph extends Graph<t.BaseGraphState, t.GraphNode> {
         config.signal = this.signal;
       }
       this.config = config;
-      const { messages } = state;
 
       let messagesToUse = messages;
       if (
@@ -730,7 +773,11 @@ export class StandardGraph extends Graph<t.BaseGraphState, t.GraphNode> {
         const bedrockOptions = agentContext.clientOptions as
           | t.BedrockAnthropicClientOptions
           | undefined;
-        if (bedrockOptions?.promptCache === true) {
+        // Both Claude and Nova models support cachePoint in system and messages
+        // (Llama, Titan, and other models do NOT support cachePoint)
+        const modelId = bedrockOptions?.model?.toLowerCase() ?? '';
+        const supportsCaching = modelId.includes('claude') || modelId.includes('anthropic') || modelId.includes('nova');
+        if (bedrockOptions?.promptCache === true && supportsCaching) {
           finalMessages = addBedrockCacheControl<BaseMessage>(finalMessages);
         }
       }
@@ -844,16 +891,6 @@ export class StandardGraph extends Graph<t.BaseGraphState, t.GraphNode> {
       throw new Error(`Agent context not found for agentId: ${agentId}`);
     }
 
-    let currentModel = this.initializeModel({
-      tools: agentContext.tools,
-      provider: agentContext.provider,
-      clientOptions: agentContext.clientOptions,
-    });
-
-    if (agentContext.systemRunnable) {
-      currentModel = agentContext.systemRunnable.pipe(currentModel);
-    }
-
     const agentNode = `${AGENT}${agentId}` as const;
     const toolNode = `${TOOLS}${agentId}` as const;
 
@@ -873,12 +910,13 @@ export class StandardGraph extends Graph<t.BaseGraphState, t.GraphNode> {
     });
 
     const workflow = new StateGraph(StateAnnotation)
-      .addNode(agentNode, this.createCallModel(agentId, currentModel))
+      .addNode(agentNode, this.createCallModel(agentId))
       .addNode(
         toolNode,
         this.initializeTools({
           currentTools: agentContext.tools,
           currentToolMap: agentContext.toolMap,
+          agentContext,
         })
       )
       .addEdge(START, agentNode)

package/src/index.ts

@@ -11,6 +11,8 @@ export * from './graphs';
 /* Tools */
 export * from './tools/Calculator';
 export * from './tools/CodeExecutor';
+export * from './tools/ProgrammaticToolCalling';
+export * from './tools/ToolSearchRegex';
 export * from './tools/handlers';
 export * from './tools/search';
 

package/src/instrumentation.ts

@@ -11,10 +11,7 @@ if (
     publicKey: process.env.LANGFUSE_PUBLIC_KEY,
     secretKey: process.env.LANGFUSE_SECRET_KEY,
     baseUrl: process.env.LANGFUSE_BASE_URL,
-    environment:
-      process.env.LANGFUSE_TRACING_ENVIRONMENT ??
-      process.env.NODE_ENV ??
-      'development',
+    environment: process.env.LANGFUSE_TRACING_ENVIRONMENT ?? process.env.NODE_ENV ?? 'development',
   });
 
   const sdk = new NodeSDK({

package/src/llm/bedrock/__tests__/bedrock-caching.test.ts

@@ -0,0 +1,473 @@
+/**
+ * Unit tests for Bedrock Prompt Caching functionality
+ * 
+ * Tests cover:
+ * 1. CustomChatBedrockConverse - Tool caching with cachePoint
+ * 2. AgentContext - System message caching with cachePoint for Bedrock
+ */
+
+import { CustomChatBedrockConverse } from '../index';
+import { AgentContext } from '@/agents/AgentContext';
+import { Providers } from '@/common';
+import type * as t from '@/types';
+
+describe('Bedrock Prompt Caching', () => {
+  describe('CustomChatBedrockConverse - Tool Caching', () => {
+    describe('invocationParams with promptCache enabled', () => {
+      it('should add cachePoint to tools array when promptCache is true', () => {
+        const model = new CustomChatBedrockConverse({
+          model: 'anthropic.claude-3-haiku-20240307-v1:0',
+          region: 'us-east-1',
+          promptCache: true,
+        });
+
+        const mockTools = [
+          {
+            toolSpec: {
+              name: 'get_weather',
+              description: 'Get weather for a location',
+              inputSchema: { json: { type: 'object', properties: {} } },
+            },
+          },
+          {
+            toolSpec: {
+              name: 'search_web',
+              description: 'Search the web',
+              inputSchema: { json: { type: 'object', properties: {} } },
+            },
+          },
+        ];
+
+        const params = model.invocationParams({
+          tools: mockTools,
+        } as any);
+
+        // Should have tools + cachePoint
+        expect(params.toolConfig?.tools).toHaveLength(3);
+        expect(params.toolConfig?.tools?.[2]).toEqual({
+          cachePoint: { type: 'default' },
+        });
+      });
+
+      it('should NOT add cachePoint when promptCache is false', () => {
+        const model = new CustomChatBedrockConverse({
+          model: 'anthropic.claude-3-haiku-20240307-v1:0',
+          region: 'us-east-1',
+          promptCache: false,
+        });
+
+        const mockTools = [
+          {
+            toolSpec: {
+              name: 'get_weather',
+              description: 'Get weather',
+              inputSchema: { json: { type: 'object', properties: {} } },
+            },
+          },
+        ];
+
+        const params = model.invocationParams({
+          tools: mockTools,
+        } as any);
+
+        // Should only have original tools, no cachePoint
+        expect(params.toolConfig?.tools).toHaveLength(1);
+        expect(params.toolConfig?.tools?.[0]).toEqual(mockTools[0]);
+      });
+
+      it('should NOT add cachePoint when promptCache is undefined (default)', () => {
+        const model = new CustomChatBedrockConverse({
+          model: 'anthropic.claude-3-haiku-20240307-v1:0',
+          region: 'us-east-1',
+        });
+
+        const mockTools = [
+          {
+            toolSpec: {
+              name: 'get_weather',
+              description: 'Get weather',
+              inputSchema: { json: { type: 'object', properties: {} } },
+            },
+          },
+        ];
+
+        const params = model.invocationParams({
+          tools: mockTools,
+        } as any);
+
+        // Should only have original tools
+        expect(params.toolConfig?.tools).toHaveLength(1);
+      });
+
+      it('should NOT add cachePoint when no tools are provided', () => {
+        const model = new CustomChatBedrockConverse({
+          model: 'anthropic.claude-3-haiku-20240307-v1:0',
+          region: 'us-east-1',
+          promptCache: true,
+        });
+
+        const params = model.invocationParams({} as any);
+
+        // toolConfig should be undefined or have no tools
+        expect(params.toolConfig?.tools).toBeUndefined();
+      });
+
+      it('should NOT add cachePoint when tools array is empty', () => {
+        const model = new CustomChatBedrockConverse({
+          model: 'anthropic.claude-3-haiku-20240307-v1:0',
+          region: 'us-east-1',
+          promptCache: true,
+        });
+
+        const params = model.invocationParams({
+          tools: [],
+        } as any);
+
+        // Empty tools array results in undefined toolConfig
+        expect(params.toolConfig).toBeUndefined();
+      });
+
+      it('should preserve other invocationParams properties', () => {
+        const model = new CustomChatBedrockConverse({
+          model: 'anthropic.claude-3-haiku-20240307-v1:0',
+          region: 'us-east-1',
+          promptCache: true,
+          temperature: 0.7,
+          maxTokens: 1000,
+        });
+
+        const mockTools = [
+          {
+            toolSpec: {
+              name: 'test_tool',
+              description: 'Test',
+              inputSchema: { json: { type: 'object', properties: {} } },
+            },
+          },
+        ];
+
+        const params = model.invocationParams({
+          tools: mockTools,
+        } as any);
+
+        // Check that other params are preserved
+        expect(params.inferenceConfig?.temperature).toBe(0.7);
+        expect(params.inferenceConfig?.maxTokens).toBe(1000);
+        // And cachePoint is still added
+        expect(params.toolConfig?.tools).toHaveLength(2);
+      });
+    });
+
+    describe('promptCache property', () => {
+      it('should store promptCache value from constructor', () => {
+        const modelWithCache = new CustomChatBedrockConverse({
+          model: 'test-model',
+          region: 'us-east-1',
+          promptCache: true,
+        });
+
+        const modelWithoutCache = new CustomChatBedrockConverse({
+          model: 'test-model',
+          region: 'us-east-1',
+          promptCache: false,
+        });
+
+        expect(modelWithCache.promptCache).toBe(true);
+        expect(modelWithoutCache.promptCache).toBe(false);
+      });
+
+      it('should default promptCache to false when not provided', () => {
+        const model = new CustomChatBedrockConverse({
+          model: 'test-model',
+          region: 'us-east-1',
+        });
+
+        expect(model.promptCache).toBe(false);
+      });
+    });
+  });
+
+  describe('AgentContext - System Message Caching', () => {
+    const createBedrockContext = (options: {
+      instructions?: string;
+      promptCache?: boolean;
+      additionalInstructions?: string;
+    }): AgentContext => {
+      const clientOptions: t.BedrockAnthropicInput = {
+        model: 'anthropic.claude-3-haiku-20240307-v1:0',
+        region: 'us-east-1',
+        promptCache: options.promptCache,
+      };
+
+      return AgentContext.fromConfig({
+        agentId: 'test-bedrock-agent',
+        provider: Providers.BEDROCK,
+        instructions: options.instructions,
+        additional_instructions: options.additionalInstructions,
+        clientOptions,
+      });
+    };
+
+    const createAnthropicContext = (options: {
+      instructions?: string;
+      promptCachingEnabled?: boolean;
+    }): AgentContext => {
+      const clientOptions: t.AnthropicClientOptions = {
+        clientOptions: {
+          defaultHeaders: options.promptCachingEnabled
+            ? { 'anthropic-beta': 'prompt-caching-2024-07-31' }
+            : undefined,
+        },
+      };
+
+      return AgentContext.fromConfig({
+        agentId: 'test-anthropic-agent',
+        provider: Providers.ANTHROPIC,
+        instructions: options.instructions,
+        clientOptions,
+      });
+    };
+
+    describe('Bedrock system message with promptCache: true', () => {
+      it('should add cachePoint to system message content', async () => {
+        const ctx = createBedrockContext({
+          instructions: 'You are a helpful assistant.',
+          promptCache: true,
+        });
+
+        const systemRunnable = ctx.systemRunnable;
+        expect(systemRunnable).toBeDefined();
+
+        // Invoke the runnable to get the messages
+        const result = await systemRunnable!.invoke([]);
+        const systemMessage = result[0];
+
+        // Check content structure has cachePoint
+        expect(systemMessage.content).toBeInstanceOf(Array);
+        const content = systemMessage.content as Array<Record<string, unknown>>;
+        
+        expect(content).toHaveLength(2);
+        expect(content[0]).toEqual({
+          type: 'text',
+          text: 'You are a helpful assistant.',
+        });
+        expect(content[1]).toEqual({
+          cachePoint: { type: 'default' },
+        });
+      });
+
+      it('should include combined instructions and additional_instructions with cachePoint', async () => {
+        const ctx = createBedrockContext({
+          instructions: 'Base instructions.',
+          additionalInstructions: 'Additional context.',
+          promptCache: true,
+        });
+
+        const result = await ctx.systemRunnable!.invoke([]);
+        const systemMessage = result[0];
+        const content = systemMessage.content as Array<Record<string, unknown>>;
+
+        expect(content).toHaveLength(2);
+        expect((content[0] as { text: string }).text).toContain('Base instructions.');
+        expect((content[0] as { text: string }).text).toContain('Additional context.');
+        expect(content[1]).toEqual({
+          cachePoint: { type: 'default' },
+        });
+      });
+    });
+
+    describe('Bedrock system message with promptCache: false', () => {
+      it('should NOT add cachePoint when promptCache is false', async () => {
+        const ctx = createBedrockContext({
+          instructions: 'You are a helpful assistant.',
+          promptCache: false,
+        });
+
+        const result = await ctx.systemRunnable!.invoke([]);
+        const systemMessage = result[0];
+
+        // Content should be plain string, not array with cachePoint
+        expect(typeof systemMessage.content).toBe('string');
+        expect(systemMessage.content).toBe('You are a helpful assistant.');
+      });
+
+      it('should NOT add cachePoint when promptCache is undefined', async () => {
+        const ctx = createBedrockContext({
+          instructions: 'You are a helpful assistant.',
+          promptCache: undefined,
+        });
+
+        const result = await ctx.systemRunnable!.invoke([]);
+        const systemMessage = result[0];
+
+        expect(typeof systemMessage.content).toBe('string');
+      });
+    });
+
+    describe('Anthropic system message caching (for comparison)', () => {
+      it('should add cache_control for Anthropic with prompt-caching beta', async () => {
+        const ctx = createAnthropicContext({
+          instructions: 'You are a helpful assistant.',
+          promptCachingEnabled: true,
+        });
+
+        const result = await ctx.systemRunnable!.invoke([]);
+        const systemMessage = result[0];
+        const content = systemMessage.content as Array<Record<string, unknown>>;
+
+        expect(content).toHaveLength(1);
+        expect(content[0]).toEqual({
+          type: 'text',
+          text: 'You are a helpful assistant.',
+          cache_control: { type: 'ephemeral' },
+        });
+      });
+
+      it('should NOT add cache_control for Anthropic without beta header', async () => {
+        const ctx = createAnthropicContext({
+          instructions: 'You are a helpful assistant.',
+          promptCachingEnabled: false,
+        });
+
+        const result = await ctx.systemRunnable!.invoke([]);
+        const systemMessage = result[0];
+
+        expect(typeof systemMessage.content).toBe('string');
+      });
+    });
+
+    describe('Provider-specific caching behavior', () => {
+      it('should use cachePoint format for Bedrock, not cache_control', async () => {
+        const bedrockCtx = createBedrockContext({
+          instructions: 'Test',
+          promptCache: true,
+        });
+
+        const result = await bedrockCtx.systemRunnable!.invoke([]);
+        const content = result[0].content as Array<Record<string, unknown>>;
+
+        // Bedrock uses cachePoint, NOT cache_control
+        expect(content.some((c) => 'cachePoint' in c)).toBe(true);
+        expect(content.some((c) => 'cache_control' in c)).toBe(false);
+      });
+
+      it('should use cache_control format for Anthropic, not cachePoint', async () => {
+        const anthropicCtx = createAnthropicContext({
+          instructions: 'Test',
+          promptCachingEnabled: true,
+        });
+
+        const result = await anthropicCtx.systemRunnable!.invoke([]);
+        const content = result[0].content as Array<Record<string, unknown>>;
+
+        // Anthropic uses cache_control, NOT cachePoint
+        expect(content.some((c) => 'cache_control' in c)).toBe(true);
+        expect(content.some((c) => 'cachePoint' in c)).toBe(false);
+      });
+
+      it('should not add any caching for OpenAI provider', async () => {
+        const openaiCtx = AgentContext.fromConfig({
+          agentId: 'test-openai-agent',
+          provider: Providers.OPENAI,
+          instructions: 'Test instructions',
+        });
+
+        const result = await openaiCtx.systemRunnable!.invoke([]);
+        const systemMessage = result[0];
+
+        // OpenAI should have plain string content
+        expect(typeof systemMessage.content).toBe('string');
+        expect(systemMessage.content).toBe('Test instructions');
+      });
+    });
+  });
+
+  describe('Integration scenarios', () => {
+    it('should handle multi-turn caching scenario', async () => {
+      // Simulate: Query 1 with tools → Query 2 with same tools
+      // Both should have cachePoint, enabling cache reuse
+      
+      const model = new CustomChatBedrockConverse({
+        model: 'anthropic.claude-3-haiku-20240307-v1:0',
+        region: 'us-east-1',
+        promptCache: true,
+      });
+
+      const tools = [
+        {
+          toolSpec: {
+            name: 'execute_code',
+            description: 'Execute Python code',
+            inputSchema: { json: { type: 'object', properties: {} } },
+          },
+        },
+        {
+          toolSpec: {
+            name: 'file_search',
+            description: 'Search files',
+            inputSchema: { json: { type: 'object', properties: {} } },
+          },
+        },
+      ];
+
+      // Query 1
+      const params1 = model.invocationParams({ tools } as any);
+      
+      // Query 2 (same tools)
+      const params2 = model.invocationParams({ tools } as any);
+
+      // Both should have identical tool configs with cachePoint
+      expect(params1.toolConfig?.tools).toEqual(params2.toolConfig?.tools);
+      expect(params1.toolConfig?.tools).toHaveLength(3); // 2 tools + cachePoint
+    });
+
+    it('should handle dynamic tool selection scenario', async () => {
+      // Simulate: Query 1 with [A, B] → Query 2 with [A, B, C]
+      // Tool set changed, so cache won't match (expected behavior)
+      
+      const model = new CustomChatBedrockConverse({
+        model: 'anthropic.claude-3-haiku-20240307-v1:0',
+        region: 'us-east-1',
+        promptCache: true,
+      });
+
+      const toolA = {
+        toolSpec: {
+          name: 'tool_a',
+          description: 'Tool A',
+          inputSchema: { json: { type: 'object', properties: {} } },
+        },
+      };
+
+      const toolB = {
+        toolSpec: {
+          name: 'tool_b',
+          description: 'Tool B',
+          inputSchema: { json: { type: 'object', properties: {} } },
+        },
+      };
+
+      const toolC = {
+        toolSpec: {
+          name: 'tool_c',
+          description: 'Tool C',
+          inputSchema: { json: { type: 'object', properties: {} } },
+        },
+      };
+
+      // Query 1: [A, B]
+      const params1 = model.invocationParams({ tools: [toolA, toolB] } as any);
+      
+      // Query 2: [A, B, C] - different tool set
+      const params2 = model.invocationParams({ tools: [toolA, toolB, toolC] } as any);
+
+      // Different lengths (cache prefix won't match beyond common prefix)
+      expect(params1.toolConfig?.tools).toHaveLength(3); // 2 tools + cachePoint
+      expect(params2.toolConfig?.tools).toHaveLength(4); // 3 tools + cachePoint
+      
+      // But both have cachePoint at the end
+      expect(params1.toolConfig?.tools?.[2]).toEqual({ cachePoint: { type: 'default' } });
+      expect(params2.toolConfig?.tools?.[3]).toEqual({ cachePoint: { type: 'default' } });
+    });
+  });
+});