illuma-agents 1.0.59 → 1.0.61

package/src/graphs/Graph.ts

@@ -785,14 +785,11 @@ export class StandardGraph extends Graph<t.BaseGraphState, t.GraphNode> {
     let lastError: Error | undefined;
     let attempts = 0;
 
-    // Create a silent config for structured output - we don't want tool call events
-    // to be emitted for the synthetic "response" tool used by withStructuredOutput()
-    // This prevents the UI from showing a fake tool call
-    const silentConfig = config ? { ...config, callbacks: undefined } : undefined;
-
     while (attempts <= maxRetries) {
       try {
-        const result = await structuredModel.invoke(finalMessages, silentConfig);
+        // Note: We pass the original config here. The stream aggregator will filter out
+        // the synthetic "response" tool call events from withStructuredOutput()
+        const result = await structuredModel.invoke(finalMessages, config);
         
         // Debug: log what we got back
         console.log('[Graph] Structured output raw result type:', typeof result);
@@ -1139,10 +1136,17 @@ export class StandardGraph extends Graph<t.BaseGraphState, t.GraphNode> {
       );
 
       // Check if structured output mode is enabled
-      if (
+      // IMPORTANT: If tools are available, we need to let the model use them first.
+      // Only use structured output when:
+      // 1. No tools are configured, OR
+      // 2. The model has already used tools and is ready to give final response
+      const hasTools = agentContext.tools && agentContext.tools.length > 0;
+      const shouldUseStructuredOutputNow = 
         agentContext.isStructuredOutputMode &&
-        agentContext.structuredOutput
-      ) {
+        agentContext.structuredOutput &&
+        !hasTools; // Only use structured output immediately if no tools
+
+      if (shouldUseStructuredOutputNow) {
         const schema = agentContext.getStructuredOutputSchema();
         if (!schema) {
           throw new Error('Structured output schema is not configured');
@@ -1193,7 +1197,7 @@ export class StandardGraph extends Graph<t.BaseGraphState, t.GraphNode> {
                 currentModel: structuredModel,
                 finalMessages,
                 schema,
-                structuredOutputConfig: agentContext.structuredOutput,
+                structuredOutputConfig: agentContext.structuredOutput!,
                 provider: agentContext.provider,
               },
               config
@@ -1466,6 +1470,107 @@ If I seem to be missing something we discussed earlier, just give me a quick rem
       if (!result) {
         throw new Error('No result after model invocation');
       }
+
+      // Check if we need to apply structured output for the final response
+      // This handles the case where tools were used, and now the model is giving its final answer
+      const resultMessage = result.messages?.[0] as AIMessageChunk | undefined;
+      const hasToolCalls = (resultMessage?.tool_calls?.length ?? 0) > 0;
+      
+      if (
+        agentContext.isStructuredOutputMode &&
+        agentContext.structuredOutput &&
+        !hasToolCalls && // Model is giving final response (no tool calls)
+        hasTools // We skipped structured output earlier because tools were available
+      ) {
+        const schema = agentContext.getStructuredOutputSchema();
+        if (schema) {
+          try {
+            console.log('[Graph] Applying structured output for final response after tool execution');
+            
+            // Get a fresh model for structured output
+            const structuredClientOptions = { ...agentContext.clientOptions } as t.ClientOptions;
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            (structuredClientOptions as any).streaming = false;
+
+            // Remove thinking configuration
+            if (agentContext.provider === Providers.BEDROCK) {
+              const bedrockOpts = structuredClientOptions as t.BedrockAnthropicClientOptions;
+              if (bedrockOpts.additionalModelRequestFields) {
+                // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                const additionalFields = Object.assign({}, bedrockOpts.additionalModelRequestFields) as any;
+                delete additionalFields.thinking;
+                delete additionalFields.budgetTokens;
+                bedrockOpts.additionalModelRequestFields = additionalFields;
+              }
+            }
+            if (agentContext.provider === Providers.ANTHROPIC) {
+              const anthropicOpts = structuredClientOptions as t.AnthropicClientOptions;
+              if (anthropicOpts.thinking) {
+                delete anthropicOpts.thinking;
+              }
+            }
+
+            const structuredModel = this.getNewModel({
+              provider: agentContext.provider,
+              clientOptions: structuredClientOptions,
+            });
+
+            // Include the current result in the messages so the model knows what it just said
+            const messagesWithResult = [...finalMessages];
+            if (resultMessage) {
+              messagesWithResult.push(resultMessage);
+            }
+
+            const { structuredResponse, rawMessage } =
+              await this.attemptStructuredInvoke(
+                {
+                  currentModel: structuredModel,
+                  finalMessages: messagesWithResult,
+                  schema,
+                  structuredOutputConfig: agentContext.structuredOutput!,
+                  provider: agentContext.provider,
+                },
+                config
+              );
+
+            // Emit structured output event
+            await safeDispatchCustomEvent(
+              GraphEvents.ON_STRUCTURED_OUTPUT,
+              {
+                structuredResponse,
+                schema,
+                raw: rawMessage,
+              },
+              config
+            );
+
+            agentContext.currentUsage = rawMessage
+              ? this.getUsageMetadata(rawMessage)
+              : undefined;
+            this.cleanupSignalListener();
+
+            // Return clean message without tool_calls
+            let cleanMessage: AIMessageChunk | undefined;
+            if (rawMessage) {
+              cleanMessage = new AIMessageChunk({
+                content: JSON.stringify(structuredResponse, null, 2),
+                id: rawMessage.id,
+                response_metadata: rawMessage.response_metadata,
+                usage_metadata: rawMessage.usage_metadata,
+              });
+            }
+
+            return {
+              messages: cleanMessage ? [cleanMessage] : [],
+              structuredResponse,
+            };
+          } catch (structuredError) {
+            console.error('[Graph] Structured output failed after tool execution:', structuredError);
+            // Fall through to return normal result
+          }
+        }
+      }
+
       agentContext.currentUsage = this.getUsageMetadata(result.messages?.[0]);
       this.cleanupSignalListener();
       return result;