@microsoft/m365-copilot-eval 1.0.1-preview.1

package/src/clients/cli/response_extractor.py

@@ -0,0 +1,589 @@
+"""
+Enhanced Response Extraction Module
+
+This module provides functionality for extracting detailed response information including
+tool calls, tool results, and message flow reconstruction from agent responses.
+
+Key Components:
+- Message flow reconstruction
+- Tool invocation parsing
+- Enhanced response data structure
+- Too                        ]
+                                          ]
+                    ]
+                })
+        
+        return reconstructed                })
+        
+        return reconstructedi                        ]
+                    ]
+                })
+        
+        return reconstructedn extraction (placeholder for future implementation)
+
+Author: GitHub Copilot
+Date: September 21, 2025
+"""
+
+import json
+import logging
+from typing import Dict, List, Any, Optional
+from datetime import datetime
+from enum import Enum
+
+# Configure logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+class MessageRole(Enum):
+    """Enumeration for message roles."""
+    USER = "user"
+    ASSISTANT = "assistant"
+    TOOL = "tool"
+
+class ContentType(Enum):
+    """Enumeration for content types."""
+    TEXT = "text"
+    TOOL_CALL = "tool_call"
+    TOOL_RESULT = "tool_result"
+
+class ToolStatus(Enum):
+    """Enumeration for tool invocation status."""
+    SUCCESS = "Success"
+    FAILURE = "Failure"
+
+class MessageType(Enum):
+    """Enumeration for different message types in conversations."""
+    USER = "user"
+    BOT = "bot"
+    INTERNAL = "Internal"
+    INTERNAL_SEARCH = "InternalSearchResult"
+
+class EnhancedResponseExtractor:
+    """Enhanced extractor for detailed response information."""
+    
+    # List of internal tool names that should be filtered out
+    INTERNAL_TOOLS = {
+        "hydrate_tool_response",
+        "meta_prioritize", 
+        "reason",
+        "generate_express_response",
+        "generate_response"
+    }
+    
+    def __init__(self):
+        self.tool_call_counter = 0
+    
+    def _generate_tool_call_id(self, tool_name: str) -> str:
+        """Generate a unique tool call ID."""
+        self.tool_call_counter += 1
+        timestamp = datetime.now().strftime("%Y%m%d")
+        return f"tool_call_{timestamp}_{self.tool_call_counter:03d}_{tool_name}"
+    
+    def _is_internal_tool(self, tool_name: str) -> bool:
+        """
+        Check if a tool is an internal tool that should be filtered out.
+        
+        Args:
+            tool_name: Name of the tool
+            
+        Returns:
+            bool: True if internal tool, False otherwise
+        """
+        return tool_name in self.INTERNAL_TOOLS
+    
+    def _is_tool_message(self, message: Dict[str, Any]) -> bool:
+        """
+        Check if a message represents a tool invocation (excluding internal tools).
+        
+        Args:
+            message: Message dictionary
+            
+        Returns:
+            bool: True if tool message and not internal tool, False otherwise
+        """
+        # First check if it's a tool message at all
+        is_tool = (
+            (message.get("messageType") == MessageType.INTERNAL.value and 
+             message.get("contentOrigin") == "OpenAPI-spec") or
+            message.get("messageType") == MessageType.INTERNAL_SEARCH.value or
+            (message.get("messageType") == MessageType.INTERNAL.value and 
+             message.get("invocation") is not None)
+        )
+        
+        if not is_tool:
+            return False
+        
+        # Check if it's an internal tool that should be filtered out
+        invocation_str = message.get("invocation", "")
+        if invocation_str:
+            tool_info = self._parse_tool_invocation(invocation_str)
+            tool_name = tool_info.get("name", "")
+            if self._is_internal_tool(tool_name):
+                return False
+        
+        return True
+    
+    def _parse_tool_invocation(self, invocation_str: str) -> Dict[str, Any]:
+        """
+        Parse tool invocation string to extract tool name and arguments.
+        
+        Args:
+            invocation_str: Tool invocation string
+            
+        Returns:
+            Dict containing tool name and arguments
+        """
+        try:
+            # Handle Flux v3 format (JSON array)
+            if invocation_str.startswith('['):
+                invocation_data = json.loads(invocation_str)
+                if isinstance(invocation_data, list) and len(invocation_data) > 0:
+                    func_data = invocation_data[0].get("function", {})
+                    tool_name = func_data.get("name", "unknown_tool")
+                    arguments = json.loads(func_data.get("arguments", "{}"))
+                    return {"name": tool_name, "arguments": arguments}
+            
+            # Handle standard format: tool_name(arg1="value1", arg2="value2")
+            if "(" in invocation_str and ")" in invocation_str:
+                tool_name = invocation_str.split("(")[0].strip()
+                args_str = invocation_str[invocation_str.find("(")+1:invocation_str.rfind(")")]
+                
+                # Parse arguments
+                arguments = {}
+                if args_str.strip():
+                    # Simple parsing for key="value" format
+                    import re
+                    matches = re.findall(r'(\w+)=(["\'])(.*?)\2', args_str)
+                    for match in matches:
+                        key, _, value = match
+                        arguments[key] = value
+                
+                return {"name": tool_name, "arguments": arguments}
+            
+            # Fallback: treat as tool name without arguments
+            return {"name": invocation_str.strip(), "arguments": {}}
+            
+        except Exception as e:
+            logger.warning(f"Failed to parse tool invocation '{invocation_str}': {e}")
+            return {"name": "unknown_tool", "arguments": {}}
+    
+    def _extract_tool_results(self, message: Dict[str, Any]) -> Any:
+        """
+        Extract tool results from a message.
+        
+        Args:
+            message: Message dictionary
+            
+        Returns:
+            Tool results or None if extraction fails
+        """
+        try:
+            text = message.get("text", "")
+            if text:
+                result_data = json.loads(text)
+                
+                # Check for search metadata errors
+                if isinstance(result_data, dict):
+                    search_metadata = result_data.get("searchMetadata", {})
+                    if "error" in search_metadata.get("status", ""):
+                        return {"error": search_metadata.get("status", "Unknown error")}
+                    
+                    # Return results or the whole object
+                    return result_data.get("results", result_data)
+                
+                return result_data
+                
+        except json.JSONDecodeError:
+            # Return raw text if not JSON
+            return message.get("text", "")
+        except Exception as e:
+            logger.warning(f"Failed to extract tool results: {e}")
+            return None
+        
+        return None
+    
+    def _extract_telemetry_tools(self, telemetry: Dict[str, Any]) -> List[Dict[str, Any]]:
+        """
+        Extract tool invocations from telemetry data.
+        
+        Args:
+            telemetry: Telemetry data dictionary
+            
+        Returns:
+            List of tool invocation details
+        """
+        tools_invoked = []
+        tools_queue = []
+        
+        for metric in telemetry.get("metrics", []):
+            service_name = metric.get("serviceName")
+            
+            # Track tool invocations from FluxToolInvoker
+            if service_name == "FluxToolInvoker" and metric.get("status") == ToolStatus.SUCCESS.value:
+                try:
+                    output = json.loads(metric["output"])
+                    for item in output:
+                        invocation = item.get("invocation", "")
+                        # Parse tool name to check if it's internal
+                        tool_info = self._parse_tool_invocation(invocation)
+                        tool_name = tool_info.get("name", "")
+                        
+                        # Skip internal tools
+                        if not self._is_internal_tool(tool_name):
+                            tools_queue.append(invocation)
+                except (json.JSONDecodeError, KeyError):
+                    continue
+            
+            # Track tool results from ExtensionRunner
+            elif service_name in ["ExtensionRunner:ext:OpenAPI-spec", "ExtensionRunner:ext:enterprise-search"]:
+                if tools_queue:
+                    invocation_str = tools_queue.pop(0)
+                    tool_info = self._parse_tool_invocation(invocation_str)
+                    
+                    tool_data = {
+                        "invocation": invocation_str,
+                        "tool_name": tool_info["name"],
+                        "arguments": tool_info["arguments"],
+                        "status": metric.get("status", ToolStatus.FAILURE.value),
+                        "results": None
+                    }
+                    
+                    if metric.get("status") == ToolStatus.SUCCESS.value:
+                        try:
+                            api_response = json.loads(metric.get("output", "")).get("responses", [])
+                            if api_response:
+                                response_data = json.loads(api_response[0].get("text", ""))
+                                
+                                # Check for errors
+                                search_metadata = response_data.get("searchMetadata", {})
+                                if "error" in search_metadata.get("status", ""):
+                                    tool_data["status"] = ToolStatus.FAILURE.value
+                                    tool_data["results"] = {"error": search_metadata.get("status", "")}
+                                else:
+                                    tool_data["results"] = response_data.get("results", response_data)
+                        except Exception as e:
+                            logger.warning(f"Failed to parse tool results from telemetry: {e}")
+                            tool_data["status"] = ToolStatus.FAILURE.value
+                    
+                    tools_invoked.append(tool_data)
+        
+        return tools_invoked
+    
+    def _extract_tool_definitions(self, telemetry: Dict[str, Any]) -> List[Dict[str, Any]]:
+        """
+        Extract tool definitions from telemetry data.
+        
+        Args:
+            telemetry: Telemetry data dictionary
+            
+        Returns:
+            List of tool definition dictionaries
+        """
+        tool_definitions = []
+        
+        for metric in telemetry.get("metrics", []):
+            service_name = metric.get("serviceName")
+            
+            # Look for DeepLeoImprovedNetworking service with function invocation
+            if (service_name == "DeepLeoImprovedNetworking" and 
+                metric.get("output", "").startswith("CallTags: fluxv3:invokingfunction,")):
+                
+                try:
+                    # Parse the input field which contains the tool definitions
+                    input_str = metric.get("input", "")
+                    if input_str:
+                        input_data = json.loads(input_str)
+                        tools = input_data.get("tools", [])
+                        
+                        if tools and isinstance(tools, list):
+                            # Add tools to our definitions list (avoid duplicates and filter out internal tools)
+                            for tool in tools:
+                                # Check if this tool is internal
+                                tool_name = tool.get("function", {}).get("name", "")
+                                if not self._is_internal_tool(tool_name) and tool not in tool_definitions:
+                                    tool_definitions.append(tool.get("function", {}))
+                            
+                            logger.info(f"Extracted {len(tools)} tool definitions from telemetry")
+                            
+                except json.JSONDecodeError as e:
+                    logger.warning(f"Failed to parse tool definitions from telemetry input: {e}")
+                except Exception as e:
+                    logger.warning(f"Error extracting tool definitions: {e}")
+        
+        return tool_definitions
+    
+    def _reconstruct_message_flow(self, messages: List[Dict[str, Any]], telemetry_tools: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        """
+        Reconstruct the message flow including tool calls and results.
+        
+        Args:
+            messages: Original messages from response
+            telemetry_tools: Tool invocations from telemetry
+            
+        Returns:
+            List of reconstructed messages
+        """
+        reconstructed = []
+        telemetry_tool_index = 0
+        
+        for message in messages:
+            timestamp = message.get("createdAt", datetime.now().isoformat() + "Z")
+            run_id = message.get("conversationId", "0")
+            author = message.get("author", "")
+            
+            # Skip user messages entirely - we only want the agent's response flow
+            if author == MessageType.USER.value:
+                continue
+            
+            # Handle tool messages
+            elif self._is_tool_message(message):
+                invocation_str = message.get("invocation", "")
+                tool_info = self._parse_tool_invocation(invocation_str)
+                tool_call_id = self._generate_tool_call_id(tool_info["name"])
+                
+                # Add tool call message
+                reconstructed.append({
+                    "createdAt": timestamp,
+                    "run_id": run_id,
+                    "role": MessageRole.ASSISTANT.value,
+                    "content": [
+                        {
+                            "type": ContentType.TOOL_CALL.value,
+                            "tool_call_id": tool_call_id,
+                            "name": tool_info["name"],
+                            "arguments": tool_info["arguments"]
+                        }
+                    ]
+                })
+                
+                # Add tool result message
+                results = self._extract_tool_results(message)
+                reconstructed.append({
+                    "createdAt": timestamp,
+                    "run_id": run_id,
+                    "role": MessageRole.TOOL.value,
+                    "tool_call_id": tool_call_id,
+                    "content": [
+                        {
+                            "type": ContentType.TOOL_RESULT.value,
+                            ContentType.TOOL_RESULT.value: results
+                        }
+                    ]
+                })
+            
+            # Handle bot response messages
+            elif author == MessageType.BOT.value and "messageType" not in message:
+                # Check if we have unused telemetry tools to add before the final response
+                while telemetry_tool_index < len(telemetry_tools):
+                    tool = telemetry_tools[telemetry_tool_index]
+                    tool_call_id = self._generate_tool_call_id(tool["tool_name"])
+                    
+                    # Add tool call
+                    reconstructed.append({
+                        "createdAt": timestamp,
+                        "run_id": run_id,
+                        "role": MessageRole.ASSISTANT.value,
+                        "content": [
+                            {
+                                "type": ContentType.TOOL_CALL.value,
+                                "tool_call_id": tool_call_id,
+                                "name": tool["tool_name"],
+                                "arguments": tool["arguments"]
+                            }
+                        ]
+                    })
+                    
+                    # Add tool result
+                    reconstructed.append({
+                        "createdAt": timestamp,
+                        "run_id": run_id,
+                        "role": MessageRole.TOOL.value,
+                        "tool_call_id": tool_call_id,
+                        "content": [
+                            {
+                                "type": ContentType.TOOL_RESULT.value,
+                                ContentType.TOOL_RESULT.value: tool["results"]
+                            }
+                        ]
+                    })
+                    
+                    telemetry_tool_index += 1
+                
+                # Add final assistant response
+                reconstructed.append({
+                    "createdAt": timestamp,
+                    "run_id": run_id,
+                    "role": MessageRole.ASSISTANT.value,
+                    "content": [
+                        {
+                            "type": ContentType.TEXT.value,
+                            "text": message.get("text", "")
+                        }
+                    ]
+                })
+        return reconstructed
+    
+    def extract_enhanced_response(self, raw_response: str) -> Dict[str, Any]:
+        """
+        Extract enhanced response information from raw agent response.
+        
+        Args:
+            raw_response: Raw response string from agent
+            
+        Returns:
+            Dict containing enhanced response data
+        """
+        try:
+            # Parse the raw response
+            response_data = json.loads(raw_response)
+            
+            # Extract basic response text (fallback to original behavior)
+            response_text = ""
+            if isinstance(response_data, dict):
+                # Look for bot response in messages array
+                messages_for_text = response_data.get("messages", [])
+                for message in messages_for_text:
+                    if (message.get("author") == "bot" and 
+                        "messageType" not in message and 
+                        message.get("text")):
+                        response_text = message.get("text", "").strip()
+                        break
+            
+            if not response_text:
+                response_text = raw_response.strip()
+            
+            # Initialize enhanced structure
+            enhanced_response = {
+                "response": [],  # Will contain reconstructed message flow
+                "tool_definitions": [],  # Placeholder for future implementation
+                "raw_response_text": response_text,  # Backward compatibility
+                "metadata": {
+                    "conversation_id": response_data.get("conversationId"),
+                    "request_id": response_data.get("requestId"),
+                    "telemetry_available": False
+                }
+            }
+            
+            # Extract messages if available
+            messages = []
+            if isinstance(response_data, dict):
+                # Messages are directly in the response_data object
+                messages = response_data.get("messages", [])
+            
+            # Extract telemetry tools if available
+            telemetry_tools = []
+            tool_definitions = []
+            telemetry = response_data.get("telemetry", {})
+            if telemetry:
+                enhanced_response["metadata"]["telemetry_available"] = True
+                telemetry_tools = self._extract_telemetry_tools(telemetry)
+                tool_definitions = self._extract_tool_definitions(telemetry)
+            
+            # Update tool definitions in the response
+            enhanced_response["tool_definitions"] = tool_definitions
+            
+            # Reconstruct message flow
+            if messages:
+                enhanced_response["response"] = self._reconstruct_message_flow(messages, telemetry_tools)
+            else:
+                # Fallback: create simple text response
+                enhanced_response["response"] = [
+                    {
+                        "createdAt": datetime.now().isoformat() + "Z",
+                        "run_id": "0",
+                        "role": MessageRole.ASSISTANT.value,
+                        "content": [
+                            {
+                                "type": ContentType.TEXT.value,
+                                "text": response_text
+                            }
+                        ]
+                    }
+                ]
+            
+            return enhanced_response
+            
+        except json.JSONDecodeError:
+            # Handle non-JSON responses
+            logger.warning("Received non-JSON response, creating simple text response")
+            return {
+                "response": [
+                    {
+                        "createdAt": datetime.now().isoformat() + "Z",
+                        "run_id": "0",
+                        "role": MessageRole.ASSISTANT.value,
+                        "content": [
+                            {
+                                "type": ContentType.TEXT.value,
+                                "text": raw_response.strip()
+                            }
+                        ]
+                    }
+                ],
+                "tool_definitions": [],
+                "raw_response_text": raw_response.strip(),
+                "metadata": {
+                    "conversation_id": None,
+                    "request_id": None,
+                    "telemetry_available": False
+                }
+            }
+        
+        except Exception as e:
+            logger.error(f"Failed to extract enhanced response: {e}")
+            # Return minimal structure on error
+            return {
+                "response": [
+                    {
+                        "createdAt": datetime.now().isoformat() + "Z",
+                        "run_id": "0",
+                        "role": MessageRole.ASSISTANT.value,
+                        "content": [
+                            {
+                                "type": ContentType.TEXT.value,
+                                "text": raw_response.strip() if raw_response else "Error processing response"
+                            }
+                        ]
+                    }
+                ],
+                "tool_definitions": [],
+                "raw_response_text": raw_response.strip() if raw_response else "",
+                "metadata": {
+                    "conversation_id": None,
+                    "request_id": None,
+                    "telemetry_available": False,
+                    "error": str(e)
+                }
+            }
+
+def extract_enhanced_responses(responses: Dict[str, str]) -> Dict[str, Dict[str, Any]]:
+    """
+    Extract enhanced response information for multiple responses.
+    
+    Args:
+        responses: Dictionary mapping prompts to raw response strings
+        
+    Returns:
+        Dictionary mapping prompts to enhanced response data
+    """
+    extractor = EnhancedResponseExtractor()
+    enhanced_responses = {}
+    
+    for prompt, raw_response in responses.items():
+        enhanced_responses[prompt] = extractor.extract_enhanced_response(raw_response)
+    
+    return enhanced_responses
+
+def get_response_text_for_evaluation(enhanced_response: Dict[str, Any]) -> str:
+    """
+    Extract simple text response for evaluation purposes (backward compatibility).
+    
+    Args:
+        enhanced_response: Enhanced response dictionary
+        
+    Returns:
+        Simple text response string
+    """
+    # Use raw_response_text for backward compatibility
+    return enhanced_response.get("raw_response_text", "")