@microsoft/m365-copilot-eval 1.0.1-preview.1

package/src/clients/cli/auth/auth_handler.py

@@ -0,0 +1,262 @@
+"""
+Uses WAM (Windows Account Manager) based authentication handler.
+
+This module provides functionality to acquire access tokens using MSAL Python
+with Windows Account Manager (WAM) as the broker. WAM is available on Windows 10+
+and Windows Server 2019+.
+
+For more information, see:
+https://learn.microsoft.com/en-us/entra/msal/python/advanced/wam
+https://github.com/AzureAD/microsoft-authentication-extensions-for-python
+"""
+
+import os
+import platform
+import logging
+from typing import Optional
+from pathlib import Path
+import jwt
+from msal import PublicClientApplication
+from msal_extensions import PersistedTokenCache, build_encrypted_persistence
+
+logger = logging.getLogger(__name__)
+
+class AuthHandler:
+    """Handler for Windows Account Manager (WAM) based authentication."""
+
+    def __init__(self, client_id: str, tenant_id: str, scopes_str: str, cache_dir: Optional[str] = None):
+        """
+        Initialize the WAM auth handler.
+
+        Args:
+            client_id: App registration client ID (required).
+            tenant_id: Directory/tenant ID (required).
+            scopes_str: Comma-separated scopes (required).
+            cache_dir: Optional directory for token cache file. Defaults to user's home directory.
+        """
+        current_os = platform.system()
+        if current_os != "Windows":
+            raise RuntimeError(
+                f"Authentication is only supported on Windows. Detected OS: {current_os}. "
+                "Support for other operating systems is coming soon."
+            )
+
+        if not client_id:
+            raise ValueError("client_id is required")
+
+        if not tenant_id:
+            raise ValueError("tenant_id is required")
+
+        if not scopes_str:
+            raise ValueError("scopes_str is required")
+
+        scopes = [s.strip() for s in scopes_str.split(",") if s.strip()]
+
+        self.client_id = client_id
+        self.authority = f"https://login.microsoftonline.com/{tenant_id}"
+        self.scopes = scopes
+
+        # Initialize the public client application with WAM broker enabled and token cache
+        try:
+            self.app = PublicClientApplication(
+                client_id=self.client_id,
+                authority=self.authority,
+                enable_broker_on_windows=True,
+                token_cache=self._setup_token_cache(cache_dir)
+            )
+            logger.info("WAM auth handler initialized successfully")
+        except Exception as e:
+            logger.error(f"Failed to initialize WAM auth handler: {e}")
+            raise
+
+    def _setup_token_cache(self, cache_dir: Optional[str] = None) -> PersistedTokenCache:
+        """
+        Setup encrypted persistent token cache using MSAL Extensions.
+
+        Creates a platform-dependent encrypted cache (DPAPI on Windows, Keychain on Mac,
+        Libsecret on Linux) and initializes self.token_cache.
+
+        Args:
+            cache_dir: Optional directory for token cache file. Defaults to ~/.msal_cache.
+
+        Returns:
+            PersistedTokenCache: The initialized encrypted token cache.
+
+        Raises:
+            Exception: If cache initialization fails.
+        """
+        if cache_dir is None:
+            cache_dir = str(Path.home() / ".msal_cache")
+        
+        cache_path = os.path.join(cache_dir, "token_cache.bin")
+        
+        try:
+            # Create cache directory if it doesn't exist
+            Path(cache_dir).mkdir(parents=True, exist_ok=True)
+            
+            # Build encrypted persistence (DPAPI on Windows, Keychain on Mac, Libsecret on Linux)
+            persistence = build_encrypted_persistence(cache_path)
+            token_cache = PersistedTokenCache(persistence)
+            logger.info(f"Encrypted token cache initialized at {cache_path} (encrypted: {persistence.is_encrypted})")
+            return token_cache
+        except Exception as e:
+            logger.error(f"Failed to initialize token cache: {e}")
+            raise
+
+    def acquire_token_interactive(self):
+        """
+        Acquire a token interactively using WAM and return the full MSAL result dict.
+
+        This method first attempts to retrieve a cached token. If no valid cached
+        token is found or if it has expired, it will prompt the user to authenticate
+        via the WAM dialog.
+
+        Returns:
+            The full result dictionary from MSAL (contains access_token, id_token, etc.)
+            or None if acquisition fails.
+
+        Raises:
+            ImportError: If broker-related dependencies are not installed.
+                        Install with: pip install 'msal[broker]>=1.20,<2'
+            Exception: If WAM communication fails or user denies the request.
+        """
+        try:
+            # First, attempt to acquire a token silently from cache
+            accounts = self.get_accounts()
+            if accounts:
+                logger.info(f"Attempting silent token acquisition from {len(accounts)} cached account(s)")
+                for account in accounts:
+                    silent_result = self.app.acquire_token_silent(
+                        scopes=self.scopes, account=account
+                    )
+                    if silent_result and "access_token" in silent_result:
+                        logger.info("Access token acquired successfully from cache")
+                        return silent_result
+                logger.info("No valid cached token found; proceeding with interactive authentication")
+            else:
+                logger.info("No cached accounts found; proceeding with interactive authentication")
+
+            # If no cached token is valid, proceed with interactive acquisition
+            result = self.app.acquire_token_interactive(
+                scopes=self.scopes,
+                parent_window_handle=self.app.CONSOLE_WINDOW_HANDLE,
+            )
+
+            if result and "access_token" in result:
+                logger.info("Access token acquired successfully via interactive authentication")
+                return result
+
+            error_msg = None
+            if result:
+                error_msg = result.get(
+                    "error_description", result.get("error", "Unknown error")
+                )
+            logger.error(f"Failed to acquire token: {error_msg or 'Unknown error'}")
+            return result
+
+        except ImportError as e:
+            logger.error(
+                f"WAM broker dependencies not installed: {e}. "
+                "Install with: pip install 'msal[broker]>=1.20,<2'"
+            )
+            raise
+        except Exception as e:
+            logger.error(f"Error during token acquisition: {e}")
+            raise
+
+    def acquire_token_silent(self, account: Optional[dict] = None) -> Optional[str]:
+        """
+        Acquire an access token silently (without user interaction).
+
+        This is useful for cached tokens or when you have a known account.
+        Returns None if a valid token is not available and interaction would be required.
+
+        Args:
+            account: Optional account object from previous interactive authentication.
+                    If None, attempts to get a cached token for any account.
+
+        Returns:
+            The access token string if successful, None if no cached token available.
+        """
+        try:
+            result = self.app.acquire_token_silent(scopes=self.scopes, account=account)
+
+            if result and "access_token" in result:
+                logger.info("Access token acquired silently")
+                return result["access_token"]
+
+            logger.debug(
+                "No cached token available; interactive acquisition would be required"
+            )
+            return None
+
+        except Exception as e:
+            logger.debug(f"Silent token acquisition failed: {e}")
+            return None
+
+    def get_accounts(self) -> list:
+        """
+        Get the list of accounts available in WAM cache.
+
+        Returns:
+            List of account objects cached by WAM.
+        """
+        try:
+            accounts = self.app.get_accounts()
+            logger.info(f"Found {len(accounts)} cached account(s)")
+            return accounts
+        except Exception as e:
+            logger.error(f"Error retrieving cached accounts: {e}")
+            return []
+
+    def clear_cache(self) -> bool:
+        """
+        Clear all cached tokens and accounts from the token cache.
+
+        This method removes all cached authentication data, forcing the user
+        to authenticate interactively on the next acquire_token_interactive call.
+
+        Returns:
+            True if cache was successfully cleared, False otherwise.
+        """
+        try:
+            # Get all cached accounts
+            accounts = self.app.get_accounts()
+
+            # Remove each account from the cache
+            for account in accounts:
+                self.app.remove_account(account)
+                logger.info(f"Removed account {account.get('username', 'unknown')} from cache")
+
+            logger.info("Token cache cleared successfully")
+            return True
+        except Exception as e:
+            logger.error(f"Error clearing token cache: {e}")
+            return False
+
+    @staticmethod
+    def extract_user_oid_from_access_token(access_token: str) -> str:
+        """
+        Extract the user OID from an access token using MSAL's JWT decoding.
+        
+        MSAL includes PyJWT under the hood, so we can use jwt.decode
+        directly without adding new dependencies.
+        
+        Args:
+            access_token: The access token string
+            
+        Returns:
+            The user OID if found, empty string otherwise
+            
+        Raises:
+            ValueError: If token format is invalid or OID not found
+        """
+        try:
+            # Decode without verification (we're just reading claims, not validating signature)
+            decoded = jwt.decode(access_token, options={"verify_signature": False})
+            oid = decoded.get('oid', '')
+            if not oid:
+                raise ValueError("OID not found in token claims")
+            return oid
+        except jwt.DecodeError as e:
+            raise ValueError(f"Failed to decode token: {e}")

package/src/clients/cli/custom_evaluators/CitationsEvaluator.py

@@ -0,0 +1,136 @@
+"""
+CitationsEvaluator - A custom evaluator for analyzing citations in M365 Copilot responses.
+
+This evaluator uses regex-based pattern matching to detect citations in two formats:
+1. New OAI format: \ue200cite\ue202turn{X}search{Y}\ue201
+2. Old format: [^i^] where i is the citation index
+
+Where X, Y, and i are natural numbers representing conversation turn, search result index, or citation index.
+"""
+
+import re
+from enum import Enum
+from typing import Dict, Any, Optional
+
+
+class CitationFormat(Enum):
+    """Enum for different citation formats supported by the evaluator."""
+    OAI_UNICODE = "oai_unicode"  # New format: \ue200cite\ue202turn{X}search{Y}\ue201
+    LEGACY_BRACKET = "legacy_bracket"  # Old format: [^i^]
+
+
+class CitationsEvaluator:
+    """
+    A custom evaluator that analyzes citations in response text without using an LLM.
+    
+    This evaluator detects citation patterns and returns:
+    - Whether at least one citation is present
+    - The number of unique citations found
+    
+    Supports both new OAI unicode format and legacy bracket format.
+    """
+    
+    def __init__(self, citation_format: CitationFormat = CitationFormat.OAI_UNICODE):
+        """
+        Initialize the CitationsEvaluator with the specified citation format.
+        
+        Args:
+            citation_format (CitationFormat): The format of citations to detect.
+                Defaults to OAI_UNICODE format.
+        """
+        self.citation_format = citation_format
+        
+        if citation_format == CitationFormat.OAI_UNICODE:
+            # Pattern to match citations: \ue200cite\ue202turn{number}search{number}\ue201
+            self.citation_pattern = r'\ue200cite\ue202turn\d+search\d+\ue201'
+        elif citation_format == CitationFormat.LEGACY_BRACKET:
+            # Pattern to match citations: [^number^]
+            self.citation_pattern = r'\[\^\d+\^\]'
+        else:
+            raise ValueError(f"Unsupported citation format: {citation_format}")
+        
+        self.compiled_pattern = re.compile(self.citation_pattern)
+    
+    def __call__(self, *, response: str, **kwargs) -> Dict[str, Any]:
+        """
+        Evaluate the response text for citations.
+        
+        Args:
+            response (str): The response text from the M365 Copilot agent
+            **kwargs: Additional keyword arguments (not used but kept for compatibility)
+        
+        Returns:
+            Dict[str, Any]: Evaluation results containing:
+                - citation_format (str): The format used for detection
+                - score (int): Number of unique citations found
+                - result (str): "pass" if citations found, "fail" otherwise
+                - threshold (int): Minimum threshold for passing (1)
+                - reason (str): Explanation of the result with citation details
+        """
+        if not isinstance(response, str):
+            response = str(response) if response is not None else ""
+        
+        # Find all citation matches
+        citation_matches = self.compiled_pattern.findall(response)
+        
+        # Get unique citations (remove duplicates)
+        unique_citations = list(set(citation_matches))
+        
+        # Extract citation identifiers for reporting
+        citation_details = []
+        for citation in unique_citations:
+            if self.citation_format == CitationFormat.OAI_UNICODE:
+                # Extract the turn and search numbers from the citation
+                turn_search_match = re.search(r'turn(\d+)search(\d+)', citation)
+                if turn_search_match:
+                    turn_num = turn_search_match.group(1)
+                    search_num = turn_search_match.group(2)
+                    citation_details.append(f"turn{turn_num}search{search_num}")
+            elif self.citation_format == CitationFormat.LEGACY_BRACKET:
+                # Extract the citation number from [^number^]
+                bracket_match = re.search(r'\[\^(\d+)\^\]', citation)
+                if bracket_match:
+                    citation_num = bracket_match.group(1)
+                    citation_details.append(f"citation{citation_num}")
+        
+        # Prepare results in a format compatible with the HTML report generator
+        results = {
+            "citation_format": self.citation_format.value,
+            # HTML report compatible fields
+            "score": len(unique_citations),  # Use citation count as the score
+            "result": "pass" if len(unique_citations) > 0 else "fail",  # Pass if citations found
+            "threshold": 1,  # Threshold of 1 citation minimum
+            "reason": f"Found {len(unique_citations)} unique citation(s): {', '.join(citation_details) if citation_details else 'None'}"
+        }
+        
+        return results
+    
+    def get_name(self) -> str:
+        """Return the name of this evaluator."""
+        return "CitationsEvaluator"
+    
+    def get_description(self) -> str:
+        """Return a description of what this evaluator does."""
+        return f"Analyzes response text for M365 Copilot citations using regex pattern matching ({self.citation_format.value} format)"
+
+
+def citations_evaluator(*, response: str, citation_format: CitationFormat = CitationFormat.OAI_UNICODE, **kwargs) -> Dict[str, Any]:
+    """
+    Standalone function wrapper for the CitationsEvaluator.
+    
+    This function provides a simple interface compatible with Azure AI Evaluation SDK.
+    
+    Args:
+        response (str): The response text to evaluate
+        citation_format (CitationFormat): The format of citations to detect
+        **kwargs: Additional keyword arguments
+    
+    Returns:
+        Dict[str, Any]: Citation evaluation results
+    """
+    evaluator = CitationsEvaluator(citation_format=citation_format)
+    return evaluator(response=response, **kwargs)
+
+
+# For convenience, export the main classes and functions
+__all__ = ['CitationsEvaluator', 'CitationFormat', 'citations_evaluator']

package/src/clients/cli/custom_evaluators/ConcisenessNonLLMEvaluator.py

@@ -0,0 +1,18 @@
+#from azure.ai.evaluation import evaluate
+
+class ConcisenessNonLLMEvaluator:
+    def __init__(self):
+        pass
+    # A class is made callable by implementing the special method __call__
+    def __call__(self, *, response: str, **kwargs):
+        length = len(response)
+        # compute raw score (example: normalized to roughly 0-5)
+        raw_score = max(0, ((100 - (length / 10)) / 20))
+        # round to nearest integer
+        rounded = int(round(raw_score))
+        return {
+            "concisenessnonllm_score": rounded,
+            "concisenessnonllm_threshold": 3,
+            "concisenessnonllm_result": "pass" if rounded >= 3 else "fail",
+            "concisenessnonllm_reason": f"Any response greater than 1000 characters is given zero score. The longer the answer, the lesser the score. The length of the response is {length} chars. And hence, the score!"
+            }

package/src/clients/cli/custom_evaluators/ExactMatchEvaluator.py

@@ -0,0 +1,25 @@
+from azure.ai.evaluation import evaluate
+
+class ExactMatchEvaluator:
+    def __init__(self):
+        pass
+
+    def __call__(self, *, response: str, expected_answer: str, **kwargs):
+        if response is None or response.strip() == "":
+            raise ValueError("Response is null, empty, or whitespace.")
+
+        if expected_answer is None:
+            raise ValueError("Expected answer cannot be None.")
+
+        # Case-sensitive exact match (mimics C# StringComparison.InvariantCulture)
+        is_match = response.strip() == expected_answer.strip()
+
+        return {
+            "exact_match": 1.0 if is_match else 0.0,
+            "exact_match_result": "pass" if is_match else "fail",
+            "exact_match_threshold": 1.0,
+            "exact_match_reason": "Exact match found" if is_match else "No exact match found"
+        }
+
+
+exact_match_evaluator = ExactMatchEvaluator()

package/src/clients/cli/custom_evaluators/PII/PII.py

@@ -0,0 +1,45 @@
+import os
+import json
+import sys
+import re
+from promptflow.client import load_flow
+
+
+class PIIEvaluator:
+    def __init__(self, model_config):
+        current_dir = os.path.dirname(__file__)
+        prompty_path = os.path.join(current_dir, "pii.prompty")
+        self._flow = load_flow(source=prompty_path, model={"configuration": model_config})
+
+    def __call__(self, *, response: str, **kwargs):
+        llm_response = self._flow(response=response)
+        try:
+            # Try to parse as JSON first
+            parsed_response = json.loads(llm_response)
+            return parsed_response
+        except Exception:
+            # If it's not JSON, try to extract the score from XML-like tags
+            if isinstance(llm_response, str):
+                # Look for <S2>score</S2> pattern
+                score_match = re.search(r'<S2>(\d+)</S2>', llm_response)
+                # Look for <S1>explanation</S1> pattern
+                explanation_match = re.search(r'<S1>(.*?)</S1>', llm_response, re.DOTALL)
+                
+                if score_match:
+                    score = int(score_match.group(1))
+                    result_dict = {
+                        'PII': score,
+                        'score': score,
+                        'raw_response': llm_response
+                    }
+                    
+                    # Add explanation if found
+                    if explanation_match:
+                        explanation = explanation_match.group(1).strip()
+                        result_dict['explanation'] = explanation
+                        result_dict['reason'] = explanation
+                        result_dict['PII_reason'] = explanation
+                    
+                    return result_dict
+            # Fallback: return the raw response
+            return {'raw_response': llm_response}

package/src/clients/cli/custom_evaluators/PartialMatchEvaluator.py

@@ -0,0 +1,39 @@
+from azure.ai.evaluation import evaluate
+
+class PartialMatchEvaluator:
+    def __init__(self, case_sensitive=False):
+        self.case_sensitive = case_sensitive
+
+    def __call__(self, *, response: str, expected_answer: str, **kwargs):
+        if response is None or response.strip() == "":
+            raise ValueError("Response cannot be null or empty.")
+        if expected_answer is None:
+            raise ValueError("Expected answer cannot be null.")
+
+        resp = response.strip()
+        exp = expected_answer.strip()
+
+        # Adjust case sensitivity
+        if not self.case_sensitive:
+            resp = resp.lower()
+            exp = exp.lower()
+
+        # Score: fraction of expected text found inside response
+        if exp in resp:
+            # Percent of expected text that matched (length-based)
+            score = len(exp) / len(resp)
+        else:
+            score = 0.0
+
+        threshold = 0.5  # 50% match threshold
+        is_pass = score >= threshold
+
+        return {
+            "partial_match": score,
+            "partial_match_result": "pass" if is_pass else "fail",
+            "partial_match_threshold": threshold,
+            "partial_match_reason": f"Match score: {score:.3f} ({'above' if is_pass else 'below'} threshold {threshold})"
+        }
+
+
+partial_match_evaluator = PartialMatchEvaluator(case_sensitive=False)

package/src/clients/cli/custom_evaluators/__init__.py

@@ -0,0 +1 @@
+# Custom evaluators package

package/src/clients/cli/demo_usage.py

@@ -0,0 +1,83 @@
+#!/usr/bin/env python3
+"""
+Example usage script for the M365 Copilot Agent Evaluation CLI
+This script demonstrates various ways to use the CLI tool.
+"""
+
+import subprocess
+import sys
+import os
+from pathlib import Path
+
+def run_command(cmd, description):
+    """Run a command and display its description."""
+    print(f"\n{'='*60}")
+    print(f"Example: {description}")
+    print(f"Command: {cmd}")
+    print(f"{'='*60}")
+    
+    try:
+        # Note: In a real scenario, you would have your Azure credentials configured
+        # This will fail without proper environment variables, but shows the CLI interface
+        result = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=30)
+        if result.returncode == 0:
+            print("✅ Command executed successfully")
+            print(result.stdout)
+        else:
+            print("❌ Command failed (likely due to missing Azure credentials)")
+            print("Error:", result.stderr)
+    except subprocess.TimeoutExpired:
+        print("⏱️ Command timed out (this is expected without Azure credentials)")
+    except Exception as e:
+        print(f"❌ Error running command: {e}")
+
+def main():
+    """Demonstrate CLI usage examples."""
+    script_dir = Path(__file__).parent
+    os.chdir(script_dir)
+    
+    print("M365 Copilot Agent Evaluation CLI - Usage Examples")
+    print("=" * 60)
+    print("Note: These examples will fail without proper Azure credentials.")
+    print("This script demonstrates the CLI interface and available options.")
+    
+    # Example 1: Show help
+    run_command(
+        "python main.py --help",
+        "Display help and all available options"
+    )
+    
+    # Example 2: Custom prompt (dry run)
+    run_command(
+        'python main.py --prompts "What is Microsoft Graph?" --expected "Microsoft Graph is a gateway to data and intelligence in Microsoft 365."',
+        "Run evaluation with custom prompt and expected response"
+    )
+    
+    # Example 3: Prompts from file
+    run_command(
+        "python main.py --prompts-file samples/example_prompts.json --output results.json",
+        "Load prompts from JSON file and save results to JSON"
+    )
+    
+    # Example 4: CSV output
+    run_command(
+        "python main.py --prompts-file samples/example_prompts.json --output results.csv --format csv",
+        "Load prompts from file and save results to CSV"
+    )
+    
+    print(f"\n{'='*60}")
+    print("Setup Instructions:")
+    print("1. Install Azure CLI and run 'az login' to authenticate")
+    print("2. Create a .env file with your Azure AI configuration")
+    print("3. Set the following environment variables:")
+    print("   - AZURE_AI_FOUNDRY_PROJECT_ENDPOINT")
+    print("   - AZURE_AI_AGENT_ID")
+    print("   - AZURE_AI_OPENAI_ENDPOINT")
+    print("   - AZURE_AI_API_KEY")
+    print("   - AZURE_AI_API_VERSION")
+    print("   - AZURE_AI_MODEL_NAME")
+    print("4. Run: python main.py")
+    print(f"{'='*60}")
+
+if __name__ == "__main__":
+    main()