freeharness 0.4.1

package/canonical/skills/agent-evaluation/scripts/utils/env_validation.py

@@ -0,0 +1,123 @@
+"""Utilities for environment variable validation and MLflow configuration."""
+
+import os
+
+from packaging import version
+
+
+def get_env_vars() -> dict[str, str | None]:
+    """Get MLflow environment variables.
+
+    Returns:
+        Dictionary with tracking_uri and experiment_id (may be None)
+    """
+    return {
+        "tracking_uri": os.getenv("MLFLOW_TRACKING_URI"),
+        "experiment_id": os.getenv("MLFLOW_EXPERIMENT_ID"),
+    }
+
+
+def validate_env_vars(
+    require_tracking_uri: bool = True, require_experiment_id: bool = True
+) -> list[str]:
+    """Validate required environment variables are set.
+
+    Args:
+        require_tracking_uri: If True, MLFLOW_TRACKING_URI must be set
+        require_experiment_id: If True, MLFLOW_EXPERIMENT_ID must be set
+
+    Returns:
+        List of error messages (empty if valid)
+    """
+    errors = []
+    env_vars = get_env_vars()
+
+    if require_tracking_uri and not env_vars["tracking_uri"]:
+        errors.append("MLFLOW_TRACKING_URI is not set")
+
+    if require_experiment_id and not env_vars["experiment_id"]:
+        errors.append("MLFLOW_EXPERIMENT_ID is not set")
+
+    return errors
+
+
+def validate_mlflow_version(min_version: str = "3.8.0") -> tuple[bool, str]:
+    """Check MLflow version compatibility.
+
+    Args:
+        min_version: Minimum required MLflow version
+
+    Returns:
+        Tuple of (is_valid, version_string)
+    """
+    try:
+        import mlflow
+
+        current_version = mlflow.__version__
+
+        # Remove dev/rc suffixes for comparison
+        clean_version = current_version.split("dev")[0].split("rc")[0]
+
+        is_valid = version.parse(clean_version) >= version.parse(min_version)
+        return is_valid, current_version
+    except ImportError:
+        return False, "not installed"
+
+
+def test_mlflow_connection(tracking_uri: str, experiment_id: str) -> tuple[bool, str]:
+    """Test connection to MLflow tracking server.
+
+    Args:
+        tracking_uri: MLflow tracking URI
+        experiment_id: MLflow experiment ID
+
+    Returns:
+        Tuple of (success, error_message_or_experiment_name)
+    """
+    try:
+        from mlflow import MlflowClient
+
+        client = MlflowClient()
+        experiment = client.get_experiment(experiment_id)
+
+        if experiment:
+            return True, experiment.name
+        else:
+            return False, f"Experiment {experiment_id} not found"
+    except Exception as e:
+        return False, str(e)[:100]
+
+
+def check_databricks_config() -> tuple[bool, str | None]:
+    """Check if running with Databricks configuration.
+
+    Returns:
+        Tuple of (is_databricks, profile_or_error_message)
+    """
+    tracking_uri = os.getenv("MLFLOW_TRACKING_URI", "")
+
+    # Check if tracking URI indicates Databricks
+    if "databricks" in tracking_uri.lower():
+        # Extract profile if present
+        if "databricks://" in tracking_uri:
+            profile = tracking_uri.split("databricks://")[1] if len(tracking_uri.split("databricks://")) > 1 else "DEFAULT"
+            return True, profile
+        return True, "databricks"
+
+    # Check for Databricks SDK/CLI
+    try:
+        import subprocess
+
+        result = subprocess.run(
+            ["databricks", "auth", "profiles"],
+            capture_output=True,
+            text=True,
+            timeout=5,
+        )
+        if result.returncode == 0:
+            profiles = [line.strip() for line in result.stdout.strip().split("\n") if line.strip()]
+            return True, profiles[0] if profiles else None
+    except (subprocess.TimeoutExpired, FileNotFoundError, subprocess.CalledProcessError):
+        pass
+
+    return False, None

package/canonical/skills/agent-evaluation/scripts/utils/tracing_utils.py

@@ -0,0 +1,108 @@
+"""Utilities for tracing-related validation.
+
+The coding agent should use Grep tool for discovery:
+- Find autolog calls: grep -r "mlflow.*autolog" . --include="*.py"
+- Find trace decorators: grep -r "@mlflow.trace" . --include="*.py"
+- Find MLflow imports: grep -r "import mlflow" . --include="*.py"
+
+This module provides validation helpers used by validation scripts.
+"""
+
+import re
+from pathlib import Path
+
+
+def check_import_order(file_path: str, import_pattern: str = None) -> tuple[bool, str]:
+    """Verify autolog is called before library/module imports.
+
+    Args:
+        file_path: Path to file containing autolog call
+        import_pattern: Optional regex pattern to match imports (e.g., r"from .* import")
+                       If None, checks for any "from ... import" after autolog
+
+    Returns:
+        Tuple of (is_correct, message)
+    """
+    try:
+        content = Path(file_path).read_text()
+        lines = content.split("\n")
+
+        autolog_line = None
+        first_import_line = None
+
+        for i, line in enumerate(lines, 1):
+            if "autolog()" in line:
+                autolog_line = i
+            # After finding autolog, look for any imports (customizable via pattern)
+            if autolog_line and "from" in line and "import" in line:
+                if import_pattern:
+                    if re.search(import_pattern, line):
+                        first_import_line = i
+                        break
+                else:
+                    first_import_line = i
+                    break
+
+        if autolog_line and first_import_line:
+            if autolog_line < first_import_line:
+                return True, f"Autolog (line {autolog_line}) before imports (line {first_import_line})"
+            else:
+                return (
+                    False,
+                    f"Autolog (line {autolog_line}) after imports (line {first_import_line})",
+                )
+        elif autolog_line:
+            return True, f"Autolog found at line {autolog_line}"
+        else:
+            return False, "Autolog not found"
+
+    except Exception as e:
+        return True, f"Could not check import order: {e}"  # Don't fail on errors
+
+
+
+
+def check_session_id_capture(file_path: str) -> bool:
+    """Check if file has session ID tracking code.
+
+    Looks for: get_last_active_trace_id(), set_trace_tag(), session_id
+
+    Args:
+        file_path: Path to file to check
+
+    Returns:
+        True if all patterns found
+    """
+    try:
+        content = Path(file_path).read_text()
+
+        session_patterns = [
+            r"mlflow\.get_last_active_trace_id\(\)",
+            r"mlflow\.set_trace_tag\(",
+            r"session_id",
+        ]
+
+        return all(re.search(pattern, content) for pattern in session_patterns)
+    except Exception:
+        return False
+
+
+def verify_mlflow_imports(file_paths: list[str]) -> dict[str, bool]:
+    """Check mlflow is imported in given files.
+
+    Args:
+        file_paths: List of file paths to check
+
+    Returns:
+        Dictionary mapping file_path to has_mlflow_import
+    """
+    results = {}
+
+    for file_path in file_paths:
+        try:
+            content = Path(file_path).read_text()
+            results[file_path] = "import mlflow" in content
+        except Exception:
+            results[file_path] = False
+
+    return results

package/canonical/skills/agent-evaluation/scripts/validate_agent_tracing.py

@@ -0,0 +1,148 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Validate MLflow tracing for your agent.
+
+This is a template script. Fill in the TODO sections before running:
+1. Update the import statement with your agent's module and function
+2. Configure any dependencies (LLM providers, config, etc.)
+3. Adjust the function call to match your agent's signature
+4. Verify environment variables are set correctly
+"""
+
+import os
+import sys
+import mlflow
+from mlflow import MlflowClient
+
+# TODO: Update these imports with your agent's module and entry point
+# Example: from my_agent.agent import run_agent
+from YOUR_MODULE import YOUR_ENTRY_POINT  # <-- UPDATE THIS LINE
+
+# Configuration
+TEST_QUERY = "What is MLflow?"
+TEST_SESSION_ID = "test-session-123"
+
+# Verify environment variables
+tracking_uri = os.getenv("MLFLOW_TRACKING_URI")
+experiment_id = os.getenv("MLFLOW_EXPERIMENT_ID")
+
+if not tracking_uri or not experiment_id:
+    print("✗ Missing required environment variables:")
+    print("  MLFLOW_TRACKING_URI:", tracking_uri or "(not set)")
+    print("  MLFLOW_EXPERIMENT_ID:", experiment_id or "(not set)")
+    print("\nRun scripts/setup_mlflow.py first")
+    sys.exit(1)
+
+print("=" * 60)
+print("MLflow Tracing Validation")
+print("=" * 60)
+print()
+print(f"Tracking URI: {tracking_uri}")
+print(f"Experiment ID: {experiment_id}")
+print()
+
+# TODO: Configure your agent's dependencies here
+# IMPORTANT: Add any required setup before calling your agent
+# Examples:
+# from your_agent.llm import LLMConfig, LLMProvider
+# llm_config = LLMConfig(model="gpt-4", temperature=0.0)
+# llm_provider = LLMProvider(config=llm_config)
+#
+# from your_agent.config import AgentConfig
+# agent_config = AgentConfig.from_env()
+
+print("Running test query...")
+print(f"  Query: {TEST_QUERY}")
+print(f"  Session ID: {TEST_SESSION_ID}")
+print()
+
+try:
+    # TODO: Update this function call to match your agent's signature
+    # Examples:
+    #   response = YOUR_ENTRY_POINT(TEST_QUERY, llm_provider)
+    #   response = YOUR_ENTRY_POINT(TEST_QUERY, session_id=TEST_SESSION_ID)
+    #   response = YOUR_ENTRY_POINT(TEST_QUERY, config=agent_config)
+
+    response = YOUR_ENTRY_POINT(TEST_QUERY)  # <-- UPDATE THIS LINE
+
+    print("✓ Agent executed successfully")
+    print()
+
+    # Capture trace
+    trace_id = mlflow.get_last_active_trace_id()
+    if not trace_id:
+        print("✗ FAILED: No trace ID captured!")
+        print("  Check that mlflow.autolog() is called before agent execution")
+        sys.exit(1)
+
+    print(f"✓ Trace captured: {trace_id}")
+
+    # Get trace details
+    client = MlflowClient()
+    trace = client.get_trace(trace_id)
+
+    # Verify trace structure
+    print()
+    print("Verifying trace structure...")
+
+    if not trace.data.spans:
+        print("✗ FAILED: No spans found in trace")
+        sys.exit(1)
+
+    print(f"✓ Top-level span: {trace.data.spans[0].name} ({trace.data.spans[0].span_type})")
+
+    # Count total spans (including nested)
+    def count_spans(spans):
+        count = len(spans)
+        for span in spans:
+            if hasattr(span, 'spans') and span.spans:
+                count += count_spans(span.spans)
+        return count
+
+    total_spans = count_spans(trace.data.spans)
+    print(f"✓ Total spans: {total_spans}")
+
+    if total_spans < 2:
+        print("⚠  WARNING: Only 1 span found - autolog may not be working")
+        print("  Expected: @mlflow.trace decorator span + autolog library spans")
+    else:
+        print("✓ Multiple spans detected - autolog appears to be working")
+
+    # Print trace hierarchy
+    def print_hierarchy(spans, indent=0):
+        for span in spans:
+            prefix = "    " + "  " * indent
+            print(f"{prefix}- {span.name} ({span.span_type})")
+            if hasattr(span, 'spans') and span.spans:
+                print_hierarchy(span.spans, indent + 1)
+
+    print()
+    print("  Trace hierarchy:")
+    print_hierarchy(trace.data.spans)
+
+    # Check session ID (optional)
+    if "session_id" in trace.info.tags:
+        actual_session_id = trace.info.tags["session_id"]
+        print()
+        if actual_session_id == TEST_SESSION_ID:
+            print(f"✓ Session ID tagged: {actual_session_id}")
+        else:
+            print(f"⚠  Session ID mismatch: expected {TEST_SESSION_ID}, got {actual_session_id}")
+    else:
+        print()
+        print("  ℹ  Note: No session_id tag found (optional for single-turn agents)")
+
+    print()
+    print("=" * 60)
+    print("✓ VALIDATION PASSED")
+    print("=" * 60)
+    print()
+    print("Your agent is properly integrated with MLflow tracing!")
+    print()
+
+except Exception as e:
+    print(f"✗ FAILED: {str(e)}")
+    import traceback
+    traceback.print_exc()
+    sys.exit(1)

package/canonical/skills/agent-evaluation/scripts/validate_auth.py

@@ -0,0 +1,240 @@
+"""
+Validate authentication for agent evaluation.
+
+This script tests authentication to required services:
+- MLflow tracking server (Databricks or local)
+- LLM provider (if configured)
+
+Performs lightweight API calls to verify credentials before expensive operations.
+
+Usage:
+    python scripts/validate_auth.py
+"""
+
+import os
+import sys
+
+from utils import check_databricks_config, validate_env_vars
+
+
+def check_databricks_auth():
+    """Test Databricks authentication."""
+    print("Testing Databricks authentication...")
+
+    is_databricks, profile = check_databricks_config()
+
+    if not is_databricks:
+        print("  ⊘ Not using Databricks (skipped)")
+        print()
+        return []
+
+    # Check for auth credentials
+    token = os.getenv("DATABRICKS_TOKEN")
+    host = os.getenv("DATABRICKS_HOST")
+
+    if not token and not host:
+        # Try using databricks SDK (more robust) or fallback to CLI
+        try:
+            # Try new Databricks SDK first
+            try:
+                from databricks import sdk
+
+                print("  ↻ Using Databricks SDK...")
+
+                # Try to create workspace client
+                try:
+                    w = sdk.WorkspaceClient()
+                    # Test with a simple API call
+                    current_user = w.current_user.me()
+                    print(f"  ✓ Authenticated as: {current_user.user_name}")
+                    print()
+                    return []
+
+                except AttributeError as e:
+                    # Handle NoneType error gracefully
+                    if "'NoneType'" in str(e):
+                        print("  ✗ Databricks configuration incomplete or corrupted")
+                        print()
+                        return ["Run: databricks auth login --profile DEFAULT"]
+                    raise
+
+            except ImportError:
+                # Fall back to old databricks-cli
+                from databricks_cli.sdk.api_client import ApiClient
+
+                print("  ↻ Using Databricks CLI profile...")
+
+                try:
+                    api_client = ApiClient()
+
+                    # Check if api_client is properly initialized
+                    if api_client is None or not hasattr(api_client, "host"):
+                        print("  ✗ Databricks CLI profile not configured")
+                        print()
+                        return ["Run: databricks auth login --profile DEFAULT"]
+
+                except (AttributeError, TypeError) as e:
+                    print(f"  ✗ Profile configuration error: {str(e)[:80]}")
+                    print()
+                    return ["Run: databricks auth login --profile DEFAULT"]
+
+            # Test with MLflow client
+            from mlflow import MlflowClient
+
+            client = MlflowClient()
+            client.search_experiments(max_results=1)
+            print("  ✓ Databricks profile authenticated")
+            print()
+            return []
+
+        except ImportError:
+            print("  ✗ Neither databricks-sdk nor databricks-cli installed")
+            print()
+            return ["Install databricks SDK: pip install databricks-sdk"]
+        except Exception as e:
+            print(f"  ✗ Authentication failed: {str(e)[:100]}")
+            print()
+            return ["Run: databricks auth login --profile DEFAULT"]
+
+    # Test with environment variables
+    try:
+        from mlflow import MlflowClient
+
+        client = MlflowClient()
+        client.search_experiments(max_results=1)
+
+        print("  ✓ Databricks token valid")
+        print()
+        return []
+
+    except Exception as e:
+        print(f"  ✗ Token validation failed: {str(e)[:100]}")
+        print()
+        return [
+            "Check DATABRICKS_TOKEN is set correctly",
+            "Run: databricks auth login --host <workspace-url>",
+        ]
+
+
+def check_mlflow_tracking():
+    """Test MLflow tracking server connectivity."""
+    print("Testing MLflow tracking server...")
+
+    # Use utility to validate env vars
+    errors = validate_env_vars()
+
+    if errors:
+        for error in errors:
+            print(f"  ✗ {error}")
+        print()
+        return [f"Set environment variable: {error}" for error in errors]
+
+    tracking_uri = os.getenv("MLFLOW_TRACKING_URI")
+    experiment_id = os.getenv("MLFLOW_EXPERIMENT_ID")
+
+    try:
+        from mlflow import MlflowClient
+
+        client = MlflowClient()
+
+        # Test connectivity by getting experiment
+        experiment = client.get_experiment(experiment_id)
+
+        print(f"  ✓ Connected to: {tracking_uri}")
+        print(f"  ✓ Experiment: {experiment.name}")
+        print()
+        return []
+
+    except Exception as e:
+        error_msg = str(e)
+        print(f"  ✗ Connection failed: {error_msg[:100]}")
+        print()
+
+        if "404" in error_msg or "not found" in error_msg.lower():
+            return [f"Experiment {experiment_id} not found - check MLFLOW_EXPERIMENT_ID"]
+        elif "401" in error_msg or "403" in error_msg or "authentication" in error_msg.lower():
+            return ["Authentication failed - check credentials"]
+        else:
+            return [f"Cannot connect to {tracking_uri} - check tracking URI and network"]
+
+
+def check_llm_provider():
+    """Check LLM provider configuration (optional)."""
+    print("Checking LLM provider configuration...")
+
+    # Check for common LLM provider env vars
+    providers_found = []
+
+    if os.getenv("OPENAI_API_KEY"):
+        providers_found.append("OpenAI")
+
+    if os.getenv("ANTHROPIC_API_KEY"):
+        providers_found.append("Anthropic")
+
+    if os.getenv("DATABRICKS_TOKEN") or os.getenv("DATABRICKS_HOST"):
+        providers_found.append("Databricks")
+
+    if providers_found:
+        print(f"  ✓ Found credentials for: {', '.join(providers_found)}")
+        print()
+    else:
+        print("  ⚠ No LLM provider credentials detected")
+        print("    This is OK if your agent uses Databricks profile auth")
+        print()
+
+    return []  # Warning only, not blocking
+
+
+def main():
+    """Main validation workflow."""
+    print("=" * 60)
+    print("Authentication Validation")
+    print("=" * 60)
+    print()
+
+    all_issues = []
+
+    # Check 1: MLflow tracking
+    tracking_issues = check_mlflow_tracking()
+    all_issues.extend(tracking_issues)
+
+    # Check 2: Databricks auth (if using Databricks)
+    databricks_issues = check_databricks_auth()
+    all_issues.extend(databricks_issues)
+
+    # Check 3: LLM provider (optional check)
+    llm_issues = check_llm_provider()
+    all_issues.extend(llm_issues)
+
+    # Summary
+    print("=" * 60)
+    print("Validation Report")
+    print("=" * 60)
+    print()
+
+    if not all_issues:
+        print("✓ ALL AUTHENTICATION CHECKS PASSED")
+        print()
+        print("Your authentication is configured correctly.")
+        print()
+        print("Next steps:")
+        print("  1. Integrate tracing: See references/tracing-integration.md")
+        print("  2. Test runtime tracing: Edit and run scripts/validate_agent_tracing.py")
+        print()
+    else:
+        print(f"✗ Found {len(all_issues)} issue(s):")
+        print()
+        for i, issue in enumerate(all_issues, 1):
+            print(f"  {i}. {issue}")
+        print()
+        print("=" * 60)
+        print("Fix the authentication issues above before proceeding.")
+        print("=" * 60)
+        print()
+        sys.exit(1)
+
+    print("=" * 60)
+
+
+if __name__ == "__main__":
+    main()