freeharness 0.4.1

package/canonical/skills/agent-evaluation/scripts/validate_environment.py

@@ -0,0 +1,185 @@
+"""
+Validate MLflow environment setup for agent evaluation.
+
+This script runs `mlflow doctor` and adds custom checks for:
+- Environment variables (MLFLOW_TRACKING_URI, MLFLOW_EXPERIMENT_ID)
+- MLflow version compatibility (>=3.8.0)
+- Agent package installation
+- Basic connectivity test
+
+Usage:
+    python scripts/validate_environment.py
+"""
+
+import importlib.util
+import subprocess
+import sys
+
+from utils import test_mlflow_connection, validate_env_vars, validate_mlflow_version
+
+
+def run_mlflow_doctor():
+    """Run mlflow doctor and return output."""
+    print("Running MLflow diagnostics...")
+    print()
+
+    try:
+        result = subprocess.run(["mlflow", "doctor"], capture_output=True, text=True, timeout=10)
+
+        # Print output (mlflow doctor goes to stderr)
+        output = result.stderr + result.stdout
+        print(output)
+
+        return result.returncode == 0
+    except subprocess.TimeoutExpired:
+        print("⚠ mlflow doctor timed out")
+        return False
+    except FileNotFoundError:
+        print("✗ mlflow command not found")
+        print("  Install: pip install mlflow")
+        return False
+
+
+def check_environment_variables():
+    """Check that required environment variables are set."""
+    print("Checking environment variables...")
+
+    errors = validate_env_vars()
+
+    if not errors:
+        env_vars = {}
+        import os
+
+        tracking_uri = os.getenv("MLFLOW_TRACKING_URI")
+        experiment_id = os.getenv("MLFLOW_EXPERIMENT_ID")
+
+        if tracking_uri:
+            print(f"  ✓ MLFLOW_TRACKING_URI: {tracking_uri}")
+        if experiment_id:
+            print(f"  ✓ MLFLOW_EXPERIMENT_ID: {experiment_id}")
+    else:
+        for error in errors:
+            print(f"  ✗ {error}")
+
+    print()
+    return ["Set environment variables" for _ in errors] if errors else []
+
+
+def check_mlflow_version():
+    """Check MLflow version is compatible."""
+    print("Checking MLflow version...")
+
+    is_valid, version_str = validate_mlflow_version("3.8.0")
+
+    if is_valid:
+        print(f"  ✓ MLflow {version_str} (>=3.8.0)")
+        print()
+        return []
+    elif version_str == "not installed":
+        print(f"  ✗ MLflow not installed")
+        print()
+        return ["Install MLflow: pip install mlflow"]
+    else:
+        print(f"  ✗ MLflow {version_str} (need >=3.8.0)")
+        print()
+        return ["Upgrade MLflow: pip install --upgrade 'mlflow>=3.8.0'"]
+
+
+def check_agent_package():
+    """Remind user to verify agent package is importable."""
+    print("Agent package check...")
+    print("  ℹ Verify your agent is importable:")
+    print("    python -c 'from your_module import your_agent'")
+    print("  Replace 'your_module' and 'your_agent' with your actual package/function names")
+    print()
+    return []  # Informational only, not blocking
+
+
+def test_connectivity():
+    """Test basic connectivity to MLflow tracking server."""
+    print("Testing MLflow connectivity...")
+
+    import os
+
+    tracking_uri = os.getenv("MLFLOW_TRACKING_URI")
+    experiment_id = os.getenv("MLFLOW_EXPERIMENT_ID")
+
+    if not tracking_uri or not experiment_id:
+        print("  ⊘ Skipped (environment variables not set)")
+        print()
+        return []
+
+    success, result = test_mlflow_connection(tracking_uri, experiment_id)
+
+    if success:
+        print(f"  ✓ Connected to experiment: {result}")
+        print()
+        return []
+    else:
+        print(f"  ✗ Connection failed: {result}")
+        print()
+        return [f"Check connectivity and authentication to {tracking_uri}"]
+
+
+def main():
+    """Main validation workflow."""
+    print("=" * 60)
+    print("MLflow Environment Validation")
+    print("=" * 60)
+    print()
+
+    all_issues = []
+
+    # Check 1: Run mlflow doctor
+    doctor_ok = run_mlflow_doctor()
+    if not doctor_ok:
+        all_issues.append("mlflow doctor reported issues")
+
+    # Check 2: Environment variables
+    env_issues = check_environment_variables()
+    all_issues.extend(env_issues)
+
+    # Check 3: MLflow version
+    version_issues = check_mlflow_version()
+    all_issues.extend(version_issues)
+
+    # Check 4: Agent package
+    agent_issues = check_agent_package()
+    all_issues.extend(agent_issues)
+
+    # Check 5: Connectivity (only if env vars set)
+    connectivity_issues = test_connectivity()
+    all_issues.extend(connectivity_issues)
+
+    # Summary
+    print("=" * 60)
+    print("Validation Report")
+    print("=" * 60)
+    print()
+
+    if not all_issues:
+        print("✓ ALL CHECKS PASSED")
+        print()
+        print("Your environment is ready for agent evaluation.")
+        print()
+        print("Next steps:")
+        print("  1. Integrate tracing: See references/tracing-integration.md")
+        print("  2. Prepare dataset: python scripts/list_datasets.py")
+        print()
+    else:
+        print(f"✗ Found {len(all_issues)} issue(s):")
+        print()
+        for i, issue in enumerate(all_issues, 1):
+            print(f"  {i}. {issue}")
+        print()
+        print("=" * 60)
+        print("Fix the issues above and re-run this script.")
+        print("=" * 60)
+        print()
+        sys.exit(1)
+
+    print("=" * 60)
+
+
+if __name__ == "__main__":
+    main()

package/canonical/skills/agent-evaluation/scripts/validate_tracing_runtime.py

@@ -0,0 +1,318 @@
+# -*- coding: utf-8 -*-
+"""
+Validate MLflow tracing by running the agent (RUNTIME VALIDATION).
+
+CRITICAL: This script REQUIRES valid authentication and LLM access.
+If this validation fails, the evaluation workflow MUST STOP until auth issues are resolved.
+
+The coding agent should discover module/entry-point/autolog using Grep first,
+then pass the discovered information to this script for runtime validation.
+
+This script verifies by actually running the agent:
+1. Traces are captured successfully
+2. Complete trace hierarchy is present (decorator + autolog spans)
+3. Session ID is tagged (if applicable)
+4. Agent execution completes without errors
+
+Usage:
+    python validate_tracing_runtime.py \
+        --module my_agent.agent \
+        --entry-point run_agent \
+        --autolog-file src/agent/__init__.py
+"""
+
+import argparse
+import importlib
+import sys
+
+from utils import validate_env_vars
+
+
+def run_test_query(
+    module_name: str,
+    entry_point_name: str,
+    test_query: str = "What is MLflow?",
+    test_session_id: str = "test-session-123",
+):
+    """Run a test query and verify trace capture."""
+    print("\nRunning test query...")
+    print(f"  Module: {module_name}")
+    print(f"  Entry point: {entry_point_name}")
+    print(f"  Query: {test_query}")
+    print(f"  Session ID: {test_session_id}")
+
+    try:
+        # Import mlflow first
+        import mlflow
+        from mlflow import MlflowClient
+
+        # Try to import the agent module
+        try:
+            agent_module = importlib.import_module(module_name)
+        except ImportError as e:
+            print(f"  ✗ Could not import module '{module_name}': {e}")
+            print("    Try: pip install -e . (from project root)")
+            return None
+
+        # Get the entry point function
+        if not hasattr(agent_module, entry_point_name):
+            print(f"  ✗ Function '{entry_point_name}' not found in {module_name}")
+            available = [name for name in dir(agent_module) if not name.startswith("_")]
+            if available:
+                print(f"    Available functions: {', '.join(available[:5])}")
+            return None
+
+        entry_point = getattr(agent_module, entry_point_name)
+        print(f"  ✓ Found entry point: {entry_point_name}")
+
+        # Try to call the entry point (be flexible with signatures)
+        print("\n  Executing agent...")
+        try:
+            # Try different call signatures
+            try:
+                entry_point(test_query, session_id=test_session_id)
+            except TypeError:
+                try:
+                    entry_point(test_query)
+                except TypeError:
+                    # Might need LLM provider or other args
+                    print(f"  ⚠ Could not call {entry_point_name} with simple args")
+                    print(
+                        "    You may need to run this validation manually with proper configuration"
+                    )
+                    return None
+
+            print("  ✓ Agent executed successfully")
+
+            # Get trace
+            trace_id = mlflow.get_last_active_trace_id()
+            if not trace_id:
+                print("  ✗ No trace ID captured!")
+                return None
+
+            print(f"  ✓ Trace captured: {trace_id}")
+
+            # Get trace details
+            client = MlflowClient()
+            return client.get_trace(trace_id)
+
+        except Exception as e:
+            print(f"  ✗ Error executing agent: {e}")
+            import traceback
+
+            traceback.print_exc()
+            return None
+
+    except Exception as e:
+        print(f"  ✗ Error: {e}")
+        import traceback
+
+        traceback.print_exc()
+        return None
+
+
+def verify_trace_structure(trace) -> tuple[bool, list[str]]:
+    """Verify the trace has the expected structure."""
+    print("\nVerifying trace structure...")
+
+    issues = []
+
+    # Check for top-level span (from @mlflow.trace decorator)
+    if not trace.data.spans:
+        issues.append("No spans found in trace")
+        return False, issues
+
+    top_span = trace.data.spans[0]
+    print(f"  ✓ Top-level span: {top_span.name} ({top_span.span_type})")
+
+    # Check for library spans (from autolog)
+    def count_spans(spans):
+        count = len(spans)
+        for span in spans:
+            if hasattr(span, "spans") and span.spans:
+                count += count_spans(span.spans)
+        return count
+
+    total_spans = count_spans(trace.data.spans)
+    print(f"  ✓ Total spans in hierarchy: {total_spans}")
+
+    if total_spans < 2:
+        issues.append("Only one span found - autolog may not be working")
+    else:
+        print("  ✓ Multiple spans detected - autolog appears to be working")
+
+    # Print hierarchy
+    def print_hierarchy(spans, indent=0):
+        for span in spans:
+            prefix = "    " + "  " * indent
+            print(f"{prefix}- {span.name} ({span.span_type})")
+            if hasattr(span, "spans") and span.spans:
+                print_hierarchy(span.spans, indent + 1)
+
+    print("\n  Trace hierarchy:")
+    print_hierarchy(trace.data.spans)
+
+    return len(issues) == 0, issues
+
+
+def verify_session_id(trace, expected_session_id: str) -> tuple[bool, str]:
+    """Verify session ID is captured in trace."""
+    print("\nVerifying session ID capture...")
+
+    if "session_id" not in trace.info.tags:
+        print("  ✗ Session ID not found in trace tags")
+        return False, "Session ID not captured"
+
+    actual_session_id = trace.info.tags["session_id"]
+    print(f"  ✓ Session ID found: {actual_session_id}")
+
+    if actual_session_id == expected_session_id:
+        print("  ✓ Session ID matches expected value")
+        return True, ""
+    else:
+        print("  ✗ Session ID mismatch!")
+        print(f"    Expected: {expected_session_id}")
+        print(f"    Got: {actual_session_id}")
+        return (
+            False,
+            f"Session ID mismatch: expected {expected_session_id}, got {actual_session_id}",
+        )
+
+
+def main():
+    """Main validation workflow."""
+    # Parse command-line arguments
+    parser = argparse.ArgumentParser(
+        description="Validate MLflow tracing integration with an agent",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  python validate_tracing_runtime.py                                    # Auto-detect everything
+  python validate_tracing_runtime.py --module my_agent.agent            # Specify module
+  python validate_tracing_runtime.py --entry-point process              # Specify entry point
+  python validate_tracing_runtime.py --module my_agent --entry-point process  # Both
+        """,
+    )
+    parser.add_argument("--module", help='Agent module name (e.g., "mlflow_agent.agent")')
+    parser.add_argument("--entry-point", help='Entry point function name (e.g., "run_agent")')
+    parser.add_argument(
+        "--autolog-file", help='File containing autolog() call (e.g., "src/agent/__init__.py")'
+    )
+    args = parser.parse_args()
+
+    print("=" * 60)
+    print("MLflow Tracing Validation")
+    print("=" * 60)
+    print()
+
+    # Track issues
+    all_issues = []
+
+    # Step 1: Check environment
+    print("Checking environment...")
+    env_errors = validate_env_vars()
+    if env_errors:
+        print()
+        print("✗ Environment issues:")
+        for error in env_errors:
+            print(f"  - {error}")
+        all_issues.extend(env_errors)
+    else:
+        import os
+
+        tracking_uri = os.getenv("MLFLOW_TRACKING_URI")
+        experiment_id = os.getenv("MLFLOW_EXPERIMENT_ID")
+        print(f"  ✓ MLFLOW_TRACKING_URI={tracking_uri}")
+        print(f"  ✓ MLFLOW_EXPERIMENT_ID={experiment_id}")
+
+    # Step 2: Get agent module (must be specified manually)
+    module_name = args.module
+    if not module_name:
+        print("\n✗ Agent module not specified")
+        print("  Use --module to specify your agent module")
+        print("  Example: --module my_agent.agent")
+        print("\n  To find your agent module:")
+        print("    grep -r 'def.*agent' . --include='*.py'")
+        sys.exit(1)
+    else:
+        print(f"\n✓ Using specified module: {module_name}")
+
+    # Step 3: Check autolog (optional - for informational purposes)
+    print("\nChecking autolog configuration...")
+    if args.autolog_file:
+        from pathlib import Path
+
+        if Path(args.autolog_file).exists():
+            print(f"  ✓ Autolog file specified: {args.autolog_file}")
+        else:
+            print(f"  ✗ Autolog file not found: {args.autolog_file}")
+            all_issues.append(f"Autolog file not found: {args.autolog_file}")
+    else:
+        print("  ⚠ No autolog file specified (use --autolog-file)")
+        print("  This is optional but recommended for full validation")
+        print("\n  To find autolog calls:")
+        print("    grep -r 'mlflow.*autolog' . --include='*.py'")
+
+    # Step 4: Get entry point (must be specified manually)
+    print("\nChecking entry point...")
+    entry_point_name = args.entry_point
+
+    if not entry_point_name:
+        print("  ✗ Entry point not specified")
+        print("  Use --entry-point to specify your agent's main function")
+        print("  Example: --entry-point run_agent")
+        print("\n  To find entry points with @mlflow.trace:")
+        print("    grep -r '@mlflow.trace' . --include='*.py'")
+        all_issues.append("No entry point specified")
+        sys.exit(1)
+    else:
+        print(f"  ✓ Using specified entry point: {entry_point_name}")
+
+    # Step 5: Run test query
+    trace = None
+    if entry_point_name:
+        trace = run_test_query(module_name, entry_point_name)
+        if not trace:
+            all_issues.append("Could not capture test trace")
+        else:
+            # Step 6: Verify trace structure
+            structure_ok, structure_issues = verify_trace_structure(trace)
+            if not structure_ok:
+                all_issues.extend(structure_issues)
+
+            # Step 7: Verify session ID (optional)
+            session_ok, session_issue = verify_session_id(trace, "test-session-123")
+            if not session_ok:
+                # Session ID is optional, so just warn
+                print(f"\n⚠ Note: {session_issue}")
+                print("  Session ID tracking is optional. Skip if not needed.")
+
+    # Final report
+    print("\n" + "=" * 60)
+    print("Validation Report")
+    print("=" * 60)
+
+    if not all_issues:
+        print("\n✓ ALL CHECKS PASSED!")
+        print("\nYour agent is properly integrated with MLflow tracing.")
+        print("You can proceed with evaluation.")
+    else:
+        print(f"\n✗ Found {len(all_issues)} issue(s):")
+        for i, issue in enumerate(all_issues, 1):
+            print(f"\n{i}. {issue}")
+
+        print("\n" + "=" * 60)
+        print("Next Steps")
+        print("=" * 60)
+        print("\n1. Fix the issues listed above")
+        print("2. Refer to references/tracing-integration.md for detailed guidance")
+        print("3. Run this script again to verify fixes")
+        print("\nDO NOT proceed with evaluation until all issues are resolved.")
+
+    print("=" * 60)
+
+    sys.exit(0 if not all_issues else 1)
+
+
+if __name__ == "__main__":
+    main()

package/canonical/skills/agentic-architecture/SKILL.md

@@ -0,0 +1,41 @@
+---
+name: agentic-architecture
+description: Use when designing or evaluating an AI agent system: topology (single agent, supervisor, subgraphs), orchestration pattern, context engineering, state schema, memory, tool contracts, reliability and observability. Standalone or inside the OpenSpec /opsx design phase with the agentic-architect agent.
+---
+
+# Agentic Architecture Protocol
+
+<philosophy>
+System capabilities stem from orchestration, context boundaries, data contracts, and state management, never from larger model parameter sizes alone. Start simple; add complexity only when mathematically justified.
+</philosophy>
+
+<modes>
+## Mode 1: Design Loop
+1. Clarify Constraints: Document goals, consumer contracts, required deterministic control, latency/cost budgets, and safety boundaries. Stop and ask before planning.
+2. Separation of Concerns: Allocate stochastic tasks to LLMs; allocate deterministic validation, business rules, auth, and calculations to standard code.
+3. Pattern Selection: Prefer simplest pattern (Deterministic Code > Single Agent + Tools > Router > Supervisor > Multi-Agent Graph). Single-agent with skills is ~10-15x cheaper than multi-agent. Document choice as an ADR.
+4. Context Strategy: Define per-step context payloads; eliminate starvation and bloat; set model-tiering (SLM for narrow/repetitive steps, LLM for open reasoning).
+5. Reliability & Observability: Implement exponential backoff, circuit breakers, strict output schema verification, and fallback routes. Instrument traces via OpenTelemetry/LangFuse.
+6. Blueprint Generation: Output blueprint using the output template.
+
+## Mode 2: Review Framework
+1. Audit Architecture: Inspect topology, tool calling sandboxes, and context boundaries. Check for the Lethal Trifecta (sensitive data + open communication + unverified execution).
+2. Report Findings: Categorize issues by severity:
+   - P0 (Blocker): Unbounded loops, untrusted execution, credential exposure.
+   - P1 (Concern): Missing circuit breakers, model tier mismatch, lack of retry backoff.
+   - P2 (Optimization): Minor schema redundancy, excessive prompt verbiage.
+3. Diagrams: Produce Mermaid C4 Component and StateGraph diagrams. End with `OK` or `FAIL`.
+</modes>
+
+<blueprint_template>
+### Architecture Blueprint
+1. Scope & Non-Goals: Explicit system boundaries.
+2. Component Architecture: Responsibilities, state schemas (Pydantic/TypedDict), and directional dependencies.
+3. Pattern Rationale: Selected pattern vs rejected alternatives.
+4. Context & Tiering: Context window budgets, memory retention strategy, and model tier allocation.
+5. Reliability & Fallbacks: Guardrails against failures, timeouts, and tool errors.
+6. Observability & SLIs: Instrumented spans, metrics, and LangFuse/OTel attributes.
+7. Security Boundaries: Trust zones, input sanitization, and permission barriers.
+8. Diagrams: Mermaid system context and component diagrams.
+9. Verification & Evals: Quantitative thresholds and gate criteria.
+</blueprint_template>

package/canonical/skills/agentic-architecture/evals/cases.yaml

@@ -0,0 +1,94 @@
+# Eval cases for the agentic-architecture skill. Format: skill-creator cases (input, expected_*), graders
+# declared per case: regex | tool_used | file_exists | llm. `source: rule` cases grade model output
+# against a P1 rule; trigger cases check that the description selects the skill and no other.
+# Runner: scripts/skill-evals.mjs. `source: dogfood` cases carry `dogfood: <date> <project>` (the
+# .freeharness/dogfood.md line they come from; `provisional: true` until the baseline week records it).
+skill: agentic-architecture
+cases:
+  - id: trigger-agentic-architecture
+    source: rule
+    rule: skill-trigger
+    input: "Design the supervisor topology for the copilot: a deterministic routing gate, one recommendation subgraph and a fallback node; compare two approaches and name the reliability mechanisms."
+    expected_skill: agentic-architecture
+    not_expected_skills:
+      - agent-evaluation
+      - code-review
+      - git-workflow
+      - local-code
+      - report-formatting
+      - subagent-driven-development
+      - systematic-debugging
+      - test-driven-development
+      - verification-before-completion
+    graders:
+      - type: tool_used
+        expected: "Skill:agentic-architecture"
+      - type: tool_used
+        not_expected:
+          - "Skill:agent-evaluation"
+          - "Skill:code-review"
+          - "Skill:git-workflow"
+          - "Skill:local-code"
+          - "Skill:report-formatting"
+          - "Skill:subagent-driven-development"
+          - "Skill:systematic-debugging"
+          - "Skill:test-driven-development"
+          - "Skill:verification-before-completion"
+    ablation: false
+
+  - id: dogfood-agentic-architecture-deterministic-gate
+    source: dogfood
+    dogfood: 2026-09-06 Copiloto
+    provisional: true
+    input: "Design the LangGraph supervisor for the Copiloto recommendation flow: the routing must be deterministic (the engine decides final_node), the LLM classifies the contact reason only when it is ambiguous and writes the phrase. Give the topology, the state schema and where the deterministic-first gate sits."
+    expected: "A thin supervisor with a deterministic-first gate before any LLM node, a typed state schema, the LLM confined to classification-on-ambiguity and phrasing, and the boundary stated explicitly."
+    graders:
+      - type: regex
+        flags: i
+        expected: "deterministic"
+      - type: regex
+        flags: i
+        expected: "state schema|TypedDict|StateGraph"
+      - type: regex
+        flags: i
+        expected: "supervisor"
+      - type: llm
+        rubric: "Pass when the design keeps the business decision in the deterministic engine, places the gate before the LLM nodes, and names the state fields; fail when the LLM chooses the final_node or the topology has no gate."
+    ablation: true
+
+  - id: dogfood-agentic-architecture-error-contract
+    source: dogfood
+    dogfood: 2026-09-06 Copiloto
+    provisional: true
+    input: "Review the tool contract of the HubGPU LLM adapter: what does the caller receive on a timeout, on an HTTP 500 and on malformed JSON from the model? Today it raises a bare Exception in all three cases."
+    expected: "An error contract with a stable code plus next action per outcome, the three outcomes as independent fields (never nested), and a degrade-to-business-error path instead of an opaque 500."
+    graders:
+      - type: regex
+        flags: i
+        expected: "code"
+      - type: regex
+        flags: i
+        expected: "next action|retry|fallback|degrade"
+      - type: regex
+        flags: i
+        expected: "timeout"
+      - type: llm
+        rubric: "Pass when each of timeout, HTTP 500 and malformed JSON gets its own code and next action and the answer rejects the bare Exception; fail when the outcomes are collapsed into one error or nested in one another."
+    ablation: true
+
+  - id: dogfood-agentic-architecture-memory-placement
+    source: dogfood
+    dogfood: 2026-09-06 STT
+    provisional: true
+    input: "The STT pipeline needs per-speaker context across segments. Should it live in the graph state or in an external store? Recommend one and give the trade-offs."
+    expected: "One recommendation with explicit trade-offs (latency, size, persistence, testability) and the consequence for the state schema, not a list of options without a choice."
+    graders:
+      - type: regex
+        flags: i
+        expected: "trade-?offs?"
+      - type: regex
+        flags: i
+        expected: "recommend"
+      - type: llm
+        rubric: "Pass when exactly one option is recommended with at least two trade-offs and the state-schema consequence; fail when both options are described without a decision."
+    ablation: true