@matt82198/aesop 0.1.0 → 0.2.0

package/ui/quality_scorecard.py

@@ -0,0 +1,232 @@
+#!/usr/bin/env python3
+"""Quality scorecard collector — per-agent-specialty success rates and retry frequencies.
+
+This module provides get_quality_scorecard() which parses the outcomes ledger
+and returns per-agent-specialty quality metrics: success rate (green/total)
+and retry/repair frequency, derived from ledger agent_type + verdict columns.
+
+Ledger format (same as cost.py):
+  | ISO timestamp | agent_type | model | duration | tokens_in | tokens_out | verdict |
+  | 2026-07-11T22:08:17 | haiku | claude-haiku-4-5-20251001 | 0 | 8 | 186 | OK |
+
+QualityScorecard JSON shape (returned by get_quality_scorecard()):
+  {
+    "specialties": {
+      "agent-type": {
+        "total_runs": int,
+        "success_count": int,
+        "failed_count": int,
+        "empty_count": int,
+        "hung_count": int,
+        "success_rate": float (0.0-1.0),
+        "repair_count": int (consecutive failures followed by success),
+        "retry_frequency": float (repairs / total_runs)
+      },
+      ...
+    },
+    "top_by_success": [
+      {"agent_type": str, "success_rate": float, "total_runs": int},
+      ...
+    ],
+    "top_by_retry": [
+      {"agent_type": str, "retry_frequency": float, "total_runs": int},
+      ...
+    ],
+    "skipped_lines": int
+  }
+
+Key behavior:
+  - Missing ledger file: returns empty summary with documented shape.
+  - Malformed lines: skipped and counted in skipped_lines field.
+  - Config read at CALL time (not import time) for test isolation.
+  - UTF-8 explicit encoding for all file operations.
+  - No external dependencies (pure stdlib).
+  - Retry frequency calculated by detecting "failure -> success" transitions (repair cycles).
+  - Rankings sorted by metric descending (highest success rate / highest retry frequency first).
+"""
+import json
+from pathlib import Path
+
+import config
+
+
+def get_quality_scorecard():
+    """Parse the outcomes ledger and return per-agent-specialty quality metrics.
+
+    Reads from the ledger path exposed by config (config.STATE_DIR/ledger/OUTCOMES-LEDGER.md).
+    Returns an empty summary with documented shape if ledger is missing or empty.
+    Malformed lines are skipped and counted in skipped_lines.
+
+    Extracts agent_type from the ledger (haiku, sonnet, opus, orchestrator, etc.) and
+    aggregates success rates and repair (retry) frequencies.
+
+    Repair cycles detected by counting consecutive failures followed by a success
+    in the same agent_type's history.
+
+    Returns:
+        dict: QualityScorecard with specialties, top_by_success, top_by_retry,
+              and skipped_lines (or error field if invalid).
+    """
+    import sys
+
+    # Read ledger path at call time
+    ledger_file = config.STATE_DIR / "ledger" / "OUTCOMES-LEDGER.md"
+
+    # Initialize result structure
+    result = {
+        "specialties": {},
+        "top_by_success": [],
+        "top_by_retry": [],
+        "skipped_lines": 0,
+    }
+
+    # If ledger file doesn't exist, return empty summary
+    if not ledger_file.exists():
+        return result
+
+    # Read and parse ledger with explicit UTF-8 encoding
+    try:
+        content = ledger_file.read_text(encoding='utf-8')
+    except Exception:
+        # Graceful: if read fails, return empty summary
+        return result
+
+    lines = content.strip().split('\n')
+
+    # Parse each line and collect per-agent-type history
+    agent_verdicts = {}  # agent_type -> [verdict1, verdict2, ...]
+
+    for line in lines:
+        line = line.strip()
+
+        # Skip empty lines
+        if not line:
+            continue
+
+        # Skip header separator lines (all dashes, pipes, and spaces)
+        if all(c in '|- ' for c in line):
+            continue
+
+        # Parse pipe-delimited row
+        if not line.startswith('|') or not line.endswith('|'):
+            result["skipped_lines"] += 1
+            continue
+
+        # Split by pipe and strip whitespace
+        parts = [p.strip() for p in line.split('|')]
+
+        if len(parts) < 9:  # Need at least 9 parts (empty + 7 columns + empty)
+            result["skipped_lines"] += 1
+            continue
+
+        # Extract columns (skip leading/trailing empty)
+        try:
+            timestamp = parts[1]  # ISO timestamp
+            agent_type = parts[2]  # "haiku", "sonnet", "orchestrator", etc.
+            model = parts[3]
+            duration_str = parts[4]
+            tokens_in_str = parts[5]
+            tokens_out_str = parts[6]
+            verdict = parts[7]
+        except IndexError:
+            result["skipped_lines"] += 1
+            continue
+
+        # Skip header line
+        if 'timestamp' in timestamp.lower() or 'iso' in timestamp.lower():
+            continue
+
+        # Validate verdict before processing
+        if verdict not in ("OK", "FAILED", "EMPTY", "HUNG"):
+            result["skipped_lines"] += 1
+            continue
+
+        # Parse and validate numeric fields
+        try:
+            tokens_in = int(tokens_in_str)
+            tokens_out = int(tokens_out_str)
+        except ValueError:
+            result["skipped_lines"] += 1
+            continue
+
+        # Initialize agent_type entry if not seen
+        if agent_type not in result["specialties"]:
+            result["specialties"][agent_type] = {
+                "total_runs": 0,
+                "success_count": 0,
+                "failed_count": 0,
+                "empty_count": 0,
+                "hung_count": 0,
+                "success_rate": 0.0,
+                "repair_count": 0,
+                "retry_frequency": 0.0,
+            }
+            agent_verdicts[agent_type] = []
+
+        # Increment verdict counters
+        result["specialties"][agent_type]["total_runs"] += 1
+        if verdict == "OK":
+            result["specialties"][agent_type]["success_count"] += 1
+        elif verdict == "FAILED":
+            result["specialties"][agent_type]["failed_count"] += 1
+        elif verdict == "EMPTY":
+            result["specialties"][agent_type]["empty_count"] += 1
+        elif verdict == "HUNG":
+            result["specialties"][agent_type]["hung_count"] += 1
+
+        # Collect verdict history for repair detection
+        agent_verdicts[agent_type].append(verdict)
+
+    # Calculate success rates and detect repair cycles
+    for agent_type, stats in result["specialties"].items():
+        total = stats["total_runs"]
+        if total > 0:
+            stats["success_rate"] = stats["success_count"] / total
+            stats["retry_frequency"] = stats["repair_count"] / total
+
+        # Detect repair cycles (consecutive failures followed by success)
+        verdicts = agent_verdicts.get(agent_type, [])
+        repair_count = 0
+        in_failure_sequence = False
+
+        for verdict in verdicts:
+            if verdict in ("FAILED", "EMPTY", "HUNG"):
+                in_failure_sequence = True
+            elif verdict == "OK" and in_failure_sequence:
+                # Success after failures = repair cycle
+                repair_count += 1
+                in_failure_sequence = False
+
+        stats["repair_count"] = repair_count
+        if total > 0:
+            stats["retry_frequency"] = repair_count / total
+
+    # Generate top-by-success ranking (highest success rate first)
+    success_ranking = sorted(
+        [
+            {
+                "agent_type": agent_type,
+                "success_rate": stats["success_rate"],
+                "total_runs": stats["total_runs"],
+            }
+            for agent_type, stats in result["specialties"].items()
+        ],
+        key=lambda x: (-x["success_rate"], -x["total_runs"]),  # Sort desc by success_rate, then runs
+    )
+    result["top_by_success"] = success_ranking
+
+    # Generate top-by-retry ranking (highest retry/repair frequency first)
+    retry_ranking = sorted(
+        [
+            {
+                "agent_type": agent_type,
+                "retry_frequency": stats["retry_frequency"],
+                "total_runs": stats["total_runs"],
+            }
+            for agent_type, stats in result["specialties"].items()
+        ],
+        key=lambda x: (-x["retry_frequency"], -x["total_runs"]),  # Sort desc by retry_frequency, then runs
+    )
+    result["top_by_retry"] = retry_ranking
+
+    return result

package/ui/wave_audit_tail.py

@@ -0,0 +1,213 @@
+#!/usr/bin/env python3
+"""
+Wave audit tail — newest audit/verification outcomes (adversarial findings, verdicts).
+
+Reads audit backlog and ledger to surface latest verification outcomes.
+Compact live tail panel showing recent audit events and findings.
+
+Returns audit tail data for the Activity view.
+"""
+import json
+import re
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Dict, List, Optional
+
+import config
+import cost
+
+
+def _parse_audit_backlog_recent() -> List[Dict]:
+    """Extract recent items from AUDIT-BACKLOG.md.
+
+    Returns the most recent 5-10 P0/P1 items (inflight, todo, or done).
+
+    Returns:
+        list: [{"status": "✅|🔵|⬜", "tier": "P0|P1", "tag": "[tag]", "title": str, ...}, ...]
+    """
+    try:
+        backlog_file = config.AUDIT_BACKLOG_FILE
+        if not backlog_file.exists():
+            return []
+
+        content = backlog_file.read_text(encoding='utf-8')
+        lines = content.split('\n')
+
+        recent_items = []
+        current_tier = None
+
+        for line in lines:
+            line_stripped = line.strip()
+
+            # Detect tier headers
+            if re.match(r'^##\s+(P0|P1|P2)', line_stripped):
+                match = re.match(r'^##\s+(P0|P1|P2)', line_stripped)
+                current_tier = match.group(1) if match else None
+                continue
+
+            if current_tier and line_stripped.startswith('- '):
+                # Extract status emoji, tag, and title
+                # Pattern: - 🔵 **[tag] Title**
+                match = re.search(r'(✅|🔵|⬜|⏸)\s+\*\*\[([^\]]+)\]\s+(.+?)\*\*', line_stripped)
+                if match:
+                    status_emoji = match.group(1)
+                    tag = match.group(2)
+                    title = match.group(3)
+
+                    recent_items.append({
+                        "status": status_emoji,
+                        "tier": current_tier,
+                        "tag": tag,
+                        "title": title,
+                        "timestamp": None,  # AUDIT-BACKLOG doesn't have timestamps
+                    })
+
+        # Return most recent (last) items
+        return recent_items[-10:] if recent_items else []
+    except Exception as e:
+        print(f"[wave_audit_tail] Error parsing audit backlog: {e}")
+        return []
+
+
+def _parse_ledger_recent_verdicts() -> List[Dict]:
+    """Extract recent verdict outcomes from OUTCOMES-LEDGER.md.
+
+    Looks for recent lines showing agent results: OK/FAILED/EMPTY/HUNG.
+
+    Ledger format (9-column, optionally 7-column legacy):
+      | ISO ts | agent_type | model | duration_sec | tokens_in | tokens_out | verdict | phase | wave |
+
+    Verdict whitelist: only OK, FAILED, EMPTY, HUNG are valid.
+    Forged/invalid verdicts are skipped.
+
+    Returns:
+        list: [{"agent": str, "verdict": "OK|FAILED|EMPTY|HUNG", "timestamp": iso, ...}, ...]
+    """
+    # Whitelist of valid verdicts
+    VALID_VERDICTS = {"OK", "FAILED", "EMPTY", "HUNG"}
+
+    try:
+        ledger_file = config.LEDGER_FILE
+        if not ledger_file.exists():
+            return []
+
+        content = ledger_file.read_text(encoding='utf-8')
+        lines = content.split('\n')
+
+        recent_verdicts = []
+
+        # Parse ledger table rows (supports 7-column and 9-column formats)
+        for line in reversed(lines[-50:]):  # Check last 50 lines
+            line_stripped = line.strip()
+            # Look for table rows with verdict info
+            # Pattern: | timestamp | agent_type | model | ... | verdict | ... |
+            if '|' not in line_stripped:
+                continue
+
+            parts = [p.strip() for p in line_stripped.split('|')]
+
+            # Accept both 7-column (9 parts) and 9-column (11 parts) formats
+            # 7 columns: ['', col1, col2, col3, col4, col5, col6, col7, '']  -> parts[7] = verdict
+            # 9 columns: ['', col1, col2, col3, col4, col5, col6, col7, col8, col9, ''] -> parts[7] = verdict
+            if len(parts) < 9:
+                continue
+
+            # Try to extract timestamp (parts[1]), agent_type (parts[2]), and verdict (parts[7])
+            try:
+                timestamp_str = parts[1]
+                agent_str = parts[2]
+                # IMPORTANT: verdict is at parts[7], not parts[3] (which is model)
+                verdict_str = parts[7].upper()
+
+                # Validate verdict against whitelist
+                if verdict_str not in VALID_VERDICTS:
+                    # Skip forged/invalid verdicts
+                    continue
+
+                # Verify timestamp looks like ISO format
+                if 'T' not in timestamp_str and '-' not in timestamp_str[:10]:
+                    continue
+
+                recent_verdicts.append({
+                    "timestamp": timestamp_str,
+                    "agent": agent_str.split('-')[-1][:13],  # Short agent ID
+                    "verdict": verdict_str,
+                })
+            except (IndexError, ValueError):
+                pass
+
+        # Return most recent (first in reversed list)
+        return recent_verdicts[-10:] if recent_verdicts else []
+    except Exception as e:
+        print(f"[wave_audit_tail] Error parsing ledger: {e}")
+        return []
+
+
+def get_wave_audit_tail() -> Dict:
+    """Get audit tail data for the current wave.
+
+    Returns latest audit/verification outcomes as a compact tail.
+
+    Returns:
+        dict: {
+            "available": bool,
+            "audit_items": [
+                {
+                    "type": "audit_backlog|verdict",
+                    "timestamp": "2026-07-21T12:34:56Z" or null,
+                    "status": "✅|🔵|⬜|...",
+                    "tier": "P0|P1|P2",
+                    "tag": "[sec]|[ui]|...",
+                    "title": str,
+                    "verdict": "OK|FAILED|...",
+                    "agent": str,
+                },
+                ...
+            ],
+            "at": "2026-07-21T12:34:56Z"
+        }
+    """
+    try:
+        audit_items = []
+
+        # Get recent backlog items
+        backlog_items = _parse_audit_backlog_recent()
+        for item in backlog_items:
+            audit_items.append({
+                "type": "audit_backlog",
+                **item,
+            })
+
+        # Get recent verdict outcomes
+        verdict_items = _parse_ledger_recent_verdicts()
+        for item in verdict_items:
+            audit_items.append({
+                "type": "verdict",
+                **item,
+            })
+
+        # Sort by timestamp (nulls last) and limit to 15 most recent
+        def sort_key(item):
+            ts = item.get("timestamp")
+            if ts is None:
+                return ""
+            return ts
+
+        audit_items.sort(key=sort_key, reverse=True)
+        audit_items = audit_items[:15]
+
+        return {
+            "available": bool(audit_items),
+            "audit_items": audit_items,
+            "at": datetime.now(timezone.utc).isoformat() + "Z",
+        }
+    except Exception as e:
+        print(f"[wave_audit_tail] Uncaught error: {e}")
+        import traceback
+        traceback.print_exc()
+        return {
+            "available": False,
+            "error": str(e),
+            "audit_items": [],
+            "at": datetime.now(timezone.utc).isoformat() + "Z"
+        }