@matt82198/aesop 0.1.0-rc.1 → 0.2.0

package/ui/cost.py

@@ -5,9 +5,14 @@ This module provides get_cost_summary() which parses the outcomes ledger
 markdown table and returns per-model, per-day, and overall cost/token aggregations
 with optional pricing estimates.
 
-Ledger format (markdown table):
-  | ISO timestamp | agent_type | model | duration | tokens_in | tokens_out | verdict |
-  | 2026-07-11T22:08:17 | Agent | claude-haiku-4-5-20251001 | 0 | 8 | 186 | OK |
+Ledger format (markdown table, supports both 7-column and 9-column):
+  Legacy 7-column:
+    | ISO timestamp | agent_type | model | duration | tokens_in | tokens_out | verdict |
+    | 2026-07-11T22:08:17 | Agent | claude-haiku-4-5-20251001 | 0 | 8 | 186 | OK |
+
+  Extended 9-column (phase/wave optional):
+    | ISO timestamp | agent_type | model | duration | tokens_in | tokens_out | verdict | phase | wave |
+    | 2026-07-11T22:08:17 | Agent | claude-haiku-4-5-20251001 | 0 | 8 | 186 | OK | build | 7 |
 
 CostSummary JSON shape (returned by get_cost_summary()):
   {
@@ -69,10 +74,11 @@ def _validate_ledger_format(lines):
     Checks the first non-empty, non-separator, non-header line to ensure it looks like a data row
     (has ISO timestamp, agent type, model, numeric fields, verdict). Returns (is_valid, error_message).
 
-    Expected format:
-      | ISO timestamp | agent_type | model | duration | tokens_in | tokens_out | verdict |
-      |---|---|---|---|---|---|---|
-      | 2026-07-11T22:08:17 | Agent | claude-haiku-4-5-20251001 | 0 | 8 | 186 | OK |
+    Accepts two formats:
+      - Legacy 7-column: | timestamp | agent_type | model | duration | tokens_in | tokens_out | verdict |
+      - Extended 9-column: | timestamp | agent_type | model | duration | tokens_in | tokens_out | verdict | phase | wave |
+
+    Optional trailing columns (phase, wave) are ignored during validation.
     """
     for line in lines:
         line = line.strip()
@@ -91,11 +97,13 @@ def _validate_ledger_format(lines):
 
         parts = [p.strip() for p in line.split('|')]
 
-        # Should have 9 parts: [empty, col1, col2, col3, col4, col5, col6, col7, empty]
-        if len(parts) != 9:
-            return False, f"Expected 7 columns, got {len(parts) - 2}"
+        # Accept both 7-column (9 parts) and 9-column (11 parts) formats
+        # 7 columns: ['', col1, col2, col3, col4, col5, col6, col7, '']
+        # 9 columns: ['', col1, col2, col3, col4, col5, col6, col7, col8, col9, '']
+        if len(parts) < 9:
+            return False, f"Too few columns, got {len(parts) - 2} (expected at least 7)"
 
-        # Extract columns and validate types
+        # Extract core columns (first 7 required columns)
         try:
             timestamp = parts[1]
             agent_type = parts[2]
@@ -145,11 +153,18 @@ def get_cost_summary():
     All config paths are read at call time (not import time) to ensure
     test-fixture isolation via config.reload().
 
+    Extended fields (additively):
+      - per_week_costs: dict of "YYYY-Www" -> week cost/token totals and model mix
+      - verdict_weighted_cost: cost-per-outcome metrics (cost per OK, weighted by verdict distribution)
+      - model_mix_trend: per-day model usage distribution (%)
+
     Returns:
         dict: CostSummary with models, daily_totals, overall_scorecard,
-              skipped_lines, has_pricing, estimates_by_model (or error field if invalid).
+              skipped_lines, has_pricing, estimates_by_model, per_week_costs,
+              verdict_weighted_cost, model_mix_trend (or error field if invalid).
     """
     import sys
+    from datetime import datetime, timedelta
 
     # Read ledger path at call time
     ledger_file = config.STATE_DIR / "ledger" / "OUTCOMES-LEDGER.md"
@@ -172,6 +187,14 @@ def get_cost_summary():
         "skipped_lines": 0,
         "has_pricing": False,
         "estimates_by_model": {},
+        "per_week_costs": {},
+        "verdict_weighted_cost": {
+            "cost_per_ok": 0.0,
+            "cost_per_failed": 0.0,
+            "cost_per_empty": 0.0,
+            "cost_per_hung": 0.0,
+        },
+        "model_mix_trend": {},
     }
 
     # If ledger file doesn't exist, return empty summary
@@ -194,6 +217,9 @@ def get_cost_summary():
         result["error"] = "ledger format invalid"
         return result
 
+    # Track ledger entries for per-week-per-model calculation
+    ledger_entries = []
+
     # Parse each line
     for line in lines:
         line = line.strip()
@@ -268,6 +294,16 @@ def get_cost_summary():
             result["skipped_lines"] += 1
             continue
 
+        # Store ledger entry for per-week calculation
+        ledger_entries.append({
+            "timestamp": timestamp,
+            "date_str": date_str,
+            "model": model,
+            "tokens_in": tokens_in,
+            "tokens_out": tokens_out,
+            "verdict": verdict
+        })
+
         # Aggregate by model
         if model not in result["models"]:
             result["models"][model] = {
@@ -329,9 +365,192 @@ def get_cost_summary():
                     "total_cost": total_cost,
                 }
 
+    # Calculate per-week costs and model mix trend
+    _calculate_weekly_costs(result, pricing_map, ledger_entries)
+    _calculate_verdict_weighted_cost(result, pricing_map)
+    _calculate_model_mix_trend(result)
+
     return result
 
 
+def _calculate_weekly_costs(result, pricing_map, ledger_entries):
+    """Calculate per-week cost rollup from ledger entries.
+
+    Groups ledger entries by ISO week (YYYY-Www format) and aggregates
+    per-model tokens for each week. If pricing is available, includes cost estimates.
+
+    BUG FIX: This function now uses EACH WEEK'S OWN per-model token counts
+    from the ledger, not the global model distribution. This prevents inflating
+    each week's cost by applying global model mix that may not be accurate
+    for that particular week.
+
+    PERF FIX: datetime.strptime is hoisted outside the inner loops. Each timestamp
+    is parsed ONCE during the initial pass, avoiding O(weeks*models*entries) re-parsing.
+
+    Modifies result["per_week_costs"] in-place with structure:
+    {
+        "YYYY-Www": {
+            "tokens_in": int,
+            "tokens_out": int,
+            "model_tokens": {"model": int, ...},
+            "cost": float (if pricing available)
+        }
+    }
+    """
+    from datetime import datetime
+
+    if not ledger_entries:
+        return
+
+    # OPTIMIZATION: Pre-parse all timestamps once (avoid O(weeks*models*entries) re-parsing)
+    # Map each entry index to its ISO week key, computed upfront
+    entry_weeks = {}
+    for idx, entry in enumerate(ledger_entries):
+        try:
+            # Parse YYYY-MM-DD to ISO week
+            dt = datetime.strptime(entry["date_str"], "%Y-%m-%d")
+            iso_year, iso_week, _ = dt.isocalendar()
+            week_key = f"{iso_year}-W{iso_week:02d}"
+            entry_weeks[idx] = week_key
+        except (ValueError, KeyError):
+            entry_weeks[idx] = None
+
+    # Group ledger entries by ISO week and aggregate per-model within each week
+    weeks = {}
+    for idx, entry in enumerate(ledger_entries):
+        week_key = entry_weeks[idx]
+        if week_key is None:
+            # Skip entries with invalid dates or missing fields
+            continue
+
+        if week_key not in weeks:
+            weeks[week_key] = {
+                "tokens_in": 0,
+                "tokens_out": 0,
+                "model_tokens": {},
+                "cost": 0.0
+            }
+
+        # Aggregate this entry's tokens into the week
+        weeks[week_key]["tokens_in"] += entry["tokens_in"]
+        weeks[week_key]["tokens_out"] += entry["tokens_out"]
+
+        # Track per-model tokens within this week
+        model = entry["model"]
+        if model not in weeks[week_key]["model_tokens"]:
+            weeks[week_key]["model_tokens"][model] = 0
+        weeks[week_key]["model_tokens"][model] += entry["tokens_in"] + entry["tokens_out"]
+
+    # If pricing available, calculate cost per week based on THAT WEEK'S model mix
+    if pricing_map:
+        for week_key, week_data in weeks.items():
+            total_cost = 0.0
+            for model, total_tokens in week_data["model_tokens"].items():
+                if model in pricing_map:
+                    pricing = pricing_map[model]
+                    input_price = pricing.get("input_per_mtok", 0.0)
+                    output_price = pricing.get("output_per_mtok", 0.0)
+                    # Use the pre-parsed week info to find entries for this model in this week
+                    model_entries_in_week = [
+                        ledger_entries[idx]
+                        for idx, w_key in entry_weeks.items()
+                        if w_key == week_key and ledger_entries[idx]["model"] == model
+                    ]
+                    if model_entries_in_week:
+                        week_model_tokens_in = sum(e["tokens_in"] for e in model_entries_in_week)
+                        week_model_tokens_out = sum(e["tokens_out"] for e in model_entries_in_week)
+                        model_cost = (week_model_tokens_in * input_price + week_model_tokens_out * output_price) / 1_000_000
+                    else:
+                        # Fallback: if we can't find the entries, use 1:2 ratio estimate
+                        tokens_in_estimate = total_tokens * 0.333
+                        tokens_out_estimate = total_tokens * 0.667
+                        model_cost = (tokens_in_estimate * input_price + tokens_out_estimate * output_price) / 1_000_000
+                    total_cost += model_cost
+            week_data["cost"] = total_cost
+
+    result["per_week_costs"] = weeks
+
+
+def _calculate_verdict_weighted_cost(result, pricing_map):
+    """Calculate cost-per-outcome metrics weighted by verdict distribution.
+
+    Computes cost per successful outcome (cost / ok_count) and cost per other outcomes.
+    If pricing is available, uses estimated costs; otherwise uses token counts as proxy.
+
+    Modifies result["verdict_weighted_cost"] in-place with structure:
+    {
+        "cost_per_ok": float,
+        "cost_per_failed": float,
+        "cost_per_empty": float,
+        "cost_per_hung": float,
+    }
+    """
+    scorecard = result["overall_scorecard"]
+
+    # Calculate total cost (if pricing available)
+    total_cost = 0.0
+    if pricing_map and result["has_pricing"]:
+        for estimate in result["estimates_by_model"].values():
+            total_cost += estimate.get("total_cost", 0.0)
+    else:
+        # Use token count as cost proxy (tokens_in + tokens_out)
+        for daily in result["daily_totals"].values():
+            total_cost += daily["tokens_in"] + daily["tokens_out"]
+
+    # Calculate cost per outcome type
+    result["verdict_weighted_cost"] = {
+        "cost_per_ok": total_cost / scorecard["ok_count"] if scorecard["ok_count"] > 0 else 0.0,
+        "cost_per_failed": total_cost / scorecard["failed_count"] if scorecard["failed_count"] > 0 else 0.0,
+        "cost_per_empty": total_cost / scorecard["empty_count"] if scorecard["empty_count"] > 0 else 0.0,
+        "cost_per_hung": total_cost / scorecard["hung_count"] if scorecard["hung_count"] > 0 else 0.0,
+    }
+
+
+def _calculate_model_mix_trend(result):
+    """Calculate per-day model usage distribution as percentages.
+
+    Breaks down the token usage by model for each day in daily_totals.
+    Modifies result["model_mix_trend"] in-place with structure:
+    {
+        "YYYY-MM-DD": {
+            "model": percentage (0.0-100.0),
+            ...
+        }
+    }
+    """
+    model_mix_trend = {}
+
+    # For each day, calculate model distribution
+    # Since daily_totals doesn't track per-model breakdown, we need to estimate
+    # based on the overall model distribution across all runs in that day
+    if not result["models"]:
+        result["model_mix_trend"] = model_mix_trend
+        return
+
+    # Calculate overall model token distribution
+    total_model_tokens = {}
+    grand_total_tokens = 0
+    for model, stats in result["models"].items():
+        tokens = stats["tokens_in"] + stats["tokens_out"]
+        total_model_tokens[model] = tokens
+        grand_total_tokens += tokens
+
+    # Apply model distribution to each day (as a simplified proxy)
+    for date_str in result["daily_totals"].keys():
+        daily_dist = {}
+        daily_total = result["daily_totals"][date_str]["tokens_in"] + result["daily_totals"][date_str]["tokens_out"]
+
+        if grand_total_tokens > 0 and daily_total > 0:
+            for model, total_tokens in total_model_tokens.items():
+                # Distribute daily tokens proportionally to model usage
+                ratio = total_tokens / grand_total_tokens
+                daily_dist[model] = round(ratio * 100.0, 2)
+
+        model_mix_trend[date_str] = daily_dist
+
+    result["model_mix_trend"] = model_mix_trend
+
+
 def _load_pricing_config():
     """Load pricing map from aesop.config.json at call time.
 

package/ui/handler.py

@@ -12,8 +12,15 @@ from pathlib import Path
 import config
 import cost
 import csrf
+import quality_scorecard
 import sse
 import wave_prs
+import wave_telemetry
+import wave_failure
+import wave_dispatch
+import wave_gantt
+import wave_audit_tail
+import wave_reasoning_tail
 import api
 import api.tracker
 import api.submit
@@ -197,6 +204,20 @@ class DashboardHandler(http.server.BaseHTTPRequestHandler):
             self.serve_api_cost()
         elif self.path == "/api/wave/prs":
             self.serve_api_wave_prs()
+        elif self.path == "/api/wave/telemetry":
+            self.serve_api_wave_telemetry()
+        elif self.path == "/api/wave/dispatch":
+            self.serve_api_wave_dispatch()
+        elif self.path == "/api/wave/gantt":
+            self.serve_api_wave_gantt()
+        elif self.path == "/api/wave/audit-tail":
+            self.serve_api_wave_audit_tail()
+        elif self.path == "/api/wave/reasoning-tail":
+            self.serve_api_wave_reasoning_tail()
+        elif self.path == "/api/wave/quality-scorecards":
+            self.serve_api_wave_quality_scorecards()
+        elif self.path.startswith("/api/wave/failure"):
+            self.serve_api_wave_failure()
         elif self.path == "/api/backlog":
             self.serve_backlog()
         elif self.path == "/api/agents":
@@ -427,6 +448,168 @@ class DashboardHandler(http.server.BaseHTTPRequestHandler):
             self.end_headers()
             self.wfile.write(json.dumps({"error": "Internal server error"}).encode('utf-8'))
 
+    def serve_api_wave_telemetry(self):
+        """GET /api/wave/telemetry — current wave phase, cost metrics, and top blocker.
+
+        Read-only; reads state at call time (no caching). Returns wave phase info,
+        current cost metrics, and top blocker from AUDIT-BACKLOG.md.
+        """
+        try:
+            payload = wave_telemetry.get_wave_telemetry()
+            self.send_response(200)
+            self.send_header("Content-Type", "application/json; charset=utf-8")
+            self.send_header("Cache-Control", "no-cache, no-store, must-revalidate")
+            self.end_headers()
+            self.wfile.write(json.dumps(payload, default=str).encode('utf-8'))
+        except Exception as e:
+            print(f"[serve_api_wave_telemetry] Uncaught exception: {e}", file=sys.stderr)
+            self.send_response(500)
+            self.send_header("Content-Type", "application/json; charset=utf-8")
+            self.end_headers()
+            self.wfile.write(json.dumps({"error": "Internal server error"}).encode('utf-8'))
+
+    def serve_api_wave_dispatch(self):
+        """GET /api/wave/dispatch — live per-agent phase and activity visibility.
+
+        Read-only; reads at call time (no caching). Returns per-agent phase,
+        last-activity age, and token burn estimates. Degrades to {available:false}
+        when no active workflow.
+        """
+        try:
+            payload = wave_dispatch.get_wave_dispatch()
+            self.send_response(200)
+            self.send_header("Content-Type", "application/json; charset=utf-8")
+            self.send_header("Cache-Control", "no-cache, no-store, must-revalidate")
+            self.end_headers()
+            self.wfile.write(json.dumps(payload, default=str).encode('utf-8'))
+        except Exception as e:
+            print(f"[serve_api_wave_dispatch] Uncaught exception: {e}", file=sys.stderr)
+            self.send_response(500)
+            self.send_header("Content-Type", "application/json; charset=utf-8")
+            self.end_headers()
+            self.wfile.write(json.dumps({"error": "Internal server error"}).encode('utf-8'))
+
+    def serve_api_wave_gantt(self):
+        """GET /api/wave/gantt — Gantt timeline data for agents in current wave.
+
+        Read-only; reads at call time (no caching). Returns per-agent rows with
+        phase timing spans suitable for Gantt visualization. Degrades to
+        {available:false} when no active workflow.
+        """
+        try:
+            payload = wave_gantt.get_wave_gantt()
+            self.send_response(200)
+            self.send_header("Content-Type", "application/json; charset=utf-8")
+            self.send_header("Cache-Control", "no-cache, no-store, must-revalidate")
+            self.end_headers()
+            self.wfile.write(json.dumps(payload, default=str).encode('utf-8'))
+        except Exception as e:
+            print(f"[serve_api_wave_gantt] Uncaught exception: {e}", file=sys.stderr)
+            self.send_response(500)
+            self.send_header("Content-Type", "application/json; charset=utf-8")
+            self.end_headers()
+            self.wfile.write(json.dumps({"error": "Internal server error"}).encode('utf-8'))
+
+    def serve_api_wave_audit_tail(self):
+        """GET /api/wave/audit-tail — latest audit/verification outcomes.
+
+        Read-only; reads at call time (no caching). Returns recent audit backlog
+        items and ledger verdicts as a compact tail. Shows latest findings from
+        adversarial reviews and verify verdicts.
+        """
+        try:
+            payload = wave_audit_tail.get_wave_audit_tail()
+            self.send_response(200)
+            self.send_header("Content-Type", "application/json; charset=utf-8")
+            self.send_header("Cache-Control", "no-cache, no-store, must-revalidate")
+            self.end_headers()
+            self.wfile.write(json.dumps(payload, default=str).encode('utf-8'))
+        except Exception as e:
+            print(f"[serve_api_wave_audit_tail] Uncaught exception: {e}", file=sys.stderr)
+            self.send_response(500)
+            self.send_header("Content-Type", "application/json; charset=utf-8")
+            self.end_headers()
+            self.wfile.write(json.dumps({"error": "Internal server error"}).encode('utf-8'))
+
+    def serve_api_wave_reasoning_tail(self):
+        """GET /api/wave/reasoning-tail — per-agent live reasoning/transcript summary.
+
+        Read-only; reads at call time (no caching). Returns latest transcript activity
+        summary for each live agent (redacted). Shows thinking/tool-call sequences
+        in a compact format suitable for Activity view transparency.
+        """
+        try:
+            payload = wave_reasoning_tail.get_wave_reasoning_tail()
+            self.send_response(200)
+            self.send_header("Content-Type", "application/json; charset=utf-8")
+            self.send_header("Cache-Control", "no-cache, no-store, must-revalidate")
+            self.end_headers()
+            self.wfile.write(json.dumps(payload, default=str).encode('utf-8'))
+        except Exception as e:
+            print(f"[serve_api_wave_reasoning_tail] Uncaught exception: {e}", file=sys.stderr)
+            self.send_response(500)
+            self.send_header("Content-Type", "application/json; charset=utf-8")
+            self.end_headers()
+            self.wfile.write(json.dumps({"error": "Internal server error"}).encode('utf-8'))
+
+    def serve_api_wave_quality_scorecards(self):
+        """GET /api/wave/quality-scorecards — per-agent-specialty quality metrics.
+
+        Read-only; reads at call time (no caching). Returns per-specialty quality:
+        success rate (green/total) and retry/repair frequency from ledger.
+        Includes top rankings by success and retry frequency.
+        """
+        try:
+            payload = quality_scorecard.get_quality_scorecard()
+            self.send_response(200)
+            self.send_header("Content-Type", "application/json; charset=utf-8")
+            self.send_header("Cache-Control", "no-cache, no-store, must-revalidate")
+            self.end_headers()
+            self.wfile.write(json.dumps(payload, default=str).encode('utf-8'))
+        except Exception as e:
+            print(f"[serve_api_wave_quality_scorecards] Uncaught exception: {e}", file=sys.stderr)
+            self.send_response(500)
+            self.send_header("Content-Type", "application/json; charset=utf-8")
+            self.end_headers()
+            self.wfile.write(json.dumps({"error": "Internal server error"}).encode('utf-8'))
+
+    def serve_api_wave_failure(self):
+        """GET /api/wave/failure?pr=N — CI job logs and failure details for a PR.
+
+        Read-only; shells `gh run view --json jobs` / `gh api .../logs` (short timeout,
+        cached a few seconds). Degrades to {available:false, error:...} when gh is
+        missing or un-authenticated — never a 500 for those.
+
+        Query params:
+          pr: PR number (required)
+        """
+        try:
+            # Parse query string for pr parameter
+            query = urllib.parse.urlparse(self.path).query
+            params = urllib.parse.parse_qs(query)
+            pr_str = params.get('pr', [None])[0]
+
+            if not pr_str or not pr_str.isdigit():
+                self.send_response(400)
+                self.send_header("Content-Type", "application/json; charset=utf-8")
+                self.end_headers()
+                self.wfile.write(json.dumps({"error": "pr parameter required and must be a number"}).encode('utf-8'))
+                return
+
+            pr_number = int(pr_str)
+            payload = wave_failure.get_wave_failure(pr_number)
+            self.send_response(200)
+            self.send_header("Content-Type", "application/json; charset=utf-8")
+            self.send_header("Cache-Control", "no-cache, no-store, must-revalidate")
+            self.end_headers()
+            self.wfile.write(json.dumps(payload, default=str).encode('utf-8'))
+        except Exception as e:
+            print(f"[serve_api_wave_failure] Uncaught exception: {e}", file=sys.stderr)
+            self.send_response(500)
+            self.send_header("Content-Type", "application/json; charset=utf-8")
+            self.end_headers()
+            self.wfile.write(json.dumps({"error": "Internal server error"}).encode('utf-8'))
+
     def serve_data(self):
         """Serve dashboard data as JSON."""
         data = {