astron-eval 0.0.1

package/skills/skill-driven-eval/scripts/aggregate_results.py

@@ -0,0 +1,681 @@
+#!/usr/bin/env python3
+"""
+Aggregate individual run results into model comparison benchmark.
+
+This script supports two workspace layouts:
+
+1. Anonymous run IDs with mapping.json (recommended for blind evaluation):
+    <workspace>/
+    ├── evals.json
+    ├── mapping.json          # Maps run IDs to models
+    └── run-001/
+        ├── outputs/
+        ├── grading.json
+        └── timing.json
+
+2. Legacy layout with model names in directories:
+    <workspace>/
+    ├── evals.json
+    └── eval-1/
+        ├── eval_metadata.json
+        ├── opus/
+        │   ├── outputs/
+        │   ├── grading.json
+        │   └── timing.json
+        └── sonnet/
+            └── ...
+
+Usage:
+    python aggregate_results.py <workspace_dir> [--mapping mapping.json]
+
+Example:
+    python aggregate_results.py pdf-eval-workspace/
+    python aggregate_results.py pdf-eval-workspace/ --mapping pdf-eval-workspace/mapping.json
+"""
+
+import argparse
+import json
+import math
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+
+
+def calculate_stats(values: list[float]) -> dict:
+    """Calculate mean, stddev, min, max for a list of values."""
+    if not values:
+        return {"mean": 0.0, "stddev": 0.0, "min": 0.0, "max": 0.0}
+
+    n = len(values)
+    mean = sum(values) / n
+
+    if n > 1:
+        variance = sum((x - mean) ** 2 for x in values) / (n - 1)
+        stddev = math.sqrt(variance)
+    else:
+        stddev = 0.0
+
+    return {
+        "mean": round(mean, 4),
+        "stddev": round(stddev, 4),
+        "min": round(min(values), 4),
+        "max": round(max(values), 4)
+    }
+
+
+def load_evals_config(workspace: Path) -> dict:
+    """Load evals.json from workspace."""
+    evals_path = workspace / "evals.json"
+    if evals_path.exists():
+        with open(evals_path) as f:
+            return json.load(f)
+    return {}
+
+
+def load_mapping(workspace: Path, mapping_path: Path | None = None) -> dict:
+    """Load mapping.json that maps run IDs to model names."""
+    if mapping_path:
+        path = mapping_path
+    else:
+        path = workspace / "mapping.json"
+
+    if path.exists():
+        with open(path) as f:
+            return json.load(f)
+    return {}
+
+
+def load_run_results_with_mapping(workspace: Path, mapping: dict) -> dict[str, list[dict]]:
+    """
+    Load run results using anonymous run IDs with mapping.
+
+    Returns dict keyed by model name, each containing a list of run results.
+    """
+    results: dict[str, list[dict]] = {}
+
+    for run_dir in sorted(workspace.glob("run-*")):
+        if not run_dir.is_dir():
+            continue
+
+        run_id = run_dir.name
+
+        # Get model and eval info from mapping
+        run_info = mapping.get(run_id, {})
+        model = run_info.get("model")
+        eval_id = run_info.get("eval_id")
+        eval_name = run_info.get("eval_name", f"Eval {eval_id}")
+
+        if not model:
+            print(f"Warning: No model mapping found for {run_id}")
+            continue
+
+        if model not in results:
+            results[model] = []
+
+        # Load grading.json
+        grading_path = run_dir / "grading.json"
+        if not grading_path.exists():
+            print(f"Warning: grading.json not found in {run_dir}")
+            continue
+
+        try:
+            with open(grading_path) as f:
+                grading = json.load(f)
+        except json.JSONDecodeError as e:
+            print(f"Warning: Invalid JSON in {grading_path}: {e}")
+            continue
+
+        # Load timing.json
+        timing_path = run_dir / "timing.json"
+        timing_data = {}
+        if timing_path.exists():
+            try:
+                with open(timing_path) as tf:
+                    timing_data = json.load(tf)
+            except json.JSONDecodeError:
+                pass
+
+        # Calculate time in seconds from timing data
+        # Support both duration_ms and total_duration_seconds formats
+        time_seconds = 0.0
+        if "duration_ms" in timing_data:
+            time_seconds = timing_data["duration_ms"] / 1000.0
+        elif "total_duration_seconds" in timing_data:
+            time_seconds = timing_data["total_duration_seconds"]
+
+        # Build result
+        result = {
+            "run_id": run_id,
+            "eval_id": eval_id,
+            "eval_name": eval_name,
+            "model": model,
+            "result": {
+                "pass_rate": grading.get("summary", {}).get("pass_rate", 0.0),
+                "passed": grading.get("summary", {}).get("passed", 0),
+                "failed": grading.get("summary", {}).get("failed", 0),
+                "total": grading.get("summary", {}).get("total", 0),
+                "time_seconds": time_seconds,
+                "tokens": timing_data.get("total_tokens", 0),
+                "tool_calls": grading.get("execution_metrics", {}).get("total_tool_calls", 0),
+                "errors": grading.get("execution_metrics", {}).get("errors_encountered", 0)
+            },
+            "expectations": grading.get("expectations", []),
+            "issues": grading.get("issues", [])
+        }
+
+        results[model].append(result)
+
+    return results
+
+
+def load_run_results_legacy(workspace: Path) -> dict[str, list[dict]]:
+    """
+    Load run results using legacy layout with model names in directories.
+
+    Returns dict keyed by model name, each containing a list of run results.
+    """
+    results: dict[str, list[dict]] = {}
+
+    # Find all eval directories
+    for eval_dir in sorted(workspace.glob("eval-*")):
+        if not eval_dir.is_dir():
+            continue
+
+        # Get eval metadata
+        metadata_path = eval_dir / "eval_metadata.json"
+        eval_id = None
+        eval_name = eval_dir.name
+
+        if metadata_path.exists():
+            try:
+                with open(metadata_path) as mf:
+                    metadata = json.load(mf)
+                    eval_id = metadata.get("eval_id")
+                    eval_name = metadata.get("eval_name", eval_dir.name)
+            except (json.JSONDecodeError, OSError):
+                pass
+
+        if eval_id is None:
+            try:
+                eval_id = int(eval_dir.name.split("-")[1])
+            except (ValueError, IndexError):
+                eval_id = 0
+
+        # Find model directories
+        for model_dir in sorted(eval_dir.iterdir()):
+            if not model_dir.is_dir():
+                continue
+            if model_dir.name in ["outputs", "inputs"]:
+                continue
+
+            model = model_dir.name
+            if model not in results:
+                results[model] = []
+
+            # Load grading.json
+            grading_path = model_dir / "grading.json"
+            if not grading_path.exists():
+                print(f"Warning: grading.json not found in {model_dir}")
+                continue
+
+            try:
+                with open(grading_path) as f:
+                    grading = json.load(f)
+            except json.JSONDecodeError as e:
+                print(f"Warning: Invalid JSON in {grading_path}: {e}")
+                continue
+
+            # Load timing.json
+            timing_path = model_dir / "timing.json"
+            timing_data = {}
+            if timing_path.exists():
+                try:
+                    with open(timing_path) as tf:
+                        timing_data = json.load(tf)
+                except json.JSONDecodeError:
+                    pass
+
+            # Build result
+            result = {
+                "run_id": f"eval-{eval_id}-{model}",
+                "eval_id": eval_id,
+                "eval_name": eval_name,
+                "model": model,
+                "result": {
+                    "pass_rate": grading.get("summary", {}).get("pass_rate", 0.0),
+                    "passed": grading.get("summary", {}).get("passed", 0),
+                    "failed": grading.get("summary", {}).get("failed", 0),
+                    "total": grading.get("summary", {}).get("total", 0),
+                    "time_seconds": timing_data.get("total_duration_seconds", 0.0),
+                    "tokens": timing_data.get("total_tokens", 0),
+                    "tool_calls": grading.get("execution_metrics", {}).get("total_tool_calls", 0),
+                    "errors": grading.get("execution_metrics", {}).get("errors_encountered", 0)
+                },
+                "expectations": grading.get("expectations", []),
+                "issues": grading.get("issues", [])
+            }
+
+            results[model].append(result)
+
+    return results
+
+
+def load_run_results(workspace: Path, mapping: dict) -> dict[str, list[dict]]:
+    """
+    Load all run results from a workspace.
+
+    Automatically detects layout: anonymous run IDs with mapping.json or legacy.
+    """
+    # If mapping is provided or exists, use anonymous layout
+    if mapping:
+        return load_run_results_with_mapping(workspace, mapping)
+
+    # Check for mapping.json
+    mapping_path = workspace / "mapping.json"
+    if mapping_path.exists():
+        mapping = load_mapping(workspace)
+        if mapping:
+            return load_run_results_with_mapping(workspace, mapping)
+
+    # Fall back to legacy layout
+    return load_run_results_legacy(workspace)
+
+
+def aggregate_model_summary(results: dict[str, list[dict]]) -> dict[str, dict]:
+    """Aggregate results into model summaries."""
+    model_summary = {}
+
+    for model, runs in results.items():
+        if not runs:
+            model_summary[model] = {
+                "pass_rate": {"mean": 0.0, "stddev": 0.0, "min": 0.0, "max": 0.0},
+                "time_seconds": {"mean": 0.0, "stddev": 0.0, "min": 0.0, "max": 0.0},
+                "tokens": {"mean": 0, "stddev": 0, "min": 0, "max": 0}
+            }
+            continue
+
+        # Safely extract values, handling missing keys
+        pass_rates = []
+        times = []
+        tokens_list = []
+
+        for r in runs:
+            result = r.get("result", {})
+            if result.get("pass_rate") is not None:
+                pass_rates.append(result["pass_rate"])
+            if result.get("time_seconds") is not None:
+                times.append(result["time_seconds"])
+            if result.get("tokens") is not None:
+                tokens_list.append(result["tokens"])
+
+        model_summary[model] = {
+            "pass_rate": calculate_stats(pass_rates) if pass_rates else {"mean": 0.0, "stddev": 0.0, "min": 0.0, "max": 0.0},
+            "time_seconds": calculate_stats(times) if times else {"mean": 0.0, "stddev": 0.0, "min": 0.0, "max": 0.0},
+            "tokens": calculate_stats(tokens_list) if tokens_list else {"mean": 0, "stddev": 0, "min": 0, "max": 0}
+        }
+
+    return model_summary
+
+
+def calculate_comparison(model_summary: dict[str, dict]) -> dict:
+    """Calculate comparison metrics between models."""
+    models = list(model_summary.keys())
+
+    if len(models) < 2:
+        return {
+            "pass_rate_delta": "N/A",
+            "time_delta": "N/A",
+            "token_delta": "N/A",
+            "cost_efficiency": {}
+        }
+
+    # Compare first model to second
+    primary = model_summary.get(models[0], {})
+    baseline = model_summary.get(models[1], {})
+
+    delta_pass_rate = primary.get("pass_rate", {}).get("mean", 0) - baseline.get("pass_rate", {}).get("mean", 0)
+    delta_time = primary.get("time_seconds", {}).get("mean", 0) - baseline.get("time_seconds", {}).get("mean", 0)
+    delta_tokens = primary.get("tokens", {}).get("mean", 0) - baseline.get("tokens", {}).get("mean", 0)
+
+    # Calculate cost efficiency: pass_rate * 1000 / (tokens / 1000)
+    cost_efficiency = {}
+    for model, summary in model_summary.items():
+        pass_rate = summary.get("pass_rate", {}).get("mean", 0)
+        tokens = summary.get("tokens", {}).get("mean", 0)
+        if tokens > 0:
+            efficiency = (pass_rate * 1000) / (tokens / 1000)
+            cost_efficiency[model] = round(efficiency, 1)
+        else:
+            cost_efficiency[model] = 0
+
+    return {
+        "pass_rate_delta": f"{delta_pass_rate:+.2f}",
+        "time_delta": f"{delta_time:+.1f}s",
+        "token_delta": f"{delta_tokens:+.0f}",
+        "cost_efficiency": cost_efficiency
+    }
+
+
+def generate_recommendations(model_summary: dict[str, dict], comparison: dict) -> list[dict]:
+    """Generate usage recommendations based on actual results."""
+    recommendations = []
+    models = list(model_summary.keys())
+
+    if len(models) < 2:
+        return recommendations
+
+    primary = models[0]
+    baseline = models[1]
+
+    primary_pr = model_summary[primary]["pass_rate"]["mean"]
+    baseline_pr = model_summary[baseline]["pass_rate"]["mean"]
+
+    primary_time = model_summary[primary]["time_seconds"]["mean"]
+    baseline_time = model_summary[baseline]["time_seconds"]["mean"]
+
+    pr_diff = primary_pr - baseline_pr
+
+    # Generate recommendations based on actual data patterns
+
+    # High accuracy recommendation
+    if pr_diff > 0.1:
+        recommendations.append({
+            "scenario": "High accuracy requirements",
+            "recommended_model": primary,
+            "reason": f"{pr_diff*100:.0f}% higher pass rate ({primary_pr*100:.0f}% vs {baseline_pr*100:.0f}%)"
+        })
+
+    # Cost efficiency recommendation
+    cost_eff = comparison.get("cost_efficiency", {})
+    baseline_eff = cost_eff.get(baseline, 0)
+    primary_eff = cost_eff.get(primary, 0)
+
+    if baseline_eff > primary_eff and baseline_pr >= 0.7:
+        recommendations.append({
+            "scenario": "Cost-conscious use",
+            "recommended_model": baseline,
+            "reason": f"Better cost efficiency ({baseline_eff:.1f} vs {primary_eff:.1f}) with acceptable quality ({baseline_pr*100:.0f}%)"
+        })
+
+    # Speed recommendation
+    if baseline_time < primary_time * 0.8 and baseline_time > 0:
+        speedup = ((primary_time - baseline_time) / primary_time) * 100
+        recommendations.append({
+            "scenario": "Speed-critical tasks",
+            "recommended_model": baseline,
+            "reason": f"{speedup:.0f}% faster ({baseline_time:.1f}s vs {primary_time:.1f}s)"
+        })
+
+    # Default fallback based on actual results
+    if not recommendations:
+        if pr_diff > 0:
+            recommendations.append({
+                "scenario": "General use",
+                "recommended_model": primary,
+                "reason": f"Higher pass rate ({primary_pr*100:.0f}% vs {baseline_pr*100:.0f}%)"
+            })
+        else:
+            recommendations.append({
+                "scenario": "General use",
+                "recommended_model": baseline,
+                "reason": f"Similar quality ({baseline_pr*100:.0f}% vs {primary_pr*100:.0f}%) with lower cost"
+            })
+
+    return recommendations
+
+
+def generate_notes(model_summary: dict[str, dict], results: dict[str, list[dict]]) -> list[str]:
+    """Generate analyst notes from the results."""
+    notes = []
+    models = list(model_summary.keys())
+
+    if len(models) < 2:
+        return notes
+
+    primary = models[0]
+    baseline = models[1]
+
+    primary_pr = model_summary[primary]["pass_rate"]["mean"]
+    baseline_pr = model_summary[baseline]["pass_rate"]["mean"]
+
+    # Pass rate comparison
+    if primary_pr > baseline_pr:
+        notes.append(f"{primary} achieves {primary_pr*100:.0f}% pass rate vs {baseline}'s {baseline_pr*100:.0f}%")
+    elif baseline_pr > primary_pr:
+        notes.append(f"{baseline} achieves {baseline_pr*100:.0f}% pass rate vs {primary}'s {primary_pr*100:.0f}%")
+    else:
+        notes.append(f"Both models achieve similar pass rate ({primary_pr*100:.0f}%)")
+
+    # Speed comparison
+    primary_time = model_summary[primary]["time_seconds"]["mean"]
+    baseline_time = model_summary[baseline]["time_seconds"]["mean"]
+    if baseline_time < primary_time and baseline_time > 0:
+        speedup = ((primary_time - baseline_time) / primary_time) * 100
+        notes.append(f"{baseline} is {speedup:.0f}% faster on average")
+    elif primary_time < baseline_time and primary_time > 0:
+        speedup = ((baseline_time - primary_time) / baseline_time) * 100
+        notes.append(f"{primary} is {speedup:.0f}% faster on average")
+
+    # Look for eval-specific patterns
+    primary_runs = results.get(primary, [])
+    baseline_runs = results.get(baseline, [])
+
+    for pr in primary_runs:
+        for br in baseline_runs:
+            if pr["eval_id"] == br["eval_id"]:
+                pr_rate = pr["result"]["pass_rate"]
+                br_rate = br["result"]["pass_rate"]
+                if pr_rate < 0.5 and br_rate < 0.5:
+                    notes.append(f"Both models struggle with '{pr['eval_name']}'")
+                elif pr_rate - br_rate > 0.3:
+                    notes.append(f"{primary} significantly outperforms on '{pr['eval_name']}'")
+                elif br_rate - pr_rate > 0.3:
+                    notes.append(f"{baseline} significantly outperforms on '{pr['eval_name']}'")
+
+    return notes[:5]  # Limit to 5 notes
+
+
+def generate_benchmark(workspace: Path, mapping: dict) -> dict:
+    """Generate complete benchmark.json from workspace results."""
+    config = load_evals_config(workspace)
+    results = load_run_results(workspace, mapping)
+    model_summary = aggregate_model_summary(results)
+    comparison = calculate_comparison(model_summary)
+    recommendations = generate_recommendations(model_summary, comparison)
+    notes = generate_notes(model_summary, results)
+
+    # Flatten runs for benchmark output
+    all_runs = []
+    for model, runs in results.items():
+        all_runs.extend(runs)
+
+    # Sort by eval_id, then model
+    all_runs.sort(key=lambda r: (r["eval_id"], r["model"]))
+
+    # Get eval IDs
+    eval_ids = sorted(set(r["eval_id"] for r in all_runs))
+
+    benchmark = {
+        "metadata": {
+            "target_skill": config.get("target_skill", "unknown"),
+            "target_skill_path": config.get("target_skill_path", ""),
+            "models_compared": list(results.keys()),
+            "timestamp": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
+            "evals_run": eval_ids,
+            "note": "Results are based on blind evaluation - graders did not know which model produced each output"
+        },
+        "runs": all_runs,
+        "model_summary": model_summary,
+        "comparison": comparison,
+        "recommendations": recommendations,
+        "notes": notes
+    }
+
+    return benchmark
+
+
+def generate_markdown(benchmark: dict) -> str:
+    """Generate human-readable benchmark.md from benchmark data."""
+    metadata = benchmark["metadata"]
+    model_summary = benchmark["model_summary"]
+    comparison = benchmark["comparison"]
+    recommendations = benchmark["recommendations"]
+    notes = benchmark["notes"]
+
+    models = list(model_summary.keys())
+    model_a = models[0] if models else "model_a"
+    model_b = models[1] if len(models) > 1 else "model_b"
+
+    lines = [
+        f"# Model Comparison: {metadata['target_skill']}",
+        "",
+        f"**Skill**: {metadata['target_skill']}",
+        f"**Date**: {metadata['timestamp']}",
+        f"**Models**: {', '.join(metadata['models_compared'])}",
+        f"**Evals**: {', '.join(map(str, metadata['evals_run']))}",
+        "",
+        "> **Note**: Results are based on blind evaluation. Graders did not know which model produced each output.",
+        "",
+        "## Summary",
+        "",
+        f"| Metric | {model_a} | {model_b} | Delta |",
+        "|--------|------------|---------------|-------|",
+    ]
+
+    a_summary = model_summary.get(model_a, {})
+    b_summary = model_summary.get(model_b, {})
+
+    # Pass rate
+    a_pr = a_summary.get("pass_rate", {})
+    b_pr = b_summary.get("pass_rate", {})
+    lines.append(f"| Pass Rate | {a_pr.get('mean', 0)*100:.0f}% ± {a_pr.get('stddev', 0)*100:.0f}% | {b_pr.get('mean', 0)*100:.0f}% ± {b_pr.get('stddev', 0)*100:.0f}% | {comparison.get('pass_rate_delta', '—')} |")
+
+    # Time
+    a_time = a_summary.get("time_seconds", {})
+    b_time = b_summary.get("time_seconds", {})
+    lines.append(f"| Time | {a_time.get('mean', 0):.1f}s ± {a_time.get('stddev', 0):.1f}s | {b_time.get('mean', 0):.1f}s ± {b_time.get('stddev', 0):.1f}s | {comparison.get('time_delta', '—')} |")
+
+    # Tokens
+    a_tokens = a_summary.get("tokens", {})
+    b_tokens = b_summary.get("tokens", {})
+    lines.append(f"| Tokens | {a_tokens.get('mean', 0):.0f} ± {a_tokens.get('stddev', 0):.0f} | {b_tokens.get('mean', 0):.0f} ± {b_tokens.get('stddev', 0):.0f} | {comparison.get('token_delta', '—')} |")
+
+    # Cost efficiency
+    cost_eff = comparison.get("cost_efficiency", {})
+    if cost_eff:
+        lines.append(f"| Cost Efficiency | {cost_eff.get(model_a, '—')} | {cost_eff.get(model_b, '—')} | Higher is better |")
+
+    # Recommendations
+    if recommendations:
+        lines.extend([
+            "",
+            "## Recommendations",
+            "",
+            "> These recommendations are derived from actual evaluation results, not pre-conceived assumptions.",
+            ""
+        ])
+        for rec in recommendations:
+            lines.append(f"- **{rec['scenario']}**: Use **{rec['recommended_model']}** — {rec['reason']}")
+
+    # Notes
+    if notes:
+        lines.extend([
+            "",
+            "## Analysis Notes",
+            ""
+        ])
+        for note in notes:
+            lines.append(f"- {note}")
+
+    # Per-eval breakdown
+    runs = benchmark.get("runs", [])
+    if runs:
+        lines.extend([
+            "",
+            "## Per-Eval Breakdown",
+            ""
+        ])
+
+        eval_ids = sorted(set(r["eval_id"] for r in runs))
+        for eval_id in eval_ids:
+            eval_runs = [r for r in runs if r["eval_id"] == eval_id]
+            eval_name = eval_runs[0]["eval_name"] if eval_runs else f"Eval {eval_id}"
+
+            lines.append(f"### {eval_name}")
+            lines.append("")
+            lines.append("| Model | Pass Rate | Time | Tokens |")
+            lines.append("|-------|-----------|------|--------|")
+
+            for run in eval_runs:
+                r = run["result"]
+                lines.append(f"| {run['model']} | {r['pass_rate']*100:.0f}% | {r['time_seconds']:.1f}s | {r['tokens']} |")
+
+            lines.append("")
+
+    return "\n".join(lines)
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Aggregate model comparison results into benchmark summary"
+    )
+    parser.add_argument(
+        "workspace",
+        type=Path,
+        help="Path to the workspace directory"
+    )
+    parser.add_argument(
+        "--mapping", "-m",
+        type=Path,
+        default=None,
+        help="Path to mapping.json (default: <workspace>/mapping.json)"
+    )
+    parser.add_argument(
+        "--output", "-o",
+        type=Path,
+        help="Output path for benchmark.json (default: <workspace>/benchmark.json)"
+    )
+
+    args = parser.parse_args()
+
+    if not args.workspace.exists():
+        print(f"Directory not found: {args.workspace}")
+        sys.exit(1)
+
+    # Load mapping if provided
+    mapping = {}
+    if args.mapping:
+        mapping = load_mapping(args.workspace, args.mapping)
+
+    # Generate benchmark
+    benchmark = generate_benchmark(args.workspace, mapping)
+
+    # Determine output paths
+    output_json = args.output or (args.workspace / "benchmark.json")
+    output_md = output_json.with_suffix(".md")
+
+    # Write benchmark.json
+    with open(output_json, "w") as f:
+        json.dump(benchmark, f, indent=2)
+    print(f"Generated: {output_json}")
+
+    # Write benchmark.md
+    markdown = generate_markdown(benchmark)
+    with open(output_md, "w") as f:
+        f.write(markdown)
+    print(f"Generated: {output_md}")
+
+    # Print summary
+    print(f"\nSummary:")
+    for model, summary in benchmark["model_summary"].items():
+        pr = summary["pass_rate"]["mean"]
+        print(f"  {model}: {pr*100:.1f}% pass rate")
+
+    if benchmark["recommendations"]:
+        print(f"\nTop recommendation:")
+        rec = benchmark["recommendations"][0]
+        print(f"  For {rec['scenario']}: use {rec['recommended_model']}")
+
+
+if __name__ == "__main__":
+    main()