@einja/dev-cli 0.1.41 → 0.1.45

package/presets/default/.claude/skills/einja-skill-creator/scripts/aggregate_benchmark.py

@@ -1,153 +1,400 @@
 #!/usr/bin/env python3
-"""複数スキルのベンチマーク結果を集約。
+"""
+ベンチマーク実行結果を集約してサマリー統計を生成。
+
+実行ディレクトリ内の grading.json ファイルを読み込み、以下を生成する:
+- 各メトリクス（平均・標準偏差・最小・最大）を含む run_summary
+- with_skill / without_skill 設定間のデルタ
+
+使用方法:
+    python aggregate_benchmark.py <benchmark_dir>
+
+例:
+    python aggregate_benchmark.py benchmarks/2026-01-15T10-30-00/
 
-複数のrun_loop.py出力を受け取り、全スキルのスコアを
-サマリーテーブルとして表示する。
+対応するディレクトリ構成:
+
+    ワークスペース構成（skill-creator イテレーションから）:
+    <benchmark_dir>/
+    └── eval-N/
+        ├── with_skill/
+        │   ├── run-1/grading.json
+        │   └── run-2/grading.json
+        └── without_skill/
+            ├── run-1/grading.json
+            └── run-2/grading.json
+
+    レガシー構成（runs/ サブディレクトリあり）:
+    <benchmark_dir>/
+    └── runs/
+        └── eval-N/
+            ├── with_skill/
+            │   └── run-1/grading.json
+            └── without_skill/
+                └── run-1/grading.json
 """
 
 import argparse
 import json
+import math
 import sys
+from datetime import datetime, timezone
 from pathlib import Path
 
 
-def aggregate_results(result_files: list[str]) -> dict:
-    """複数のrun_loop.py出力ファイルを集約する。"""
-    skills = []
+def calculate_stats(values: list[float]) -> dict:
+    """値リストの平均・標準偏差・最小・最大を計算する。"""
+    if not values:
+        return {"mean": 0.0, "stddev": 0.0, "min": 0.0, "max": 0.0}
 
-    for filepath in result_files:
-        try:
-            data = json.loads(Path(filepath).read_text())
-        except (json.JSONDecodeError, FileNotFoundError) as e:
-            print(f"警告: {filepath} の読み込みに失敗しました: {e}", file=sys.stderr)
-            continue
+    n = len(values)
+    mean = sum(values) / n
+
+    if n > 1:
+        variance = sum((x - mean) ** 2 for x in values) / (n - 1)
+        stddev = math.sqrt(variance)
+    else:
+        stddev = 0.0
+
+    return {
+        "mean": round(mean, 4),
+        "stddev": round(stddev, 4),
+        "min": round(min(values), 4),
+        "max": round(max(values), 4)
+    }
+
+
+def load_run_results(benchmark_dir: Path) -> dict:
+    """
+    ベンチマークディレクトリから全実行結果を読み込む。
+
+    設定名（例: "with_skill"/"without_skill" または "new_skill"/"old_skill"）を
+    キーとして実行結果リストを格納した dict を返す。
+    """
+    # eval ディレクトリが benchmark_dir 直下または runs/ 以下の両方に対応
+    runs_dir = benchmark_dir / "runs"
+    if runs_dir.exists():
+        search_dir = runs_dir
+    elif list(benchmark_dir.glob("eval-*")):
+        search_dir = benchmark_dir
+    else:
+        print(f"No eval directories found in {benchmark_dir} or {benchmark_dir / 'runs'}")
+        return {}
+
+    results: dict[str, list] = {}
+
+    for eval_idx, eval_dir in enumerate(sorted(search_dir.glob("eval-*"))):
+        metadata_path = eval_dir / "eval_metadata.json"
+        if metadata_path.exists():
+            try:
+                with open(metadata_path) as mf:
+                    eval_id = json.load(mf).get("eval_id", eval_idx)
+            except (json.JSONDecodeError, OSError):
+                eval_id = eval_idx
+        else:
+            try:
+                eval_id = int(eval_dir.name.split("-")[1])
+            except ValueError:
+                eval_id = eval_idx
+
+        # 設定ディレクトリをハードコードせず動的に検出
+        for config_dir in sorted(eval_dir.iterdir()):
+            if not config_dir.is_dir():
+                continue
+            # run-* を含まないディレクトリ（inputs, outputs 等）はスキップ
+            if not list(config_dir.glob("run-*")):
+                continue
+            config = config_dir.name
+            if config not in results:
+                results[config] = []
+
+            for run_dir in sorted(config_dir.glob("run-*")):
+                run_number = int(run_dir.name.split("-")[1])
+                grading_file = run_dir / "grading.json"
+
+                if not grading_file.exists():
+                    print(f"Warning: grading.json not found in {run_dir}")
+                    continue
+
+                try:
+                    with open(grading_file) as f:
+                        grading = json.load(f)
+                except json.JSONDecodeError as e:
+                    print(f"Warning: Invalid JSON in {grading_file}: {e}")
+                    continue
+
+                # メトリクスを抽出
+                result = {
+                    "eval_id": eval_id,
+                    "run_number": run_number,
+                    "pass_rate": grading.get("summary", {}).get("pass_rate", 0.0),
+                    "passed": grading.get("summary", {}).get("passed", 0),
+                    "failed": grading.get("summary", {}).get("failed", 0),
+                    "total": grading.get("summary", {}).get("total", 0),
+                }
+
+                # タイミングを抽出（grading.json 優先、なければ timing.json を参照）
+                timing = grading.get("timing", {})
+                result["time_seconds"] = timing.get("total_duration_seconds", 0.0)
+                timing_file = run_dir / "timing.json"
+                if result["time_seconds"] == 0.0 and timing_file.exists():
+                    try:
+                        with open(timing_file) as tf:
+                            timing_data = json.load(tf)
+                        result["time_seconds"] = timing_data.get("total_duration_seconds", 0.0)
+                        result["tokens"] = timing_data.get("total_tokens", 0)
+                    except json.JSONDecodeError:
+                        pass
+
+                # 実行メトリクスを抽出
+                metrics = grading.get("execution_metrics", {})
+                result["tool_calls"] = metrics.get("total_tool_calls", 0)
+                if not result.get("tokens"):
+                    result["tokens"] = metrics.get("output_chars", 0)
+                result["errors"] = metrics.get("errors_encountered", 0)
+
+                # expectations を抽出（viewer に必要なフィールド: text, passed, evidence）
+                raw_expectations = grading.get("expectations", [])
+                for exp in raw_expectations:
+                    if "text" not in exp or "passed" not in exp:
+                        print(f"Warning: expectation in {grading_file} missing required fields (text, passed, evidence): {exp}")
+                result["expectations"] = raw_expectations
+
+                # user_notes_summary からノートを抽出
+                notes_summary = grading.get("user_notes_summary", {})
+                notes = []
+                notes.extend(notes_summary.get("uncertainties", []))
+                notes.extend(notes_summary.get("needs_review", []))
+                notes.extend(notes_summary.get("workarounds", []))
+                result["notes"] = notes
+
+                results[config].append(result)
+
+    return results
+
+
+def aggregate_results(results: dict) -> dict:
+    """
+    実行結果をサマリー統計に集約する。
+
+    各設定のスタットと設定間のデルタを含む run_summary を返す。
+    """
+    run_summary = {}
+    configs = list(results.keys())
 
-        history = data.get("history", [])
-        if not history:
-            print(f"警告: {filepath} に履歴がありません", file=sys.stderr)
+    for config in configs:
+        runs = results.get(config, [])
+
+        if not runs:
+            run_summary[config] = {
+                "pass_rate": {"mean": 0.0, "stddev": 0.0, "min": 0.0, "max": 0.0},
+                "time_seconds": {"mean": 0.0, "stddev": 0.0, "min": 0.0, "max": 0.0},
+                "tokens": {"mean": 0, "stddev": 0, "min": 0, "max": 0}
+            }
             continue
 
-        # 最良のイテレーションを見つける（テスト > トレーニングで優先）
-        best_idx = 0
-        best_test = -1
-        best_train = -1
-        for i, h in enumerate(history):
-            t_passed = h.get("test_passed", -1)
-            tr_passed = h.get("train_passed", h.get("passed", 0))
-            if t_passed > best_test or (t_passed == best_test and tr_passed > best_train):
-                best_test = t_passed
-                best_train = tr_passed
-                best_idx = i
-
-        best = history[best_idx]
-        original = history[0]
-
-        skill_entry = {
-            "skill_name": data.get("skill_name", Path(filepath).stem),
-            "file": filepath,
-            "iterations": len(history),
-            "best_iteration": best_idx,
-            "original_description": data.get("original_description", ""),
-            "best_description": best.get("description", ""),
-            "original_train_score": f"{original.get('train_passed', original.get('passed', 0))}/{original.get('train_total', original.get('total', 0))}",
-            "best_train_score": f"{best.get('train_passed', best.get('passed', 0))}/{best.get('train_total', best.get('total', 0))}",
-            "original_train_passed": original.get("train_passed", original.get("passed", 0)),
-            "original_train_total": original.get("train_total", original.get("total", 0)),
-            "best_train_passed": best.get("train_passed", best.get("passed", 0)),
-            "best_train_total": best.get("train_total", best.get("total", 0)),
+        pass_rates = [r["pass_rate"] for r in runs]
+        times = [r["time_seconds"] for r in runs]
+        tokens = [r.get("tokens", 0) for r in runs]
+
+        run_summary[config] = {
+            "pass_rate": calculate_stats(pass_rates),
+            "time_seconds": calculate_stats(times),
+            "tokens": calculate_stats(tokens)
         }
 
-        # テストスコア（存在する場合）
-        if best.get("test_passed") is not None:
-            skill_entry["original_test_score"] = f"{original.get('test_passed', '?')}/{original.get('test_total', '?')}"
-            skill_entry["best_test_score"] = f"{best.get('test_passed', '?')}/{best.get('test_total', '?')}"
-            skill_entry["best_test_passed"] = best.get("test_passed", 0)
-            skill_entry["best_test_total"] = best.get("test_total", 0)
-
-        skills.append(skill_entry)
-
-    # トレーニングスコアでソート（降順）
-    skills.sort(
-        key=lambda s: (
-            s.get("best_test_passed", 0) / max(s.get("best_test_total", 1), 1),
-            s["best_train_passed"] / max(s["best_train_total"], 1),
-        ),
-        reverse=True,
-    )
+    # 最初の2設定間のデルタを計算
+    if len(configs) >= 2:
+        primary = run_summary.get(configs[0], {})
+        baseline = run_summary.get(configs[1], {})
+    else:
+        primary = run_summary.get(configs[0], {}) if configs else {}
+        baseline = {}
 
-    # 全体サマリーの計算
-    total_train_passed = sum(s["best_train_passed"] for s in skills)
-    total_train_total = sum(s["best_train_total"] for s in skills)
-    total_test_passed = sum(s.get("best_test_passed", 0) for s in skills if "best_test_passed" in s)
-    total_test_total = sum(s.get("best_test_total", 0) for s in skills if "best_test_total" in s)
+    delta_pass_rate = primary.get("pass_rate", {}).get("mean", 0) - baseline.get("pass_rate", {}).get("mean", 0)
+    delta_time = primary.get("time_seconds", {}).get("mean", 0) - baseline.get("time_seconds", {}).get("mean", 0)
+    delta_tokens = primary.get("tokens", {}).get("mean", 0) - baseline.get("tokens", {}).get("mean", 0)
 
-    return {
-        "skills": skills,
-        "summary": {
-            "total_skills": len(skills),
-            "total_train_passed": total_train_passed,
-            "total_train_total": total_train_total,
-            "total_train_score": f"{total_train_passed}/{total_train_total}",
-            "total_test_passed": total_test_passed,
-            "total_test_total": total_test_total,
-            "total_test_score": f"{total_test_passed}/{total_test_total}" if total_test_total > 0 else None,
+    run_summary["delta"] = {
+        "pass_rate": f"{delta_pass_rate:+.2f}",
+        "time_seconds": f"{delta_time:+.1f}",
+        "tokens": f"{delta_tokens:+.0f}"
+    }
+
+    return run_summary
+
+
+def generate_benchmark(benchmark_dir: Path, skill_name: str = "", skill_path: str = "") -> dict:
+    """
+    実行結果から完全な benchmark.json を生成する。
+    """
+    results = load_run_results(benchmark_dir)
+    run_summary = aggregate_results(results)
+
+    # benchmark.json 用の runs 配列を構築
+    runs = []
+    for config in results:
+        for result in results[config]:
+            runs.append({
+                "eval_id": result["eval_id"],
+                "configuration": config,
+                "run_number": result["run_number"],
+                "result": {
+                    "pass_rate": result["pass_rate"],
+                    "passed": result["passed"],
+                    "failed": result["failed"],
+                    "total": result["total"],
+                    "time_seconds": result["time_seconds"],
+                    "tokens": result.get("tokens", 0),
+                    "tool_calls": result.get("tool_calls", 0),
+                    "errors": result.get("errors", 0)
+                },
+                "expectations": result["expectations"],
+                "notes": result["notes"]
+            })
+
+    # 結果から eval ID を決定
+    eval_ids = sorted(set(
+        r["eval_id"]
+        for config in results.values()
+        for r in config
+    ))
+
+    benchmark = {
+        "metadata": {
+            "skill_name": skill_name or "<skill-name>",
+            "skill_path": skill_path or "<path/to/skill>",
+            "executor_model": "<model-name>",
+            "analyzer_model": "<model-name>",
+            "timestamp": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
+            "evals_run": eval_ids,
+            "runs_per_configuration": 3
         },
+        "runs": runs,
+        "run_summary": run_summary,
+        "notes": []  # アナライザーが埋める
     }
 
+    return benchmark
+
+
+def generate_markdown(benchmark: dict) -> str:
+    """benchmark データから人間が読みやすい benchmark.md を生成する。"""
+    metadata = benchmark["metadata"]
+    run_summary = benchmark["run_summary"]
+
+    # 設定名を取得（"delta" を除く）
+    configs = [k for k in run_summary if k != "delta"]
+    config_a = configs[0] if len(configs) >= 1 else "config_a"
+    config_b = configs[1] if len(configs) >= 2 else "config_b"
+    label_a = config_a.replace("_", " ").title()
+    label_b = config_b.replace("_", " ").title()
+
+    lines = [
+        f"# Skill Benchmark: {metadata['skill_name']}",
+        "",
+        f"**Model**: {metadata['executor_model']}",
+        f"**Date**: {metadata['timestamp']}",
+        f"**Evals**: {', '.join(map(str, metadata['evals_run']))} ({metadata['runs_per_configuration']} runs each per configuration)",
+        "",
+        "## Summary",
+        "",
+        f"| Metric | {label_a} | {label_b} | Delta |",
+        "|--------|------------|---------------|-------|",
+    ]
 
-def print_table(aggregated: dict, verbose: bool = False) -> None:
-    """集約結果をテーブル形式でstderrに出力する。"""
-    skills = aggregated["skills"]
-    summary = aggregated["summary"]
-
-    has_test = any("best_test_score" in s for s in skills)
-
-    # ヘッダー
-    header = f"{'スキル名':<30} {'トレーニング(元)':<14} {'トレーニング(最良)':<14}"
-    if has_test:
-        header += f" {'テスト(元)':<12} {'テスト(最良)':<12}"
-    header += f" {'回数':<6} {'最良回':<6}"
-    print(header, file=sys.stderr)
-    print("-" * len(header), file=sys.stderr)
-
-    # 各スキル
-    for s in skills:
-        line = f"{s['skill_name']:<30} {s['original_train_score']:<14} {s['best_train_score']:<14}"
-        if has_test:
-            orig_test = s.get("original_test_score", "-")
-            best_test = s.get("best_test_score", "-")
-            line += f" {orig_test:<12} {best_test:<12}"
-        line += f" {s['iterations']:<6} {s['best_iteration']:<6}"
-        print(line, file=sys.stderr)
-
-        if verbose:
-            print(f"  オリジナル: {s['original_description'][:80]}...", file=sys.stderr)
-            print(f"  最良:       {s['best_description'][:80]}...", file=sys.stderr)
-
-    # サマリー
-    print("-" * len(header), file=sys.stderr)
-    total_line = f"{'合計':<30} {'':<14} {summary['total_train_score']:<14}"
-    if has_test and summary.get("total_test_score"):
-        total_line += f" {'':<12} {summary['total_test_score']:<12}"
-    total_line += f" {summary['total_skills']} スキル"
-    print(total_line, file=sys.stderr)
+    a_summary = run_summary.get(config_a, {})
+    b_summary = run_summary.get(config_b, {})
+    delta = run_summary.get("delta", {})
+
+    # パスレートのフォーマット
+    a_pr = a_summary.get("pass_rate", {})
+    b_pr = b_summary.get("pass_rate", {})
+    lines.append(f"| Pass Rate | {a_pr.get('mean', 0)*100:.0f}% ± {a_pr.get('stddev', 0)*100:.0f}% | {b_pr.get('mean', 0)*100:.0f}% ± {b_pr.get('stddev', 0)*100:.0f}% | {delta.get('pass_rate', '—')} |")
+
+    # タイムのフォーマット
+    a_time = a_summary.get("time_seconds", {})
+    b_time = b_summary.get("time_seconds", {})
+    lines.append(f"| Time | {a_time.get('mean', 0):.1f}s ± {a_time.get('stddev', 0):.1f}s | {b_time.get('mean', 0):.1f}s ± {b_time.get('stddev', 0):.1f}s | {delta.get('time_seconds', '—')}s |")
+
+    # トークン数のフォーマット
+    a_tokens = a_summary.get("tokens", {})
+    b_tokens = b_summary.get("tokens", {})
+    lines.append(f"| Tokens | {a_tokens.get('mean', 0):.0f} ± {a_tokens.get('stddev', 0):.0f} | {b_tokens.get('mean', 0):.0f} ± {b_tokens.get('stddev', 0):.0f} | {delta.get('tokens', '—')} |")
+
+    # ノートセクション
+    if benchmark.get("notes"):
+        lines.extend([
+            "",
+            "## Notes",
+            ""
+        ])
+        for note in benchmark["notes"]:
+            lines.append(f"- {note}")
+
+    return "\n".join(lines)
 
 
 def main():
-    parser = argparse.ArgumentParser(description="複数スキルのベンチマーク結果を集約")
-    parser.add_argument("files", nargs="+", help="run_loop.pyのJSON出力ファイル（複数指定可）")
-    parser.add_argument("--verbose", action="store_true", help="各スキルのdescriptionも表示")
-    parser.add_argument("--json", action="store_true", help="JSON形式で標準出力に出力")
+    parser = argparse.ArgumentParser(
+        description="ベンチマーク実行結果を集約してサマリー統計を生成"
+    )
+    parser.add_argument(
+        "benchmark_dir",
+        type=Path,
+        help="ベンチマークディレクトリへのパス"
+    )
+    parser.add_argument(
+        "--skill-name",
+        default="",
+        help="ベンチマーク対象スキルの名前"
+    )
+    parser.add_argument(
+        "--skill-path",
+        default="",
+        help="ベンチマーク対象スキルへのパス"
+    )
+    parser.add_argument(
+        "--output", "-o",
+        type=Path,
+        help="benchmark.json の出力パス（デフォルト: <benchmark_dir>/benchmark.json）"
+    )
+
     args = parser.parse_args()
 
-    aggregated = aggregate_results(args.files)
+    if not args.benchmark_dir.exists():
+        print(f"Directory not found: {args.benchmark_dir}")
+        sys.exit(1)
+
+    # ベンチマーク生成
+    benchmark = generate_benchmark(args.benchmark_dir, args.skill_name, args.skill_path)
+
+    # 出力パスを決定
+    output_json = args.output or (args.benchmark_dir / "benchmark.json")
+    output_md = output_json.with_suffix(".md")
+
+    # benchmark.json を書き込み
+    with open(output_json, "w") as f:
+        json.dump(benchmark, f, indent=2)
+    print(f"Generated: {output_json}")
+
+    # benchmark.md を書き込み
+    markdown = generate_markdown(benchmark)
+    with open(output_md, "w") as f:
+        f.write(markdown)
+    print(f"Generated: {output_md}")
 
-    # テーブル表示
-    print_table(aggregated, verbose=args.verbose)
+    # サマリーを表示
+    run_summary = benchmark["run_summary"]
+    configs = [k for k in run_summary if k != "delta"]
+    delta = run_summary.get("delta", {})
 
-    # JSON出力
-    if args.json:
-        print(json.dumps(aggregated, indent=2, ensure_ascii=False))
+    print(f"\nSummary:")
+    for config in configs:
+        pr = run_summary[config]["pass_rate"]["mean"]
+        label = config.replace("_", " ").title()
+        print(f"  {label}: {pr*100:.1f}% pass rate")
+    print(f"  Delta:         {delta.get('pass_rate', '—')}")
 
 
 if __name__ == "__main__":