@einja/dev-cli 0.1.40 → 0.1.44

package/presets/default/.claude/skills/einja-skill-creator/scripts/aggregate_benchmark.py

@@ -0,0 +1,401 @@
+#!/usr/bin/env python3
+"""
+ベンチマーク実行結果を集約してサマリー統計を生成。
+
+実行ディレクトリ内の grading.json ファイルを読み込み、以下を生成する:
+- 各メトリクス（平均・標準偏差・最小・最大）を含む run_summary
+- with_skill / without_skill 設定間のデルタ
+
+使用方法:
+    python aggregate_benchmark.py <benchmark_dir>
+
+例:
+    python aggregate_benchmark.py benchmarks/2026-01-15T10-30-00/
+
+対応するディレクトリ構成:
+
+    ワークスペース構成（skill-creator イテレーションから）:
+    <benchmark_dir>/
+    └── eval-N/
+        ├── with_skill/
+        │   ├── run-1/grading.json
+        │   └── run-2/grading.json
+        └── without_skill/
+            ├── run-1/grading.json
+            └── run-2/grading.json
+
+    レガシー構成（runs/ サブディレクトリあり）:
+    <benchmark_dir>/
+    └── runs/
+        └── eval-N/
+            ├── with_skill/
+            │   └── run-1/grading.json
+            └── without_skill/
+                └── run-1/grading.json
+"""
+
+import argparse
+import json
+import math
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+
+
+def calculate_stats(values: list[float]) -> dict:
+    """値リストの平均・標準偏差・最小・最大を計算する。"""
+    if not values:
+        return {"mean": 0.0, "stddev": 0.0, "min": 0.0, "max": 0.0}
+
+    n = len(values)
+    mean = sum(values) / n
+
+    if n > 1:
+        variance = sum((x - mean) ** 2 for x in values) / (n - 1)
+        stddev = math.sqrt(variance)
+    else:
+        stddev = 0.0
+
+    return {
+        "mean": round(mean, 4),
+        "stddev": round(stddev, 4),
+        "min": round(min(values), 4),
+        "max": round(max(values), 4)
+    }
+
+
+def load_run_results(benchmark_dir: Path) -> dict:
+    """
+    ベンチマークディレクトリから全実行結果を読み込む。
+
+    設定名（例: "with_skill"/"without_skill" または "new_skill"/"old_skill"）を
+    キーとして実行結果リストを格納した dict を返す。
+    """
+    # eval ディレクトリが benchmark_dir 直下または runs/ 以下の両方に対応
+    runs_dir = benchmark_dir / "runs"
+    if runs_dir.exists():
+        search_dir = runs_dir
+    elif list(benchmark_dir.glob("eval-*")):
+        search_dir = benchmark_dir
+    else:
+        print(f"No eval directories found in {benchmark_dir} or {benchmark_dir / 'runs'}")
+        return {}
+
+    results: dict[str, list] = {}
+
+    for eval_idx, eval_dir in enumerate(sorted(search_dir.glob("eval-*"))):
+        metadata_path = eval_dir / "eval_metadata.json"
+        if metadata_path.exists():
+            try:
+                with open(metadata_path) as mf:
+                    eval_id = json.load(mf).get("eval_id", eval_idx)
+            except (json.JSONDecodeError, OSError):
+                eval_id = eval_idx
+        else:
+            try:
+                eval_id = int(eval_dir.name.split("-")[1])
+            except ValueError:
+                eval_id = eval_idx
+
+        # 設定ディレクトリをハードコードせず動的に検出
+        for config_dir in sorted(eval_dir.iterdir()):
+            if not config_dir.is_dir():
+                continue
+            # run-* を含まないディレクトリ（inputs, outputs 等）はスキップ
+            if not list(config_dir.glob("run-*")):
+                continue
+            config = config_dir.name
+            if config not in results:
+                results[config] = []
+
+            for run_dir in sorted(config_dir.glob("run-*")):
+                run_number = int(run_dir.name.split("-")[1])
+                grading_file = run_dir / "grading.json"
+
+                if not grading_file.exists():
+                    print(f"Warning: grading.json not found in {run_dir}")
+                    continue
+
+                try:
+                    with open(grading_file) as f:
+                        grading = json.load(f)
+                except json.JSONDecodeError as e:
+                    print(f"Warning: Invalid JSON in {grading_file}: {e}")
+                    continue
+
+                # メトリクスを抽出
+                result = {
+                    "eval_id": eval_id,
+                    "run_number": run_number,
+                    "pass_rate": grading.get("summary", {}).get("pass_rate", 0.0),
+                    "passed": grading.get("summary", {}).get("passed", 0),
+                    "failed": grading.get("summary", {}).get("failed", 0),
+                    "total": grading.get("summary", {}).get("total", 0),
+                }
+
+                # タイミングを抽出（grading.json 優先、なければ timing.json を参照）
+                timing = grading.get("timing", {})
+                result["time_seconds"] = timing.get("total_duration_seconds", 0.0)
+                timing_file = run_dir / "timing.json"
+                if result["time_seconds"] == 0.0 and timing_file.exists():
+                    try:
+                        with open(timing_file) as tf:
+                            timing_data = json.load(tf)
+                        result["time_seconds"] = timing_data.get("total_duration_seconds", 0.0)
+                        result["tokens"] = timing_data.get("total_tokens", 0)
+                    except json.JSONDecodeError:
+                        pass
+
+                # 実行メトリクスを抽出
+                metrics = grading.get("execution_metrics", {})
+                result["tool_calls"] = metrics.get("total_tool_calls", 0)
+                if not result.get("tokens"):
+                    result["tokens"] = metrics.get("output_chars", 0)
+                result["errors"] = metrics.get("errors_encountered", 0)
+
+                # expectations を抽出（viewer に必要なフィールド: text, passed, evidence）
+                raw_expectations = grading.get("expectations", [])
+                for exp in raw_expectations:
+                    if "text" not in exp or "passed" not in exp:
+                        print(f"Warning: expectation in {grading_file} missing required fields (text, passed, evidence): {exp}")
+                result["expectations"] = raw_expectations
+
+                # user_notes_summary からノートを抽出
+                notes_summary = grading.get("user_notes_summary", {})
+                notes = []
+                notes.extend(notes_summary.get("uncertainties", []))
+                notes.extend(notes_summary.get("needs_review", []))
+                notes.extend(notes_summary.get("workarounds", []))
+                result["notes"] = notes
+
+                results[config].append(result)
+
+    return results
+
+
+def aggregate_results(results: dict) -> dict:
+    """
+    実行結果をサマリー統計に集約する。
+
+    各設定のスタットと設定間のデルタを含む run_summary を返す。
+    """
+    run_summary = {}
+    configs = list(results.keys())
+
+    for config in configs:
+        runs = results.get(config, [])
+
+        if not runs:
+            run_summary[config] = {
+                "pass_rate": {"mean": 0.0, "stddev": 0.0, "min": 0.0, "max": 0.0},
+                "time_seconds": {"mean": 0.0, "stddev": 0.0, "min": 0.0, "max": 0.0},
+                "tokens": {"mean": 0, "stddev": 0, "min": 0, "max": 0}
+            }
+            continue
+
+        pass_rates = [r["pass_rate"] for r in runs]
+        times = [r["time_seconds"] for r in runs]
+        tokens = [r.get("tokens", 0) for r in runs]
+
+        run_summary[config] = {
+            "pass_rate": calculate_stats(pass_rates),
+            "time_seconds": calculate_stats(times),
+            "tokens": calculate_stats(tokens)
+        }
+
+    # 最初の2設定間のデルタを計算
+    if len(configs) >= 2:
+        primary = run_summary.get(configs[0], {})
+        baseline = run_summary.get(configs[1], {})
+    else:
+        primary = run_summary.get(configs[0], {}) if configs else {}
+        baseline = {}
+
+    delta_pass_rate = primary.get("pass_rate", {}).get("mean", 0) - baseline.get("pass_rate", {}).get("mean", 0)
+    delta_time = primary.get("time_seconds", {}).get("mean", 0) - baseline.get("time_seconds", {}).get("mean", 0)
+    delta_tokens = primary.get("tokens", {}).get("mean", 0) - baseline.get("tokens", {}).get("mean", 0)
+
+    run_summary["delta"] = {
+        "pass_rate": f"{delta_pass_rate:+.2f}",
+        "time_seconds": f"{delta_time:+.1f}",
+        "tokens": f"{delta_tokens:+.0f}"
+    }
+
+    return run_summary
+
+
+def generate_benchmark(benchmark_dir: Path, skill_name: str = "", skill_path: str = "") -> dict:
+    """
+    実行結果から完全な benchmark.json を生成する。
+    """
+    results = load_run_results(benchmark_dir)
+    run_summary = aggregate_results(results)
+
+    # benchmark.json 用の runs 配列を構築
+    runs = []
+    for config in results:
+        for result in results[config]:
+            runs.append({
+                "eval_id": result["eval_id"],
+                "configuration": config,
+                "run_number": result["run_number"],
+                "result": {
+                    "pass_rate": result["pass_rate"],
+                    "passed": result["passed"],
+                    "failed": result["failed"],
+                    "total": result["total"],
+                    "time_seconds": result["time_seconds"],
+                    "tokens": result.get("tokens", 0),
+                    "tool_calls": result.get("tool_calls", 0),
+                    "errors": result.get("errors", 0)
+                },
+                "expectations": result["expectations"],
+                "notes": result["notes"]
+            })
+
+    # 結果から eval ID を決定
+    eval_ids = sorted(set(
+        r["eval_id"]
+        for config in results.values()
+        for r in config
+    ))
+
+    benchmark = {
+        "metadata": {
+            "skill_name": skill_name or "<skill-name>",
+            "skill_path": skill_path or "<path/to/skill>",
+            "executor_model": "<model-name>",
+            "analyzer_model": "<model-name>",
+            "timestamp": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
+            "evals_run": eval_ids,
+            "runs_per_configuration": 3
+        },
+        "runs": runs,
+        "run_summary": run_summary,
+        "notes": []  # アナライザーが埋める
+    }
+
+    return benchmark
+
+
+def generate_markdown(benchmark: dict) -> str:
+    """benchmark データから人間が読みやすい benchmark.md を生成する。"""
+    metadata = benchmark["metadata"]
+    run_summary = benchmark["run_summary"]
+
+    # 設定名を取得（"delta" を除く）
+    configs = [k for k in run_summary if k != "delta"]
+    config_a = configs[0] if len(configs) >= 1 else "config_a"
+    config_b = configs[1] if len(configs) >= 2 else "config_b"
+    label_a = config_a.replace("_", " ").title()
+    label_b = config_b.replace("_", " ").title()
+
+    lines = [
+        f"# Skill Benchmark: {metadata['skill_name']}",
+        "",
+        f"**Model**: {metadata['executor_model']}",
+        f"**Date**: {metadata['timestamp']}",
+        f"**Evals**: {', '.join(map(str, metadata['evals_run']))} ({metadata['runs_per_configuration']} runs each per configuration)",
+        "",
+        "## Summary",
+        "",
+        f"| Metric | {label_a} | {label_b} | Delta |",
+        "|--------|------------|---------------|-------|",
+    ]
+
+    a_summary = run_summary.get(config_a, {})
+    b_summary = run_summary.get(config_b, {})
+    delta = run_summary.get("delta", {})
+
+    # パスレートのフォーマット
+    a_pr = a_summary.get("pass_rate", {})
+    b_pr = b_summary.get("pass_rate", {})
+    lines.append(f"| Pass Rate | {a_pr.get('mean', 0)*100:.0f}% ± {a_pr.get('stddev', 0)*100:.0f}% | {b_pr.get('mean', 0)*100:.0f}% ± {b_pr.get('stddev', 0)*100:.0f}% | {delta.get('pass_rate', '—')} |")
+
+    # タイムのフォーマット
+    a_time = a_summary.get("time_seconds", {})
+    b_time = b_summary.get("time_seconds", {})
+    lines.append(f"| Time | {a_time.get('mean', 0):.1f}s ± {a_time.get('stddev', 0):.1f}s | {b_time.get('mean', 0):.1f}s ± {b_time.get('stddev', 0):.1f}s | {delta.get('time_seconds', '—')}s |")
+
+    # トークン数のフォーマット
+    a_tokens = a_summary.get("tokens", {})
+    b_tokens = b_summary.get("tokens", {})
+    lines.append(f"| Tokens | {a_tokens.get('mean', 0):.0f} ± {a_tokens.get('stddev', 0):.0f} | {b_tokens.get('mean', 0):.0f} ± {b_tokens.get('stddev', 0):.0f} | {delta.get('tokens', '—')} |")
+
+    # ノートセクション
+    if benchmark.get("notes"):
+        lines.extend([
+            "",
+            "## Notes",
+            ""
+        ])
+        for note in benchmark["notes"]:
+            lines.append(f"- {note}")
+
+    return "\n".join(lines)
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="ベンチマーク実行結果を集約してサマリー統計を生成"
+    )
+    parser.add_argument(
+        "benchmark_dir",
+        type=Path,
+        help="ベンチマークディレクトリへのパス"
+    )
+    parser.add_argument(
+        "--skill-name",
+        default="",
+        help="ベンチマーク対象スキルの名前"
+    )
+    parser.add_argument(
+        "--skill-path",
+        default="",
+        help="ベンチマーク対象スキルへのパス"
+    )
+    parser.add_argument(
+        "--output", "-o",
+        type=Path,
+        help="benchmark.json の出力パス（デフォルト: <benchmark_dir>/benchmark.json）"
+    )
+
+    args = parser.parse_args()
+
+    if not args.benchmark_dir.exists():
+        print(f"Directory not found: {args.benchmark_dir}")
+        sys.exit(1)
+
+    # ベンチマーク生成
+    benchmark = generate_benchmark(args.benchmark_dir, args.skill_name, args.skill_path)
+
+    # 出力パスを決定
+    output_json = args.output or (args.benchmark_dir / "benchmark.json")
+    output_md = output_json.with_suffix(".md")
+
+    # benchmark.json を書き込み
+    with open(output_json, "w") as f:
+        json.dump(benchmark, f, indent=2)
+    print(f"Generated: {output_json}")
+
+    # benchmark.md を書き込み
+    markdown = generate_markdown(benchmark)
+    with open(output_md, "w") as f:
+        f.write(markdown)
+    print(f"Generated: {output_md}")
+
+    # サマリーを表示
+    run_summary = benchmark["run_summary"]
+    configs = [k for k in run_summary if k != "delta"]
+    delta = run_summary.get("delta", {})
+
+    print(f"\nSummary:")
+    for config in configs:
+        pr = run_summary[config]["pass_rate"]["mean"]
+        label = config.replace("_", " ").title()
+        print(f"  {label}: {pr*100:.1f}% pass rate")
+    print(f"  Delta:         {delta.get('pass_rate', '—')}")
+
+
+if __name__ == "__main__":
+    main()

package/presets/default/.claude/skills/einja-skill-creator/scripts/compare_runs.py

@@ -0,0 +1,154 @@
+#!/usr/bin/env python3
+"""複数スキルのベンチマーク結果を集約。
+
+複数のrun_loop.py出力を受け取り、全スキルのスコアを
+サマリーテーブルとして表示する。
+"""
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+
+def aggregate_results(result_files: list[str]) -> dict:
+    """複数のrun_loop.py出力ファイルを集約する。"""
+    skills = []
+
+    for filepath in result_files:
+        try:
+            data = json.loads(Path(filepath).read_text())
+        except (json.JSONDecodeError, FileNotFoundError) as e:
+            print(f"警告: {filepath} の読み込みに失敗しました: {e}", file=sys.stderr)
+            continue
+
+        history = data.get("history", [])
+        if not history:
+            print(f"警告: {filepath} に履歴がありません", file=sys.stderr)
+            continue
+
+        # 最良のイテレーションを見つける（テスト > トレーニングで優先）
+        best_idx = 0
+        best_test = -1
+        best_train = -1
+        for i, h in enumerate(history):
+            t_passed = h.get("test_passed", -1)
+            tr_passed = h.get("train_passed", h.get("passed", 0))
+            if t_passed > best_test or (t_passed == best_test and tr_passed > best_train):
+                best_test = t_passed
+                best_train = tr_passed
+                best_idx = i
+
+        best = history[best_idx]
+        original = history[0]
+
+        skill_entry = {
+            "skill_name": data.get("skill_name", Path(filepath).stem),
+            "file": filepath,
+            "iterations": len(history),
+            "best_iteration": best_idx,
+            "original_description": data.get("original_description", ""),
+            "best_description": best.get("description", ""),
+            "original_train_score": f"{original.get('train_passed', original.get('passed', 0))}/{original.get('train_total', original.get('total', 0))}",
+            "best_train_score": f"{best.get('train_passed', best.get('passed', 0))}/{best.get('train_total', best.get('total', 0))}",
+            "original_train_passed": original.get("train_passed", original.get("passed", 0)),
+            "original_train_total": original.get("train_total", original.get("total", 0)),
+            "best_train_passed": best.get("train_passed", best.get("passed", 0)),
+            "best_train_total": best.get("train_total", best.get("total", 0)),
+        }
+
+        # テストスコア（存在する場合）
+        if best.get("test_passed") is not None:
+            skill_entry["original_test_score"] = f"{original.get('test_passed', '?')}/{original.get('test_total', '?')}"
+            skill_entry["best_test_score"] = f"{best.get('test_passed', '?')}/{best.get('test_total', '?')}"
+            skill_entry["best_test_passed"] = best.get("test_passed", 0)
+            skill_entry["best_test_total"] = best.get("test_total", 0)
+
+        skills.append(skill_entry)
+
+    # トレーニングスコアでソート（降順）
+    skills.sort(
+        key=lambda s: (
+            s.get("best_test_passed", 0) / max(s.get("best_test_total", 1), 1),
+            s["best_train_passed"] / max(s["best_train_total"], 1),
+        ),
+        reverse=True,
+    )
+
+    # 全体サマリーの計算
+    total_train_passed = sum(s["best_train_passed"] for s in skills)
+    total_train_total = sum(s["best_train_total"] for s in skills)
+    total_test_passed = sum(s.get("best_test_passed", 0) for s in skills if "best_test_passed" in s)
+    total_test_total = sum(s.get("best_test_total", 0) for s in skills if "best_test_total" in s)
+
+    return {
+        "skills": skills,
+        "summary": {
+            "total_skills": len(skills),
+            "total_train_passed": total_train_passed,
+            "total_train_total": total_train_total,
+            "total_train_score": f"{total_train_passed}/{total_train_total}",
+            "total_test_passed": total_test_passed,
+            "total_test_total": total_test_total,
+            "total_test_score": f"{total_test_passed}/{total_test_total}" if total_test_total > 0 else None,
+        },
+    }
+
+
+def print_table(aggregated: dict, verbose: bool = False) -> None:
+    """集約結果をテーブル形式でstderrに出力する。"""
+    skills = aggregated["skills"]
+    summary = aggregated["summary"]
+
+    has_test = any("best_test_score" in s for s in skills)
+
+    # ヘッダー
+    header = f"{'スキル名':<30} {'トレーニング(元)':<14} {'トレーニング(最良)':<14}"
+    if has_test:
+        header += f" {'テスト(元)':<12} {'テスト(最良)':<12}"
+    header += f" {'回数':<6} {'最良回':<6}"
+    print(header, file=sys.stderr)
+    print("-" * len(header), file=sys.stderr)
+
+    # 各スキル
+    for s in skills:
+        line = f"{s['skill_name']:<30} {s['original_train_score']:<14} {s['best_train_score']:<14}"
+        if has_test:
+            orig_test = s.get("original_test_score", "-")
+            best_test = s.get("best_test_score", "-")
+            line += f" {orig_test:<12} {best_test:<12}"
+        line += f" {s['iterations']:<6} {s['best_iteration']:<6}"
+        print(line, file=sys.stderr)
+
+        if verbose:
+            print(f"  オリジナル: {s['original_description'][:80]}...", file=sys.stderr)
+            print(f"  最良:       {s['best_description'][:80]}...", file=sys.stderr)
+
+    # サマリー
+    print("-" * len(header), file=sys.stderr)
+    total_line = f"{'合計':<30} {'':<14} {summary['total_train_score']:<14}"
+    if has_test and summary.get("total_test_score"):
+        total_line += f" {'':<12} {summary['total_test_score']:<12}"
+    total_line += f" {summary['total_skills']} スキル"
+    print(total_line, file=sys.stderr)
+
+
+def main():
+    parser = argparse.ArgumentParser(description="複数スキルのベンチマーク結果を集約")
+    parser.add_argument("files", nargs="+", help="run_loop.pyのJSON出力ファイル（複数指定可）")
+    parser.add_argument("--verbose", action="store_true", help="各スキルのdescriptionも表示")
+    parser.add_argument("--json", action="store_true", help="JSON形式で標準出力に出力")
+    args = parser.parse_args()
+
+    aggregated = aggregate_results(args.files)
+
+    # テーブル表示
+    print_table(aggregated, verbose=args.verbose)
+
+    # JSON出力
+    if args.json:
+        print(json.dumps(aggregated, indent=2, ensure_ascii=False))
+
+
+if __name__ == "__main__":
+    main()