@einja/dev-cli 0.1.39 → 0.1.41

package/presets/default/.claude/skills/einja-skill-creator/scripts/run_eval.py

@@ -0,0 +1,310 @@
+#!/usr/bin/env python3
+"""スキルdescriptionのトリガー評価を実行。
+
+スキルのdescriptionが一連のクエリに対してClaudeのスキル使用を
+トリガーするかどうかをテストする。結果をJSONで出力。
+"""
+
+import argparse
+import json
+import os
+import select
+import subprocess
+import sys
+import time
+import uuid
+from concurrent.futures import ProcessPoolExecutor, as_completed
+from pathlib import Path
+
+try:
+    from scripts.utils import parse_skill_md
+except ImportError:
+    from utils import parse_skill_md
+
+
+def find_project_root() -> Path:
+    """cwdから上方向にウォークし、.claude/を探してプロジェクトルートを見つける。
+
+    Claude Codeがプロジェクトルートを発見する方法を模倣し、
+    作成するコマンドファイルがclaude -pの検索対象に入るようにする。
+    """
+    current = Path.cwd()
+    for parent in [current, *current.parents]:
+        if (parent / ".claude").is_dir():
+            return parent
+    return current
+
+
+def run_single_query(
+    query: str,
+    skill_name: str,
+    skill_description: str,
+    timeout: int,
+    project_root: str,
+    model: str | None = None,
+) -> bool:
+    """単一のクエリを実行し、スキルがトリガーされたかどうかを返す。
+
+    .claude/commands/にコマンドファイルを作成してClaudeのavailable_skillsリストに
+    表示させ、`claude -p`で生のクエリを実行する。
+    --include-partial-messagesを使用してストリームイベント（content_block_start）から
+    早期にトリガーを検出する。
+    """
+    unique_id = uuid.uuid4().hex[:8]
+    clean_name = f"{skill_name}-skill-{unique_id}"
+    project_commands_dir = Path(project_root) / ".claude" / "commands"
+    command_file = project_commands_dir / f"{clean_name}.md"
+
+    try:
+        project_commands_dir.mkdir(parents=True, exist_ok=True)
+        # description内のクォートでの破損を避けるためYAMLブロックスカラーを使用
+        indented_desc = "\n  ".join(skill_description.split("\n"))
+        command_content = (
+            f"---\n"
+            f"description: |\n"
+            f"  {indented_desc}\n"
+            f"---\n\n"
+            f"# {skill_name}\n\n"
+            f"This skill handles: {skill_description}\n"
+        )
+        command_file.write_text(command_content)
+
+        cmd = [
+            "claude",
+            "-p", query,
+            "--output-format", "stream-json",
+            "--verbose",
+            "--include-partial-messages",
+        ]
+        if model:
+            cmd.extend(["--model", model])
+
+        # Claude Codeセッション内でclaude -pのネストを許可するためCLAUDECODE環境変数を除去
+        env = {k: v for k, v in os.environ.items() if k != "CLAUDECODE"}
+
+        process = subprocess.Popen(
+            cmd,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.DEVNULL,
+            cwd=project_root,
+            env=env,
+        )
+
+        triggered = False
+        start_time = time.time()
+        buffer = ""
+        # ストリームイベント検出用の状態追跡
+        pending_tool_name = None
+        accumulated_json = ""
+
+        try:
+            while time.time() - start_time < timeout:
+                if process.poll() is not None:
+                    remaining = process.stdout.read()
+                    if remaining:
+                        buffer += remaining.decode("utf-8", errors="replace")
+                    break
+
+                ready, _, _ = select.select([process.stdout], [], [], 1.0)
+                if not ready:
+                    continue
+
+                chunk = os.read(process.stdout.fileno(), 8192)
+                if not chunk:
+                    break
+                buffer += chunk.decode("utf-8", errors="replace")
+
+                while "\n" in buffer:
+                    line, buffer = buffer.split("\n", 1)
+                    line = line.strip()
+                    if not line:
+                        continue
+
+                    try:
+                        event = json.loads(line)
+                    except json.JSONDecodeError:
+                        continue
+
+                    # ストリームイベントによる早期検出
+                    if event.get("type") == "stream_event":
+                        se = event.get("event", {})
+                        se_type = se.get("type", "")
+
+                        if se_type == "content_block_start":
+                            cb = se.get("content_block", {})
+                            if cb.get("type") == "tool_use":
+                                tool_name = cb.get("name", "")
+                                if tool_name in ("Skill", "Read"):
+                                    pending_tool_name = tool_name
+                                    accumulated_json = ""
+                                else:
+                                    return False
+
+                        elif se_type == "content_block_delta" and pending_tool_name:
+                            delta = se.get("delta", {})
+                            if delta.get("type") == "input_json_delta":
+                                accumulated_json += delta.get("partial_json", "")
+                                if clean_name in accumulated_json:
+                                    return True
+
+                        elif se_type in ("content_block_stop", "message_stop"):
+                            if pending_tool_name:
+                                return clean_name in accumulated_json
+                            if se_type == "message_stop":
+                                return False
+
+                    # フォールバック: 完全なassistantメッセージ
+                    elif event.get("type") == "assistant":
+                        message = event.get("message", {})
+                        for content_item in message.get("content", []):
+                            if content_item.get("type") != "tool_use":
+                                continue
+                            tool_name = content_item.get("name", "")
+                            tool_input = content_item.get("input", {})
+                            if tool_name == "Skill" and clean_name in tool_input.get("skill", ""):
+                                triggered = True
+                            elif tool_name == "Read" and clean_name in tool_input.get("file_path", ""):
+                                triggered = True
+                            return triggered
+
+                    elif event.get("type") == "result":
+                        return triggered
+        finally:
+            # 任意の終了パス（return、例外、タイムアウト）でプロセスをクリーンアップ
+            if process.poll() is None:
+                process.kill()
+                process.wait()
+
+        return triggered
+    finally:
+        if command_file.exists():
+            command_file.unlink()
+
+
+def run_eval(
+    eval_set: list[dict],
+    skill_name: str,
+    description: str,
+    num_workers: int,
+    timeout: int,
+    project_root: Path,
+    runs_per_query: int = 1,
+    trigger_threshold: float = 0.5,
+    model: str | None = None,
+) -> dict:
+    """評価セット全体を実行し、結果を返す。"""
+    results = []
+
+    with ProcessPoolExecutor(max_workers=num_workers) as executor:
+        future_to_info = {}
+        for item in eval_set:
+            for run_idx in range(runs_per_query):
+                future = executor.submit(
+                    run_single_query,
+                    item["query"],
+                    skill_name,
+                    description,
+                    timeout,
+                    str(project_root),
+                    model,
+                )
+                future_to_info[future] = (item, run_idx)
+
+        query_triggers: dict[str, list[bool]] = {}
+        query_items: dict[str, dict] = {}
+        for future in as_completed(future_to_info):
+            item, _ = future_to_info[future]
+            query = item["query"]
+            query_items[query] = item
+            if query not in query_triggers:
+                query_triggers[query] = []
+            try:
+                query_triggers[query].append(future.result())
+            except Exception as e:
+                print(f"警告: クエリが失敗しました: {e}", file=sys.stderr)
+                query_triggers[query].append(False)
+
+    for query, triggers in query_triggers.items():
+        item = query_items[query]
+        trigger_rate = sum(triggers) / len(triggers)
+        should_trigger = item["should_trigger"]
+        if should_trigger:
+            did_pass = trigger_rate >= trigger_threshold
+        else:
+            did_pass = trigger_rate < trigger_threshold
+        results.append({
+            "query": query,
+            "should_trigger": should_trigger,
+            "trigger_rate": trigger_rate,
+            "triggers": sum(triggers),
+            "runs": len(triggers),
+            "pass": did_pass,
+        })
+
+    passed = sum(1 for r in results if r["pass"])
+    total = len(results)
+
+    return {
+        "skill_name": skill_name,
+        "description": description,
+        "results": results,
+        "summary": {
+            "total": total,
+            "passed": passed,
+            "failed": total - passed,
+        },
+    }
+
+
+def main():
+    parser = argparse.ArgumentParser(description="スキルdescriptionのトリガー評価を実行")
+    parser.add_argument("--eval-set", required=True, help="評価セットJSONファイルへのパス")
+    parser.add_argument("--skill-path", required=True, help="スキルディレクトリへのパス")
+    parser.add_argument("--description", default=None, help="テスト用descriptionの上書き")
+    parser.add_argument("--num-workers", type=int, default=10, help="並行ワーカー数")
+    parser.add_argument("--timeout", type=int, default=30, help="クエリごとのタイムアウト（秒）")
+    parser.add_argument("--runs-per-query", type=int, default=3, help="クエリごとの実行回数")
+    parser.add_argument("--trigger-threshold", type=float, default=0.5, help="トリガー率の閾値")
+    parser.add_argument("--model", default=None, help="claude -pに使用するモデル（デフォルト: ユーザー設定のモデル）")
+    parser.add_argument("--verbose", action="store_true", help="進捗をstderrに出力")
+    args = parser.parse_args()
+
+    eval_set = json.loads(Path(args.eval_set).read_text())
+    skill_path = Path(args.skill_path)
+
+    if not (skill_path / "SKILL.md").exists():
+        print(f"エラー: {skill_path} にSKILL.mdが見つかりません", file=sys.stderr)
+        sys.exit(1)
+
+    name, original_description, content = parse_skill_md(skill_path)
+    description = args.description or original_description
+    project_root = find_project_root()
+
+    if args.verbose:
+        print(f"評価中: {description}", file=sys.stderr)
+
+    output = run_eval(
+        eval_set=eval_set,
+        skill_name=name,
+        description=description,
+        num_workers=args.num_workers,
+        timeout=args.timeout,
+        project_root=project_root,
+        runs_per_query=args.runs_per_query,
+        trigger_threshold=args.trigger_threshold,
+        model=args.model,
+    )
+
+    if args.verbose:
+        summary = output["summary"]
+        print(f"結果: {summary['passed']}/{summary['total']} パス", file=sys.stderr)
+        for r in output["results"]:
+            status = "PASS" if r["pass"] else "FAIL"
+            rate_str = f"{r['triggers']}/{r['runs']}"
+            print(f"  [{status}] rate={rate_str} expected={r['should_trigger']}: {r['query'][:70]}", file=sys.stderr)
+
+    print(json.dumps(output, indent=2))
+
+
+if __name__ == "__main__":
+    main()

package/presets/default/.claude/skills/einja-skill-creator/scripts/run_loop.py

@@ -0,0 +1,295 @@
+#!/usr/bin/env python3
+"""評価＋改善ループを実行。全パスまたは最大イテレーション到達まで繰り返す。
+
+run_eval.pyとimprove_description.pyをループで組み合わせ、
+履歴を追跡し最良のdescriptionを返す。
+過学習防止のためtrain/test分割に対応。
+"""
+
+import argparse
+import json
+import random
+import sys
+import time
+from pathlib import Path
+
+try:
+    from scripts.generate_report import generate_html
+    from scripts.improve_description import improve_description
+    from scripts.run_eval import find_project_root, run_eval
+    from scripts.utils import parse_skill_md
+except ImportError:
+    from generate_report import generate_html
+    from improve_description import improve_description
+    from run_eval import find_project_root, run_eval
+    from utils import parse_skill_md
+
+import anthropic
+
+
+def split_eval_set(
+    eval_set: list[dict],
+    holdout: int,
+    seed: int | None = None,
+) -> tuple[list[dict], list[dict]]:
+    """評価セットをトレーニングとテストに分割する。
+
+    holdoutが0の場合、全データをトレーニングに使用する。
+    should_trigger=Trueとshould_trigger=Falseの両方から
+    均等にホールドアウトする。
+    """
+    if holdout <= 0:
+        return eval_set, []
+
+    rng = random.Random(seed)
+
+    positive = [item for item in eval_set if item.get("should_trigger", True)]
+    negative = [item for item in eval_set if not item.get("should_trigger", True)]
+
+    # 正例と負例から均等にホールドアウト
+    pos_holdout = holdout // 2
+    neg_holdout = holdout - pos_holdout
+
+    # 上限調整
+    pos_holdout = min(pos_holdout, len(positive) - 1) if len(positive) > 1 else 0
+    neg_holdout = min(neg_holdout, len(negative) - 1) if len(negative) > 1 else 0
+
+    rng.shuffle(positive)
+    rng.shuffle(negative)
+
+    test_set = positive[:pos_holdout] + negative[:neg_holdout]
+    train_set = positive[pos_holdout:] + negative[neg_holdout:]
+
+    return train_set, test_set
+
+
+def run_loop(
+    eval_set_path: str,
+    skill_path: str,
+    max_iterations: int = 10,
+    num_workers: int = 10,
+    timeout: int = 30,
+    runs_per_query: int = 3,
+    trigger_threshold: float = 0.5,
+    holdout: int = 0,
+    seed: int | None = None,
+    model: str | None = None,
+    improve_model: str = "claude-sonnet-4-20250514",
+    verbose: bool = False,
+    report_path: str | None = None,
+    log_dir: str | None = None,
+) -> dict:
+    """評価＋改善ループのメイン関数。"""
+    eval_set = json.loads(Path(eval_set_path).read_text())
+    skill_dir = Path(skill_path)
+
+    if not (skill_dir / "SKILL.md").exists():
+        print(f"エラー: {skill_dir} にSKILL.mdが見つかりません", file=sys.stderr)
+        sys.exit(1)
+
+    name, original_description, content = parse_skill_md(skill_dir)
+    project_root = find_project_root()
+
+    # train/test分割
+    train_set, test_set = split_eval_set(eval_set, holdout, seed)
+
+    if verbose:
+        print(f"スキル: {name}", file=sys.stderr)
+        print(f"トレーニングクエリ: {len(train_set)}, テストクエリ: {len(test_set)}", file=sys.stderr)
+        print(f"最大イテレーション: {max_iterations}", file=sys.stderr)
+        print(f"オリジナルdescription: {original_description}", file=sys.stderr)
+
+    client = anthropic.Anthropic()
+    current_description = original_description
+    history: list[dict] = []
+    improve_history: list[dict] = []
+    log_path = Path(log_dir) if log_dir else None
+
+    for iteration in range(max_iterations):
+        if verbose:
+            print(f"\n--- イテレーション {iteration} ---", file=sys.stderr)
+            print(f"description: {current_description[:100]}...", file=sys.stderr)
+
+        # トレーニング評価
+        train_results = run_eval(
+            eval_set=train_set,
+            skill_name=name,
+            description=current_description,
+            num_workers=num_workers,
+            timeout=timeout,
+            project_root=project_root,
+            runs_per_query=runs_per_query,
+            trigger_threshold=trigger_threshold,
+            model=model,
+        )
+
+        # テスト評価（テストセットがある場合）
+        test_results = None
+        if test_set:
+            test_results = run_eval(
+                eval_set=test_set,
+                skill_name=name,
+                description=current_description,
+                num_workers=num_workers,
+                timeout=timeout,
+                project_root=project_root,
+                runs_per_query=runs_per_query,
+                trigger_threshold=trigger_threshold,
+                model=model,
+            )
+
+        # 履歴エントリの構築
+        entry: dict = {
+            "description": current_description,
+            "train_passed": train_results["summary"]["passed"],
+            "train_failed": train_results["summary"]["failed"],
+            "train_total": train_results["summary"]["total"],
+            "train_results": train_results["results"],
+        }
+        if test_results:
+            entry["test_passed"] = test_results["summary"]["passed"]
+            entry["test_failed"] = test_results["summary"]["failed"]
+            entry["test_total"] = test_results["summary"]["total"]
+            entry["test_results"] = test_results["results"]
+
+        history.append(entry)
+
+        if verbose:
+            train_s = f"{train_results['summary']['passed']}/{train_results['summary']['total']}"
+            msg = f"トレーニングスコア: {train_s}"
+            if test_results:
+                test_s = f"{test_results['summary']['passed']}/{test_results['summary']['total']}"
+                msg += f", テストスコア: {test_s}"
+            print(msg, file=sys.stderr)
+
+        # レポート更新
+        if report_path:
+            output_data = {"history": history, "holdout": holdout}
+            report_html = generate_html(output_data, auto_refresh=True, skill_name=name)
+            Path(report_path).write_text(report_html)
+            if verbose:
+                print(f"レポートを更新しました: {report_path}", file=sys.stderr)
+
+        # 全パスなら終了
+        if train_results["summary"]["failed"] == 0:
+            if test_results is None or test_results["summary"]["failed"] == 0:
+                if verbose:
+                    print("全クエリパス。ループを終了します。", file=sys.stderr)
+                break
+
+        # 最終イテレーションでなければ改善
+        if iteration < max_iterations - 1:
+            if verbose:
+                print("descriptionを改善中...", file=sys.stderr)
+
+            new_description = improve_description(
+                client=client,
+                skill_name=name,
+                skill_content=content,
+                current_description=current_description,
+                eval_results=train_results,
+                history=improve_history,
+                model=improve_model,
+                test_results=test_results,
+                log_dir=log_path,
+                iteration=iteration,
+            )
+
+            # 改善履歴を更新
+            improve_entry: dict = {
+                "description": current_description,
+                "train_passed": train_results["summary"]["passed"],
+                "train_total": train_results["summary"]["total"],
+                "results": train_results["results"],
+            }
+            if test_results:
+                improve_entry["test_passed"] = test_results["summary"]["passed"]
+                improve_entry["test_total"] = test_results["summary"]["total"]
+            improve_history.append(improve_entry)
+
+            current_description = new_description
+
+            if verbose:
+                print(f"新しいdescription: {new_description[:100]}...", file=sys.stderr)
+
+    # 最良のdescriptionを選択（テスト > トレーニングで優先）
+    best_idx = 0
+    best_test = -1
+    best_train = -1
+    for i, h in enumerate(history):
+        t_passed = h.get("test_passed", -1)
+        tr_passed = h.get("train_passed", h.get("passed", 0))
+        if t_passed > best_test or (t_passed == best_test and tr_passed > best_train):
+            best_test = t_passed
+            best_train = tr_passed
+            best_idx = i
+
+    best = history[best_idx]
+
+    # 最終レポート（auto_refreshオフ）
+    if report_path:
+        output_data = {"history": history, "holdout": holdout}
+        report_html = generate_html(output_data, auto_refresh=False, skill_name=name)
+        Path(report_path).write_text(report_html)
+
+    output = {
+        "skill_name": name,
+        "original_description": original_description,
+        "best_description": best["description"],
+        "best_iteration": best_idx,
+        "history": history,
+        "holdout": holdout,
+    }
+
+    if verbose:
+        print(f"\n最良のdescription (イテレーション {best_idx}): {best['description']}", file=sys.stderr)
+        train_s = f"{best['train_passed']}/{best['train_total']}"
+        msg = f"最良スコア - トレーニング: {train_s}"
+        if best.get("test_passed") is not None:
+            test_s = f"{best['test_passed']}/{best['test_total']}"
+            msg += f", テスト: {test_s}"
+        print(msg, file=sys.stderr)
+
+    return output
+
+
+def main():
+    parser = argparse.ArgumentParser(description="評価＋改善ループを実行")
+    parser.add_argument("--eval-set", required=True, help="評価セットJSONファイルへのパス")
+    parser.add_argument("--skill-path", required=True, help="スキルディレクトリへのパス")
+    parser.add_argument("--max-iterations", type=int, default=10, help="最大イテレーション数（デフォルト: 10）")
+    parser.add_argument("--num-workers", type=int, default=10, help="並行ワーカー数（デフォルト: 10）")
+    parser.add_argument("--timeout", type=int, default=30, help="クエリごとのタイムアウト秒数（デフォルト: 30）")
+    parser.add_argument("--runs-per-query", type=int, default=3, help="クエリごとの実行回数（デフォルト: 3）")
+    parser.add_argument("--trigger-threshold", type=float, default=0.5, help="トリガー率の閾値（デフォルト: 0.5）")
+    parser.add_argument("--holdout", type=int, default=0, help="テスト用ホールドアウトクエリ数（デフォルト: 0）")
+    parser.add_argument("--seed", type=int, default=None, help="train/test分割のランダムシード")
+    parser.add_argument("--model", default=None, help="評価時にclaude -pに使用するモデル")
+    parser.add_argument("--improve-model", default="claude-sonnet-4-20250514", help="description改善に使用するモデル（デフォルト: claude-sonnet-4-20250514）")
+    parser.add_argument("--verbose", action="store_true", help="進捗をstderrに出力")
+    parser.add_argument("--report", default=None, help="HTMLレポートの出力先パス（ライブ更新あり）")
+    parser.add_argument("--log-dir", default=None, help="改善トランスクリプトのログディレクトリ")
+    args = parser.parse_args()
+
+    output = run_loop(
+        eval_set_path=args.eval_set,
+        skill_path=args.skill_path,
+        max_iterations=args.max_iterations,
+        num_workers=args.num_workers,
+        timeout=args.timeout,
+        runs_per_query=args.runs_per_query,
+        trigger_threshold=args.trigger_threshold,
+        holdout=args.holdout,
+        seed=args.seed,
+        model=args.model,
+        improve_model=args.improve_model,
+        verbose=args.verbose,
+        report_path=args.report,
+        log_dir=args.log_dir,
+    )
+
+    print(json.dumps(output, indent=2))
+
+
+if __name__ == "__main__":
+    main()

package/presets/default/.claude/skills/einja-skill-creator/scripts/utils.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+"""スキルクリエイタースクリプト共通ユーティリティ。"""
+
+from pathlib import Path
+
+
+
+def parse_skill_md(skill_path: Path) -> tuple[str, str, str]:
+    """SKILL.mdファイルをパースし、(name, description, full_content)を返す。"""
+    content = (skill_path / "SKILL.md").read_text()
+    lines = content.split("\n")
+
+    if lines[0].strip() != "---":
+        raise ValueError("SKILL.mdにフロントマターがありません（開始の---がありません）")
+
+    end_idx = None
+    for i, line in enumerate(lines[1:], start=1):
+        if line.strip() == "---":
+            end_idx = i
+            break
+
+    if end_idx is None:
+        raise ValueError("SKILL.mdにフロントマターがありません（終了の---がありません）")
+
+    name = ""
+    description = ""
+    frontmatter_lines = lines[1:end_idx]
+    i = 0
+    while i < len(frontmatter_lines):
+        line = frontmatter_lines[i]
+        if line.startswith("name:"):
+            name = line[len("name:"):].strip().strip('"').strip("'")
+        elif line.startswith("description:"):
+            value = line[len("description:"):].strip()
+            # YAMLマルチラインインジケータ（>, |, >-, |-）の処理
+            if value in (">", "|", ">-", "|-"):
+                continuation_lines: list[str] = []
+                i += 1
+                while i < len(frontmatter_lines) and (frontmatter_lines[i].startswith("  ") or frontmatter_lines[i].startswith("\t")):
+                    continuation_lines.append(frontmatter_lines[i].strip())
+                    i += 1
+                description = " ".join(continuation_lines)
+                continue
+            else:
+                description = value.strip('"').strip("'")
+        i += 1
+
+    return name, description, content

package/presets/default/.claude/skills/einja-spec-context-loader/SKILL.md

@@ -175,3 +175,7 @@ interface Example {
 ---
 
 **最終更新**: 2025-01-10
+
+<!-- @einja:project-private:start id="einja-spec-context-loader-project" -->
+<!-- プロジェクト固有の情報を記入 -->
+<!-- @einja:project-private:end -->

package/presets/default/.claude/skills/einja-task-commit/SKILL.md

@@ -267,3 +267,7 @@ EOF
 ---
 
 **最終更新**: 2026-01-10
+
+<!-- @einja:project-private:start id="einja-task-commit-project" -->
+<!-- プロジェクト固有の情報を記入 -->
+<!-- @einja:project-private:end -->

package/presets/default/.claude/skills/einja-task-qa/SKILL.md

@@ -306,3 +306,7 @@ AskUserQuestion:
 ---
 
 **最終更新**: 2025-12-20
+
+<!-- @einja:project-private:start id="einja-task-qa-project" -->
+<!-- プロジェクト固有の情報を記入 -->
+<!-- @einja:project-private:end -->