ma-agents 3.12.2 → 3.13.0

package/lib/bmad-cache/bmb/skills/bmad-eval-runner/scripts/run_evals.py

@@ -0,0 +1,492 @@
+#!/usr/bin/env python3
+# /// script
+# requires-python = ">=3.9"
+# ///
+"""Run a skill's artifact evals in isolated workspaces.
+
+For each eval, the runner:
+  1. Stages a fresh workspace (Docker container or local tmp dir under ~/bmad-evals).
+  2. Applies the setup overlay (base then per-eval) so _bmad/ config and dependency
+     skills land in the workspace BEFORE the skill is staged — the skill's own copy
+     always wins over overlay content.
+  3. Copies the skill into .claude/skills/ so it is discoverable by claude.
+  4. Stages any fixture files declared in the eval's `files` list.
+  5. Runs `claude -p '<prompt>' --output-format stream-json --verbose`, capturing
+     the transcript. The Skill tool is available in -p mode and fires for installed
+     skills, so dependency skills provided by the setup overlay are properly invokable.
+  6. Rsyncs any files claude wrote into `<run-dir>/<eval-id>/artifacts/`.
+  7. Writes `metrics.json` (tool-call counts, timing, output sizes).
+
+Grading is performed separately by the parent skill's grader subagents.
+
+Usage:
+  python3 run_evals.py \\
+    --skill-path PATH \\
+    --evals-file PATH/evals.json \\
+    --project-root PATH \\
+    --output-dir PATH \\
+    --isolation docker|local \\
+    [--workers N] [--timeout SECS] [--eval-ids A1,B3] [--quiet]
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import shutil
+import subprocess
+import sys
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from pathlib import Path
+
+SCRIPT_DIR = Path(__file__).resolve().parent
+sys.path.insert(0, str(SCRIPT_DIR))
+
+from utils import (  # noqa: E402
+    apply_setup_overlay,
+    discover_setup_dirs,
+    new_run_id,
+    parse_skill_md,
+    read_json,
+    read_macos_keychain_credentials,
+    stage_credentials,
+    utc_now_iso,
+    write_json,
+)
+
+DOCKER_IMAGE = "bmad-eval-runner:latest"
+_KEYCHAIN_CREDS: str | None = read_macos_keychain_credentials()
+RSYNC_EXCLUDES = (
+    ".git", ".bare", "node_modules", ".venv", "__pycache__",
+    ".pytest_cache", ".next", "dist", "build", ".cache",
+    ".DS_Store", "*.pyc",
+)
+
+
+def stage_workspace_local(
+    workspace: Path,
+    project_root: Path,
+    skill_path: Path,
+    fixtures: list[tuple[Path, str]],
+    setup_dirs: list[Path] | None = None,
+) -> Path:
+    """Build a clean local workspace. Returns the project root inside workspace."""
+    workspace.mkdir(parents=True, exist_ok=True)
+    project_dest = workspace / "project"
+    home_dir = workspace / ".home"
+    (home_dir / ".claude").mkdir(parents=True, exist_ok=True)
+
+    excludes: list[str] = []
+    for pat in RSYNC_EXCLUDES:
+        excludes.extend(["--exclude", pat])
+
+    if shutil.which("rsync"):
+        subprocess.run(
+            ["rsync", "-a", *excludes, f"{project_root}/", f"{project_dest}/"],
+            check=True,
+        )
+    else:
+        shutil.copytree(project_root, project_dest, dirs_exist_ok=True,
+                        ignore=shutil.ignore_patterns(*RSYNC_EXCLUDES))
+
+    # Apply setup overlay before staging the skill — the skill's own copy wins.
+    if setup_dirs:
+        apply_setup_overlay(setup_dirs, project_dest)
+
+    skill_link_dir = project_dest / ".claude" / "skills"
+    skill_link_dir.mkdir(parents=True, exist_ok=True)
+    skill_dest = skill_link_dir / skill_path.name
+    if not skill_dest.exists():
+        try:
+            os.symlink(skill_path, skill_dest)
+        except OSError:
+            shutil.copytree(skill_path, skill_dest, dirs_exist_ok=True)
+
+    for src, dest_rel in fixtures:
+        dest = project_dest / dest_rel
+        dest.parent.mkdir(parents=True, exist_ok=True)
+        shutil.copy2(src, dest)
+
+    return project_dest
+
+
+def run_eval_local(
+    eval_item: dict,
+    run_dir: Path,
+    skill_path: Path,
+    project_root: Path,
+    timeout: int,
+    setup_dirs: list[Path] | None = None,
+) -> dict:
+    eval_id = str(eval_item.get("id", "unnamed"))
+    eval_dir = run_dir / eval_id
+    workspace_root = eval_dir / "workspace"
+    artifacts_dir = eval_dir / "artifacts"
+    transcript_path = eval_dir / "transcript.jsonl"
+
+    eval_dir.mkdir(parents=True, exist_ok=True)
+    artifacts_dir.mkdir(parents=True, exist_ok=True)
+
+    fixtures = resolve_fixtures(eval_item.get("files", []), project_root)
+    workspace_project = stage_workspace_local(
+        workspace_root, project_root, skill_path, fixtures, setup_dirs
+    )
+
+    (eval_dir / "prompt.txt").write_text(eval_item["prompt"], encoding="utf-8")
+    workspace_snapshot_before = snapshot_files(workspace_project)
+
+    home_dir = workspace_root / ".home"
+    stage_credentials(home_dir / ".claude", _KEYCHAIN_CREDS)
+    env = {
+        "HOME": str(home_dir),
+        "CLAUDE_CONFIG_DIR": str(home_dir / ".claude"),
+        "PATH": os.environ.get("PATH", ""),
+        "ANTHROPIC_API_KEY": os.environ.get("ANTHROPIC_API_KEY", ""),
+    }
+
+    cmd = [
+        "claude",
+        "-p", eval_item["prompt"],
+        "--output-format", "stream-json",
+        "--verbose",
+        "--dangerously-skip-permissions",
+    ]
+
+    start = time.time()
+    try:
+        with transcript_path.open("wb") as out:
+            proc = subprocess.run(
+                cmd,
+                stdout=out,
+                stderr=subprocess.PIPE,
+                cwd=str(workspace_project),
+                env=env,
+                timeout=timeout,
+            )
+        elapsed = time.time() - start
+        return_code = proc.returncode
+        stderr_tail = (proc.stderr or b"").decode("utf-8", errors="replace")[-2000:]
+    except subprocess.TimeoutExpired as e:
+        elapsed = time.time() - start
+        return_code = -1
+        stderr_tail = f"TIMEOUT after {timeout}s"
+        if e.stderr:
+            stderr_tail += "\n" + e.stderr.decode("utf-8", errors="replace")[-2000:]
+
+    new_files = diff_workspace(workspace_project, workspace_snapshot_before)
+    sync_artifacts(workspace_project, new_files, artifacts_dir)
+
+    metrics = compute_metrics(transcript_path, artifacts_dir, elapsed, return_code, stderr_tail)
+    write_json(eval_dir / "metrics.json", metrics)
+
+    return {
+        "eval_id": eval_id,
+        "elapsed_s": elapsed,
+        "return_code": return_code,
+        "transcript": str(transcript_path.relative_to(run_dir)),
+        "artifacts_dir": str(artifacts_dir.relative_to(run_dir)),
+        "metrics": metrics,
+    }
+
+
+def run_eval_docker(
+    eval_item: dict,
+    run_dir: Path,
+    skill_path: Path,
+    project_root: Path,
+    timeout: int,
+    setup_dirs: list[Path] | None = None,
+) -> dict:
+    eval_id = str(eval_item.get("id", "unnamed"))
+    eval_dir = run_dir / eval_id
+    artifacts_dir = eval_dir / "artifacts"
+    transcript_path = eval_dir / "transcript.jsonl"
+
+    eval_dir.mkdir(parents=True, exist_ok=True)
+    artifacts_dir.mkdir(parents=True, exist_ok=True)
+    fixtures_staging = eval_dir / "fixtures_in"
+    fixtures_staging.mkdir(parents=True, exist_ok=True)
+
+    fixtures = resolve_fixtures(eval_item.get("files", []), project_root)
+    for src, dest_rel in fixtures:
+        dest = fixtures_staging / dest_rel
+        dest.parent.mkdir(parents=True, exist_ok=True)
+        shutil.copy2(src, dest)
+
+    (eval_dir / "prompt.txt").write_text(eval_item["prompt"], encoding="utf-8")
+
+    # Pre-merge setup overlay dirs on the host; mount as /setup:ro in the container.
+    setup_merged: Path | None = None
+    if setup_dirs:
+        setup_merged = eval_dir / "setup_merged"
+        apply_setup_overlay(setup_dirs, setup_merged)
+        if not any(setup_merged.iterdir()):
+            setup_merged = None
+
+    creds_dir: Path | None = None
+    if _KEYCHAIN_CREDS:
+        creds_dir = eval_dir / "creds"
+        creds_dir.mkdir(parents=True, exist_ok=True)
+        (creds_dir / ".credentials.json").write_text(_KEYCHAIN_CREDS, encoding="utf-8")
+
+    container_script = r"""
+set -e
+mkdir -p /workspace
+rsync -a \
+  --exclude=.git --exclude=.bare --exclude=node_modules --exclude=.venv \
+  --exclude=__pycache__ --exclude=.pytest_cache --exclude=.next \
+  --exclude=dist --exclude=build --exclude=.cache --exclude=.DS_Store \
+  /project/ /workspace/
+if [ -d /setup ]; then
+  rsync -a /setup/ /workspace/
+fi
+mkdir -p /workspace/.claude/skills
+cp -R "$SKILL_SRC" "/workspace/.claude/skills/$SKILL_NAME"
+if [ -d /fixtures ]; then
+  cp -R /fixtures/. /workspace/
+fi
+if [ -f /creds/.credentials.json ]; then
+  mkdir -p /home/evaluator/.claude
+  cp /creds/.credentials.json /home/evaluator/.claude/.credentials.json
+fi
+cd /workspace
+claude -p "$EVAL_PROMPT" \
+  --output-format stream-json --verbose \
+  --dangerously-skip-permissions \
+  > /output/transcript.jsonl 2> /output/stderr.log || true
+mkdir -p /output/artifacts
+rsync -a --exclude=.claude --exclude=node_modules --exclude=.git \
+  --filter='+ */' --filter='+ *' \
+  /workspace/ /output/artifacts/
+"""
+
+    skill_name = skill_path.name
+    cmd = [
+        "docker", "run", "--rm",
+        "-v", f"{project_root}:/project:ro",
+        "-v", f"{skill_path}:/skill_src:ro",
+        "-v", f"{eval_dir}:/output",
+        "-e", "ANTHROPIC_API_KEY",
+        "-e", f"EVAL_PROMPT={eval_item['prompt']}",
+        "-e", f"SKILL_SRC=/skill_src",
+        "-e", f"SKILL_NAME={skill_name}",
+    ]
+    if creds_dir:
+        cmd += ["-v", f"{creds_dir}:/creds:ro"]
+    if fixtures:
+        cmd += ["-v", f"{fixtures_staging}:/fixtures:ro"]
+    if setup_merged:
+        cmd += ["-v", f"{setup_merged}:/setup:ro"]
+    cmd += [DOCKER_IMAGE, "bash", "-c", container_script]
+
+    start = time.time()
+    try:
+        proc = subprocess.run(
+            cmd,
+            capture_output=True,
+            timeout=timeout + 30,
+        )
+        elapsed = time.time() - start
+        return_code = proc.returncode
+        stderr_tail = proc.stderr.decode("utf-8", errors="replace")[-2000:]
+        if proc.stdout:
+            (eval_dir / "docker.stdout.log").write_bytes(proc.stdout)
+    except subprocess.TimeoutExpired as e:
+        elapsed = time.time() - start
+        return_code = -1
+        stderr_tail = f"TIMEOUT after {timeout}s"
+        if e.stderr:
+            stderr_tail += "\n" + e.stderr.decode("utf-8", errors="replace")[-2000:]
+
+    metrics = compute_metrics(transcript_path, artifacts_dir, elapsed, return_code, stderr_tail)
+    write_json(eval_dir / "metrics.json", metrics)
+    shutil.rmtree(fixtures_staging, ignore_errors=True)
+
+    return {
+        "eval_id": eval_id,
+        "elapsed_s": elapsed,
+        "return_code": return_code,
+        "transcript": str(transcript_path.relative_to(run_dir)),
+        "artifacts_dir": str(artifacts_dir.relative_to(run_dir)),
+        "metrics": metrics,
+    }
+
+
+def resolve_fixtures(files: list[str], project_root: Path) -> list[tuple[Path, str]]:
+    out: list[tuple[Path, str]] = []
+    for entry in files:
+        candidate = (project_root / entry).resolve()
+        if not candidate.is_file():
+            alt = Path(entry).resolve()
+            if alt.is_file():
+                candidate = alt
+            else:
+                print(f"Warning: fixture not found: {entry}", file=sys.stderr)
+                continue
+        out.append((candidate, entry))
+    return out
+
+
+def snapshot_files(root: Path) -> set[str]:
+    snap: set[str] = set()
+    for p in root.rglob("*"):
+        if p.is_file():
+            snap.add(str(p.relative_to(root)))
+    return snap
+
+
+def diff_workspace(root: Path, before: set[str]) -> list[str]:
+    after = snapshot_files(root)
+    return sorted(after - before)
+
+
+def sync_artifacts(workspace: Path, new_files: list[str], dest: Path) -> None:
+    for rel in new_files:
+        src = workspace / rel
+        if not src.is_file():
+            continue
+        if any(part in (".claude", "node_modules", ".git", ".venv") for part in src.parts):
+            continue
+        target = dest / rel
+        target.parent.mkdir(parents=True, exist_ok=True)
+        shutil.copy2(src, target)
+
+
+def compute_metrics(transcript: Path, artifacts: Path, elapsed: float,
+                    rc: int, stderr_tail: str) -> dict:
+    tool_calls: dict[str, int] = {}
+    total_steps = 0
+    if transcript.is_file():
+        for raw in transcript.read_text(encoding="utf-8", errors="replace").splitlines():
+            raw = raw.strip()
+            if not raw:
+                continue
+            try:
+                evt = json.loads(raw)
+            except json.JSONDecodeError:
+                continue
+            if evt.get("type") == "assistant":
+                total_steps += 1
+                for item in evt.get("message", {}).get("content", []):
+                    if item.get("type") == "tool_use":
+                        name = item.get("name", "?")
+                        tool_calls[name] = tool_calls.get(name, 0) + 1
+
+    output_chars = 0
+    for f in artifacts.rglob("*"):
+        if f.is_file():
+            try:
+                output_chars += f.stat().st_size
+            except OSError:
+                pass
+
+    return {
+        "elapsed_s": round(elapsed, 2),
+        "return_code": rc,
+        "tool_calls": tool_calls,
+        "total_tool_calls": sum(tool_calls.values()),
+        "total_steps": total_steps,
+        "output_chars": output_chars,
+        "transcript_chars": transcript.stat().st_size if transcript.is_file() else 0,
+        "stderr_tail": stderr_tail,
+    }
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Run a skill's artifact evals in isolation")
+    parser.add_argument("--skill-path", required=True, type=Path)
+    parser.add_argument("--evals-file", required=True, type=Path)
+    parser.add_argument("--project-root", required=True, type=Path)
+    parser.add_argument("--output-dir", required=True, type=Path)
+    parser.add_argument("--isolation", choices=("docker", "local"), required=True)
+    parser.add_argument("--workers", type=int, default=8)
+    parser.add_argument("--timeout", type=int, default=600)
+    parser.add_argument("--eval-ids", default=None, help="Comma-separated subset of eval ids to run")
+    parser.add_argument("--quiet", action="store_true")
+    args = parser.parse_args()
+
+    skill_path = args.skill_path.resolve()
+    project_root = args.project_root.resolve()
+    evals_file = args.evals_file.resolve()
+    if not evals_file.is_file():
+        print(f"evals file not found: {evals_file}", file=sys.stderr)
+        return 2
+
+    skill_name, _, _ = parse_skill_md(skill_path)
+    data = read_json(evals_file)
+    evals = data["evals"] if isinstance(data, dict) and "evals" in data else data
+
+    if args.eval_ids:
+        wanted = {x.strip() for x in args.eval_ids.split(",") if x.strip()}
+        evals = [e for e in evals if str(e.get("id")) in wanted]
+
+    run_id = new_run_id(skill_name)
+    run_dir = (args.output_dir / run_id).resolve()
+    run_dir.mkdir(parents=True, exist_ok=True)
+
+    write_json(run_dir / "run.json", {
+        "run_id": run_id,
+        "skill_name": skill_name,
+        "skill_path": str(skill_path),
+        "project_root": str(project_root),
+        "evals_file": str(evals_file),
+        "isolation": args.isolation,
+        "started_at": utc_now_iso(),
+        "eval_count": len(evals),
+    })
+
+    runner = run_eval_docker if args.isolation == "docker" else run_eval_local
+
+    results: list[dict] = []
+    if not args.quiet:
+        print(
+            f"[run_evals] {len(evals)} evals, isolation={args.isolation}, run_dir={run_dir}",
+            file=sys.stderr,
+        )
+
+    with ThreadPoolExecutor(max_workers=args.workers) as pool:
+        future_to_eval = {
+            pool.submit(
+                runner,
+                item,
+                run_dir,
+                skill_path,
+                project_root,
+                int(item.get("timeout", args.timeout)),
+                discover_setup_dirs(evals_file, str(item.get("id", ""))),
+            ): item
+            for item in evals
+        }
+        for fut in as_completed(future_to_eval):
+            item = future_to_eval[fut]
+            try:
+                res = fut.result()
+            except Exception as e:
+                res = {"eval_id": str(item.get("id")), "error": str(e), "return_code": -1}
+            results.append(res)
+            if not args.quiet:
+                rc = res.get("return_code")
+                status = "ok" if rc == 0 else f"rc={rc}"
+                print(
+                    f"  [{status}] eval {res.get('eval_id')} ({res.get('elapsed_s', 0):.1f}s)",
+                    file=sys.stderr,
+                )
+
+    summary = {
+        "run_id": run_id,
+        "completed_at": utc_now_iso(),
+        "total": len(evals),
+        "executed": len(results),
+        "exec_failures": sum(1 for r in results if r.get("return_code") != 0),
+        "run_dir": str(run_dir),
+        "results": results,
+    }
+    write_json(run_dir / "execution-summary.json", summary)
+    print(json.dumps(summary, indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())