@event4u/agent-config 6.0.0 → 7.0.0

package/src/scripts/skill_trigger_eval.py

@@ -1,682 +0,0 @@
-#!/usr/bin/env python3
-"""Skill trigger evaluation runner.
-
-Phase 1 of agents/roadmaps/road-to-trigger-evals.md — measures whether a
-pilot skill's frontmatter description actually causes Claude to route to
-the skill for queries that should trigger it, and to avoid routing for
-queries that should not.
-
-Input:  one skill name + its evals/triggers.json (5 should-trigger +
-        5 should-not-trigger queries).
-Output: evals/last-run.json with per-query observed vs expected,
-        aggregate precision/recall, model id, timestamp, cost estimate.
-
-Design notes:
-- The real Anthropic client is a **soft** dependency. If the `anthropic`
-  package is not installed, only --dry-run works (mock router).
-- The router is injectable — tests use a `MockRouter` that returns a
-  canned list per query. CI never makes real API calls.
-- The full set of skill frontmatter (name + description) is passed in
-  every routing call. That is the actual production routing condition;
-  anything less is cheating.
-
-Budget per roadmap: ≤500 LoC single file, no framework.
-"""
-from __future__ import annotations
-
-import argparse
-import json
-import stat
-import sys
-from dataclasses import asdict, dataclass, field
-from datetime import datetime, timezone
-from pathlib import Path
-from typing import Callable, IO, Protocol
-
-PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent
-SKILLS_SOURCE = PROJECT_ROOT / ".agent-src.uncondensed" / "skills"
-RESULTS_DIR = PROJECT_ROOT / "internal" / "evals" / "results"
-DEFAULT_MODEL = "claude-sonnet-4-5"
-
-# Approximate Anthropic API pricing (USD per 1M tokens). Used for the
-# cost estimate only — exact billing comes from the API response headers
-# once we run with a real key.
-PRICE_PER_MTOK_IN = {"claude-sonnet-4-5": 3.0, "claude-opus-4": 15.0}
-PRICE_PER_MTOK_OUT = {"claude-sonnet-4-5": 15.0, "claude-opus-4": 75.0}
-
-# On-disk key file. Companion: scripts/install_anthropic_key.sh writes it
-# with mode 0600; load_anthropic_key() refuses to read anything else.
-# Resolution prefers the new namespace (``~/.event4u/agent-config/``) and
-# falls back to the legacy ``~/.config/agent-config/`` so pre-2.4 keys
-# stay usable until the user runs the migration shim.
-from scripts._lib import user_global_paths  # noqa: E402
-
-ANTHROPIC_KEY_FILENAME = "anthropic.key"
-ANTHROPIC_KEY_PATH = (
-    user_global_paths.resolve_with_fallback(ANTHROPIC_KEY_FILENAME)
-    or user_global_paths.write_target(ANTHROPIC_KEY_FILENAME)
-)
-# Token heuristics used for the *pre-run* cost preview. Real billing
-# comes from the API response once the user has confirmed.
-TOKENS_PER_CHAR = 0.25          # ~4 chars per token, industry rule of thumb.
-PROMPT_OVERHEAD_TOKENS = 200    # routing instructions above the catalogue.
-OUTPUT_TOKENS_PER_QUERY = 60    # JSON `{"would_load": [...]}` is short.
-
-
-class KeyGateError(RuntimeError):
-    """Raised when the on-disk key file fails any safety check."""
-
-
-class ConfirmationAborted(RuntimeError):
-    """Raised when the user declines at the confirmation prompt or stdin
-    is non-interactive."""
-
-
-@dataclass
-class SkillMeta:
-    """Name + description of one skill, loaded from SKILL.md frontmatter."""
-
-    name: str
-    description: str
-
-
-@dataclass
-class Query:
-    q: str
-    trigger: bool
-
-
-@dataclass
-class QueryResult:
-    q: str
-    expected: bool
-    observed: bool
-    loaded_skills: list[str]
-    passed: bool
-
-
-@dataclass
-class Metrics:
-    true_positive: int = 0
-    false_positive: int = 0
-    true_negative: int = 0
-    false_negative: int = 0
-    precision: float = 0.0
-    recall: float = 0.0
-
-
-@dataclass
-class EvalResult:
-    skill: str
-    model: str
-    timestamp: str
-    router: str
-    queries: list[QueryResult] = field(default_factory=list)
-    metrics: Metrics = field(default_factory=Metrics)
-    input_tokens: int = 0
-    output_tokens: int = 0
-    cost_usd_estimate: float = 0.0
-
-
-class TriggerRouter(Protocol):
-    """Contract: given a user query and the full skill catalogue, return
-    the list of skill names the model would load. Implementations decide
-    whether that means a live API call or a canned response."""
-
-    name: str
-
-    def route(self, query: str, skills: list[SkillMeta]) -> tuple[list[str], int, int]:
-        """Returns (loaded_skill_names, input_tokens, output_tokens)."""
-        ...
-
-
-class MockRouter:
-    """Deterministic router for tests and dry-runs.
-
-    Constructed with a callable `decide(query, skills) -> list[str]`.
-    Token counts are faked as len(query)//4 + len(skills)*20 for input
-    and 16 for output, which keeps the cost-estimate math testable
-    without inventing numbers that look real.
-    """
-
-    name = "mock"
-
-    def __init__(self, decide: Callable[[str, list[SkillMeta]], list[str]]):
-        self._decide = decide
-
-    def route(self, query: str, skills: list[SkillMeta]) -> tuple[list[str], int, int]:
-        loaded = self._decide(query, skills)
-        return loaded, len(query) // 4 + len(skills) * 20, 16
-
-
-def load_skill_metas(root: Path | None = None) -> list[SkillMeta]:
-    """Parse name + description from every SKILL.md frontmatter under root.
-
-    If ``root`` is None, the package's ``artefact_roots()`` are scanned so
-    discovery works across the monorepo's per-pack ``.agent-src.uncondensed/``
-    trees. A single explicit ``root`` keeps tests that mock a sub-tree working
-    unchanged.
-    """
-    if root is not None:
-        roots = [root]
-    else:
-        from _lib.agent_src import artefact_roots
-        roots = [r / "skills" for r in artefact_roots()]
-    metas: list[SkillMeta] = []
-    seen: set[str] = set()
-    for skills_dir in roots:
-        if not skills_dir.is_dir():
-            continue
-        for skill_dir in sorted(p for p in skills_dir.iterdir() if p.is_dir()):
-            skill_md = skill_dir / "SKILL.md"
-            if not skill_md.exists():
-                continue
-            meta = _parse_frontmatter(skill_md)
-            if meta is not None and meta.name not in seen:
-                metas.append(meta)
-                seen.add(meta.name)
-    return metas
-
-
-def _parse_frontmatter(path: Path) -> SkillMeta | None:
-    text = path.read_text(encoding="utf-8")
-    if not text.startswith("---"):
-        return None
-    end = text.find("\n---", 3)
-    if end < 0:
-        return None
-    block = text[3:end]
-    name = _extract_field(block, "name")
-    desc = _extract_field(block, "description")
-    if name is None or desc is None:
-        return None
-    return SkillMeta(name=name, description=desc)
-
-
-def _extract_field(block: str, field_name: str) -> str | None:
-    """Minimal YAML-ish frontmatter field extractor — supports quoted
-    and unquoted single-line values. We do not pull PyYAML in here; the
-    audit script already proved stdlib suffices for our frontmatter."""
-    prefix = f"{field_name}:"
-    for line in block.splitlines():
-        stripped = line.lstrip()
-        if not stripped.startswith(prefix):
-            continue
-        value = stripped[len(prefix):].strip()
-        if value.startswith('"') and value.endswith('"'):
-            value = value[1:-1]
-        elif value.startswith("'") and value.endswith("'"):
-            value = value[1:-1]
-        return value
-    return None
-
-
-def load_triggers(path: Path) -> tuple[str, list[Query]]:
-    """Read evals/triggers.json. Returns (skill_name, queries)."""
-    data = json.loads(path.read_text(encoding="utf-8"))
-    skill = data["skill"]
-    queries = [Query(q=item["q"], trigger=bool(item["trigger"])) for item in data["queries"]]
-    if not queries:
-        raise ValueError(f"{path} has zero queries; roadmap minimum is 10")
-    return skill, queries
-
-
-def run_eval(
-    skill_name: str,
-    queries: list[Query],
-    router: TriggerRouter,
-    skills: list[SkillMeta],
-    model: str = DEFAULT_MODEL,
-) -> EvalResult:
-    """Execute every query through `router` and aggregate into EvalResult."""
-    result = EvalResult(
-        skill=skill_name,
-        model=model,
-        timestamp=datetime.now(timezone.utc).isoformat(timespec="seconds"),
-        router=router.name,
-    )
-    for q in queries:
-        loaded, in_tok, out_tok = router.route(q.q, skills)
-        observed = skill_name in loaded
-        passed = observed == q.trigger
-        result.queries.append(
-            QueryResult(
-                q=q.q,
-                expected=q.trigger,
-                observed=observed,
-                loaded_skills=sorted(loaded),
-                passed=passed,
-            )
-        )
-        result.input_tokens += in_tok
-        result.output_tokens += out_tok
-    result.metrics = compute_metrics(result.queries)
-    result.cost_usd_estimate = estimate_cost(model, result.input_tokens, result.output_tokens)
-    return result
-
-
-def compute_metrics(results: list[QueryResult]) -> Metrics:
-    tp = sum(1 for r in results if r.expected and r.observed)
-    fp = sum(1 for r in results if not r.expected and r.observed)
-    tn = sum(1 for r in results if not r.expected and not r.observed)
-    fn = sum(1 for r in results if r.expected and not r.observed)
-    precision = tp / (tp + fp) if (tp + fp) else 0.0
-    recall = tp / (tp + fn) if (tp + fn) else 0.0
-    return Metrics(
-        true_positive=tp,
-        false_positive=fp,
-        true_negative=tn,
-        false_negative=fn,
-        precision=round(precision, 3),
-        recall=round(recall, 3),
-    )
-
-
-def estimate_cost(model: str, in_tokens: int, out_tokens: int) -> float:
-    """Rough pre-invoice cost estimate. Real figure comes from response
-    headers once we wire a real key — this is only used to sanity-check
-    the roadmap's ≤$5-per-run budget before launching a batch."""
-    price_in = PRICE_PER_MTOK_IN.get(model, 3.0)
-    price_out = PRICE_PER_MTOK_OUT.get(model, 15.0)
-    cost = (in_tokens / 1_000_000) * price_in + (out_tokens / 1_000_000) * price_out
-    return round(cost, 6)
-
-
-def pre_estimate_cost(
-    model: str,
-    skills: list[SkillMeta],
-    queries: list[Query],
-) -> tuple[int, int, float]:
-    """Pre-run token + cost estimate for the confirmation prompt.
-
-    Returns (input_tokens, output_tokens, cost_usd) — approximate,
-    because the real tokeniser runs server-side. Calibration is
-    deliberately slightly high so the prompt never understates cost.
-    """
-    catalogue_chars = sum(len(s.name) + len(s.description) + 6 for s in skills)
-    per_query_chars = catalogue_chars + PROMPT_OVERHEAD_TOKENS * 4
-    in_tokens_per_q = int(per_query_chars * TOKENS_PER_CHAR) + PROMPT_OVERHEAD_TOKENS
-    avg_query_chars = sum(len(q.q) for q in queries) // max(len(queries), 1)
-    in_tokens_per_q += int(avg_query_chars * TOKENS_PER_CHAR)
-    in_tokens = in_tokens_per_q * len(queries)
-    out_tokens = OUTPUT_TOKENS_PER_QUERY * len(queries)
-    return in_tokens, out_tokens, estimate_cost(model, in_tokens, out_tokens)
-
-
-# ── Key gate ─────────────────────────────────────────────────────────────
-#
-# No environment-variable fallback, no keychain fallback. The key only
-# ever comes from a 0600 file written by scripts/install_anthropic_key.sh.
-# Drift from that contract is a hard abort.
-
-def load_anthropic_key(path: Path = ANTHROPIC_KEY_PATH) -> str:
-    """Load an Anthropic key from `path` with strict safety checks.
-
-    Enforced invariants:
-    - File exists.
-    - Mode is exactly 0o600 (owner-only read/write).
-    - Content is non-empty after strip.
-    - Content starts with `sk-ant-`.
-    """
-    if not path.exists():
-        raise KeyGateError(
-            f"Anthropic key not found at {path}.\n"
-            f"    Install it with: bash scripts/install_anthropic_key.sh"
-        )
-    st = path.stat()
-    mode = stat.S_IMODE(st.st_mode)
-    if mode != 0o600:
-        raise KeyGateError(
-            f"Unsafe permissions on {path}: got {oct(mode)}, expected 0o600.\n"
-            f"    Fix:  chmod 600 {path}"
-        )
-    key = path.read_text(encoding="utf-8").strip()
-    if not key:
-        raise KeyGateError(f"{path} is empty.")
-    if not key.startswith("sk-ant-"):
-        raise KeyGateError(
-            f"{path} does not contain an Anthropic key "
-            f"(expected 'sk-ant-' prefix)."
-        )
-    return key
-
-
-# ── Confirmation gate ────────────────────────────────────────────────────
-#
-# Every live invocation must pass through this. No --force, no --yes,
-# no env-var bypass. Non-tty stdin is rejected outright so the runner
-# cannot be scheduled, piped, or wrapped by an agent.
-
-def build_confirmation_summary(
-    *,
-    model: str,
-    skill: str,
-    query_count: int,
-    catalogue_size: int,
-    input_tokens: int,
-    output_tokens: int,
-    cost_usd: float,
-    key_path: Path,
-) -> str:
-    bar = "═" * 56
-    return (
-        f"{bar}\n"
-        f"  Trigger Eval — Confirmation Required\n"
-        f"{bar}\n"
-        f"  Model:       {model}\n"
-        f"  Skill:       {skill}\n"
-        f"  Queries:     {query_count}\n"
-        f"  Catalogue:   {catalogue_size} skills in routing prompt\n"
-        f"  Est. tokens: in≈{input_tokens:,}  out≈{output_tokens:,}\n"
-        f"  Est. cost:   ~${cost_usd:.2f} USD (actual via API headers)\n"
-        f"  Key source:  {key_path}\n"
-        f"{bar}"
-    )
-
-
-def require_confirmation(
-    summary: str,
-    *,
-    stdin: IO[str] | None = None,
-    stdout: IO[str] | None = None,
-) -> None:
-    """Print `summary`, require exactly `yes` from the controlling terminal.
-
-    Production path (stdin/stdout both None) reads from /dev/tty and
-    writes to /dev/tty, not from `sys.stdin` / `sys.stdout`. That makes
-    the gate immune to any wrapper that rebinds stdin (task runners,
-    nohup, sudo, agents) and guarantees every keystroke comes from the
-    user's real keyboard.
-
-    Tests inject explicit streams to bypass /dev/tty. When a test
-    passes an object, it must supply both `stdin` and `stdout` so the
-    isatty check covers the injected path too. `yes` is case-sensitive
-    to block accidents from auto-expanded `y`.
-    """
-    if stdin is None and stdout is None:
-        # Production path: controlling-terminal-only. If there is no
-        # /dev/tty (CI, cron, non-interactive agent) this is a hard
-        # abort before any API call.
-        try:
-            tty_in = open("/dev/tty", "r", encoding="utf-8")  # noqa: SIM115
-            tty_out = open("/dev/tty", "w", encoding="utf-8")  # noqa: SIM115
-        except OSError as exc:
-            raise ConfirmationAborted(
-                "Confirmation requires a controlling terminal (/dev/tty). "
-                "Refusing to run under automation."
-            ) from exc
-        try:
-            tty_out.write(summary + "\n")
-            tty_out.write(
-                "Proceed? [type 'yes' exactly to run, anything else aborts]: "
-            )
-            tty_out.flush()
-            answer = tty_in.readline().rstrip("\n")
-        finally:
-            tty_in.close()
-            tty_out.close()
-    else:
-        # Test path \u2014 both streams must be supplied.
-        assert stdin is not None and stdout is not None, (
-            "require_confirmation: stdin and stdout must both be supplied "
-            "when overriding defaults (test-only path)."
-        )
-        tty = getattr(stdin, "isatty", lambda: False)()
-        if not tty:
-            raise ConfirmationAborted(
-                "Confirmation requires an interactive tty on stdin. "
-                "Refusing non-interactive, piped, or redirected input."
-            )
-        stdout.write(summary + "\n")
-        stdout.write(
-            "Proceed? [type 'yes' exactly to run, anything else aborts]: "
-        )
-        stdout.flush()
-        answer = stdin.readline().rstrip("\n")
-
-    if answer != "yes":
-        raise ConfirmationAborted(f"Aborted at confirmation (got {answer!r}).")
-
-
-def write_result(result: EvalResult, output_path: Path) -> None:
-    output_path.parent.mkdir(parents=True, exist_ok=True)
-    payload = asdict(result)
-    output_path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8")
-
-
-def format_summary(result: EvalResult) -> str:
-    m = result.metrics
-    total = len(result.queries)
-    pass_count = sum(1 for r in result.queries if r.passed)
-    fail_count = total - pass_count
-    lines = [
-        f"Skill:     {result.skill}",
-        f"Router:    {result.router}    Model: {result.model}",
-        f"Queries:   {total}  ({pass_count} pass, {fail_count} fail)",
-        f"Precision: {m.precision}  (TP={m.true_positive} FP={m.false_positive})",
-        f"Recall:    {m.recall}  (TP={m.true_positive} FN={m.false_negative})",
-        f"Tokens:    in={result.input_tokens}  out={result.output_tokens}  "
-        f"cost~${result.cost_usd_estimate}",
-    ]
-    if fail_count:
-        lines.append("")
-        lines.append("Failures:")
-        for r in result.queries:
-            if r.passed:
-                continue
-            lines.append(
-                f"  [{'FN' if r.expected else 'FP'}] expected={r.expected} "
-                f"observed={r.observed} :: {r.q}"
-            )
-    return "\n".join(lines)
-
-
-ROUTING_PROMPT_HEADER = """You are a skill-routing oracle. Given the catalogue below
-and a single user query, return ONLY the JSON object {"would_load": [...]}
-listing the skill names whose bodies you would load to answer the query.
-
-Rules:
-- Use the skill frontmatter description verbatim as the only routing signal.
-- Return at most 4 skill names.
-- If no skill applies, return {"would_load": []}.
-- Output ONLY the JSON. No prose, no code fences.
-
-Skill catalogue (name :: description):
-"""
-
-
-class AnthropicRouter:
-    """Real-API router. Builds a routing prompt with the full skill
-    catalogue, asks the model for structured JSON output, parses the
-    `would_load` list. Token counts come from the usage field of the
-    SDK response."""
-
-    name = "anthropic"
-
-    def __init__(
-        self,
-        model: str = DEFAULT_MODEL,
-        client=None,
-        max_tokens: int = 256,
-        api_key: str | None = None,
-    ):
-        self._model = model
-        self._max_tokens = max_tokens
-        if client is not None:
-            self._client = client
-            return
-        if api_key is None:
-            raise RuntimeError(
-                "AnthropicRouter requires an explicit api_key or an injected client. "
-                "Load the key with load_anthropic_key() — no env-var fallback."
-            )
-        try:
-            import anthropic  # type: ignore[import-not-found]
-        except ImportError as exc:  # pragma: no cover - exercised only with real key
-            raise RuntimeError(
-                "anthropic package not installed. "
-                "`pip install anthropic` or run with --dry-run."
-            ) from exc
-        self._client = anthropic.Anthropic(api_key=api_key)
-
-    def route(self, query: str, skills: list[SkillMeta]) -> tuple[list[str], int, int]:
-        catalogue = "\n".join(f"- {s.name} :: {s.description}" for s in skills)
-        prompt = ROUTING_PROMPT_HEADER + catalogue + "\n"
-        response = self._client.messages.create(
-            model=self._model,
-            max_tokens=self._max_tokens,
-            system=prompt,
-            messages=[{"role": "user", "content": query}],
-        )
-        text = _first_text_block(response)
-        loaded = _parse_would_load(text)
-        usage = getattr(response, "usage", None)
-        in_tok = getattr(usage, "input_tokens", 0) if usage else 0
-        out_tok = getattr(usage, "output_tokens", 0) if usage else 0
-        return loaded, in_tok, out_tok
-
-
-def _first_text_block(response) -> str:
-    """Extract the text from the first content block of an Anthropic
-    Messages API response."""
-    content = getattr(response, "content", None)
-    if not content:
-        return ""
-    first = content[0]
-    return getattr(first, "text", "") or ""
-
-
-def _parse_would_load(text: str) -> list[str]:
-    """Parse `{"would_load": [...]}` out of a model response. Tolerates
-    leading/trailing whitespace and code fences even though the prompt
-    forbids them — models occasionally ignore that instruction."""
-    stripped = text.strip()
-    if stripped.startswith("```"):
-        stripped = stripped.strip("`").lstrip("json").strip()
-    try:
-        data = json.loads(stripped)
-    except json.JSONDecodeError:
-        return []
-    loaded = data.get("would_load", [])
-    if not isinstance(loaded, list):
-        return []
-    return [str(name) for name in loaded]
-
-
-def build_arg_parser() -> argparse.ArgumentParser:
-    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
-    parser.add_argument("--skill", required=True, help="Skill name (e.g. eloquent)")
-    parser.add_argument(
-        "--triggers",
-        type=Path,
-        default=None,
-        help="Path to evals/triggers.json. Default: .agent-src.uncondensed/skills/<skill>/evals/triggers.json",
-    )
-    parser.add_argument(
-        "--output",
-        type=Path,
-        default=None,
-        help=(
-            "Path to write the result. Default: internal/evals/results/"
-            "<timestamp>-<skill>-<model>.json (live) or "
-            "<triggers-dir>/last-run.json (dry-run)."
-        ),
-    )
-    parser.add_argument("--model", default=DEFAULT_MODEL)
-    parser.add_argument(
-        "--dry-run",
-        action="store_true",
-        help="Use MockRouter (no API call). Returns the pilot skill only for should-trigger queries.",
-    )
-    parser.add_argument(
-        "--key-path",
-        type=Path,
-        default=ANTHROPIC_KEY_PATH,
-        help=(
-            "Override the key file location. Default: "
-            "~/.event4u/agent-config/anthropic.key (legacy "
-            "~/.config/agent-config/anthropic.key read as fallback). "
-            "Mode 0600 required."
-        ),
-    )
-    return parser
-
-
-def _default_triggers_path(skill: str) -> Path:
-    from _lib.agent_src import resolve_logical
-    resolved = resolve_logical(f"skills/{skill}/evals/triggers.json")
-    if resolved is not None:
-        return resolved
-    return SKILLS_SOURCE / skill / "evals" / "triggers.json"
-
-
-def _default_live_output(skill: str, model: str) -> Path:
-    ts = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H%M%SZ")
-    return RESULTS_DIR / f"{ts}-{skill}-{model}.json"
-
-
-def main(argv: list[str] | None = None) -> int:
-    args = build_arg_parser().parse_args(argv)
-    triggers_path = args.triggers or _default_triggers_path(args.skill)
-    if not triggers_path.exists():
-        print(f"❌  triggers.json not found: {triggers_path}", file=sys.stderr)
-        return 2
-
-    skill_from_file, queries = load_triggers(triggers_path)
-    if skill_from_file != args.skill:
-        print(
-            f"❌  skill mismatch: --skill={args.skill} but triggers.json says {skill_from_file}",
-            file=sys.stderr,
-        )
-        return 2
-
-    skills = load_skill_metas()
-    if args.dry_run:
-        expected = {q.q: q.trigger for q in queries}
-
-        def decide(query: str, _skills: list[SkillMeta]) -> list[str]:
-            return [args.skill] if expected.get(query, False) else []
-
-        router: TriggerRouter = MockRouter(decide)
-        default_output = triggers_path.parent / "last-run.json"
-    else:
-        # Live path: key gate → cost preview → confirmation → router.
-        # Any failure here aborts before a single API call is made.
-        try:
-            api_key = load_anthropic_key(args.key_path)
-        except KeyGateError as exc:
-            print(f"❌  {exc}", file=sys.stderr)
-            return 2
-
-        in_tok, out_tok, cost = pre_estimate_cost(args.model, skills, queries)
-        summary = build_confirmation_summary(
-            model=args.model,
-            skill=args.skill,
-            query_count=len(queries),
-            catalogue_size=len(skills),
-            input_tokens=in_tok,
-            output_tokens=out_tok,
-            cost_usd=cost,
-            key_path=args.key_path,
-        )
-        try:
-            require_confirmation(summary)
-        except ConfirmationAborted as exc:
-            print(f"⏹   {exc}", file=sys.stderr)
-            return 2
-
-        router = AnthropicRouter(model=args.model, api_key=api_key)
-        default_output = _default_live_output(args.skill, args.model)
-
-    result = run_eval(args.skill, queries, router, skills, model=args.model)
-    output_path = args.output or default_output
-    write_result(result, output_path)
-    print(format_summary(result))
-    print(f"\nWrote: {output_path}")
-    fail_count = sum(1 for r in result.queries if not r.passed)
-    return 1 if fail_count else 0
-
-
-if __name__ == "__main__":
-    sys.exit(main())
-