@event4u/agent-config 6.0.0 → 7.0.0

package/src/scripts/router_telemetry.py

@@ -1,470 +0,0 @@
-#!/usr/bin/env python3
-"""Local replay of the router against a corpus — pure, no API spend.
-
-Phase 3 of `agents/roadmaps/road-to-value-dashboard-netto-cuts.md`.
-
-For each prompt in a corpus, applies the same trigger-match logic
-agent hosts would apply at runtime against `dist/router.json`:
-
-- kernel rules: always active (no triggers, always-on by definition).
-- tier_1 + tier_2 rules: active iff any trigger matches the prompt
-  text (plus optional open-files / command context).
-
-Trigger semantics implemented:
-
-  | Type           | Match rule                                                       |
-  |----------------|------------------------------------------------------------------|
-  | `keyword`      | case-insensitive substring inside the prompt text                |
-  | `phrase`       | case-insensitive substring (multi-word) inside the prompt text   |
-  | `command`      | case-sensitive prefix on `command:` field (optional context)     |
-  | `intent`       | informational only — never auto-matches; counted separately      |
-  | `path_prefix`  | prefix match against any path in `open_files` (optional context) |
-  | `file_pattern` | fnmatch against any path in `open_files` (optional context)      |
-
-Rules a task expects to fire only via `intent` (or a router coverage
-gap) the static replay cannot see are declared in the corpus field
-`replay_opaque_triggers`. They surface in `intended_vs_observed_match`
-under `replay_opaque` and are excluded from both `missed_intended`
-(no false drift) and `unintended_activations`.
-
-Reports go to `internal/bench/reports/router-telemetry/<UTC>.json`
-with three blocks:
-
-  - `per_trigger_hits`         — count of times each trigger fired
-  - `per_rule_activations`     — count of times each rule activated
-  - `panel_b_untouchable_rules` — tier-1 rules that activated on ≥ 1
-                                  Track B task; hard floor for Phase 5
-
-Sample size is capped per corpus (`--sample-cap`, default 200).
-Larger corpora are replayed deterministically over the first N
-sorted-by-id prompts.
-
-Honours `--quiet` per the script-output convention.
-"""
-from __future__ import annotations
-
-import argparse
-import fnmatch
-import json
-import sys
-from datetime import datetime, timezone
-from pathlib import Path
-from typing import Any, Dict, Iterable, List, Optional, Tuple
-
-try:
-    import yaml
-except ImportError:
-    yaml = None  # type: ignore[assignment]
-
-
-REPO_ROOT = Path(__file__).resolve().parent.parent.parent
-ROUTER_JSON = REPO_ROOT / "dist" / "router.json"
-DEFAULT_OUT_DIR = REPO_ROOT / "internal" / "bench" / "reports" / "router-telemetry"
-DEFAULT_SAMPLE_CAP = 200
-
-# Track B corpus = the Panel B evidence basis; rules that fire on its
-# tasks are the attribution map and become the untouchable set.
-TRACK_B_CORPUS_REL = "internal/bench/corpora/ab-trackb.yaml"
-
-
-def _log(msg: str, quiet: bool, *, err: bool = False) -> None:
-    if err:
-        print(msg, file=sys.stderr)
-    elif not quiet:
-        print(msg)
-
-
-def _utc_iso() -> str:
-    return datetime.now(timezone.utc).isoformat(timespec="seconds")
-
-
-# ── Trigger matching ────────────────────────────────────────────────────
-
-
-def trigger_matches(
-    trigger: Dict[str, Any],
-    prompt: str,
-    open_files: Optional[Iterable[str]] = None,
-    command: Optional[str] = None,
-) -> bool:
-    """Apply one trigger to a prompt + context; return True on match."""
-    prompt_lower = prompt.lower()
-    if "keyword" in trigger:
-        return str(trigger["keyword"]).lower() in prompt_lower
-    if "phrase" in trigger:
-        return str(trigger["phrase"]).lower() in prompt_lower
-    if "command" in trigger:
-        if not command:
-            return False
-        return command.startswith(str(trigger["command"]))
-    if "path_prefix" in trigger:
-        if not open_files:
-            return False
-        pref = str(trigger["path_prefix"])
-        return any(str(p).startswith(pref) for p in open_files)
-    if "file_pattern" in trigger:
-        if not open_files:
-            return False
-        pat = str(trigger["file_pattern"])
-        return any(fnmatch.fnmatch(str(p), pat) for p in open_files)
-    if "intent" in trigger:
-        # Intent triggers are informational and never auto-match.
-        return False
-    return False
-
-
-def match_prompt(
-    router: Dict[str, Any],
-    prompt: str,
-    profile: str = "full",
-    open_files: Optional[Iterable[str]] = None,
-    command: Optional[str] = None,
-) -> Dict[str, Any]:
-    """Return the matched-triggers + activated-rules for one prompt.
-
-    Kernel rules are always active. tier_1 always considered. tier_2
-    only considered when `profile == 'full'`.
-    """
-    tiers = [("tier_1", router.get("tier_1", []) or [])]
-    if profile == "full":
-        tiers.append(("tier_2", router.get("tier_2", []) or []))
-
-    matched_triggers: List[Dict[str, Any]] = []
-    activated_rules: List[Dict[str, Any]] = []
-
-    for tier_name, rules in tiers:
-        for rule in rules:
-            rule_id = rule.get("id")
-            rule_triggers = rule.get("triggers", []) or []
-            rule_hit = False
-            for trig in rule_triggers:
-                if trigger_matches(trig, prompt, open_files, command):
-                    matched_triggers.append({"tier": tier_name, "rule": rule_id, "trigger": trig})
-                    rule_hit = True
-            if rule_hit:
-                activated_rules.append({"tier": tier_name, "rule": rule_id})
-
-    # Kernel rules are always active.
-    for kid in router.get("kernel", []) or []:
-        activated_rules.append({"tier": "kernel", "rule": kid})
-
-    return {
-        "matched_triggers": matched_triggers,
-        "activated_rules": activated_rules,
-    }
-
-
-# ── Corpus loading ──────────────────────────────────────────────────────
-
-
-def _safe_yaml_load(path: Path) -> Optional[Dict[str, Any]]:
-    if yaml is None or not path.exists():
-        return None
-    try:
-        return yaml.safe_load(path.read_text()) or {}
-    except yaml.YAMLError:
-        return None
-
-
-def load_corpus_prompts(
-    corpus_path: Path, sample_cap: int
-) -> List[Dict[str, Any]]:
-    """Return per-prompt entries capped at sample_cap, sorted by id.
-
-    Each entry: `{id, text, intended_triggers, open_files, command}`.
-    All context fields beyond id/text are optional; missing → defaults.
-    """
-    data = _safe_yaml_load(corpus_path)
-    if not data:
-        return []
-    out: List[Dict[str, Any]] = []
-    # Track B uses `tasks:`, dev uses `prompts:`.
-    for key in ("tasks", "prompts"):
-        for entry in data.get(key, []) or []:
-            pid = str(entry.get("id", ""))
-            text = entry.get("prompt") or entry.get("text") or ""
-            intended = entry.get("intended_triggers") or []
-            opaque = entry.get("replay_opaque_triggers") or []
-            open_files = entry.get("open_files") or []
-            command = entry.get("command") or None
-            if not isinstance(intended, list):
-                intended = []
-            if not isinstance(opaque, list):
-                opaque = []
-            if not isinstance(open_files, list):
-                open_files = []
-            if pid and text:
-                out.append(
-                    {
-                        "id": pid,
-                        "text": str(text),
-                        "intended_triggers": [str(t) for t in intended],
-                        "replay_opaque_triggers": [str(t) for t in opaque],
-                        "open_files": [str(p) for p in open_files],
-                        "command": str(command) if command else None,
-                    }
-                )
-    out.sort(key=lambda x: x["id"])
-    return out[:sample_cap]
-
-
-# ── Aggregation ─────────────────────────────────────────────────────────
-
-
-def aggregate_replay(
-    router: Dict[str, Any],
-    corpora: List[Tuple[str, Path]],
-    sample_cap: int,
-    profile: str,
-) -> Dict[str, Any]:
-    """Replay every corpus through the router; aggregate hits."""
-    per_trigger_hits: Dict[str, int] = {}
-    per_rule_activations: Dict[str, Dict[str, int]] = {}
-    panel_b_seen_tier1: set = set()
-    panel_b_seen_tier2: set = set()
-    per_corpus_summary: List[Dict[str, Any]] = []
-    intended_vs_observed: List[Dict[str, Any]] = []
-    unintended_histogram: Dict[str, int] = {}
-
-    for corpus_name, corpus_path in corpora:
-        prompts = load_corpus_prompts(corpus_path, sample_cap)
-        corpus_rule_hits: Dict[str, int] = {}
-        for entry in prompts:
-            pid = entry["id"]
-            text = entry["text"]
-            intended = entry["intended_triggers"]
-            opaque = entry["replay_opaque_triggers"]
-            result = match_prompt(
-                router,
-                text,
-                profile=profile,
-                open_files=entry["open_files"] or None,
-                command=entry["command"],
-            )
-            for hit in result["matched_triggers"]:
-                key = f"{hit['rule']}::{json.dumps(hit['trigger'], sort_keys=True)}"
-                per_trigger_hits[key] = per_trigger_hits.get(key, 0) + 1
-            seen_in_prompt: set = set()
-            for act in result["activated_rules"]:
-                rid = act["rule"]
-                if rid is None or act["tier"] == "kernel":
-                    # Skip kernel — always-on by definition, no signal.
-                    continue
-                seen_in_prompt.add((act["tier"], rid))
-            activated_ids = {rid for _t, rid in seen_in_prompt}
-            for tier, rid in seen_in_prompt:
-                per_rule_activations.setdefault(tier, {})
-                per_rule_activations[tier][rid] = (
-                    per_rule_activations[tier].get(rid, 0) + 1
-                )
-                corpus_rule_hits[rid] = corpus_rule_hits.get(rid, 0) + 1
-                if corpus_name == "ab-trackb":
-                    if tier == "tier_1":
-                        panel_b_seen_tier1.add(rid)
-                    elif tier == "tier_2":
-                        panel_b_seen_tier2.add(rid)
-            # Council R3 honesty floor: surface intended vs observed.
-            # `replay_opaque` rules fire at runtime only via `intent`
-            # triggers (or router gaps) the deterministic replay cannot
-            # see — they are NOT counted as `missed_intended` (that would
-            # be false drift) and NOT counted as `unintended_activations`.
-            if intended or opaque:
-                intended_set = set(intended)
-                opaque_set = set(opaque)
-                hit = sorted(intended_set & activated_ids)
-                miss = sorted(intended_set - activated_ids)
-                unintended = sorted(activated_ids - intended_set - opaque_set)
-                intended_vs_observed.append(
-                    {
-                        "corpus": corpus_name,
-                        "task": pid,
-                        "intended": sorted(intended),
-                        "replay_opaque": sorted(opaque),
-                        "hit": hit,
-                        "missed_intended": miss,
-                        "unintended_activations": unintended,
-                    }
-                )
-                # Council R3 #3: inter-rule conflict histogram.
-                for rid in unintended:
-                    unintended_histogram[rid] = unintended_histogram.get(rid, 0) + 1
-        per_corpus_summary.append(
-            {
-                "corpus": corpus_name,
-                "prompts_replayed": len(prompts),
-                "unique_rules_activated": len(corpus_rule_hits),
-                "top_rules": sorted(
-                    corpus_rule_hits.items(), key=lambda x: -x[1]
-                )[:10],
-            }
-        )
-
-    panel_b_untouchable = sorted(panel_b_seen_tier1)
-    return {
-        "per_trigger_hits": per_trigger_hits,
-        "per_rule_activations": per_rule_activations,
-        "panel_b_untouchable_rules": panel_b_untouchable,
-        "panel_b_tier2_drivers": sorted(panel_b_seen_tier2),
-        "per_corpus_summary": per_corpus_summary,
-        "intended_vs_observed_match": intended_vs_observed,
-        "unintended_activation_histogram": sorted(
-            unintended_histogram.items(), key=lambda x: -x[1]
-        ),
-    }
-
-
-# ── Reports ─────────────────────────────────────────────────────────────
-
-
-def write_report(
-    aggregate: Dict[str, Any],
-    out_dir: Path,
-    corpora_paths: List[Path],
-    sample_cap: int,
-    profile: str,
-) -> Path:
-    out_dir.mkdir(parents=True, exist_ok=True)
-    stamp = _utc_iso().replace(":", "-")
-    out_path = out_dir / f"{stamp}.json"
-    latest = out_dir / "latest.json"
-    payload = {
-        "schema_version": 1,
-        "schema_id": "router-telemetry-v1",
-        "generated_at": _utc_iso(),
-        "config": {
-            "router": "dist/router.json",
-            "profile": profile,
-            "sample_cap_per_corpus": sample_cap,
-            "corpora": [str(p.relative_to(REPO_ROOT)) for p in corpora_paths],
-        },
-        **aggregate,
-    }
-    text = json.dumps(payload, indent=2, ensure_ascii=False) + "\n"
-    out_path.write_text(text)
-    latest.write_text(text)
-    return out_path
-
-
-def find_never_matched_tier1(router: Dict[str, Any], activations: Dict[str, Any]) -> List[str]:
-    """Tier-1 rules with zero activations across all corpora — dead-rule candidates."""
-    tier_1_activations = activations.get("tier_1", {}) or {}
-    all_tier_1_ids = [r.get("id") for r in router.get("tier_1", []) if r.get("id")]
-    return sorted([rid for rid in all_tier_1_ids if rid not in tier_1_activations])
-
-
-# ── Entry point ─────────────────────────────────────────────────────────
-
-
-def run(
-    corpora: List[Tuple[str, Path]],
-    out_dir: Path = DEFAULT_OUT_DIR,
-    sample_cap: int = DEFAULT_SAMPLE_CAP,
-    profile: str = "full",
-    quiet: bool = False,
-) -> int:
-    if not ROUTER_JSON.exists():
-        _log(f"router not found: {ROUTER_JSON}", quiet, err=True)
-        return 1
-    try:
-        router = json.loads(ROUTER_JSON.read_text())
-    except json.JSONDecodeError as exc:
-        _log(f"failed to parse router: {exc}", quiet, err=True)
-        return 1
-
-    _log(
-        f"router_telemetry: replaying {len(corpora)} corpora · "
-        f"cap={sample_cap} prompts/corpus · profile={profile}",
-        quiet,
-    )
-    agg = aggregate_replay(router, corpora, sample_cap, profile)
-    never_matched = find_never_matched_tier1(router, agg["per_rule_activations"])
-    agg["never_matched_tier1"] = never_matched
-
-    out_path = write_report(
-        agg, out_dir, [p for _name, p in corpora], sample_cap, profile
-    )
-    relpath = out_path.relative_to(REPO_ROOT)
-    _log(
-        f"router_telemetry: wrote {relpath} · "
-        f"panel_b_untouchable={len(agg['panel_b_untouchable_rules'])} · "
-        f"never_matched_tier1={len(never_matched)}",
-        quiet=False,
-    )
-    return 0
-
-
-def parse_args(argv: List[str]) -> argparse.Namespace:
-    p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
-    p.add_argument(
-        "--corpus",
-        action="append",
-        default=[],
-        metavar="NAME:PATH",
-        help="Repeatable. NAME tags the corpus in the report; PATH is the YAML.",
-    )
-    p.add_argument(
-        "--sample-cap",
-        type=int,
-        default=DEFAULT_SAMPLE_CAP,
-        help="Max prompts per corpus (default %(default)s).",
-    )
-    p.add_argument(
-        "--profile",
-        choices=["balanced", "full"],
-        default="full",
-        help="Routing profile (default 'full' — includes tier-2 rules).",
-    )
-    p.add_argument(
-        "--out",
-        type=Path,
-        default=DEFAULT_OUT_DIR,
-        help="Output directory (default %(default)s).",
-    )
-    p.add_argument("--quiet", action="store_true")
-    return p.parse_args(argv)
-
-
-def _default_corpora() -> List[Tuple[str, Path]]:
-    """The default manifest: original 3 corpora + every router-coverage file.
-
-    Phase 3 of road-to-corpus-expansion-evidence-based-cuts: adding a new
-    corpus file under `internal/bench/corpora/router-coverage/` no longer
-    requires editing this script — the manifest auto-discovers them.
-    """
-    corpora: List[Tuple[str, Path]] = [
-        ("ab-trackb", REPO_ROOT / TRACK_B_CORPUS_REL),
-        ("dev", REPO_ROOT / "tests/eval/corpus-dev.yaml"),
-        ("non-dev", REPO_ROOT / "tests/eval/corpus-non-dev.yaml"),
-    ]
-    coverage_dir = REPO_ROOT / "internal" / "bench" / "corpora" / "router-coverage"
-    if coverage_dir.is_dir():
-        for p in sorted(coverage_dir.glob("*.yaml")):
-            # Tag name: "router-coverage:<stem>" so the report distinguishes
-            # them from the original 3 corpora at a glance.
-            corpora.append((f"router-coverage:{p.stem}", p))
-    return corpora
-
-
-def main(argv: List[str] | None = None) -> int:
-    args = parse_args(argv if argv is not None else sys.argv[1:])
-    if not args.corpus:
-        corpora = _default_corpora()
-    else:
-        corpora = []
-        for spec in args.corpus:
-            if ":" not in spec:
-                print(f"--corpus expects NAME:PATH, got {spec!r}", file=sys.stderr)
-                return 1
-            name, path = spec.split(":", 1)
-            corpora.append((name.strip(), Path(path.strip())))
-    return run(
-        corpora,
-        out_dir=args.out,
-        sample_cap=args.sample_cap,
-        profile=args.profile,
-        quiet=args.quiet,
-    )
-
-
-if __name__ == "__main__":
-    raise SystemExit(main())

package/src/scripts/run_skill_evals.py

@@ -1,185 +0,0 @@
-#!/usr/bin/env python3
-"""Quantitative skill-eval orchestrator (skill-writing § 7).
-
-Scaffolds, aggregates, and reports sub-agent eval runs for a skill.
-
-Sub-agent SPAWNING is per-environment (Claude Code, Augment Code,
-council) and is left as a stub `_spawn_subagent(...)` that authors
-implement once for their environment. The rest of the loop —
-scaffold / aggregate / report — works out of the box and reads /
-writes JSON files in `runs/`.
-
-Layout per skill:
-
-    .agent-src.uncondensed/skills/{name}/evals/
-        evals.json
-        runs/                              # gitignored
-            {timestamp}-baseline/{scenario_id}/output.txt
-            {timestamp}-baseline/{scenario_id}/grade.json
-            {timestamp}-with-skill/{scenario_id}/output.txt
-            {timestamp}-with-skill/{scenario_id}/grade.json
-            {timestamp}-benchmark.json
-"""
-
-from __future__ import annotations
-
-import argparse
-import json
-import sys
-from datetime import datetime, timezone
-from pathlib import Path
-from typing import Any
-
-REPO_ROOT = Path(__file__).resolve().parent.parent.parent
-SKILLS_ROOT = REPO_ROOT / ".agent-src.uncondensed" / "skills"
-
-
-def _skill_dir(skill: str) -> Path:
-    p = SKILLS_ROOT / skill
-    if not p.is_dir():
-        sys.exit(f"error: skill {skill!r} not found at {p}")
-    return p
-
-
-def _evals_dir(skill: str) -> Path:
-    return _skill_dir(skill) / "evals"
-
-
-def _load_evals(skill: str) -> dict[str, Any]:
-    f = _evals_dir(skill) / "evals.json"
-    if not f.exists():
-        sys.exit(f"error: {f} not found — create it before scaffolding")
-    return json.loads(f.read_text(encoding="utf-8"))
-
-
-def _timestamp() -> str:
-    return datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
-
-
-def _spawn_subagent(prompt: str, *, load_skill: str | None) -> dict[str, Any]:
-    """STUB — implement per environment.
-
-    Must return {"output": str, "elapsed_s": float, "tokens_in": int,
-    "tokens_out": int}. When load_skill is None, run baseline; when
-    set, load that skill into the sub-agent's context.
-    """
-    raise NotImplementedError(
-        "implement _spawn_subagent for this environment (Claude Code, "
-        "Augment, council, ...) — see docstring contract"
-    )
-
-
-def _grade_assertions(output: str, run_dir: Path, assertions: list[dict[str, Any]]) -> list[dict[str, Any]]:
-    results: list[dict[str, Any]] = []
-    for a in assertions:
-        kind = a.get("kind")
-        if kind == "contains":
-            ok = a["value"] in output
-            results.append({"kind": kind, "value": a["value"], "pass": ok})
-        elif kind == "file_exists":
-            ok = (run_dir / a["path"]).exists() or Path(a["path"]).exists()
-            results.append({"kind": kind, "path": a["path"], "pass": ok})
-        elif kind == "rubric":
-            results.append({"kind": kind, "criterion": a["criterion"], "pass": None,
-                            "note": "rubric grading requires sub-agent — fill in manually or via grader"})
-        else:
-            results.append({"kind": kind, "pass": False, "note": f"unknown assertion kind {kind!r}"})
-    return results
-
-
-def cmd_scaffold(skill: str) -> int:
-    spec = _load_evals(skill)
-    scenarios = spec.get("scenarios", [])
-    if not scenarios:
-        sys.exit("error: evals.json has no scenarios")
-    ts = _timestamp()
-    runs = _evals_dir(skill) / "runs"
-    for arm in ("baseline", "with-skill"):
-        for sc in scenarios:
-            d = runs / f"{ts}-{arm}" / sc["id"]
-            d.mkdir(parents=True, exist_ok=True)
-            (d / "meta.json").write_text(json.dumps({
-                "skill": skill, "arm": arm, "scenario_id": sc["id"],
-                "prompt": sc["prompt"], "assertions": sc.get("assertions", []),
-                "timestamp": ts,
-            }, indent=2) + "\n", encoding="utf-8")
-    print(f"scaffolded {len(scenarios)} scenarios × 2 arms at runs/{ts}-{{baseline,with-skill}}/")
-    print(f"timestamp: {ts}")
-    return 0
-
-
-def cmd_aggregate(skill: str, run: str) -> int:
-    runs = _evals_dir(skill) / "runs"
-    spec = _load_evals(skill)
-    bench: dict[str, Any] = {"skill": skill, "run": run, "generated_at": _timestamp(), "scenarios": []}
-    totals = {"baseline_pass": 0, "with_skill_pass": 0, "scenarios": 0}
-    for sc in spec.get("scenarios", []):
-        row: dict[str, Any] = {"id": sc["id"], "arms": {}}
-        for arm in ("baseline", "with-skill"):
-            run_dir = runs / f"{run}-{arm}" / sc["id"]
-            grade_f = run_dir / "grade.json"
-            if not grade_f.exists():
-                row["arms"][arm] = {"status": "missing", "pass_count": 0, "total": 0}
-                continue
-            g = json.loads(grade_f.read_text(encoding="utf-8"))
-            results = g.get("results", [])
-            passed = sum(1 for r in results if r.get("pass") is True)
-            row["arms"][arm] = {"status": "graded", "pass_count": passed, "total": len(results),
-                                 "elapsed_s": g.get("elapsed_s"), "tokens_in": g.get("tokens_in"),
-                                 "tokens_out": g.get("tokens_out")}
-            if arm == "baseline" and passed == len(results) and results:
-                totals["baseline_pass"] += 1
-            if arm == "with-skill" and passed == len(results) and results:
-                totals["with_skill_pass"] += 1
-        bench["scenarios"].append(row)
-        totals["scenarios"] += 1
-    bench["totals"] = totals
-    out = runs / f"{run}-benchmark.json"
-    out.write_text(json.dumps(bench, indent=2) + "\n", encoding="utf-8")
-    print(f"wrote {out.relative_to(REPO_ROOT)}")
-    print(f"  baseline pass: {totals['baseline_pass']}/{totals['scenarios']}")
-    print(f"  with-skill pass: {totals['with_skill_pass']}/{totals['scenarios']}")
-    return 0
-
-
-def cmd_report(skill: str, run: str) -> int:
-    bench_f = _evals_dir(skill) / "runs" / f"{run}-benchmark.json"
-    if not bench_f.exists():
-        sys.exit(f"error: {bench_f} not found — run aggregate first")
-    bench = json.loads(bench_f.read_text(encoding="utf-8"))
-    print(f"# Skill eval report — {skill} @ {run}\n")
-    print("| Scenario | Baseline | With skill | Δ tokens_out | Δ elapsed_s |")
-    print("|---|---|---|---|---|")
-    for sc in bench["scenarios"]:
-        b = sc["arms"].get("baseline", {})
-        w = sc["arms"].get("with-skill", {})
-        bp = f"{b.get('pass_count', 0)}/{b.get('total', 0)}"
-        wp = f"{w.get('pass_count', 0)}/{w.get('total', 0)}"
-        dt = (w.get("tokens_out") or 0) - (b.get("tokens_out") or 0)
-        de = (w.get("elapsed_s") or 0) - (b.get("elapsed_s") or 0)
-        print(f"| {sc['id']} | {bp} | {wp} | {dt:+d} | {de:+.2f} |")
-    t = bench["totals"]
-    print(f"\n**Totals:** baseline {t['baseline_pass']}/{t['scenarios']} · with-skill {t['with_skill_pass']}/{t['scenarios']}")
-    return 0
-
-
-def main() -> int:
-    p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
-    sub = p.add_subparsers(dest="cmd", required=True)
-    for name in ("scaffold", "aggregate", "report"):
-        sp = sub.add_parser(name)
-        sp.add_argument("skill")
-        if name != "scaffold":
-            sp.add_argument("--run", required=True, help="run timestamp (from scaffold output)")
-    args = p.parse_args()
-    if args.cmd == "scaffold":
-        return cmd_scaffold(args.skill)
-    if args.cmd == "aggregate":
-        return cmd_aggregate(args.skill, args.run)
-    if args.cmd == "report":
-        return cmd_report(args.skill, args.run)
-    return 1
-
-
-if __name__ == "__main__":
-    sys.exit(main())