@event4u/agent-config 6.1.0 → 7.0.0

package/src/scripts/bench_ab_task_runner.py

@@ -1,814 +0,0 @@
-#!/usr/bin/env python3
-"""Track B — task runner for the package-impact A/B bench.
-
-Phase 4 Step 2 of `agents/roadmaps/road-to-package-impact-benchmark.md`.
-
-For each task in `internal/bench/corpora/ab-trackb.yaml`, in each variant:
-
-1. Snapshot the variant clone's file tree.
-2. Invoke the `claude` CLI with the task prompt — OR dry-run, depending
-   on `--mode`.
-3. Capture the transcript, tool-call events, wall-time, and (if available)
-   token + cost counts.
-4. Snapshot the post-run tree.
-5. Score the task via scripts/_lib/bench_ab_scoring.py.
-
-Modes:
-
-- `dry-run` (default) — record the would-run shell command, write a stub
-  transcript naming the variant, score against the unchanged tree. The
-  result is structural-zero for every check that requires a file write,
-  but the scoring + reporting pipeline runs end-to-end. This is what the
-  bench produces in CI by default — fast, free, repeatable.
-- `live` — actually invoke the `claude` CLI with `--print` (one-shot
-  mode) and the task prompt. Reads `CLAUDE_CLI` from env if set, falls
-  back to `claude` on PATH. Captures stdout as the transcript. Honors
-  `--samples N` for repeated runs.
-
-The runner ALWAYS resets the clone to a clean state before each task and
-ALWAYS records the mode in the report header so a reader can never mistake
-a dry-run report for a real measurement.
-"""
-from __future__ import annotations
-
-import argparse
-import hashlib
-import json
-import os
-import shutil
-import subprocess
-import sys
-import threading
-import time
-from datetime import datetime, timezone
-from pathlib import Path
-
-REPO_ROOT = Path(__file__).resolve().parent.parent.parent
-sys.path.insert(0, str(REPO_ROOT / "src" / "scripts"))
-
-from _lib import bench_ab_cache  # type: ignore[import-not-found]  # noqa: E402
-from _lib import bench_ab_scoring  # type: ignore[import-not-found]  # noqa: E402
-
-try:
-    import yaml
-except ImportError:
-    sys.stderr.write("bench_ab_task_runner: PyYAML required (pip install pyyaml)\n")
-    raise SystemExit(2)
-
-CORPUS_PATH = REPO_ROOT / "internal" / "bench" / "corpora" / "ab-trackb.yaml"
-CLONES_DIR = REPO_ROOT / "internal" / "bench" / "ab" / "clones"
-REPORTS_DIR = REPO_ROOT / "internal" / "bench" / "reports" / "ab"
-
-# How far we descend into a clone when snapshotting. The fixture is shallow.
-SNAPSHOT_MAX_DEPTH = 6
-
-# --- Activation (proven mechanism) ---
-# agent-config is a GLOBAL Claude Code plugin (enabledPlugins in ~/.claude
-# settings), so plain `claude --print` already runs WITH the package. The clean
-# control is `--setting-sources project,local`, which excludes the user settings
-# where `enabledPlugins` lives → plugin OFF, but auth survives. Measured proof:
-# plain --print = ~35.5k input tokens; --setting-sources project,local = ~11.9k
-# → the ~24k delta IS the package's always-on footprint. So:
-#   without  = `--setting-sources project,local`  (plugin OFF, base model)
-#   with     = plain `--print`                     (the real installed plugin = package)
-#   with-rdp = plain `--print` + RDP rules injected (RDP not yet in the release plugin)
-# (`--bare` is NOT used — it disables auth too.)
-RDP_EXTRA_FILES = (
-    REPO_ROOT / "src" / "rules" / "notes-first-reasoning.md",
-    REPO_ROOT / "src" / "agent-src" / "contexts" / "execution" / "rdp-gate.md",
-)
-
-
-def _concat_rules(paths) -> str:
-    parts: list[str] = []
-    for p in paths:
-        try:
-            parts.append(p.read_text(encoding="utf-8"))
-        except OSError:
-            continue
-    return "\n\n---\n\n".join(parts)
-
-
-def system_prompt_for(variant: str) -> str | None:
-    """Extra rules injected on top of the plugin. Only `with-rdp` injects (the RDP
-    artifacts aren't in the released plugin yet); `with` uses the real plugin,
-    `without` runs plugin-off."""
-    if variant == "with-rdp":
-        return _concat_rules([p for p in RDP_EXTRA_FILES if p.exists()])
-    return None
-
-
-def setting_sources_for(variant: str) -> str | None:
-    """`without` excludes user settings to drop the global plugin (auth survives)."""
-    return "project,local" if variant == "without" else None
-
-
-def utc_stamp() -> str:
-    return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H-%M-%SZ")
-
-
-def snapshot_clone(clone_root: Path, *, max_depth: int = SNAPSHOT_MAX_DEPTH) -> dict[str, str]:
-    """Return {relpath: sha256-short} for every fixture file under the clone.
-
-    Skips the agent-config surface (.claude, .augment, AGENTS.md, CLAUDE.md, manifest)
-    because that's the variant axis, not the task surface.
-    """
-    skip_roots = {".claude", ".augment"}
-    skip_files = {"AGENTS.md", "CLAUDE.md", ".bench-ab-manifest.json"}
-    out: dict[str, str] = {}
-    for path in sorted(clone_root.rglob("*")):
-        if not path.is_file():
-            continue
-        rel = path.relative_to(clone_root)
-        parts = rel.parts
-        if parts and parts[0] in skip_roots:
-            continue
-        if rel.as_posix() in skip_files:
-            continue
-        if len(parts) > max_depth:
-            continue
-        h = hashlib.sha256()
-        try:
-            h.update(path.read_bytes())
-        except OSError:
-            continue
-        out[rel.as_posix()] = h.hexdigest()[:16]
-    return out
-
-
-def reset_clone(variant: str) -> Path:
-    """Rebuild the clone so each task starts from the same state."""
-    import importlib.util
-
-    spec = importlib.util.spec_from_file_location(
-        "bench_ab_clone", REPO_ROOT / "src" / "scripts" / "bench_ab_clone.py"
-    )
-    if spec is None or spec.loader is None:
-        raise RuntimeError("cannot load bench_ab_clone helper")
-    module = importlib.util.module_from_spec(spec)
-    spec.loader.exec_module(module)
-    return module.clone(variant, refresh=True, quiet=True)  # type: ignore[attr-defined]
-
-
-def claude_executable() -> str | None:
-    """Resolve the claude CLI binary (env override → PATH)."""
-    override = os.environ.get("CLAUDE_CLI")
-    if override:
-        return override
-    # Resolve to an absolute path so the subprocess (run with cwd=clone_root)
-    # cannot miss it on a PATH/cwd quirk — the failure that showed up as a
-    # spurious "claude CLI not found" on a later arm of the first full run.
-    return shutil.which("claude")
-
-
-def run_live(
-    task: dict,
-    clone_root: Path,
-    *,
-    timeout_s: int,
-    sysprompt_file: "Path | None" = None,
-    setting_sources: "str | None" = None,
-    max_budget: "float | None" = None,
-    model: "str | None" = None,
-) -> dict:
-    """Invoke claude in print/one-shot mode against the task prompt.
-
-    `setting_sources` (e.g. "project,local") drops the global plugin for the
-    `without` arm while keeping auth. `sysprompt_file` injects extra rules
-    (the `with-rdp` arm). `with` passes neither → the real installed plugin.
-    """
-    binary = claude_executable()
-    if binary is None:
-        return {
-            "mode": "live-skipped",
-            "reason": "claude CLI not found; set CLAUDE_CLI or install it",
-            "transcript": "",
-            "exit_code": None,
-            "wall_time_seconds": 0.0,
-            "tokens": 0,
-            "tokens_breakdown": {},
-            "errored": True,
-        }
-    prompt = task.get("prompt", "")
-    # --output-format json yields a `usage` block for token counts. The global
-    # plugin is dropped per-arm via --setting-sources (NOT --bare, which kills auth).
-    # bypassPermissions on EVERY arm: the clone is a throwaway fixture, and this
-    # equalizes file-edit capability across arms (else `without`, which excludes
-    # user settings, would lack edit perms and fail tasks for the wrong reason).
-    cmd = [binary, "--print", "--output-format", "json", "--permission-mode", "bypassPermissions"]
-    if model:
-        # Pin ONE model across every arm. The session default here is Opus-4.8-1M,
-        # whose ~$1.78 first-turn cache-creation trips any sane budget cap instantly
-        # and makes a full corpus run blow the account quota. Holding the model
-        # constant is also a validity requirement: the bench measures the package
-        # LIFT on a fixed host, not model-vs-model.
-        cmd += ["--model", model]
-    if max_budget:
-        # Caps per-task API spend so one runaway agentic loop can't exhaust the
-        # account quota (the failure mode that starved later arms on the first run).
-        cmd += ["--max-budget-usd", str(max_budget)]
-    if setting_sources:
-        cmd += ["--setting-sources", setting_sources]
-    if sysprompt_file is not None:
-        cmd += ["--append-system-prompt-file", str(sysprompt_file)]
-    cmd += ["--", prompt]
-    started = time.monotonic()
-    try:
-        proc = subprocess.run(
-            cmd,
-            cwd=clone_root,
-            capture_output=True,
-            text=True,
-            timeout=timeout_s,
-            check=False,
-        )
-    except subprocess.TimeoutExpired as exc:
-        return {
-            "mode": "live",
-            "reason": f"timeout after {timeout_s}s",
-            "transcript": (exc.stdout or "") + "\n[TIMEOUT]",
-            "exit_code": -1,
-            "wall_time_seconds": round(time.monotonic() - started, 3),
-            "tokens": 0,
-            "tokens_breakdown": {},
-            "errored": True,
-        }
-    duration = time.monotonic() - started
-    # Parse the JSON envelope: `result` is the model text; `usage` holds tokens.
-    transcript = proc.stdout
-    tokens = 0
-    is_error = False
-    err_reason = "ok"
-    num_turns = 0
-    subtype = ""
-    breakdown = {
-        "input_tokens": 0,
-        "output_tokens": 0,
-        "cache_read_input_tokens": 0,
-        "cache_creation_input_tokens": 0,
-    }
-    try:
-        obj = json.loads(proc.stdout)
-        is_error = bool(obj.get("is_error"))
-        transcript = obj.get("result") or obj.get("text") or proc.stdout
-        usage = obj.get("usage") or {}
-        breakdown = {
-            k: int(usage.get(k, 0) or 0)
-            for k in (
-                "input_tokens",
-                "output_tokens",
-                "cache_read_input_tokens",
-                "cache_creation_input_tokens",
-            )
-        }
-        tokens = sum(breakdown.values())
-        # The top-level `usage` block is zeroed on a budget-capped / errored run
-        # (and unreliable even on some completions). `modelUsage` carries the
-        # authoritative per-model counts — sum it as the fallback so token deltas
-        # survive even when a task hits its cap mid-flight.
-        if tokens == 0:
-            mu = obj.get("modelUsage") or {}
-            agg = {
-                "input_tokens": 0,
-                "output_tokens": 0,
-                "cache_read_input_tokens": 0,
-                "cache_creation_input_tokens": 0,
-            }
-            for stats in mu.values():
-                agg["input_tokens"] += int(stats.get("inputTokens", 0) or 0)
-                agg["output_tokens"] += int(stats.get("outputTokens", 0) or 0)
-                agg["cache_read_input_tokens"] += int(
-                    stats.get("cacheReadInputTokens", 0) or 0
-                )
-                agg["cache_creation_input_tokens"] += int(
-                    stats.get("cacheCreationInputTokens", 0) or 0
-                )
-            mu_total = sum(agg.values())
-            if mu_total > 0:
-                breakdown = agg
-                tokens = mu_total
-        num_turns = int(obj.get("num_turns", 0) or 0)
-        subtype = str(obj.get("subtype") or "")
-        # Surface WHY a task errored (budget cap vs. other) without leaking $.
-        if is_error:
-            err_reason = obj.get("subtype") or "error"
-    except (json.JSONDecodeError, AttributeError, ValueError):
-        transcript = proc.stdout
-    return {
-        "mode": "live",
-        "reason": err_reason if is_error else ("ok" if proc.returncode == 0 else f"exit {proc.returncode}"),
-        "transcript": str(transcript) + "\n" + proc.stderr,
-        "exit_code": proc.returncode,
-        "wall_time_seconds": round(duration, 3),
-        "tokens": tokens,
-        "tokens_breakdown": breakdown,
-        "errored": is_error or proc.returncode != 0,
-        "num_turns": num_turns,
-        "subtype": subtype,
-    }
-
-
-def run_dry(task: dict, clone_root: Path, variant: str) -> dict:
-    """Record what would have run; produce a deterministic stub transcript.
-
-    The stub deliberately does NOT echo the user prompt: doing so would let
-    transcript-keyword criteria spuriously match against the prompt text
-    instead of the agent's response. The stub is therefore inert for every
-    `transcript_contains_*` criterion, which is the honest dry-run signal.
-    """
-    stub_transcript = (
-        "[bench_ab_task_runner dry-run]\n"
-        f"variant={variant}\n"
-        f"clone={clone_root}\n"
-        f"task_id={task.get('id')}\n"
-        "[no claude invocation; --mode live to execute for real]\n"
-    )
-    return {
-        "mode": "dry-run",
-        "reason": "ok",
-        "transcript": stub_transcript,
-        "exit_code": 0,
-        "wall_time_seconds": 0.0,
-    }
-
-
-def count_ask_events(transcript: str) -> dict[str, int]:
-    """Crude ask-vs-act heuristic over the transcript."""
-    if not transcript:
-        return {"asked": 0, "acted_with_commit": 0, "ratio": 0}
-    lt = transcript.lower()
-    ask_markers = ["should i", "do you want", "shall i", "soll ich", "möchtest du"]
-    asked = sum(lt.count(m) for m in ask_markers)
-    commit_markers = ["git commit", "git push", "gh pr create", "gh pr merge"]
-    acted = sum(lt.count(m) for m in commit_markers)
-    total = asked + acted
-    ratio = round(asked / total, 3) if total else 0
-    return {"asked": asked, "acted_with_commit": acted, "ratio": ratio}
-
-
-PROGRESS_PATH = REPORTS_DIR / ".progress.json"
-
-
-def _write_progress(state: dict) -> None:
-    """Mirror live state to .progress.json for `task bench:ab:watch` (best-effort)."""
-    try:
-        REPORTS_DIR.mkdir(parents=True, exist_ok=True)
-        PROGRESS_PATH.write_text(json.dumps(state, indent=2) + "\n")
-    except OSError:
-        pass
-
-
-class Progress:
-    """Live per-task progress. stdlib-only, TTY-aware, log-safe.
-
-    style: auto (bar if stderr is a TTY, else one plain line per task) | bar |
-    plain | none. Mirrors state to .progress.json regardless of style.
-    """
-
-    BAR_WIDTH = 24
-
-    def __init__(self, total: int, *, mode: str, style: str = "auto", stream=sys.stderr) -> None:
-        self.total = max(total, 1)
-        self.mode = mode
-        self.stream = stream
-        self.done = 0
-        self.started = time.monotonic()
-        if style in ("bar", "plain", "none"):
-            self.kind = style
-        else:  # auto
-            self.kind = "bar" if getattr(stream, "isatty", lambda: False)() else "plain"
-        self._cur = ""
-        self._task_started = 0.0
-        self._hb_stop: "threading.Event | None" = None
-        self._hb_thread: "threading.Thread | None" = None
-
-    def _elapsed(self, since: float) -> str:
-        s = int(time.monotonic() - since)
-        return f"{s // 60}m{s % 60:02d}s" if s >= 60 else f"{s}s"
-
-    def _bar(self) -> str:
-        filled = int(self.BAR_WIDTH * self.done / self.total)
-        return "█" * filled + "░" * (self.BAR_WIDTH - filled)
-
-    def _render_bar(self, suffix: str = "") -> None:
-        line = f"\r[{self._bar()}] {self.done}/{self.total} · {self._cur} · {self._elapsed(self.started)}{suffix}"
-        self.stream.write(line.ljust(90)[:160])
-        self.stream.flush()
-
-    def _start_heartbeat(self) -> None:
-        if self.kind != "bar" or self.mode != "live":
-            return
-        self._hb_stop = threading.Event()
-
-        def _tick() -> None:
-            assert self._hb_stop is not None
-            while not self._hb_stop.wait(1.0):
-                self._render_bar(suffix=f" · {self._elapsed(self._task_started)}…")
-
-        self._hb_thread = threading.Thread(target=_tick, daemon=True)
-        self._hb_thread.start()
-
-    def _stop_heartbeat(self) -> None:
-        if self._hb_stop is not None:
-            self._hb_stop.set()
-        if self._hb_thread is not None:
-            self._hb_thread.join(timeout=2.0)
-        self._hb_stop = self._hb_thread = None
-
-    def start_task(self, variant: str, idx: int, count: int, task_id: str) -> None:
-        self._cur = f"{variant} {idx}/{count} · {task_id}"
-        self._task_started = time.monotonic()
-        _write_progress({
-            "mode": self.mode, "variant": variant, "task_idx": idx, "task_count": count,
-            "total_done": self.done, "total": self.total, "current_id": task_id,
-            "started_at": utc_stamp(), "last_result": None,
-        })
-        if self.kind == "none":
-            return
-        if self.kind == "bar":
-            self._render_bar(suffix=" · running…" if self.mode == "live" else "")
-            self._start_heartbeat()
-        elif self.mode == "live":  # plain: a start marker so a long task isn't mistaken for a hang
-            self.stream.write(f"[{self.done + 1}/{self.total}] ▶ {self._cur}\n")
-            self.stream.flush()
-
-    def end_task(self, *, passed: bool, wall: float, variant: str, task_id: str) -> None:
-        self._stop_heartbeat()
-        self.done += 1
-        mark = "✓" if passed else "✗"
-        _write_progress({
-            "mode": self.mode, "variant": variant, "total_done": self.done,
-            "total": self.total, "current_id": task_id, "updated_at": utc_stamp(),
-            "last_result": "pass" if passed else "fail",
-        })
-        if self.kind == "none":
-            return
-        if self.kind == "bar":
-            self._render_bar(suffix=f" · {mark}")
-        else:
-            self.stream.write(f"[{self.done}/{self.total}] {mark} {variant} · {task_id} · {wall:.1f}s\n")
-            self.stream.flush()
-
-    def variant_done(self, line: str) -> None:
-        """Print a per-variant summary line without corrupting an active bar."""
-        if self.kind == "bar":
-            self.stream.write("\n")
-        self.stream.write(line if line.endswith("\n") else line + "\n")
-        self.stream.flush()
-
-    def finish(self) -> None:
-        if self.kind == "bar":
-            self.stream.write("\n")
-        if self.kind != "none":
-            self.stream.write(
-                f"bench progress: {self.done}/{self.total} tasks · total {self._elapsed(self.started)}\n"
-            )
-            self.stream.flush()
-
-
-def per_category_aggregate(per_task: list[dict]) -> dict[str, dict]:
-    by_cat: dict[str, list[dict]] = {}
-    for entry in per_task:
-        by_cat.setdefault(entry.get("category", "unknown"), []).append(entry)
-    out: dict[str, dict] = {}
-    for cat, entries in by_cat.items():
-        done = [e for e in entries if not e.get("errored")]
-        passed = sum(1 for e in done if e.get("score", {}).get("passed"))
-        total = len(entries)
-        completed = len(done)
-        out[cat] = {
-            "passed": passed,
-            "total": total,
-            "completed": completed,
-            "errored": total - completed,
-            "completion_rate": round(passed / completed, 4) if completed else 0,
-            "mean_wall_time": round(
-                sum(e.get("wall_time_seconds", 0) for e in done) / completed, 3
-            )
-            if completed
-            else 0,
-            "mean_tokens": round(sum(e.get("tokens", 0) for e in done) / completed)
-            if completed
-            else 0,
-        }
-    return out
-
-
-def per_cell_aggregate(per_task: list[dict]) -> dict[str, dict]:
-    """Aggregate by the 2×2 (duration × cognitive) cell — the value-benchmark axis.
-
-    Compared across conditions this answers "are short tasks more expensive?"
-    (cell `short/mechanical`) and "do long tasks get cheaper / better?"
-    (cell `long/reasoning-heavy`). Cell key is `"<duration>/<cognitive>"`.
-    """
-    by_cell: dict[str, list[dict]] = {}
-    for entry in per_task:
-        cell = f"{entry.get('duration', 'untagged')}/{entry.get('cognitive', 'untagged')}"
-        by_cell.setdefault(cell, []).append(entry)
-    out: dict[str, dict] = {}
-    for cell, entries in by_cell.items():
-        done = [e for e in entries if not e.get("errored")]
-        passed = sum(1 for e in done if e.get("score", {}).get("passed"))
-        total = len(entries)
-        completed = len(done)
-        out[cell] = {
-            "passed": passed,
-            "total": total,
-            "completed": completed,
-            "errored": total - completed,
-            "completion_rate": round(passed / completed, 4) if completed else 0,
-            "mean_wall_time": round(
-                sum(e.get("wall_time_seconds", 0) for e in done) / completed, 3
-            )
-            if completed
-            else 0,
-            "mean_tokens": round(sum(e.get("tokens", 0) for e in done) / completed)
-            if completed
-            else 0,
-        }
-    return out
-
-
-def write_report(
-    variant: str,
-    *,
-    mode: str,
-    per_task: list[dict],
-    duration: float,
-) -> Path:
-    REPORTS_DIR.mkdir(parents=True, exist_ok=True)
-    cache_key = bench_ab_cache.CacheKey(
-        corpus_hash=bench_ab_cache.hash_file(CORPUS_PATH),
-        claude_cli_version=bench_ab_cache.claude_cli_version(),
-        target_shape_hash=bench_ab_cache.target_shape_hash(),
-    )
-    total = len(per_task)
-    done = [e for e in per_task if not e.get("errored")]
-    completed = len(done)
-    errored = total - completed
-    passed = sum(1 for e in done if e.get("score", {}).get("passed"))
-    results = {
-        "mode": mode,
-        # Hit-rate is over COMPLETED tasks only — errored (rate-limit / budget /
-        # timeout / CLI-fail) tasks are excluded so a transient quota trip does
-        # not read as a content failure of the package.
-        "completion_rate": round(passed / completed, 4) if completed else 0,
-        "passed": passed,
-        "completed": completed,
-        "errored": errored,
-        "total": total,
-        "per_category": per_category_aggregate(per_task),
-        "per_cell": per_cell_aggregate(per_task),
-        "mean_wall_time": round(
-            sum(e.get("wall_time_seconds", 0) for e in done) / completed, 3
-        )
-        if completed
-        else 0,
-        "total_tokens": sum(e.get("tokens", 0) for e in done),
-        "mean_tokens": round(sum(e.get("tokens", 0) for e in done) / completed)
-        if completed
-        else 0,
-        "ask_vs_act_ratio": round(
-            sum(e.get("ask_events", {}).get("ratio", 0) for e in done) / completed, 3
-        )
-        if completed
-        else 0,
-        "per_task": per_task,
-    }
-    stamp = utc_stamp()
-    payload = {
-        "schema": "ab-bench/0.1",
-        "stamp": stamp,
-        "variant": variant,
-        "corpus": "ab-trackb",
-        "cache_key": cache_key.to_dict(),
-        "duration_seconds": round(duration, 3),
-        "results": results,
-    }
-    path = REPORTS_DIR / f"{stamp}-ab-trackb-{variant}.json"
-    path.write_text(json.dumps(payload, indent=2) + "\n")
-    md = path.with_suffix(".md")
-    md.write_text(
-        f"# Track B · {variant} · {mode}\n\n"
-        f"- Stamp: `{stamp}`\n"
-        f"- Completion rate: **{results['completion_rate'] * 100:.1f}%**"
-        f" ({passed}/{completed} completed; {errored} errored of {total})\n"
-        f"- Mean wall-time: {results['mean_wall_time']}s\n"
-        f"- Ask vs. act ratio: {results['ask_vs_act_ratio']}\n"
-        f"\n## Per-category\n\n"
-        + "\n".join(
-            f"- `{cat}` — {info['passed']}/{info['total']} "
-            f"({info['completion_rate'] * 100:.1f}%)"
-            for cat, info in results["per_category"].items()
-        )
-        + "\n"
-    )
-    return path
-
-
-def run_variant(
-    variant: str,
-    tasks: list[dict],
-    *,
-    mode: str,
-    timeout_s: int,
-    max_budget: "float | None" = None,
-    model: "str | None" = None,
-    progress: "Progress | None" = None,
-) -> dict:
-    started = time.monotonic()
-    # Build the injected rule corpus once per variant (live only).
-    sp_file: "Path | None" = None
-    if mode == "live":
-        sp_text = system_prompt_for(variant)
-        if sp_text:
-            REPORTS_DIR.mkdir(parents=True, exist_ok=True)
-            sp_file = REPORTS_DIR / f".sysprompt-{variant}.txt"
-            sp_file.write_text(sp_text, encoding="utf-8")
-    per_task: list[dict] = []
-    for i, task in enumerate(tasks):
-        if progress is not None:
-            progress.start_task(variant, i + 1, len(tasks), str(task.get("id")))
-        # Fixture-only working dir, identical for every arm — the package is NOT
-        # in the clone files; activation is the injected system prompt (sp_file).
-        clone_root = reset_clone("without")
-        pre = snapshot_clone(clone_root)
-        if mode == "live":
-            run_result = run_live(
-                task,
-                clone_root,
-                timeout_s=timeout_s,
-                sysprompt_file=sp_file,
-                setting_sources=setting_sources_for(variant),
-                max_budget=max_budget,
-                model=model,
-            )
-        else:
-            run_result = run_dry(task, clone_root, variant)
-        post = snapshot_clone(clone_root)
-        score = bench_ab_scoring.score_task(
-            task,
-            pre_snapshot=pre,
-            post_snapshot=post,
-            clone_root=clone_root,
-            transcript=run_result.get("transcript", ""),
-        )
-        per_task.append(
-            {
-                "id": task.get("id"),
-                "category": task.get("category"),
-                "duration": task.get("duration"),
-                "cognitive": task.get("cognitive"),
-                "score": score,
-                # `errored` = the run did not complete on merit (rate-limit,
-                # budget-cap, timeout, CLI failure). Distinct from a content
-                # fail (`score.passed == False`). Errored tasks are excluded
-                # from the hit-rate so a transient quota trip can't masquerade
-                # as the package "not working".
-                "errored": bool(run_result.get("errored", False)),
-                "wall_time_seconds": run_result.get("wall_time_seconds", 0.0),
-                "tokens": run_result.get("tokens", 0),
-                "tokens_breakdown": run_result.get("tokens_breakdown", {}),
-                "exit_code": run_result.get("exit_code"),
-                "mode": run_result.get("mode", mode),
-                "reason": run_result.get("reason", ""),
-                "ask_events": count_ask_events(run_result.get("transcript", "")),
-            }
-        )
-        if progress is not None:
-            progress.end_task(
-                passed=bool(score.get("passed")),
-                wall=float(run_result.get("wall_time_seconds", 0.0) or 0.0),
-                variant=variant,
-                task_id=str(task.get("id")),
-            )
-    duration = time.monotonic() - started
-    path = write_report(variant, mode=mode, per_task=per_task, duration=duration)
-    summary = (
-        f"bench_ab_task_runner: {variant} ({mode}) → "
-        f"{sum(1 for e in per_task if e['score']['passed'])}/{len(per_task)} "
-        f"passed — {path.relative_to(REPO_ROOT)}"
-    )
-    if progress is not None:
-        progress.variant_done(summary)
-    else:
-        sys.stdout.write(summary + "\n")
-    return {"path": path, "per_task": per_task, "duration": duration}
-
-
-def parse_args(argv: list[str]) -> argparse.Namespace:
-    parser = argparse.ArgumentParser(description="Run Track B tasks per variant.")
-    parser.add_argument(
-        "--variant",
-        choices=("with", "without", "with-rdp", "both", "all"),
-        default="both",
-        help="with | without | with-rdp | both (=with+without, back-compat "
-        "default) | all (=the 3-condition value-benchmark set).",
-    )
-    parser.add_argument(
-        "--mode",
-        choices=("dry-run", "live"),
-        default="dry-run",
-        help=(
-            "dry-run: stub transcript, no CLI invocation (fast, free). "
-            "live: invoke `claude --print` per task (cost-bearing)."
-        ),
-    )
-    parser.add_argument(
-        "--timeout",
-        type=int,
-        default=120,
-        help="Live mode: per-task timeout in seconds (default 120).",
-    )
-    parser.add_argument(
-        "--progress",
-        choices=("auto", "bar", "plain", "none"),
-        default="auto",
-        help="Live display: auto (TTY→bar, else plain line-per-task) | bar | plain | none.",
-    )
-    parser.add_argument(
-        "--limit",
-        type=int,
-        default=0,
-        help="Run only the first N tasks per variant (0 = all). For cheap smoke tests.",
-    )
-    parser.add_argument(
-        "--tasks",
-        default="",
-        help=(
-            "Comma-separated task IDs to run (e.g. trackb-bugfix-01,trackb-refactor-01). "
-            "Overrides --limit. Use to span the 2×2 cells in a bounded run instead of "
-            "taking the first-N in file order."
-        ),
-    )
-    parser.add_argument(
-        "--model",
-        default="claude-sonnet-4-6",
-        help=(
-            "Pin ONE model across all arms (live mode). Default claude-sonnet-4-6 — "
-            "capable enough to complete the coding tasks, ~2.3x cheaper per turn than "
-            "the Opus-4.8-1M session default whose cache-creation blows the quota. "
-            "Empty string = inherit the session default (expensive)."
-        ),
-    )
-    parser.add_argument(
-        "--budget",
-        type=float,
-        default=2.0,
-        help=(
-            "Live mode: per-task API spend cap in USD (passed to "
-            "`claude --max-budget-usd`). Stops a runaway agentic loop from "
-            "exhausting the account quota and starving later arms. 0 = uncapped. "
-            "Default 2.0."
-        ),
-    )
-    return parser.parse_args(argv)
-
-
-def main(argv: list[str] | None = None) -> int:
-    args = parse_args(argv if argv is not None else sys.argv[1:])
-    if not CORPUS_PATH.exists():
-        sys.stderr.write(f"bench_ab_task_runner: corpus missing at {CORPUS_PATH}\n")
-        return 1
-    data = yaml.safe_load(CORPUS_PATH.read_text())
-    tasks = data.get("tasks") or []
-    if not tasks:
-        sys.stderr.write("bench_ab_task_runner: corpus has no tasks\n")
-        return 1
-    if args.tasks.strip():
-        wanted = [s.strip() for s in args.tasks.split(",") if s.strip()]
-        by_id = {t.get("id"): t for t in tasks}
-        missing = [w for w in wanted if w not in by_id]
-        if missing:
-            sys.stderr.write(
-                f"bench_ab_task_runner: unknown task id(s): {', '.join(missing)}\n"
-            )
-            return 1
-        tasks = [by_id[w] for w in wanted]
-    elif args.limit and args.limit > 0:
-        tasks = tasks[: args.limit]
-    if args.variant == "both":
-        variants = ("with", "without")
-    elif args.variant == "all":
-        variants = ("with", "without", "with-rdp")
-    else:
-        variants = (args.variant,)
-    max_budget = args.budget if args.budget and args.budget > 0 else None
-    model = args.model or None
-    progress = Progress(len(variants) * len(tasks), mode=args.mode, style=args.progress)
-    for variant in variants:
-        run_variant(
-            variant,
-            tasks,
-            mode=args.mode,
-            timeout_s=args.timeout,
-            max_budget=max_budget,
-            model=model,
-            progress=progress,
-        )
-    progress.finish()
-    return 0
-
-
-if __name__ == "__main__":
-    raise SystemExit(main())