bigpowers 2.91.1 → 2.93.0

package/scripts/generate-context-bundle.sh

@@ -51,7 +51,8 @@ while IFS= read -r row; do
 
   adapter_script="$SCRIPT_DIR/adapters/${adapter}.sh"
   # shellcheck source=/dev/null
-  source "$adapter_script"
+  # stdin is this loop's target list: an adapter that reads stdin when sourced would swallow it.
+  source "$adapter_script" </dev/null
   case "$mode" in
     symlink) wire_context_mode symlink "$file" ;;
     copy) wire_context_mode copy "$file" ;;

package/scripts/lib/ab-eval.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# story: e58s01 e58s02 e58s03 e58s05 e58s06 e58s07
+# story: e58s01 e58s02 e58s03 e58s05 e58s06 e58s07 e58s08 e58s09
 """A/B eval: run the same tasks with and without bigpowers and grade only the outcome.
 
 Each run gets a fresh temp git workdir (outside the repo) copied from the task fixture.
@@ -25,7 +25,9 @@ from pathlib import Path
 import yaml
 
 sys.path.insert(0, str(Path(__file__).resolve().parent))
-from ab_metrics import parse_metrics, tap_score  # type: ignore[import]  # noqa: E402
+from ab_metrics import parse_metrics, tap_score, tap_failures, tap_failed_names  # type: ignore[import]  # noqa: E402
+from ab_vsm import vsm_from  # type: ignore[import]  # noqa: E402
+from ab_waste import waste_for  # type: ignore[import]  # noqa: E402
 from ab_report import (  # type: ignore[import]  # noqa: E402
     BOOTSTRAP_RESAMPLES, BOOTSTRAP_SEED, aggregate, bigpowers_sha, print_summary, write_reports,
 )
@@ -180,6 +182,8 @@ def run_once(task: dict, arm: str, setup: str, adapter: dict, model: str | None,
 
         tampered, protected_added = protected_changes(workdir, before)
         diff = changed_lines(workdir, base)
+        waste = waste_for(workdir, base, stdout, adapter.get("parser", "none"), task["dir"],
+                          trace_dir / f"{task['id']}-{arm}-{run_idx}.patch.gz" if trace_dir else None)
         shutil.copytree(task["dir"] / "hidden", workdir, dirs_exist_ok=True)
         cmd, genv = grade_cmd(adapter, task["grade"], env)
         passed = sh(cmd, workdir, genv, timeout=300).returncode == 0
@@ -187,12 +191,22 @@ def run_once(task: dict, arm: str, setup: str, adapter: dict, model: str | None,
         if task.get("score"):
             cmd, genv = grade_cmd(adapter, task["score"], env)
             score = tap_score(sh(cmd, workdir, genv, timeout=300).stdout)
+        bugs_opened = 0
+        bug_details = []
+        if task.get("audit"):
+            if (task["dir"] / "audit").is_dir():
+                shutil.copytree(task["dir"] / "audit", workdir, dirs_exist_ok=True)
+            cmd, genv = grade_cmd(adapter, task["audit"], env)
+            out = sh(cmd, workdir, genv, timeout=300).stdout
+            bugs_opened = tap_failures(out)
+            bug_details = tap_failed_names(out)
         metrics = parse_metrics(adapter.get("parser", "none"), stdout)
         return {
-            "task": task["id"], "arm": arm, "run": run_idx, "pass": passed, "score": score, "tampered": tampered,
+            "task": task["id"], "arm": arm, "run": run_idx, "pass": passed, "score": score,
+            "bugs_opened": bugs_opened, "bug_details": bug_details, "tampered": tampered,
             "protected_lines_added": protected_added, "diff_lines": diff, "duration_s": duration, "agent_exit": exit_code,
             "agent_error": timed_out or exit_code != 0 or bool(metrics.pop("agent_reported_error")),
-            "timed_out": timed_out, **metrics,
+            "timed_out": timed_out, "vsm": vsm_from(adapter.get("parser", "none"), stdout), "waste": waste, **metrics,
         }
     finally:
         shutil.rmtree(home, ignore_errors=True)
@@ -262,7 +276,7 @@ def main(argv: list[str] | None = None) -> int:
                     with live.open("a") as f:
                         f.write(json.dumps({**rec, "bigpowers_sha": sha, "model": model}) + "\n")
                     print(f"  {task['id']:<8} {arm:<10} run {i}: {'PASS' if rec['pass'] else 'FAIL'}"
-                          f"{' TAMPERED' if rec['tampered'] else ''}{' AGENT-ERROR' if rec['agent_error'] else ''}")
+                          f"{' TAMPERED' if rec['tampered'] else ''}{' AGENT-ERROR' if rec['agent_error'] else ''}", flush=True)
     except ConfigError as exc:
         print(f"ERROR: {exc}")
         return 2

package/scripts/lib/ab_lean.py

@@ -0,0 +1,116 @@
+#!/usr/bin/env python3
+# story: e58s10
+"""Lean views of an A/B run: each arm's mean value stream (stage time and share) and waste metrics,
+deltas against vanilla with seeded bootstrap CIs, and text VSM bars plus a waste table.
+
+Reads the `vsm` (e58s08) and `waste` (e58s09) blocks of run records; runs without them are skipped.
+"""
+from __future__ import annotations
+
+from ab_report import bootstrap_delta_ci, mean  # type: ignore[import]
+
+STAGES = ["plan", "design", "build", "test", "deploy", "maintain"]
+LETTER = {"plan": "P", "design": "D", "build": "B", "test": "T", "deploy": "R", "maintain": "M"}
+WASTE_KEYS = ["test_runs", "redundant_test_runs", "rework_loops", "reedits", "rereads",
+              "repeated_commands", "review_load", "diff_ratio", "pce"]
+# (key, label, per-run extractor) for the deltas against vanilla.
+DELTAS = [
+    ("lead_time_s", "lead time (s)", lambda r: r["vsm"]["lead_time_s"]),
+    ("time_to_first_build_s", "first build (s)", lambda r: r["vsm"]["time_to_first_build_s"]),
+    ("switches", "stage switches", lambda r: r["vsm"]["switches"]),
+    ("build_share", "build share", lambda r: share(r, "build")),
+    ("test_share", "test share", lambda r: share(r, "test")),
+    ("test_runs", "test runs", lambda r: r["waste"].get("test_runs")),
+    ("redundant_test_runs", "redundant test runs", lambda r: r["waste"].get("redundant_test_runs")),
+    ("rework_loops", "rework loops", lambda r: r["waste"].get("rework_loops")),
+    ("reedits", "re-edits", lambda r: r["waste"].get("reedits")),
+    ("review_load", "review load", lambda r: r["waste"].get("review_load")),
+    ("diff_ratio", "diff ratio", lambda r: r["waste"].get("diff_ratio")),
+    ("ftt", "first-time-through", lambda r: None if r["waste"].get("ftt") is None else float(r["waste"]["ftt"])),
+    ("pce", "PCE", lambda r: r["waste"].get("pce")),
+]
+
+
+def share(run: dict, stage: str) -> float | None:
+    lead = run["vsm"]["lead_time_s"]
+    return run["vsm"]["stages"].get(stage, {}).get("s", 0) / lead if lead else None
+
+
+def lean_runs(records: list[dict], arm: str) -> list[dict]:
+    return [r for r in records if r["arm"] == arm and r.get("vsm") and r.get("waste")]
+
+
+def arm_lean(runs: list[dict], all_runs: list[dict]) -> dict:
+    passes = sum(r["pass"] for r in all_runs)
+    cost = sum(r.get("cost_usd") or 0 for r in all_runs)
+    return {
+        "runs": len(runs),
+        "lead_time_s": mean([r["vsm"]["lead_time_s"] for r in runs]),
+        "time_to_first_build_s": mean([r["vsm"]["time_to_first_build_s"] for r in runs]),
+        "switches": mean([r["vsm"]["switches"] for r in runs]),
+        "stage_s": {s: mean([r["vsm"]["stages"].get(s, {}).get("s", 0) for r in runs]) for s in STAGES},
+        "stage_share": {s: mean([share(r, s) for r in runs]) for s in STAGES},
+        "waste": {k: mean([r["waste"].get(k) for r in runs]) for k in WASTE_KEYS},
+        "ftt_rate": mean([None if r["waste"].get("ftt") is None else float(r["waste"]["ftt"]) for r in runs]),
+        "leftover_rate": mean([float(bool(r["waste"].get("leftovers"))) for r in runs]),
+        "cost_per_pass": round(cost / passes, 6) if passes else None,
+    }
+
+
+def lean_summary(records: list[dict], arms: list[str]) -> dict | None:
+    by_arm = {a: lean_runs(records, a) for a in arms}
+    if not any(by_arm.values()):
+        return None
+    summary = {a: arm_lean(runs, [r for r in records if r["arm"] == a]) for a, runs in by_arm.items() if runs}
+    deltas = {}
+    if "vanilla" in summary:
+        for arm in summary:
+            if arm == "vanilla":
+                continue
+            deltas[arm] = {}
+            for key, _, get in DELTAS:
+                tr = [v for v in map(get, by_arm[arm]) if v is not None]
+                va = [v for v in map(get, by_arm["vanilla"]) if v is not None]
+                if tr and va:
+                    deltas[arm][key] = {"delta": round(sum(tr) / len(tr) - sum(va) / len(va), 4),
+                                        "ci95": bootstrap_delta_ci(tr, va)}
+    return {"arms": summary, "deltas": deltas}
+
+
+def bar(arm: dict, scale: float) -> str:
+    """Stage letters, one per `scale` seconds of mean stage time: the bar's length is the lead time."""
+    return "".join(LETTER[s] * round((arm["stage_s"][s] or 0) / scale) for s in STAGES)
+
+
+def fmt(value, places=2) -> str:
+    return "-" if value is None else f"{value:.{places}f}"
+
+
+def lean_lines(summary: dict | None, width: int = 50) -> list[str]:
+    if not summary:
+        return []
+    arms = summary["arms"]
+    scale = max((a["lead_time_s"] or 0) for a in arms.values()) / width or 1
+    out = ["", "value stream (bar length = mean lead time; P plan, D design, B build, T test, R release, M maintain):"]
+    for name, a in arms.items():
+        out.append(f"  {name:<10} |{bar(a, scale):<{width}}| {fmt(a['lead_time_s'], 0):>5} s  "
+                   f"first build {fmt(a['time_to_first_build_s'], 0)} s  PCE {fmt(a['waste']['pce'])}  "
+                   f"FTT {fmt(a['ftt_rate'])}  $/pass {fmt(a['cost_per_pass'], 4)}")
+    out += ["", f"  {'waste (mean per run)':<22}" + "".join(f"{n:>11}" for n in arms) + "   delta vs vanilla [95% CI]"]
+    for key, label, _ in DELTAS:
+        if key in ("lead_time_s", "time_to_first_build_s"):
+            continue
+        values = "".join(f"{fmt(row_value(a, key)):>11}" for a in arms.values())
+        ds = "  ".join(f"{n} {d[key]['delta']:+.2f} {d[key]['ci95']}" for n, d in summary["deltas"].items() if key in d)
+        out.append(f"  {label:<22}{values}   {ds}")
+    return out
+
+
+def row_value(arm: dict, key: str):
+    if key == "switches":
+        return arm["switches"]
+    if key in ("build_share", "test_share"):
+        return arm["stage_share"][key.split("_")[0]]
+    if key == "ftt":
+        return arm["ftt_rate"]
+    return arm["waste"].get(key)

package/scripts/lib/ab_metrics.py

@@ -121,3 +121,11 @@ def tap_score(stdout: str) -> float | None:
     passed = sum(int(n) for n in re.findall(r"^# pass (\d+)", stdout, re.M))
     failed = sum(int(n) for n in re.findall(r"^# (?:fail|cancelled) (\d+)", stdout, re.M))
     return round(passed / (passed + failed), 4) if passed + failed else None
+
+
+def tap_failures(stdout: str) -> int:
+    return sum(int(n) for n in re.findall(r"^# (?:fail|cancelled) (\d+)", stdout, re.M))
+
+
+def tap_failed_names(stdout: str) -> list[str]:
+    return [m.strip() for m in re.findall(r"^not ok \d+ - (.+)$", stdout, re.M)]

package/scripts/lib/ab_progress.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# story: e58s05
+# story: e58s05 e58s10
 """Live view of an A/B eval while it runs: reads every ab-live.jsonl under the given folders
 (one per shard) and prints runs, pass rate, score and cost per task and arm, each arm's delta
 against vanilla with its bootstrap CI, and how those deltas evolved as runs accumulated.
@@ -16,6 +16,7 @@ from datetime import datetime
 from pathlib import Path
 
 sys.path.insert(0, str(Path(__file__).resolve().parent))
+from ab_lean import lean_lines  # type: ignore[import]  # noqa: E402
 from ab_report import aggregate, mean  # type: ignore[import]  # noqa: E402
 
 CHECKPOINTS = (5, 10, 20, 30, 50, 75, 100)
@@ -79,6 +80,7 @@ def render(records: list[dict]) -> str:
         out += ["", "deltas (all runs so far, 95% CI):"]
         out += [delta_line(n, d) for n, d in result["deltas_vs_vanilla"].items()]
         out += ["", "evolution (pass and score delta by run index):"] + evolution(records, tasks, arms)
+    out += lean_lines(result.get("lean"))
     return "\n".join(out)
 
 

package/scripts/lib/ab_report.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# story: e58s01 e58s03 e58s04
+# story: e58s01 e58s03 e58s04 e58s10
 """A/B eval aggregation and reports — pure functions over run records (no agent, no subprocess
 except reading the bigpowers git SHA). Split from ab-eval.py to keep both under the 300-line cap."""
 from __future__ import annotations
@@ -47,6 +47,7 @@ def arm_summary(runs: list[dict]) -> dict:
         "runs": len(runs),
         "pass_rate": mean([float(r["pass"]) for r in runs]),
         "mean_score": mean([r.get("score") for r in runs]),
+        "mean_bugs_opened": mean([r.get("bugs_opened", 0) for r in runs]),
         "tamper_rate": mean([float(r["tampered"]) for r in runs]),
         "agent_error_rate": mean([float(r["agent_error"]) for r in runs]),
         "mean_cost_usd": mean([r["cost_usd"] for r in runs]),
@@ -66,8 +67,10 @@ def aggregate(records: list[dict], arms: list[str], tasks: list[str]) -> dict:
     per_task_score = {t: {a: mean([r.get("score") for r in by_arm[a] if r["task"] == t]) for a in arms}
                       for t in tasks}
     deltas = {a: arm_delta(by_arm, summary, a) for a in arms if a != "vanilla"} if "vanilla" in arms else {}
+    from ab_lean import lean_summary  # type: ignore[import]  # lazy: ab_lean imports this module
     return {"arms_summary": summary, "per_task": per_task, "per_task_score": per_task_score,
-            "delta": deltas.get("bigpowers"), "deltas_vs_vanilla": deltas}
+            "delta": deltas.get("bigpowers"), "deltas_vs_vanilla": deltas,
+            "lean": lean_summary(records, arms)}
 
 
 def arm_delta(by_arm: dict, summary: dict, arm: str) -> dict:
@@ -86,6 +89,8 @@ def arm_delta(by_arm: dict, summary: dict, arm: str) -> dict:
     if scores[arm] and scores["vanilla"]:
         delta["score"] = round(tr["mean_score"] - va["mean_score"], 4)
         delta["score_ci95"] = bootstrap_delta_ci(scores[arm], scores["vanilla"])
+    if tr.get("mean_bugs_opened") is not None and va.get("mean_bugs_opened") is not None:
+        delta["bugs_opened"] = round(tr["mean_bugs_opened"] - va["mean_bugs_opened"], 4)
     return delta
 
 
@@ -111,13 +116,19 @@ def write_reports(out: Path, meta: dict, result: dict, records: list[dict]) -> P
 
 
 def print_summary(result: dict) -> None:
-    print(f"{'arm':<10} {'pass':>6} {'score':>6} {'tamper':>7} {'cost$':>8} {'turns':>6} {'secs':>7}")
+    print(f"{'arm':<10} {'pass':>6} {'score':>6} {'bugs':>6} {'tamper':>7} {'cost$':>8} {'turns':>6} {'secs':>7}")
     for arm, s in result["arms_summary"].items():
         fmt = lambda v, p=2: "-" if v is None else f"{v:.{p}f}"
-        print(f"{arm:<10} {fmt(s['pass_rate']):>6} {fmt(s['mean_score']):>6} {fmt(s['tamper_rate']):>7} "
+        b_str = "-" if s.get("mean_bugs_opened") is None else f"{s['mean_bugs_opened']:.1f}"
+        print(f"{arm:<10} {fmt(s['pass_rate']):>6} {fmt(s['mean_score']):>6} {b_str:>6} {fmt(s['tamper_rate']):>7} "
               f"{fmt(s['mean_cost_usd'], 3):>8} {fmt(s['mean_turns'], 1):>6} {fmt(s['mean_duration_s'], 0):>7}")
     for arm, d in result.get("deltas_vs_vanilla", {}).items():
         line = f"delta {arm:<10} - vanilla: pass {d['pass_rate']:+.2f} CI {d['ci95']}"
         if "score" in d:
             line += f"  score {d['score']:+.2f} CI {d['score_ci95']}"
+        if "bugs_opened" in d:
+            line += f"  bugs {d['bugs_opened']:+.1f}"
+        print(line)
+    from ab_lean import lean_lines  # type: ignore[import]
+    for line in lean_lines(result.get("lean")):
         print(line)

package/scripts/lib/ab_vsm.py

@@ -0,0 +1,122 @@
+#!/usr/bin/env python3
+# story: e58s08
+"""Value stream map of one A/B run: credit each model turn's time, tokens and cost to the lean stage
+of the action it takes (plan, design, build, test, deploy, maintain; wrap-up when it takes none).
+
+Tools are only 1-3% of a run's time, so time is credited per turn, not per tool call. pi stamps an
+assistant message when it starts, so a turn lasts until the next assistant message starts: the model
+time plus the tools it ran.
+Rules: specs/benchmarks/ab/vsm.yaml.
+
+CLI: python3 scripts/lib/ab_vsm.py <trace.jsonl.gz>...   (one VSM line per saved transcript)
+"""
+from __future__ import annotations
+
+import gzip
+import json
+import re
+import sys
+from pathlib import Path
+
+import yaml
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+RULES_PATH = REPO_ROOT / "specs" / "benchmarks" / "ab" / "vsm.yaml"
+WRAP_UP = "wrap-up"
+
+
+def load_rules(path: Path = RULES_PATH) -> dict:
+    rules = yaml.safe_load(path.read_text())
+    rules["compiled"] = [(r["stage"], re.compile(r["tools"]), re.compile(r["target"])) for r in rules["rules"]]
+    return rules
+
+
+def classify_call(tool: str, args: dict | None, rules: dict) -> str:
+    args = args or {}
+    target = str(args.get("command") or args.get("path") or args.get("file_path") or "")
+    for stage, tools, pattern in rules["compiled"]:
+        if tools.search(tool or "") and pattern.search(target):
+            return stage
+    return "design"
+
+
+def turn_stage(message: dict, rules: dict) -> str:
+    calls = [c for c in message.get("content") or [] if isinstance(c, dict) and c.get("type") == "toolCall"]
+    stages = {classify_call(c.get("name", ""), c.get("arguments"), rules) for c in calls}
+    return next((s for s in rules["turn_priority"] if s in stages), WRAP_UP)
+
+
+def pi_messages(stdout: str) -> list[dict]:
+    messages = []
+    for line in stdout.splitlines():
+        try:
+            event = json.loads(line)
+        except json.JSONDecodeError:
+            continue
+        if isinstance(event, dict) and event.get("type") == "message_end":
+            message = event.get("message") or {}
+            if isinstance(message.get("timestamp"), (int, float)):
+                messages.append(message)
+    return messages
+
+
+def vsm_from_pi(stdout: str, rules: dict) -> dict | None:
+    messages = pi_messages(stdout)
+    if not any(m.get("role") == "assistant" for m in messages):
+        return None
+    start = messages[0]["timestamp"]
+    stages: dict[str, dict] = {}
+    sequence, first_build = [], None
+    turns = [m for m in messages if m.get("role") == "assistant"]
+    for i, message in enumerate(turns):
+        stage = turn_stage(message, rules)
+        end = turns[i + 1]["timestamp"] if i + 1 < len(turns) else messages[-1]["timestamp"]
+        usage = message.get("usage") or {}
+        slot = stages.setdefault(stage, {"s": 0.0, "cost_usd": 0.0, "output_tokens": 0, "turns": 0})
+        slot["s"] = round(slot["s"] + (end - message["timestamp"]) / 1000, 3)
+        slot["cost_usd"] = round(slot["cost_usd"] + ((usage.get("cost") or {}).get("total") or 0), 6)
+        slot["output_tokens"] += usage.get("output", 0)
+        slot["turns"] += 1
+        sequence.append(stage)
+        if stage == "build" and first_build is None:
+            first_build = round((message["timestamp"] - start) / 1000, 3)
+    return {
+        "lead_time_s": round((messages[-1]["timestamp"] - start) / 1000, 3),
+        "time_to_first_build_s": first_build,
+        "switches": sum(1 for a, b in zip(sequence, sequence[1:]) if a != b),
+        "sequence": sequence,
+        "stages": stages,
+    }
+
+
+def vsm_from(parser: str, stdout: str, rules: dict | None = None) -> dict | None:
+    """vsm block for a run record; None when the adapter's transcript format is not supported."""
+    if parser != "pi-json":
+        return None
+    return vsm_from_pi(stdout, rules or load_rules())
+
+
+def describe(name: str, block: dict | None) -> str:
+    if not block:
+        return f"{name:<32} no VSM (unsupported or empty transcript)"
+    lead = block["lead_time_s"] or 1
+    parts = sorted(block["stages"].items(), key=lambda kv: -kv[1]["s"])
+    shares = "  ".join(f"{stage} {v['s']:.0f}s {100 * v['s'] / lead:.0f}%" for stage, v in parts if v["turns"])
+    return f"{name:<32} lead {block['lead_time_s']:6.1f}s  first build {block['time_to_first_build_s']}s  " \
+           f"switches {block['switches']:>2}  | {shares}"
+
+
+def main(argv: list[str]) -> int:
+    if not argv:
+        print("usage: python3 scripts/lib/ab_vsm.py <trace.jsonl.gz>...")
+        return 2
+    rules = load_rules()
+    for name in argv:
+        path = Path(name)
+        text = gzip.decompress(path.read_bytes()).decode() if path.suffix == ".gz" else path.read_text()
+        print(describe(path.name, vsm_from_pi(text, rules)))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))

package/scripts/lib/ab_waste.py

@@ -0,0 +1,173 @@
+#!/usr/bin/env python3
+# story: e58s09 e58s10
+"""Lean waste metrics for one A/B run, from its final patch and its pi transcript.
+
+Patch (any adapter): changed lines per file class, the reference solution's size, extra files,
+leftover scratch files and the review load a person would face. Transcript (pi): relearning
+(rereads, repeated commands), defects and rework (failing test runs after the first build edit,
+re-edits), first-time-through and process cycle efficiency. See LEAN-VSM_LATEST.md and
+specs/benchmarks/ab/vsm.yaml (file classes reuse the stage rules; `leftovers` regex).
+"""
+from __future__ import annotations
+
+import functools
+import gzip
+import json
+import re
+import shutil
+import subprocess
+import tempfile
+from collections import Counter
+from pathlib import Path
+
+from ab_vsm import classify_call, load_rules, vsm_from_pi  # type: ignore[import]
+
+CLASS_OF_STAGE = {"build": "product", "test": "test", "maintain": "doc", "plan": "plan"}
+FAILED = re.compile(r'"exit_code":\s*[1-9]|(?:^|\\n|\n)\s*(?:ℹ|#)\s*fail\s+[1-9]|(?:^|\\n|\n)not ok ')
+
+
+def git(cwd: Path, *args: str) -> str:
+    res = subprocess.run(["git", "-c", "user.name=ab", "-c", "user.email=ab@local", *args],
+                         cwd=cwd, capture_output=True, text=True)
+    return res.stdout
+
+
+def file_class(path: str, rules: dict) -> str:
+    cls = CLASS_OF_STAGE.get(classify_call("edit", {"path": path}, rules), "product")
+    return "doc" if cls == "product" and path.endswith(".md") else cls
+
+
+def numstat(cwd: Path, base: str) -> dict[str, int]:
+    changed = {}
+    for line in git(cwd, "diff", "--cached", "--numstat", base).splitlines():
+        added, deleted, path = line.split("\t", 2)
+        changed[path] = sum(int(n) for n in (added, deleted) if n.isdigit())
+    return changed
+
+
+@functools.lru_cache(maxsize=None)
+def solution_files(task_dir: str) -> dict[str, int] | None:
+    """Changed lines per file of the reference solution, applied over the fixture."""
+    task = Path(task_dir)
+    if not (task / "solution").is_dir():
+        return None
+    work = Path(tempfile.mkdtemp(prefix="ab-solution-"))
+    try:
+        shutil.copytree(task / "fixture", work, dirs_exist_ok=True)
+        git(work, "init", "-q")
+        git(work, "add", "-A")
+        git(work, "commit", "-q", "--no-verify", "-m", "fixture")
+        shutil.copytree(task / "solution", work, dirs_exist_ok=True)
+        git(work, "add", "-A")
+        return numstat(work, "HEAD")
+    finally:
+        shutil.rmtree(work, ignore_errors=True)
+
+
+def patch_waste(workdir: Path, base: str, task_dir: Path, patch_path: Path | None, rules: dict) -> dict:
+    git(workdir, "add", "-A")
+    patch = git(workdir, "diff", "--cached", base)
+    if patch_path is not None:
+        patch_path.parent.mkdir(parents=True, exist_ok=True)
+        patch_path.write_bytes(gzip.compress(patch.encode()))
+    changed = numstat(workdir, base)
+    lines = Counter()
+    for path, n in changed.items():
+        lines[file_class(path, rules)] += n
+    added = [line[2:] for line in git(workdir, "diff", "--cached", "--name-status", base).splitlines()
+             if line.startswith("A\t")]
+    solution = solution_files(str(task_dir))
+    solution_lines = sum(solution.values()) if solution else None
+    product = [p for p in changed if file_class(p, rules) == "product"]
+    doc_words = sum(len(line[1:].split()) for line in md_added_lines(patch))
+    return {
+        "product_lines": lines["product"], "test_lines": lines["test"],
+        "doc_lines": lines["doc"], "plan_lines": lines["plan"],
+        "solution_lines": solution_lines,
+        "diff_ratio": round(lines["product"] / solution_lines, 3) if solution_lines else None,
+        "extra_files": sorted(p for p in product if solution is not None and p not in solution),
+        "leftovers": sorted(p for p in added if re.search(rules["leftovers"], p)),
+        "review_load": sum(changed.values()) + len(changed) + doc_words,
+    }
+
+
+def md_added_lines(patch: str) -> list[str]:
+    out, in_md = [], False
+    for line in patch.splitlines():
+        if line.startswith("+++ "):
+            in_md = line.endswith(".md")
+        elif in_md and line.startswith("+"):
+            out.append(line)
+    return out
+
+
+def tool_calls(stdout: str) -> list[dict]:
+    """Ordered tool calls with their stage and whether a test run failed."""
+    calls, by_id = [], {}
+    for line in stdout.splitlines():
+        try:
+            event = json.loads(line)
+        except json.JSONDecodeError:
+            continue
+        if not isinstance(event, dict):
+            continue
+        if event.get("type") == "tool_execution_start":
+            call = {"tool": event.get("toolName", ""), "args": event.get("args") or {}, "failed": False}
+            by_id[event.get("toolCallId")] = call
+            calls.append(call)
+        elif event.get("type") == "tool_execution_end" and event.get("toolCallId") in by_id:
+            by_id[event["toolCallId"]]["failed"] = bool(FAILED.search(json.dumps(event.get("result"), ensure_ascii=False)))
+    return calls
+
+
+def transcript_waste(stdout: str, rules: dict | None = None) -> dict | None:
+    rules = rules or load_rules()
+    vsm = vsm_from_pi(stdout, rules)
+    calls = tool_calls(stdout)
+    if vsm is None and not calls:
+        return None
+    for call in calls:
+        call["stage"] = classify_call(call["tool"], call["args"], rules)
+    reads = Counter(c["args"].get("path") for c in calls if c["tool"] == "read")
+    commands = Counter(c["args"].get("command") for c in calls if c["tool"] == "bash" and c["stage"] != "test")
+    edits = Counter(c["args"].get("path") for c in calls if c["tool"] in ("edit", "write"))
+    first_build = next((i for i, c in enumerate(calls) if c["stage"] == "build"), None)
+    runs = [c for c in calls if c["tool"] == "bash" and c["stage"] == "test"]
+    after = [c for c in calls[first_build:] if c in runs] if first_build is not None else []
+    rework = sum(c["failed"] for c in after)
+    redundant = redundant_runs(calls)
+    stages = (vsm or {}).get("stages", {})
+    lead = (vsm or {}).get("lead_time_s") or 0
+    value_add = sum(stages.get(s, {}).get("s", 0) for s in ("build", "test"))
+    return {
+        "rereads": sum(n - 1 for n in reads.values()),
+        "repeated_commands": sum(n - 1 for n in commands.values()),
+        "test_runs": len(runs), "redundant_test_runs": redundant, "rework_loops": rework,
+        "reedits": sum(n - 1 for n in edits.values()),
+        "ftt": bool(after) and rework == 0,
+        "pce": round(value_add / lead, 3) if lead else None,
+    }
+
+
+def redundant_runs(calls: list[dict]) -> int:
+    """Passing test runs with no file edited since the previous passing run (overprocessing, e58s10)."""
+    redundant, green, dirty = 0, False, False
+    for call in calls:
+        if call["tool"] in ("edit", "write"):
+            dirty = True
+        elif call["tool"] == "bash" and call["stage"] == "test" and not call["failed"]:
+            redundant += green and not dirty
+            green, dirty = True, False
+    return redundant
+
+
+EMPTY_TRANSCRIPT = dict.fromkeys(["rereads", "repeated_commands", "test_runs", "redundant_test_runs",
+                                  "rework_loops", "reedits", "ftt", "pce"])
+
+
+def waste_for(workdir: Path, base: str, stdout: str, parser: str, task_dir: Path,
+              patch_path: Path | None = None) -> dict:
+    """The run record's waste block; transcript metrics are None for non-pi adapters."""
+    rules = load_rules()
+    transcript = transcript_waste(stdout, rules) if parser == "pi-json" else None
+    return {**patch_waste(workdir, base, task_dir, patch_path, rules), **(transcript or EMPTY_TRANSCRIPT)}

package/scripts/test-adapter-render.sh

@@ -60,7 +60,7 @@ for adapter in scripts/adapters/*.sh; do
         IR_DESC_ESCAPED="test desc"; IR_MODEL=""
         declare -f render_skill >/dev/null || exit 1
         render_skill
-      ' _ "$PWD/$adapter" "$nm" >/dev/null 2>&1 )
+      ' _ "$PWD/$adapter" "$nm" </dev/null >/dev/null 2>&1 )
   }
 
   dest="$TMP_ROOT/$id"

package/scripts/test-test-lock-hook.sh

@@ -10,7 +10,7 @@ HOOK="$REPO_ROOT/hooks/pre-tool-use.sh"
 pass_count=0
 fail_count=0
 
-assert_denied() {
+lock_assert_denied() {
   local label="$1"
   local input="$2"
   local expected_substr="${3:-}"
@@ -30,7 +30,7 @@ assert_denied() {
   fi
 }
 
-assert_allowed() {
+lock_assert_allowed() {
   local label="$1"
   local input="$2"
   local output exit_code=0
@@ -64,7 +64,7 @@ git add .
 git commit -q -m "feat(app): initial commit"
 
 # 1. Without a RED commit yet during fix_bug, test edits should be allowed (to author the RED test)
-assert_allowed "Test file edit allowed when authoring RED test" \
+lock_assert_allowed "Test file edit allowed when authoring RED test" \
   '{"tool": "Edit", "file_path": "test/app.test.js"}'
 
 # 2. Commit a RED test
@@ -73,23 +73,23 @@ git add test/app.test.js
 git commit -q -m "test(app): red test reproducing defect"
 
 # 3. Now that a RED commit exists and active_flow is fix_bug, test edits must be LOCKED
-assert_denied "Test file edit denied after RED commit in fix_bug" \
+lock_assert_denied "Test file edit denied after RED commit in fix_bug" \
   '{"tool": "Edit", "file_path": "test/app.test.js"}' \
   "locked"
 
-assert_denied "Test file write denied after RED commit in fix_bug" \
+lock_assert_denied "Test file write denied after RED commit in fix_bug" \
   '{"tool": "Write", "file_path": "tests/test_other.py"}' \
   "locked"
 
 # 4. Source file edits must remain allowed
-assert_allowed "Source file edit allowed after RED commit in fix_bug" \
+lock_assert_allowed "Source file edit allowed after RED commit in fix_bug" \
   '{"tool": "Edit", "file_path": "src/app.js"}'
 
 # 5. When active_flow is not fix_bug, test edits are allowed
 cat > specs/state.yaml <<'YAML'
 active_flow: build_epic
 YAML
-assert_allowed "Test file edit allowed when active_flow is not fix_bug" \
+lock_assert_allowed "Test file edit allowed when active_flow is not fix_bug" \
   '{"tool": "Edit", "file_path": "test/app.test.js"}'
 
 echo ""