bigpowers 2.91.1 → 2.92.0

package/.gemini/extensions/bigpowers/gemini-extension.json

@@ -1,6 +1,6 @@
 {
   "name": "bigpowers",
-  "version": "2.91.1",
+  "version": "2.92.0",
   "description": "80 skills — 73 agent skills synthesizing 17 years of software engineering discipline into a prescriptive methodology for solo developers",
   "keywords": [
     "gemini-extension"

package/.pi/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bigpowers",
-  "version": "2.91.1",
+  "version": "2.92.0",
   "description": "80 skills — 73 agent skills synthesizing 17 years of software engineering discipline into a prescriptive methodology for solo developers",
   "keywords": [
     "pi-package"

package/CHANGELOG.md

@@ -1,3 +1,10 @@
+# [2.92.0](https://github.com/danielvm-git/bigpowers/compare/v2.91.1...v2.92.0) (2026-11-03)
+
+
+### Features
+
+* **benchmarks:** lean value-stream and waste metrics for the A/B eval, via pi in the Docker sandbox (e58s05–e58s11) ([#195](https://github.com/danielvm-git/bigpowers/issues/195)) ([11f5c8f](https://github.com/danielvm-git/bigpowers/commit/11f5c8f4b6407c67c4131620861345b49fce6193)), closes [#191](https://github.com/danielvm-git/bigpowers/issues/191)
+
 ## [2.91.1](https://github.com/danielvm-git/bigpowers/compare/v2.91.0...v2.91.1) (2026-11-03)
 
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bigpowers",
-  "version": "2.91.1",
+  "version": "2.92.0",
   "description": "73 agent skills synthesizing 17 years of software engineering discipline into a prescriptive methodology for solo developers",
   "main": "index.js",
   "scripts": {

package/scripts/ab-sandbox.sh

@@ -15,7 +15,7 @@ INTERNAL_NET="bp-ab-internal"
 EGRESS_NET="bp-ab-egress"
 ASSETS="$(cd "$(dirname "${BASH_SOURCE[0]}")/lib/ab-sandbox" && pwd)"
 
-die() { echo "ERROR: $*" >&2; exit 1; }
+sandbox_die() { echo "ERROR: $*" >&2; exit 1; }
 
 have_image() { docker image inspect "$1" >/dev/null 2>&1; }
 have_net() { docker network inspect "$1" >/dev/null 2>&1; }
@@ -43,7 +43,7 @@ wait_for_proxy() {
     sleep 1
   done
   docker logs "$PROXY_NAME" >&2 || true
-  die "proxy did not start listening"
+  sandbox_die "proxy did not start listening"
 }
 
 cmd_up() {
@@ -81,5 +81,5 @@ case "${1:-status}" in
   up) cmd_up ;;
   down) cmd_down ;;
   status) cmd_status ;;
-  *) die "usage: $(basename "$0") up|down|status" ;;
+  *) sandbox_die "usage: $(basename "$0") up|down|status" ;;
 esac

package/scripts/adapters/agy.sh

@@ -38,7 +38,7 @@ wire_context() {
 }
 
 # If stdin is a pipe/redirect, read the SkillIR JSON and render
-if [[ ! -t 0 ]]; then
+if [[ "${BASH_SOURCE[0]}" == "${0}" ]] && [[ ! -t 0 ]]; then
   JSON_INPUT=$(cat)
   if [[ -n "$JSON_INPUT" ]]; then
     IR_NAME=$(echo "$JSON_INPUT" | jq -r '.name')

package/scripts/adapters/cursor.sh

@@ -29,7 +29,7 @@ wire_context() {
 }
 
 # If stdin is a pipe/redirect, read the SkillIR JSON and render
-if [[ ! -t 0 ]]; then
+if [[ "${BASH_SOURCE[0]}" == "${0}" ]] && [[ ! -t 0 ]]; then
   JSON_INPUT=$(cat)
   if [[ -n "$JSON_INPUT" ]]; then
     IR_NAME=$(echo "$JSON_INPUT" | jq -r '.name')

package/scripts/adapters/gemini.sh

@@ -101,7 +101,7 @@ if [[ "${1:-}" == "--list-hook-events" ]]; then
 fi
 
 # If stdin is a pipe/redirect, read the SkillIR JSON and render
-if [[ ! -t 0 ]]; then
+if [[ "${BASH_SOURCE[0]}" == "${0}" ]] && [[ ! -t 0 ]]; then
   JSON_INPUT=$(cat)
   if [[ -n "$JSON_INPUT" ]]; then
     IR_NAME=$(echo "$JSON_INPUT" | jq -r '.name')

package/scripts/adapters/mimo.sh

@@ -30,7 +30,7 @@ wire_context() {
 }
 
 # If stdin is a pipe/redirect, read the SkillIR JSON and render
-if [[ ! -t 0 ]]; then
+if [[ "${BASH_SOURCE[0]}" == "${0}" ]] && [[ ! -t 0 ]]; then
   JSON_INPUT=$(cat)
   if [[ -n "$JSON_INPUT" ]]; then
     IR_NAME=$(echo "$JSON_INPUT" | jq -r '.name')

package/scripts/adapters/pi.sh

@@ -41,7 +41,7 @@ wire_context() {
 }
 
 # If stdin is a pipe/redirect, read the SkillIR JSON and render
-if [[ ! -t 0 ]]; then
+if [[ "${BASH_SOURCE[0]}" == "${0}" ]] && [[ ! -t 0 ]]; then
   JSON_INPUT=$(cat)
   if [[ -n "$JSON_INPUT" ]]; then
     IR_NAME=$(echo "$JSON_INPUT" | jq -r '.name')

package/scripts/adapters/zcode.sh

@@ -27,7 +27,7 @@ wire_context() {
 }
 
 # If stdin is a pipe/redirect, read the SkillIR JSON and render
-if [[ ! -t 0 ]]; then
+if [[ "${BASH_SOURCE[0]}" == "${0}" ]] && [[ ! -t 0 ]]; then
   JSON_INPUT=$(cat)
   if [[ -n "$JSON_INPUT" ]]; then
     IR_NAME=$(echo "$JSON_INPUT" | jq -r '.name')

package/scripts/generate-context-bundle.sh

@@ -51,7 +51,8 @@ while IFS= read -r row; do
 
   adapter_script="$SCRIPT_DIR/adapters/${adapter}.sh"
   # shellcheck source=/dev/null
-  source "$adapter_script"
+  # stdin is this loop's target list: an adapter that reads stdin when sourced would swallow it.
+  source "$adapter_script" </dev/null
   case "$mode" in
     symlink) wire_context_mode symlink "$file" ;;
     copy) wire_context_mode copy "$file" ;;

package/scripts/lib/ab-eval.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# story: e58s01 e58s02 e58s03 e58s05 e58s06 e58s07
+# story: e58s01 e58s02 e58s03 e58s05 e58s06 e58s07 e58s08 e58s09
 """A/B eval: run the same tasks with and without bigpowers and grade only the outcome.
 
 Each run gets a fresh temp git workdir (outside the repo) copied from the task fixture.
@@ -25,7 +25,9 @@ from pathlib import Path
 import yaml
 
 sys.path.insert(0, str(Path(__file__).resolve().parent))
-from ab_metrics import parse_metrics, tap_score  # type: ignore[import]  # noqa: E402
+from ab_metrics import parse_metrics, tap_score, tap_failures, tap_failed_names  # type: ignore[import]  # noqa: E402
+from ab_vsm import vsm_from  # type: ignore[import]  # noqa: E402
+from ab_waste import waste_for  # type: ignore[import]  # noqa: E402
 from ab_report import (  # type: ignore[import]  # noqa: E402
     BOOTSTRAP_RESAMPLES, BOOTSTRAP_SEED, aggregate, bigpowers_sha, print_summary, write_reports,
 )
@@ -180,6 +182,8 @@ def run_once(task: dict, arm: str, setup: str, adapter: dict, model: str | None,
 
         tampered, protected_added = protected_changes(workdir, before)
         diff = changed_lines(workdir, base)
+        waste = waste_for(workdir, base, stdout, adapter.get("parser", "none"), task["dir"],
+                          trace_dir / f"{task['id']}-{arm}-{run_idx}.patch.gz" if trace_dir else None)
         shutil.copytree(task["dir"] / "hidden", workdir, dirs_exist_ok=True)
         cmd, genv = grade_cmd(adapter, task["grade"], env)
         passed = sh(cmd, workdir, genv, timeout=300).returncode == 0
@@ -187,12 +191,22 @@ def run_once(task: dict, arm: str, setup: str, adapter: dict, model: str | None,
         if task.get("score"):
             cmd, genv = grade_cmd(adapter, task["score"], env)
             score = tap_score(sh(cmd, workdir, genv, timeout=300).stdout)
+        bugs_opened = 0
+        bug_details = []
+        if task.get("audit"):
+            if (task["dir"] / "audit").is_dir():
+                shutil.copytree(task["dir"] / "audit", workdir, dirs_exist_ok=True)
+            cmd, genv = grade_cmd(adapter, task["audit"], env)
+            out = sh(cmd, workdir, genv, timeout=300).stdout
+            bugs_opened = tap_failures(out)
+            bug_details = tap_failed_names(out)
         metrics = parse_metrics(adapter.get("parser", "none"), stdout)
         return {
-            "task": task["id"], "arm": arm, "run": run_idx, "pass": passed, "score": score, "tampered": tampered,
+            "task": task["id"], "arm": arm, "run": run_idx, "pass": passed, "score": score,
+            "bugs_opened": bugs_opened, "bug_details": bug_details, "tampered": tampered,
             "protected_lines_added": protected_added, "diff_lines": diff, "duration_s": duration, "agent_exit": exit_code,
             "agent_error": timed_out or exit_code != 0 or bool(metrics.pop("agent_reported_error")),
-            "timed_out": timed_out, **metrics,
+            "timed_out": timed_out, "vsm": vsm_from(adapter.get("parser", "none"), stdout), "waste": waste, **metrics,
         }
     finally:
         shutil.rmtree(home, ignore_errors=True)
@@ -262,7 +276,7 @@ def main(argv: list[str] | None = None) -> int:
                     with live.open("a") as f:
                         f.write(json.dumps({**rec, "bigpowers_sha": sha, "model": model}) + "\n")
                     print(f"  {task['id']:<8} {arm:<10} run {i}: {'PASS' if rec['pass'] else 'FAIL'}"
-                          f"{' TAMPERED' if rec['tampered'] else ''}{' AGENT-ERROR' if rec['agent_error'] else ''}")
+                          f"{' TAMPERED' if rec['tampered'] else ''}{' AGENT-ERROR' if rec['agent_error'] else ''}", flush=True)
     except ConfigError as exc:
         print(f"ERROR: {exc}")
         return 2

package/scripts/lib/ab_lean.py

@@ -0,0 +1,116 @@
+#!/usr/bin/env python3
+# story: e58s10
+"""Lean views of an A/B run: each arm's mean value stream (stage time and share) and waste metrics,
+deltas against vanilla with seeded bootstrap CIs, and text VSM bars plus a waste table.
+
+Reads the `vsm` (e58s08) and `waste` (e58s09) blocks of run records; runs without them are skipped.
+"""
+from __future__ import annotations
+
+from ab_report import bootstrap_delta_ci, mean  # type: ignore[import]
+
+STAGES = ["plan", "design", "build", "test", "deploy", "maintain"]
+LETTER = {"plan": "P", "design": "D", "build": "B", "test": "T", "deploy": "R", "maintain": "M"}
+WASTE_KEYS = ["test_runs", "redundant_test_runs", "rework_loops", "reedits", "rereads",
+              "repeated_commands", "review_load", "diff_ratio", "pce"]
+# (key, label, per-run extractor) for the deltas against vanilla.
+DELTAS = [
+    ("lead_time_s", "lead time (s)", lambda r: r["vsm"]["lead_time_s"]),
+    ("time_to_first_build_s", "first build (s)", lambda r: r["vsm"]["time_to_first_build_s"]),
+    ("switches", "stage switches", lambda r: r["vsm"]["switches"]),
+    ("build_share", "build share", lambda r: share(r, "build")),
+    ("test_share", "test share", lambda r: share(r, "test")),
+    ("test_runs", "test runs", lambda r: r["waste"].get("test_runs")),
+    ("redundant_test_runs", "redundant test runs", lambda r: r["waste"].get("redundant_test_runs")),
+    ("rework_loops", "rework loops", lambda r: r["waste"].get("rework_loops")),
+    ("reedits", "re-edits", lambda r: r["waste"].get("reedits")),
+    ("review_load", "review load", lambda r: r["waste"].get("review_load")),
+    ("diff_ratio", "diff ratio", lambda r: r["waste"].get("diff_ratio")),
+    ("ftt", "first-time-through", lambda r: None if r["waste"].get("ftt") is None else float(r["waste"]["ftt"])),
+    ("pce", "PCE", lambda r: r["waste"].get("pce")),
+]
+
+
+def share(run: dict, stage: str) -> float | None:
+    lead = run["vsm"]["lead_time_s"]
+    return run["vsm"]["stages"].get(stage, {}).get("s", 0) / lead if lead else None
+
+
+def lean_runs(records: list[dict], arm: str) -> list[dict]:
+    return [r for r in records if r["arm"] == arm and r.get("vsm") and r.get("waste")]
+
+
+def arm_lean(runs: list[dict], all_runs: list[dict]) -> dict:
+    passes = sum(r["pass"] for r in all_runs)
+    cost = sum(r.get("cost_usd") or 0 for r in all_runs)
+    return {
+        "runs": len(runs),
+        "lead_time_s": mean([r["vsm"]["lead_time_s"] for r in runs]),
+        "time_to_first_build_s": mean([r["vsm"]["time_to_first_build_s"] for r in runs]),
+        "switches": mean([r["vsm"]["switches"] for r in runs]),
+        "stage_s": {s: mean([r["vsm"]["stages"].get(s, {}).get("s", 0) for r in runs]) for s in STAGES},
+        "stage_share": {s: mean([share(r, s) for r in runs]) for s in STAGES},
+        "waste": {k: mean([r["waste"].get(k) for r in runs]) for k in WASTE_KEYS},
+        "ftt_rate": mean([None if r["waste"].get("ftt") is None else float(r["waste"]["ftt"]) for r in runs]),
+        "leftover_rate": mean([float(bool(r["waste"].get("leftovers"))) for r in runs]),
+        "cost_per_pass": round(cost / passes, 6) if passes else None,
+    }
+
+
+def lean_summary(records: list[dict], arms: list[str]) -> dict | None:
+    by_arm = {a: lean_runs(records, a) for a in arms}
+    if not any(by_arm.values()):
+        return None
+    summary = {a: arm_lean(runs, [r for r in records if r["arm"] == a]) for a, runs in by_arm.items() if runs}
+    deltas = {}
+    if "vanilla" in summary:
+        for arm in summary:
+            if arm == "vanilla":
+                continue
+            deltas[arm] = {}
+            for key, _, get in DELTAS:
+                tr = [v for v in map(get, by_arm[arm]) if v is not None]
+                va = [v for v in map(get, by_arm["vanilla"]) if v is not None]
+                if tr and va:
+                    deltas[arm][key] = {"delta": round(sum(tr) / len(tr) - sum(va) / len(va), 4),
+                                        "ci95": bootstrap_delta_ci(tr, va)}
+    return {"arms": summary, "deltas": deltas}
+
+
+def bar(arm: dict, scale: float) -> str:
+    """Stage letters, one per `scale` seconds of mean stage time: the bar's length is the lead time."""
+    return "".join(LETTER[s] * round((arm["stage_s"][s] or 0) / scale) for s in STAGES)
+
+
+def fmt(value, places=2) -> str:
+    return "-" if value is None else f"{value:.{places}f}"
+
+
+def lean_lines(summary: dict | None, width: int = 50) -> list[str]:
+    if not summary:
+        return []
+    arms = summary["arms"]
+    scale = max((a["lead_time_s"] or 0) for a in arms.values()) / width or 1
+    out = ["", "value stream (bar length = mean lead time; P plan, D design, B build, T test, R release, M maintain):"]
+    for name, a in arms.items():
+        out.append(f"  {name:<10} |{bar(a, scale):<{width}}| {fmt(a['lead_time_s'], 0):>5} s  "
+                   f"first build {fmt(a['time_to_first_build_s'], 0)} s  PCE {fmt(a['waste']['pce'])}  "
+                   f"FTT {fmt(a['ftt_rate'])}  $/pass {fmt(a['cost_per_pass'], 4)}")
+    out += ["", f"  {'waste (mean per run)':<22}" + "".join(f"{n:>11}" for n in arms) + "   delta vs vanilla [95% CI]"]
+    for key, label, _ in DELTAS:
+        if key in ("lead_time_s", "time_to_first_build_s"):
+            continue
+        values = "".join(f"{fmt(row_value(a, key)):>11}" for a in arms.values())
+        ds = "  ".join(f"{n} {d[key]['delta']:+.2f} {d[key]['ci95']}" for n, d in summary["deltas"].items() if key in d)
+        out.append(f"  {label:<22}{values}   {ds}")
+    return out
+
+
+def row_value(arm: dict, key: str):
+    if key == "switches":
+        return arm["switches"]
+    if key in ("build_share", "test_share"):
+        return arm["stage_share"][key.split("_")[0]]
+    if key == "ftt":
+        return arm["ftt_rate"]
+    return arm["waste"].get(key)

package/scripts/lib/ab_metrics.py

@@ -121,3 +121,11 @@ def tap_score(stdout: str) -> float | None:
     passed = sum(int(n) for n in re.findall(r"^# pass (\d+)", stdout, re.M))
     failed = sum(int(n) for n in re.findall(r"^# (?:fail|cancelled) (\d+)", stdout, re.M))
     return round(passed / (passed + failed), 4) if passed + failed else None
+
+
+def tap_failures(stdout: str) -> int:
+    return sum(int(n) for n in re.findall(r"^# (?:fail|cancelled) (\d+)", stdout, re.M))
+
+
+def tap_failed_names(stdout: str) -> list[str]:
+    return [m.strip() for m in re.findall(r"^not ok \d+ - (.+)$", stdout, re.M)]

package/scripts/lib/ab_progress.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# story: e58s05
+# story: e58s05 e58s10
 """Live view of an A/B eval while it runs: reads every ab-live.jsonl under the given folders
 (one per shard) and prints runs, pass rate, score and cost per task and arm, each arm's delta
 against vanilla with its bootstrap CI, and how those deltas evolved as runs accumulated.
@@ -16,6 +16,7 @@ from datetime import datetime
 from pathlib import Path
 
 sys.path.insert(0, str(Path(__file__).resolve().parent))
+from ab_lean import lean_lines  # type: ignore[import]  # noqa: E402
 from ab_report import aggregate, mean  # type: ignore[import]  # noqa: E402
 
 CHECKPOINTS = (5, 10, 20, 30, 50, 75, 100)
@@ -79,6 +80,7 @@ def render(records: list[dict]) -> str:
         out += ["", "deltas (all runs so far, 95% CI):"]
         out += [delta_line(n, d) for n, d in result["deltas_vs_vanilla"].items()]
         out += ["", "evolution (pass and score delta by run index):"] + evolution(records, tasks, arms)
+    out += lean_lines(result.get("lean"))
     return "\n".join(out)
 
 

package/scripts/lib/ab_report.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# story: e58s01 e58s03 e58s04
+# story: e58s01 e58s03 e58s04 e58s10
 """A/B eval aggregation and reports — pure functions over run records (no agent, no subprocess
 except reading the bigpowers git SHA). Split from ab-eval.py to keep both under the 300-line cap."""
 from __future__ import annotations
@@ -47,6 +47,7 @@ def arm_summary(runs: list[dict]) -> dict:
         "runs": len(runs),
         "pass_rate": mean([float(r["pass"]) for r in runs]),
         "mean_score": mean([r.get("score") for r in runs]),
+        "mean_bugs_opened": mean([r.get("bugs_opened", 0) for r in runs]),
         "tamper_rate": mean([float(r["tampered"]) for r in runs]),
         "agent_error_rate": mean([float(r["agent_error"]) for r in runs]),
         "mean_cost_usd": mean([r["cost_usd"] for r in runs]),
@@ -66,8 +67,10 @@ def aggregate(records: list[dict], arms: list[str], tasks: list[str]) -> dict:
     per_task_score = {t: {a: mean([r.get("score") for r in by_arm[a] if r["task"] == t]) for a in arms}
                       for t in tasks}
     deltas = {a: arm_delta(by_arm, summary, a) for a in arms if a != "vanilla"} if "vanilla" in arms else {}
+    from ab_lean import lean_summary  # type: ignore[import]  # lazy: ab_lean imports this module
     return {"arms_summary": summary, "per_task": per_task, "per_task_score": per_task_score,
-            "delta": deltas.get("bigpowers"), "deltas_vs_vanilla": deltas}
+            "delta": deltas.get("bigpowers"), "deltas_vs_vanilla": deltas,
+            "lean": lean_summary(records, arms)}
 
 
 def arm_delta(by_arm: dict, summary: dict, arm: str) -> dict:
@@ -86,6 +89,8 @@ def arm_delta(by_arm: dict, summary: dict, arm: str) -> dict:
     if scores[arm] and scores["vanilla"]:
         delta["score"] = round(tr["mean_score"] - va["mean_score"], 4)
         delta["score_ci95"] = bootstrap_delta_ci(scores[arm], scores["vanilla"])
+    if tr.get("mean_bugs_opened") is not None and va.get("mean_bugs_opened") is not None:
+        delta["bugs_opened"] = round(tr["mean_bugs_opened"] - va["mean_bugs_opened"], 4)
     return delta
 
 
@@ -111,13 +116,19 @@ def write_reports(out: Path, meta: dict, result: dict, records: list[dict]) -> P
 
 
 def print_summary(result: dict) -> None:
-    print(f"{'arm':<10} {'pass':>6} {'score':>6} {'tamper':>7} {'cost$':>8} {'turns':>6} {'secs':>7}")
+    print(f"{'arm':<10} {'pass':>6} {'score':>6} {'bugs':>6} {'tamper':>7} {'cost$':>8} {'turns':>6} {'secs':>7}")
     for arm, s in result["arms_summary"].items():
         fmt = lambda v, p=2: "-" if v is None else f"{v:.{p}f}"
-        print(f"{arm:<10} {fmt(s['pass_rate']):>6} {fmt(s['mean_score']):>6} {fmt(s['tamper_rate']):>7} "
+        b_str = "-" if s.get("mean_bugs_opened") is None else f"{s['mean_bugs_opened']:.1f}"
+        print(f"{arm:<10} {fmt(s['pass_rate']):>6} {fmt(s['mean_score']):>6} {b_str:>6} {fmt(s['tamper_rate']):>7} "
               f"{fmt(s['mean_cost_usd'], 3):>8} {fmt(s['mean_turns'], 1):>6} {fmt(s['mean_duration_s'], 0):>7}")
     for arm, d in result.get("deltas_vs_vanilla", {}).items():
         line = f"delta {arm:<10} - vanilla: pass {d['pass_rate']:+.2f} CI {d['ci95']}"
         if "score" in d:
             line += f"  score {d['score']:+.2f} CI {d['score_ci95']}"
+        if "bugs_opened" in d:
+            line += f"  bugs {d['bugs_opened']:+.1f}"
+        print(line)
+    from ab_lean import lean_lines  # type: ignore[import]
+    for line in lean_lines(result.get("lean")):
         print(line)

package/scripts/lib/ab_vsm.py

@@ -0,0 +1,122 @@
+#!/usr/bin/env python3
+# story: e58s08
+"""Value stream map of one A/B run: credit each model turn's time, tokens and cost to the lean stage
+of the action it takes (plan, design, build, test, deploy, maintain; wrap-up when it takes none).
+
+Tools are only 1-3% of a run's time, so time is credited per turn, not per tool call. pi stamps an
+assistant message when it starts, so a turn lasts until the next assistant message starts: the model
+time plus the tools it ran.
+Rules: specs/benchmarks/ab/vsm.yaml.
+
+CLI: python3 scripts/lib/ab_vsm.py <trace.jsonl.gz>...   (one VSM line per saved transcript)
+"""
+from __future__ import annotations
+
+import gzip
+import json
+import re
+import sys
+from pathlib import Path
+
+import yaml
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+RULES_PATH = REPO_ROOT / "specs" / "benchmarks" / "ab" / "vsm.yaml"
+WRAP_UP = "wrap-up"
+
+
+def load_rules(path: Path = RULES_PATH) -> dict:
+    rules = yaml.safe_load(path.read_text())
+    rules["compiled"] = [(r["stage"], re.compile(r["tools"]), re.compile(r["target"])) for r in rules["rules"]]
+    return rules
+
+
+def classify_call(tool: str, args: dict | None, rules: dict) -> str:
+    args = args or {}
+    target = str(args.get("command") or args.get("path") or args.get("file_path") or "")
+    for stage, tools, pattern in rules["compiled"]:
+        if tools.search(tool or "") and pattern.search(target):
+            return stage
+    return "design"
+
+
+def turn_stage(message: dict, rules: dict) -> str:
+    calls = [c for c in message.get("content") or [] if isinstance(c, dict) and c.get("type") == "toolCall"]
+    stages = {classify_call(c.get("name", ""), c.get("arguments"), rules) for c in calls}
+    return next((s for s in rules["turn_priority"] if s in stages), WRAP_UP)
+
+
+def pi_messages(stdout: str) -> list[dict]:
+    messages = []
+    for line in stdout.splitlines():
+        try:
+            event = json.loads(line)
+        except json.JSONDecodeError:
+            continue
+        if isinstance(event, dict) and event.get("type") == "message_end":
+            message = event.get("message") or {}
+            if isinstance(message.get("timestamp"), (int, float)):
+                messages.append(message)
+    return messages
+
+
+def vsm_from_pi(stdout: str, rules: dict) -> dict | None:
+    messages = pi_messages(stdout)
+    if not any(m.get("role") == "assistant" for m in messages):
+        return None
+    start = messages[0]["timestamp"]
+    stages: dict[str, dict] = {}
+    sequence, first_build = [], None
+    turns = [m for m in messages if m.get("role") == "assistant"]
+    for i, message in enumerate(turns):
+        stage = turn_stage(message, rules)
+        end = turns[i + 1]["timestamp"] if i + 1 < len(turns) else messages[-1]["timestamp"]
+        usage = message.get("usage") or {}
+        slot = stages.setdefault(stage, {"s": 0.0, "cost_usd": 0.0, "output_tokens": 0, "turns": 0})
+        slot["s"] = round(slot["s"] + (end - message["timestamp"]) / 1000, 3)
+        slot["cost_usd"] = round(slot["cost_usd"] + ((usage.get("cost") or {}).get("total") or 0), 6)
+        slot["output_tokens"] += usage.get("output", 0)
+        slot["turns"] += 1
+        sequence.append(stage)
+        if stage == "build" and first_build is None:
+            first_build = round((message["timestamp"] - start) / 1000, 3)
+    return {
+        "lead_time_s": round((messages[-1]["timestamp"] - start) / 1000, 3),
+        "time_to_first_build_s": first_build,
+        "switches": sum(1 for a, b in zip(sequence, sequence[1:]) if a != b),
+        "sequence": sequence,
+        "stages": stages,
+    }
+
+
+def vsm_from(parser: str, stdout: str, rules: dict | None = None) -> dict | None:
+    """vsm block for a run record; None when the adapter's transcript format is not supported."""
+    if parser != "pi-json":
+        return None
+    return vsm_from_pi(stdout, rules or load_rules())
+
+
+def describe(name: str, block: dict | None) -> str:
+    if not block:
+        return f"{name:<32} no VSM (unsupported or empty transcript)"
+    lead = block["lead_time_s"] or 1
+    parts = sorted(block["stages"].items(), key=lambda kv: -kv[1]["s"])
+    shares = "  ".join(f"{stage} {v['s']:.0f}s {100 * v['s'] / lead:.0f}%" for stage, v in parts if v["turns"])
+    return f"{name:<32} lead {block['lead_time_s']:6.1f}s  first build {block['time_to_first_build_s']}s  " \
+           f"switches {block['switches']:>2}  | {shares}"
+
+
+def main(argv: list[str]) -> int:
+    if not argv:
+        print("usage: python3 scripts/lib/ab_vsm.py <trace.jsonl.gz>...")
+        return 2
+    rules = load_rules()
+    for name in argv:
+        path = Path(name)
+        text = gzip.decompress(path.read_bytes()).decode() if path.suffix == ".gz" else path.read_text()
+        print(describe(path.name, vsm_from_pi(text, rules)))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))

package/scripts/lib/ab_waste.py

@@ -0,0 +1,173 @@
+#!/usr/bin/env python3
+# story: e58s09 e58s10
+"""Lean waste metrics for one A/B run, from its final patch and its pi transcript.
+
+Patch (any adapter): changed lines per file class, the reference solution's size, extra files,
+leftover scratch files and the review load a person would face. Transcript (pi): relearning
+(rereads, repeated commands), defects and rework (failing test runs after the first build edit,
+re-edits), first-time-through and process cycle efficiency. See LEAN-VSM_LATEST.md and
+specs/benchmarks/ab/vsm.yaml (file classes reuse the stage rules; `leftovers` regex).
+"""
+from __future__ import annotations
+
+import functools
+import gzip
+import json
+import re
+import shutil
+import subprocess
+import tempfile
+from collections import Counter
+from pathlib import Path
+
+from ab_vsm import classify_call, load_rules, vsm_from_pi  # type: ignore[import]
+
+CLASS_OF_STAGE = {"build": "product", "test": "test", "maintain": "doc", "plan": "plan"}
+FAILED = re.compile(r'"exit_code":\s*[1-9]|(?:^|\\n|\n)\s*(?:ℹ|#)\s*fail\s+[1-9]|(?:^|\\n|\n)not ok ')
+
+
+def git(cwd: Path, *args: str) -> str:
+    res = subprocess.run(["git", "-c", "user.name=ab", "-c", "user.email=ab@local", *args],
+                         cwd=cwd, capture_output=True, text=True)
+    return res.stdout
+
+
+def file_class(path: str, rules: dict) -> str:
+    cls = CLASS_OF_STAGE.get(classify_call("edit", {"path": path}, rules), "product")
+    return "doc" if cls == "product" and path.endswith(".md") else cls
+
+
+def numstat(cwd: Path, base: str) -> dict[str, int]:
+    changed = {}
+    for line in git(cwd, "diff", "--cached", "--numstat", base).splitlines():
+        added, deleted, path = line.split("\t", 2)
+        changed[path] = sum(int(n) for n in (added, deleted) if n.isdigit())
+    return changed
+
+
+@functools.lru_cache(maxsize=None)
+def solution_files(task_dir: str) -> dict[str, int] | None:
+    """Changed lines per file of the reference solution, applied over the fixture."""
+    task = Path(task_dir)
+    if not (task / "solution").is_dir():
+        return None
+    work = Path(tempfile.mkdtemp(prefix="ab-solution-"))
+    try:
+        shutil.copytree(task / "fixture", work, dirs_exist_ok=True)
+        git(work, "init", "-q")
+        git(work, "add", "-A")
+        git(work, "commit", "-q", "--no-verify", "-m", "fixture")
+        shutil.copytree(task / "solution", work, dirs_exist_ok=True)
+        git(work, "add", "-A")
+        return numstat(work, "HEAD")
+    finally:
+        shutil.rmtree(work, ignore_errors=True)
+
+
+def patch_waste(workdir: Path, base: str, task_dir: Path, patch_path: Path | None, rules: dict) -> dict:
+    git(workdir, "add", "-A")
+    patch = git(workdir, "diff", "--cached", base)
+    if patch_path is not None:
+        patch_path.parent.mkdir(parents=True, exist_ok=True)
+        patch_path.write_bytes(gzip.compress(patch.encode()))
+    changed = numstat(workdir, base)
+    lines = Counter()
+    for path, n in changed.items():
+        lines[file_class(path, rules)] += n
+    added = [line[2:] for line in git(workdir, "diff", "--cached", "--name-status", base).splitlines()
+             if line.startswith("A\t")]
+    solution = solution_files(str(task_dir))
+    solution_lines = sum(solution.values()) if solution else None
+    product = [p for p in changed if file_class(p, rules) == "product"]
+    doc_words = sum(len(line[1:].split()) for line in md_added_lines(patch))
+    return {
+        "product_lines": lines["product"], "test_lines": lines["test"],
+        "doc_lines": lines["doc"], "plan_lines": lines["plan"],
+        "solution_lines": solution_lines,
+        "diff_ratio": round(lines["product"] / solution_lines, 3) if solution_lines else None,
+        "extra_files": sorted(p for p in product if solution is not None and p not in solution),
+        "leftovers": sorted(p for p in added if re.search(rules["leftovers"], p)),
+        "review_load": sum(changed.values()) + len(changed) + doc_words,
+    }
+
+
+def md_added_lines(patch: str) -> list[str]:
+    out, in_md = [], False
+    for line in patch.splitlines():
+        if line.startswith("+++ "):
+            in_md = line.endswith(".md")
+        elif in_md and line.startswith("+"):
+            out.append(line)
+    return out
+
+
+def tool_calls(stdout: str) -> list[dict]:
+    """Ordered tool calls with their stage and whether a test run failed."""
+    calls, by_id = [], {}
+    for line in stdout.splitlines():
+        try:
+            event = json.loads(line)
+        except json.JSONDecodeError:
+            continue
+        if not isinstance(event, dict):
+            continue
+        if event.get("type") == "tool_execution_start":
+            call = {"tool": event.get("toolName", ""), "args": event.get("args") or {}, "failed": False}
+            by_id[event.get("toolCallId")] = call
+            calls.append(call)
+        elif event.get("type") == "tool_execution_end" and event.get("toolCallId") in by_id:
+            by_id[event["toolCallId"]]["failed"] = bool(FAILED.search(json.dumps(event.get("result"), ensure_ascii=False)))
+    return calls
+
+
+def transcript_waste(stdout: str, rules: dict | None = None) -> dict | None:
+    rules = rules or load_rules()
+    vsm = vsm_from_pi(stdout, rules)
+    calls = tool_calls(stdout)
+    if vsm is None and not calls:
+        return None
+    for call in calls:
+        call["stage"] = classify_call(call["tool"], call["args"], rules)
+    reads = Counter(c["args"].get("path") for c in calls if c["tool"] == "read")
+    commands = Counter(c["args"].get("command") for c in calls if c["tool"] == "bash" and c["stage"] != "test")
+    edits = Counter(c["args"].get("path") for c in calls if c["tool"] in ("edit", "write"))
+    first_build = next((i for i, c in enumerate(calls) if c["stage"] == "build"), None)
+    runs = [c for c in calls if c["tool"] == "bash" and c["stage"] == "test"]
+    after = [c for c in calls[first_build:] if c in runs] if first_build is not None else []
+    rework = sum(c["failed"] for c in after)
+    redundant = redundant_runs(calls)
+    stages = (vsm or {}).get("stages", {})
+    lead = (vsm or {}).get("lead_time_s") or 0
+    value_add = sum(stages.get(s, {}).get("s", 0) for s in ("build", "test"))
+    return {
+        "rereads": sum(n - 1 for n in reads.values()),
+        "repeated_commands": sum(n - 1 for n in commands.values()),
+        "test_runs": len(runs), "redundant_test_runs": redundant, "rework_loops": rework,
+        "reedits": sum(n - 1 for n in edits.values()),
+        "ftt": bool(after) and rework == 0,
+        "pce": round(value_add / lead, 3) if lead else None,
+    }
+
+
+def redundant_runs(calls: list[dict]) -> int:
+    """Passing test runs with no file edited since the previous passing run (overprocessing, e58s10)."""
+    redundant, green, dirty = 0, False, False
+    for call in calls:
+        if call["tool"] in ("edit", "write"):
+            dirty = True
+        elif call["tool"] == "bash" and call["stage"] == "test" and not call["failed"]:
+            redundant += green and not dirty
+            green, dirty = True, False
+    return redundant
+
+
+EMPTY_TRANSCRIPT = dict.fromkeys(["rereads", "repeated_commands", "test_runs", "redundant_test_runs",
+                                  "rework_loops", "reedits", "ftt", "pce"])
+
+
+def waste_for(workdir: Path, base: str, stdout: str, parser: str, task_dir: Path,
+              patch_path: Path | None = None) -> dict:
+    """The run record's waste block; transcript metrics are None for non-pi adapters."""
+    rules = load_rules()
+    transcript = transcript_waste(stdout, rules) if parser == "pi-json" else None
+    return {**patch_waste(workdir, base, task_dir, patch_path, rules), **(transcript or EMPTY_TRANSCRIPT)}

package/scripts/test-adapter-render.sh

@@ -60,7 +60,7 @@ for adapter in scripts/adapters/*.sh; do
         IR_DESC_ESCAPED="test desc"; IR_MODEL=""
         declare -f render_skill >/dev/null || exit 1
         render_skill
-      ' _ "$PWD/$adapter" "$nm" >/dev/null 2>&1 )
+      ' _ "$PWD/$adapter" "$nm" </dev/null >/dev/null 2>&1 )
   }
 
   dest="$TMP_ROOT/$id"

package/scripts/test-test-lock-hook.sh

@@ -10,7 +10,7 @@ HOOK="$REPO_ROOT/hooks/pre-tool-use.sh"
 pass_count=0
 fail_count=0
 
-assert_denied() {
+lock_assert_denied() {
   local label="$1"
   local input="$2"
   local expected_substr="${3:-}"
@@ -30,7 +30,7 @@ assert_denied() {
   fi
 }
 
-assert_allowed() {
+lock_assert_allowed() {
   local label="$1"
   local input="$2"
   local output exit_code=0
@@ -64,7 +64,7 @@ git add .
 git commit -q -m "feat(app): initial commit"
 
 # 1. Without a RED commit yet during fix_bug, test edits should be allowed (to author the RED test)
-assert_allowed "Test file edit allowed when authoring RED test" \
+lock_assert_allowed "Test file edit allowed when authoring RED test" \
   '{"tool": "Edit", "file_path": "test/app.test.js"}'
 
 # 2. Commit a RED test
@@ -73,23 +73,23 @@ git add test/app.test.js
 git commit -q -m "test(app): red test reproducing defect"
 
 # 3. Now that a RED commit exists and active_flow is fix_bug, test edits must be LOCKED
-assert_denied "Test file edit denied after RED commit in fix_bug" \
+lock_assert_denied "Test file edit denied after RED commit in fix_bug" \
   '{"tool": "Edit", "file_path": "test/app.test.js"}' \
   "locked"
 
-assert_denied "Test file write denied after RED commit in fix_bug" \
+lock_assert_denied "Test file write denied after RED commit in fix_bug" \
   '{"tool": "Write", "file_path": "tests/test_other.py"}' \
   "locked"
 
 # 4. Source file edits must remain allowed
-assert_allowed "Source file edit allowed after RED commit in fix_bug" \
+lock_assert_allowed "Source file edit allowed after RED commit in fix_bug" \
   '{"tool": "Edit", "file_path": "src/app.js"}'
 
 # 5. When active_flow is not fix_bug, test edits are allowed
 cat > specs/state.yaml <<'YAML'
 active_flow: build_epic
 YAML
-assert_allowed "Test file edit allowed when active_flow is not fix_bug" \
+lock_assert_allowed "Test file edit allowed when active_flow is not fix_bug" \
   '{"tool": "Edit", "file_path": "test/app.test.js"}'
 
 echo ""

package/scripts/verify-adversarial.sh

@@ -109,9 +109,16 @@ fi
 DEFECTS_FOUND=0
 mkdir -p specs/verifications
 
+# A JS/TS source file that exists and is not a test: the files the boundary scan inspects.
+is_scannable_source() {
+  local file="$1"
+  [[ "$file" =~ \.(js|ts|mjs)$ && -f "$file" ]] || return 1
+  [[ ! "$file" =~ (^|/)test/ && ! "$file" =~ \.test\. ]]
+}
+
 # 2. Check for common unsafe property access patterns on JS/TS files
 for file in $CHANGED_FILES; do
-  if [[ "$file" =~ \.(js|ts|mjs)$ ]] && [[ -f "$file" ]] && [[ ! "$file" =~ (^|/)test/ ]] && [[ ! "$file" =~ \.test\. ]]; then
+  if is_scannable_source "$file"; then
     # Look for unsafe chained property access like item.brand.toLowerCase() or item.prop.subprop
     if grep -nE '\.[a-zA-Z0-9_]+\.(toLowerCase|toUpperCase|trim|length|slice)\(' "$file" | grep -v '\?\.' >/dev/null 2>&1; then
       echo "WARNING [boundary-mutation]: Potential unhandled undefined access without optional chaining in $file:"

package/tests/test_ab_eval.py

@@ -9,7 +9,6 @@ import importlib.util
 import io
 import json
 import shutil
-import subprocess
 import sys
 import tempfile
 import unittest
@@ -249,49 +248,6 @@ class TestCli(Sandbox):
         self.assertIsNone(report["delta"])
 
 
-@unittest.skipUnless(shutil.which("node"), "node is needed for the starter tasks")
-class TestStarterTasks(unittest.TestCase):
-    """Each shipped task must fail as given and pass with its reference solution."""
-
-    TASKS = REPO_ROOT / "specs" / "benchmarks" / "ab" / "tasks"
-
-    def grade_copy(self, task_dir: Path, with_solution: bool) -> bool:
-        task = yaml.safe_load((task_dir / "task.yaml").read_text())
-        work = Path(tempfile.mkdtemp(prefix="ab-starter-"))
-        try:
-            shutil.copytree(task_dir / "fixture", work, dirs_exist_ok=True)
-            if with_solution:
-                shutil.copytree(task_dir / "solution", work, dirs_exist_ok=True)
-            shutil.copytree(task_dir / "hidden", work, dirs_exist_ok=True)
-            res = subprocess.run(["bash", "-c", task["grade"]], cwd=work, capture_output=True)
-            return res.returncode == 0
-        finally:
-            shutil.rmtree(work, ignore_errors=True)
-
-    def test_eight_tasks_ship(self):
-        self.assertGreaterEqual(len(list(self.TASKS.glob("*/task.yaml"))), 8)
-
-    def test_fixture_fails_and_solution_passes(self):
-        for task_yaml in sorted(self.TASKS.glob("*/task.yaml")):
-            with self.subTest(task=task_yaml.parent.name):
-                self.assertFalse(self.grade_copy(task_yaml.parent, with_solution=False))
-                self.assertTrue(self.grade_copy(task_yaml.parent, with_solution=True))
-
-    def test_prompts_never_name_bigpowers(self):
-        for task_yaml in self.TASKS.glob("*/task.yaml"):
-            self.assertNotIn("bigpowers", yaml.safe_load(task_yaml.read_text())["prompt"].lower())
-
-    def test_grade_commands_never_pass_a_bare_directory(self):
-        """`node --test test/` exits 1 on node 24 (it resolves the directory as a module path), so
-        such a task can never pass however good the solution is, while a hidden-only `score`
-        command still looks healthy (BUG found in e58s07: ab-11 and ab-13 read pass=0 forever)."""
-        for task_yaml in self.TASKS.glob("*/task.yaml"):
-            with self.subTest(task=task_yaml.parent.name):
-                for token in yaml.safe_load(task_yaml.read_text())["grade"].split():
-                    self.assertNotIn(token.rstrip("/"), ("test", "test-hidden"),
-                                     f"{task_yaml.parent.name} grades a directory, not a glob")
-
-
 class TestRunBenchmarkHonesty(unittest.TestCase):
     def test_without_skill_arm_is_unmeasured_not_zero(self):
         sc = {"id": "s1", "grader": {"type": "code", "command": "true"}}

package/tests/test_ab_lean.py

@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+# story: e58s10
+"""tests/test_ab_lean.py — lean views: per-arm VSM and waste summary, deltas, bars, redundant tests.
+
+Run: python3 tests/test_ab_lean.py
+"""
+import sys
+import unittest
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from test_ab_eval import load_module, rep  # noqa: E402
+from test_ab_waste import Stream, waste  # noqa: E402
+
+lean = load_module("ab_lean", "ab_lean.py")
+
+
+def run(arm, lead, build, test, passed=True, cost=0.01, test_runs=3, rework=0):
+    stages = {"plan": {"s": lead * 0.1}, "design": {"s": lead * 0.2}, "build": {"s": build}, "test": {"s": test}}
+    vsm = {"lead_time_s": lead, "time_to_first_build_s": lead * 0.3, "switches": 4, "stages": stages}
+    w = {"test_runs": test_runs, "redundant_test_runs": 1, "rework_loops": rework, "reedits": 0,
+         "rereads": 0, "repeated_commands": 0, "review_load": 50, "diff_ratio": 1.2, "ftt": rework == 0,
+         "pce": round((build + test) / lead, 3), "leftovers": [], "extra_files": []}
+    return {"task": "t", "arm": arm, "pass": passed, "score": None, "tampered": False, "agent_error": False,
+            "cost_usd": cost, "input_tokens": None, "output_tokens": None, "turns": 5, "duration_s": lead,
+            "diff_lines": 5, "vsm": vsm, "waste": w}
+
+
+def records():
+    return ([run("vanilla", 70, 28, 21) for _ in range(5)]
+            + [run("bigpowers", 140, 20, 70, cost=0.02, test_runs=8, rework=1) for _ in range(5)])
+
+
+class TestLeanSummary(unittest.TestCase):
+    def test_per_arm_summary_and_deltas(self):
+        summary = lean.lean_summary(records(), ["vanilla", "bigpowers"])
+        va, bp = summary["arms"]["vanilla"], summary["arms"]["bigpowers"]
+        self.assertEqual(va["lead_time_s"], 70)
+        self.assertEqual(round(va["stage_share"]["build"], 2), 0.4)
+        self.assertEqual(bp["waste"]["test_runs"], 8)
+        self.assertEqual(bp["ftt_rate"], 0.0)
+        self.assertEqual(va["cost_per_pass"], 0.01)
+        d = summary["deltas"]["bigpowers"]
+        self.assertEqual(d["lead_time_s"]["delta"], 70)
+        self.assertEqual(d["test_runs"]["delta"], 5)
+        self.assertEqual(len(d["build_share"]["ci95"]), 2)
+
+    def test_old_records_give_none(self):
+        old = [{k: v for k, v in r.items() if k not in ("vsm", "waste")} for r in records()]
+        self.assertIsNone(lean.lean_summary(old, ["vanilla", "bigpowers"]))
+        self.assertEqual(lean.lean_lines(None), [])
+
+    def test_report_carries_lean(self):
+        result = rep.aggregate(records(), ["vanilla", "bigpowers"], ["t"])
+        self.assertIn("bigpowers", result["lean"]["deltas"])
+
+
+class TestViews(unittest.TestCase):
+    def test_bars_scale_with_lead_time(self):
+        lines = lean.lean_lines(lean.lean_summary(records(), ["vanilla", "bigpowers"]))
+        bars = {line.split()[0]: line for line in lines if line.split() and line.split()[0] in ("vanilla", "bigpowers")}
+        va_len = bars["vanilla"].count("|") and len(bars["vanilla"].split("|")[1].rstrip())
+        bp_len = len(bars["bigpowers"].split("|")[1].rstrip())
+        self.assertAlmostEqual(bp_len / va_len, 2, delta=0.2)
+        self.assertIn("T", bars["bigpowers"])
+        self.assertTrue(any("rework" in line for line in lines))
+
+
+class TestRedundantTestRuns(unittest.TestCase):
+    def test_repeated_green_run_without_edit_is_redundant(self):
+        stream = (Stream()
+                  .call("edit", {"path": "/work/src/a.js"})
+                  .call("bash", {"command": "npm test"}, "# pass 3\n# fail 0\n")
+                  .call("bash", {"command": "npm test 2>&1 | tail -3"}, "# pass 3\n# fail 0\n")
+                  .call("edit", {"path": "/work/src/a.js"})
+                  .call("bash", {"command": "npm test"}, "# pass 3\n# fail 0\n"))
+        self.assertEqual(waste.transcript_waste(stream.text())["redundant_test_runs"], 1)
+
+
+if __name__ == "__main__":
+    unittest.main(verbosity=1)

package/tests/test_ab_score.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# story: e58s03
+# story: e58s03 e58s11
 """tests/test_ab_score.py — partial-credit scoring and the hard tasks (ab-11, ab-13).
 
 Run: python3 tests/test_ab_score.py
@@ -18,7 +18,17 @@ sys.path.insert(0, str(Path(__file__).resolve().parent))
 from test_ab_eval import REPO_ROOT, Sandbox, met, rep  # noqa: E402
 
 TASKS = REPO_ROOT / "specs" / "benchmarks" / "ab" / "tasks"
-HARD_TASKS = ["ab-11-ledger-rounding", "ab-13-orders-split"]
+HARD_TASKS = [
+    "ab-09-inventory-reservations",
+    "ab-11-ledger-rounding",
+    "ab-12-todo-due-dates",
+    "ab-13-orders-split",
+    "ab-15-config-precedence",
+    "ab-16-site-tags-rss",
+    "ab-17-payment-alias",
+    "ab-19-catalog-segregation",
+    "ab-20-brand-discount",
+]
 
 
 class TestTapScore(unittest.TestCase):

package/tests/test_ab_tasks.py

@@ -0,0 +1,75 @@
+#!/usr/bin/env python3
+# story: e58s01 e58s02 e58s07 e58s09
+"""tests/test_ab_tasks.py — every shipped A/B task is calibrated: its fixture fails, its reference
+solution passes, its prompt never names bigpowers, and its grade command uses file globs.
+
+Run: python3 tests/test_ab_tasks.py
+Split from test_ab_eval.py (300-line cap).
+"""
+import json
+import shutil
+import subprocess
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+import yaml
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from test_ab_eval import REPO_ROOT  # noqa: E402
+
+
+@unittest.skipUnless(shutil.which("node"), "node is needed for the starter tasks")
+class TestStarterTasks(unittest.TestCase):
+    """Each shipped task must fail as given and pass with its reference solution."""
+
+    TASKS = REPO_ROOT / "specs" / "benchmarks" / "ab" / "tasks"
+
+    def grade_copy(self, task_dir: Path, with_solution: bool) -> bool:
+        task = yaml.safe_load((task_dir / "task.yaml").read_text())
+        work = Path(tempfile.mkdtemp(prefix="ab-starter-"))
+        try:
+            shutil.copytree(task_dir / "fixture", work, dirs_exist_ok=True)
+            if with_solution:
+                shutil.copytree(task_dir / "solution", work, dirs_exist_ok=True)
+            shutil.copytree(task_dir / "hidden", work, dirs_exist_ok=True)
+            res = subprocess.run(["bash", "-c", task["grade"]], cwd=work, capture_output=True)
+            return res.returncode == 0
+        finally:
+            shutil.rmtree(work, ignore_errors=True)
+
+    def test_eight_tasks_ship(self):
+        self.assertGreaterEqual(len(list(self.TASKS.glob("*/task.yaml"))), 8)
+
+    def test_fixture_fails_and_solution_passes(self):
+        for task_yaml in sorted(self.TASKS.glob("*/task.yaml")):
+            with self.subTest(task=task_yaml.parent.name):
+                self.assertFalse(self.grade_copy(task_yaml.parent, with_solution=False))
+                self.assertTrue(self.grade_copy(task_yaml.parent, with_solution=True))
+
+    def test_prompts_never_name_bigpowers(self):
+        for task_yaml in self.TASKS.glob("*/task.yaml"):
+            self.assertNotIn("bigpowers", yaml.safe_load(task_yaml.read_text())["prompt"].lower())
+
+    def test_fixture_test_scripts_never_pass_a_bare_directory(self):
+        """`npm test` must work in every fixture: a `node --test test/` script fails on node 24, and
+        agents then spend the run repairing the harness (found by e58s09 waste metrics)."""
+        for package in self.TASKS.glob("*/fixture/package.json"):
+            with self.subTest(task=package.parent.parent.name):
+                script = json.loads(package.read_text()).get("scripts", {}).get("test", "")
+                self.assertNotRegex(script, r"--test\s+\S*/(\s|$)", script)
+
+    def test_grade_commands_never_pass_a_bare_directory(self):
+        """`node --test test/` exits 1 on node 24 (it resolves the directory as a module path), so
+        such a task can never pass however good the solution is, while a hidden-only `score`
+        command still looks healthy (BUG found in e58s07: ab-11 and ab-13 read pass=0 forever)."""
+        for task_yaml in self.TASKS.glob("*/task.yaml"):
+            with self.subTest(task=task_yaml.parent.name):
+                for token in yaml.safe_load(task_yaml.read_text())["grade"].split():
+                    self.assertNotIn(token.rstrip("/"), ("test", "test-hidden"),
+                                     f"{task_yaml.parent.name} grades a directory, not a glob")
+
+
+if __name__ == "__main__":
+    unittest.main(verbosity=1)

package/tests/test_ab_vsm.py

@@ -0,0 +1,109 @@
+#!/usr/bin/env python3
+# story: e58s08
+"""tests/test_ab_vsm.py — value stream map per run: stage rules, per-turn time, run records.
+
+Run: python3 tests/test_ab_vsm.py
+"""
+import json
+import sys
+import unittest
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from test_ab_eval import Sandbox, load_module  # noqa: E402
+
+vsm = load_module("ab_vsm", "ab_vsm.py")
+STAGES = {"plan", "design", "build", "test", "deploy", "maintain", "wrap-up"}
+
+
+def assistant(ts_s, cost=0.001, out=10, calls=()):
+    content = [{"type": "toolCall", "name": tool, "arguments": args} for tool, args in calls]
+    usage = {"input": 100, "output": out, "cacheRead": 0, "cacheWrite": 0, "cost": {"total": cost}}
+    return {"type": "message_end", "message": {"role": "assistant", "timestamp": int(ts_s * 1000),
+                                               "content": content, "usage": usage}}
+
+
+def tool_result(ts_s):
+    return {"type": "message_end", "message": {"role": "toolResult", "timestamp": int(ts_s * 1000)}}
+
+
+def transcript():
+    events = [
+        {"type": "session", "id": "s"},
+        {"type": "message_end", "message": {"role": "user", "timestamp": 0}},
+        assistant(0, calls=[("read", {"path": "/work/ISSUE.md"})]), tool_result(9),
+        assistant(10, calls=[("bash", {"command": "cat src/reports.js"})]), tool_result(24),
+        assistant(25, calls=[("edit", {"path": "/work/src/reports.js"})]), tool_result(39),
+        assistant(40, calls=[("bash", {"command": "npm test 2>&1 | tail"})]), tool_result(49),
+        assistant(50, calls=[("bash", {"command": "git commit -am fix"})]), tool_result(54),
+        assistant(55, out=40),
+        {"type": "agent_settled", "aborted": False},
+    ]
+    return "\n".join(json.dumps(e) for e in events)
+
+
+class TestRules(unittest.TestCase):
+    def setUp(self):
+        self.rules = vsm.load_rules()
+
+    def test_rules_file_uses_known_stages(self):
+        self.assertTrue({r["stage"] for r in self.rules["rules"]} <= STAGES)
+        self.assertTrue(set(self.rules["turn_priority"]) <= STAGES)
+
+    def test_calls_are_classified(self):
+        cases = [
+            ("read", {"path": "/work/ISSUE.md"}, "plan"),
+            ("bash", {"command": "sed -n 1,80p .agents/skills/fix-bug/SKILL.md"}, "plan"),
+            ("read", {"path": "/work/src/reports.js"}, "design"),
+            ("edit", {"path": "/work/src/reports.js"}, "build"),
+            ("edit", {"path": "/work/test/ledger.test.js"}, "test"),
+            ("bash", {"command": "npm test 2>&1 | tail -20"}, "test"),
+            ("bash", {"command": "node --test test/"}, "test"),
+            ("bash", {"command": "git checkout -b fix/rounding"}, "deploy"),
+            ("bash", {"command": "git commit -am 'fix'"}, "deploy"),
+            ("write", {"path": "/work/specs/bugs/BUG-001.md"}, "maintain"),
+            ("edit", {"path": "/work/docs/LEDGER.md"}, "maintain"),
+            ("write", {"path": "/work/specs/plan.md"}, "plan"),
+        ]
+        for tool, args, expected in cases:
+            with self.subTest(tool=tool, args=args):
+                self.assertEqual(vsm.classify_call(tool, args, self.rules), expected)
+
+
+class TestStageTime(unittest.TestCase):
+    def test_turn_time_goes_to_the_stage_of_its_action(self):
+        block = vsm.vsm_from("pi-json", transcript())
+        seconds = {stage: round(v["s"], 1) for stage, v in block["stages"].items()}
+        self.assertEqual(seconds, {"plan": 10.0, "design": 15.0, "build": 15.0, "test": 10.0,
+                                   "deploy": 5.0, "wrap-up": 0.0})
+        self.assertEqual(block["lead_time_s"], 55.0)
+        self.assertEqual(block["time_to_first_build_s"], 25.0)
+        self.assertEqual(block["switches"], 5)
+        self.assertEqual(block["sequence"], ["plan", "design", "build", "test", "deploy", "wrap-up"])
+        self.assertEqual(block["stages"]["wrap-up"]["output_tokens"], 40)
+        self.assertAlmostEqual(sum(v["cost_usd"] for v in block["stages"].values()), 0.006)
+
+    def test_build_wins_a_turn_with_several_calls(self):
+        events = [assistant(0, calls=[("read", {"path": "/work/ISSUE.md"}), ("edit", {"path": "/work/src/a.js"})]),
+                  tool_result(5)]
+        block = vsm.vsm_from("pi-json", "\n".join(json.dumps(e) for e in events))
+        self.assertEqual(block["sequence"], ["build"])
+
+    def test_unsupported_or_empty_is_none(self):
+        self.assertIsNone(vsm.vsm_from("none", transcript()))
+        self.assertIsNone(vsm.vsm_from("pi-json", "not json"))
+
+
+class TestRunRecord(Sandbox):
+    def test_record_carries_vsm(self):
+        script = self.tmp / "transcript.jsonl"
+        script.write_text(transcript() + "\n")
+        self.run_main("--adapter-cmd", f"cat {script}; echo 42 > answer.txt", "--parser", "pi-json",
+                      "--arms", "vanilla")
+        rec = self.records()[0]
+        self.assertEqual(rec["vsm"]["stages"]["build"]["turns"], 1)
+        self.assertEqual(rec["vsm"]["lead_time_s"], 55.0)
+
+
+if __name__ == "__main__":
+    unittest.main(verbosity=1)

package/tests/test_ab_waste.py

@@ -0,0 +1,106 @@
+#!/usr/bin/env python3
+# story: e58s09
+"""tests/test_ab_waste.py — lean waste metrics: relearning, rework, first-time-through, patch metrics.
+
+Run: python3 tests/test_ab_waste.py
+"""
+import gzip
+import json
+import sys
+import unittest
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from test_ab_eval import Sandbox, load_module  # noqa: E402
+
+waste = load_module("ab_waste", "ab_waste.py")
+
+
+class Stream:
+    """Builds a pi JSON transcript: assistant turns with one tool call each, plus tool results."""
+
+    def __init__(self):
+        self.events, self.t, self.n = [], 0, 0
+
+    def call(self, tool, args, result_text="ok", exit_code=0):
+        self.n += 1
+        cid = f"c{self.n}"
+        self.events.append({"type": "message_end", "message": {
+            "role": "assistant", "timestamp": self.t * 1000, "usage": {"output": 5, "cost": {"total": 0.001}},
+            "content": [{"type": "toolCall", "name": tool, "arguments": args}]}})
+        self.events.append({"type": "tool_execution_start", "toolCallId": cid, "toolName": tool, "args": args})
+        result = {"content": [{"type": "text", "text": result_text}], "details": {"exit_code": exit_code}}
+        self.events.append({"type": "tool_execution_end", "toolCallId": cid, "toolName": tool,
+                            "result": result, "isError": False})
+        self.events.append({"type": "message_end", "message": {"role": "toolResult",
+                                                               "timestamp": (self.t + 1) * 1000}})
+        self.t += 10
+        return self
+
+    def text(self):
+        return "\n".join(json.dumps(e) for e in self.events)
+
+
+class TestTranscriptWaste(unittest.TestCase):
+    def test_relearning_and_rework_are_counted(self):
+        stream = (Stream()
+                  .call("read", {"path": "/work/src/a.js"})
+                  .call("read", {"path": "/work/src/a.js"})
+                  .call("bash", {"command": "ls"})
+                  .call("bash", {"command": "ls"})
+                  .call("edit", {"path": "/work/src/a.js"})
+                  .call("bash", {"command": "npm test 2>&1 | tail"}, "ℹ pass 3\nℹ fail 1\n")
+                  .call("edit", {"path": "/work/src/a.js"})
+                  .call("bash", {"command": "npm test 2>&1 | tail"}, "ℹ pass 4\nℹ fail 0\n"))
+        got = waste.transcript_waste(stream.text())
+        self.assertEqual((got["rereads"], got["repeated_commands"], got["test_runs"], got["rework_loops"],
+                          got["reedits"], got["ftt"]), (1, 1, 2, 1, 1, False))
+
+    def test_non_zero_exit_is_a_failing_run(self):
+        stream = Stream().call("edit", {"path": "/work/src/a.js"}).call(
+            "bash", {"command": "node --test test/"}, "boom", exit_code=1)
+        self.assertEqual(waste.transcript_waste(stream.text())["rework_loops"], 1)
+
+    def test_clean_run_is_first_time_through(self):
+        stream = Stream().call("edit", {"path": "/work/src/a.js"}).call(
+            "bash", {"command": "npm test"}, "# pass 5\n# fail 0\n")
+        got = waste.transcript_waste(stream.text())
+        self.assertEqual((got["rework_loops"], got["ftt"]), (0, True))
+        self.assertGreater(got["pce"], 0)
+
+    def test_no_transcript_gives_none(self):
+        self.assertIsNone(waste.transcript_waste("not json"))
+
+
+class TestPatchWaste(Sandbox):
+    AGENT = ("echo 42 > answer.txt; printf 'a\\nb\\n' > extra.js; "
+             "printf 'one two three\\n' > notes.md; echo x > debug.tmp.js")
+
+    def setUp(self):
+        super().setUp()
+        solution = self.tasks_dir / "t-01" / "solution"
+        solution.mkdir()
+        (solution / "answer.txt").write_text("42\n")
+
+    def test_patch_metrics_and_saved_patch(self):
+        self.run_main("--adapter-cmd", self.AGENT, "--arms", "vanilla")
+        got = self.records()[0]["waste"]
+        self.assertEqual(got["solution_lines"], 1)
+        self.assertIn("extra.js", got["extra_files"])
+        self.assertNotIn("answer.txt", got["extra_files"])
+        self.assertEqual(got["leftovers"], ["debug.tmp.js"])
+        self.assertEqual(got["doc_lines"], 1)
+        self.assertGreater(got["diff_ratio"], 1)
+        self.assertGreaterEqual(got["review_load"], got["product_lines"] + 3)
+        patch = self.out / "traces" / "t-01-vanilla-1.patch.gz"
+        self.assertIn("extra.js", gzip.decompress(patch.read_bytes()).decode())
+
+    def test_non_pi_adapter_has_patch_metrics_only(self):
+        self.run_main("--adapter-cmd", self.AGENT, "--arms", "vanilla")
+        got = self.records()[0]["waste"]
+        self.assertIsNone(got["rework_loops"])
+        self.assertIsNotNone(got["review_load"])
+
+
+if __name__ == "__main__":
+    unittest.main(verbosity=1)