bigpowers 2.91.0 → 2.92.0

package/scripts/lib/ab_lean.py

@@ -0,0 +1,116 @@
+#!/usr/bin/env python3
+# story: e58s10
+"""Lean views of an A/B run: each arm's mean value stream (stage time and share) and waste metrics,
+deltas against vanilla with seeded bootstrap CIs, and text VSM bars plus a waste table.
+
+Reads the `vsm` (e58s08) and `waste` (e58s09) blocks of run records; runs without them are skipped.
+"""
+from __future__ import annotations
+
+from ab_report import bootstrap_delta_ci, mean  # type: ignore[import]
+
+STAGES = ["plan", "design", "build", "test", "deploy", "maintain"]
+LETTER = {"plan": "P", "design": "D", "build": "B", "test": "T", "deploy": "R", "maintain": "M"}
+WASTE_KEYS = ["test_runs", "redundant_test_runs", "rework_loops", "reedits", "rereads",
+              "repeated_commands", "review_load", "diff_ratio", "pce"]
+# (key, label, per-run extractor) for the deltas against vanilla.
+DELTAS = [
+    ("lead_time_s", "lead time (s)", lambda r: r["vsm"]["lead_time_s"]),
+    ("time_to_first_build_s", "first build (s)", lambda r: r["vsm"]["time_to_first_build_s"]),
+    ("switches", "stage switches", lambda r: r["vsm"]["switches"]),
+    ("build_share", "build share", lambda r: share(r, "build")),
+    ("test_share", "test share", lambda r: share(r, "test")),
+    ("test_runs", "test runs", lambda r: r["waste"].get("test_runs")),
+    ("redundant_test_runs", "redundant test runs", lambda r: r["waste"].get("redundant_test_runs")),
+    ("rework_loops", "rework loops", lambda r: r["waste"].get("rework_loops")),
+    ("reedits", "re-edits", lambda r: r["waste"].get("reedits")),
+    ("review_load", "review load", lambda r: r["waste"].get("review_load")),
+    ("diff_ratio", "diff ratio", lambda r: r["waste"].get("diff_ratio")),
+    ("ftt", "first-time-through", lambda r: None if r["waste"].get("ftt") is None else float(r["waste"]["ftt"])),
+    ("pce", "PCE", lambda r: r["waste"].get("pce")),
+]
+
+
+def share(run: dict, stage: str) -> float | None:
+    lead = run["vsm"]["lead_time_s"]
+    return run["vsm"]["stages"].get(stage, {}).get("s", 0) / lead if lead else None
+
+
+def lean_runs(records: list[dict], arm: str) -> list[dict]:
+    return [r for r in records if r["arm"] == arm and r.get("vsm") and r.get("waste")]
+
+
+def arm_lean(runs: list[dict], all_runs: list[dict]) -> dict:
+    passes = sum(r["pass"] for r in all_runs)
+    cost = sum(r.get("cost_usd") or 0 for r in all_runs)
+    return {
+        "runs": len(runs),
+        "lead_time_s": mean([r["vsm"]["lead_time_s"] for r in runs]),
+        "time_to_first_build_s": mean([r["vsm"]["time_to_first_build_s"] for r in runs]),
+        "switches": mean([r["vsm"]["switches"] for r in runs]),
+        "stage_s": {s: mean([r["vsm"]["stages"].get(s, {}).get("s", 0) for r in runs]) for s in STAGES},
+        "stage_share": {s: mean([share(r, s) for r in runs]) for s in STAGES},
+        "waste": {k: mean([r["waste"].get(k) for r in runs]) for k in WASTE_KEYS},
+        "ftt_rate": mean([None if r["waste"].get("ftt") is None else float(r["waste"]["ftt"]) for r in runs]),
+        "leftover_rate": mean([float(bool(r["waste"].get("leftovers"))) for r in runs]),
+        "cost_per_pass": round(cost / passes, 6) if passes else None,
+    }
+
+
+def lean_summary(records: list[dict], arms: list[str]) -> dict | None:
+    by_arm = {a: lean_runs(records, a) for a in arms}
+    if not any(by_arm.values()):
+        return None
+    summary = {a: arm_lean(runs, [r for r in records if r["arm"] == a]) for a, runs in by_arm.items() if runs}
+    deltas = {}
+    if "vanilla" in summary:
+        for arm in summary:
+            if arm == "vanilla":
+                continue
+            deltas[arm] = {}
+            for key, _, get in DELTAS:
+                tr = [v for v in map(get, by_arm[arm]) if v is not None]
+                va = [v for v in map(get, by_arm["vanilla"]) if v is not None]
+                if tr and va:
+                    deltas[arm][key] = {"delta": round(sum(tr) / len(tr) - sum(va) / len(va), 4),
+                                        "ci95": bootstrap_delta_ci(tr, va)}
+    return {"arms": summary, "deltas": deltas}
+
+
+def bar(arm: dict, scale: float) -> str:
+    """Stage letters, one per `scale` seconds of mean stage time: the bar's length is the lead time."""
+    return "".join(LETTER[s] * round((arm["stage_s"][s] or 0) / scale) for s in STAGES)
+
+
+def fmt(value, places=2) -> str:
+    return "-" if value is None else f"{value:.{places}f}"
+
+
+def lean_lines(summary: dict | None, width: int = 50) -> list[str]:
+    if not summary:
+        return []
+    arms = summary["arms"]
+    scale = max((a["lead_time_s"] or 0) for a in arms.values()) / width or 1
+    out = ["", "value stream (bar length = mean lead time; P plan, D design, B build, T test, R release, M maintain):"]
+    for name, a in arms.items():
+        out.append(f"  {name:<10} |{bar(a, scale):<{width}}| {fmt(a['lead_time_s'], 0):>5} s  "
+                   f"first build {fmt(a['time_to_first_build_s'], 0)} s  PCE {fmt(a['waste']['pce'])}  "
+                   f"FTT {fmt(a['ftt_rate'])}  $/pass {fmt(a['cost_per_pass'], 4)}")
+    out += ["", f"  {'waste (mean per run)':<22}" + "".join(f"{n:>11}" for n in arms) + "   delta vs vanilla [95% CI]"]
+    for key, label, _ in DELTAS:
+        if key in ("lead_time_s", "time_to_first_build_s"):
+            continue
+        values = "".join(f"{fmt(row_value(a, key)):>11}" for a in arms.values())
+        ds = "  ".join(f"{n} {d[key]['delta']:+.2f} {d[key]['ci95']}" for n, d in summary["deltas"].items() if key in d)
+        out.append(f"  {label:<22}{values}   {ds}")
+    return out
+
+
+def row_value(arm: dict, key: str):
+    if key == "switches":
+        return arm["switches"]
+    if key in ("build_share", "test_share"):
+        return arm["stage_share"][key.split("_")[0]]
+    if key == "ftt":
+        return arm["ftt_rate"]
+    return arm["waste"].get(key)

package/scripts/lib/ab_metrics.py

@@ -121,3 +121,11 @@ def tap_score(stdout: str) -> float | None:
     passed = sum(int(n) for n in re.findall(r"^# pass (\d+)", stdout, re.M))
     failed = sum(int(n) for n in re.findall(r"^# (?:fail|cancelled) (\d+)", stdout, re.M))
     return round(passed / (passed + failed), 4) if passed + failed else None
+
+
+def tap_failures(stdout: str) -> int:
+    return sum(int(n) for n in re.findall(r"^# (?:fail|cancelled) (\d+)", stdout, re.M))
+
+
+def tap_failed_names(stdout: str) -> list[str]:
+    return [m.strip() for m in re.findall(r"^not ok \d+ - (.+)$", stdout, re.M)]

package/scripts/lib/ab_progress.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# story: e58s05
+# story: e58s05 e58s10
 """Live view of an A/B eval while it runs: reads every ab-live.jsonl under the given folders
 (one per shard) and prints runs, pass rate, score and cost per task and arm, each arm's delta
 against vanilla with its bootstrap CI, and how those deltas evolved as runs accumulated.
@@ -16,6 +16,7 @@ from datetime import datetime
 from pathlib import Path
 
 sys.path.insert(0, str(Path(__file__).resolve().parent))
+from ab_lean import lean_lines  # type: ignore[import]  # noqa: E402
 from ab_report import aggregate, mean  # type: ignore[import]  # noqa: E402
 
 CHECKPOINTS = (5, 10, 20, 30, 50, 75, 100)
@@ -79,6 +80,7 @@ def render(records: list[dict]) -> str:
         out += ["", "deltas (all runs so far, 95% CI):"]
         out += [delta_line(n, d) for n, d in result["deltas_vs_vanilla"].items()]
         out += ["", "evolution (pass and score delta by run index):"] + evolution(records, tasks, arms)
+    out += lean_lines(result.get("lean"))
     return "\n".join(out)
 
 

package/scripts/lib/ab_report.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# story: e58s01 e58s03 e58s04
+# story: e58s01 e58s03 e58s04 e58s10
 """A/B eval aggregation and reports — pure functions over run records (no agent, no subprocess
 except reading the bigpowers git SHA). Split from ab-eval.py to keep both under the 300-line cap."""
 from __future__ import annotations
@@ -47,6 +47,7 @@ def arm_summary(runs: list[dict]) -> dict:
         "runs": len(runs),
         "pass_rate": mean([float(r["pass"]) for r in runs]),
         "mean_score": mean([r.get("score") for r in runs]),
+        "mean_bugs_opened": mean([r.get("bugs_opened", 0) for r in runs]),
         "tamper_rate": mean([float(r["tampered"]) for r in runs]),
         "agent_error_rate": mean([float(r["agent_error"]) for r in runs]),
         "mean_cost_usd": mean([r["cost_usd"] for r in runs]),
@@ -66,8 +67,10 @@ def aggregate(records: list[dict], arms: list[str], tasks: list[str]) -> dict:
     per_task_score = {t: {a: mean([r.get("score") for r in by_arm[a] if r["task"] == t]) for a in arms}
                       for t in tasks}
     deltas = {a: arm_delta(by_arm, summary, a) for a in arms if a != "vanilla"} if "vanilla" in arms else {}
+    from ab_lean import lean_summary  # type: ignore[import]  # lazy: ab_lean imports this module
     return {"arms_summary": summary, "per_task": per_task, "per_task_score": per_task_score,
-            "delta": deltas.get("bigpowers"), "deltas_vs_vanilla": deltas}
+            "delta": deltas.get("bigpowers"), "deltas_vs_vanilla": deltas,
+            "lean": lean_summary(records, arms)}
 
 
 def arm_delta(by_arm: dict, summary: dict, arm: str) -> dict:
@@ -86,6 +89,8 @@ def arm_delta(by_arm: dict, summary: dict, arm: str) -> dict:
     if scores[arm] and scores["vanilla"]:
         delta["score"] = round(tr["mean_score"] - va["mean_score"], 4)
         delta["score_ci95"] = bootstrap_delta_ci(scores[arm], scores["vanilla"])
+    if tr.get("mean_bugs_opened") is not None and va.get("mean_bugs_opened") is not None:
+        delta["bugs_opened"] = round(tr["mean_bugs_opened"] - va["mean_bugs_opened"], 4)
     return delta
 
 
@@ -111,13 +116,19 @@ def write_reports(out: Path, meta: dict, result: dict, records: list[dict]) -> P
 
 
 def print_summary(result: dict) -> None:
-    print(f"{'arm':<10} {'pass':>6} {'score':>6} {'tamper':>7} {'cost$':>8} {'turns':>6} {'secs':>7}")
+    print(f"{'arm':<10} {'pass':>6} {'score':>6} {'bugs':>6} {'tamper':>7} {'cost$':>8} {'turns':>6} {'secs':>7}")
     for arm, s in result["arms_summary"].items():
         fmt = lambda v, p=2: "-" if v is None else f"{v:.{p}f}"
-        print(f"{arm:<10} {fmt(s['pass_rate']):>6} {fmt(s['mean_score']):>6} {fmt(s['tamper_rate']):>7} "
+        b_str = "-" if s.get("mean_bugs_opened") is None else f"{s['mean_bugs_opened']:.1f}"
+        print(f"{arm:<10} {fmt(s['pass_rate']):>6} {fmt(s['mean_score']):>6} {b_str:>6} {fmt(s['tamper_rate']):>7} "
               f"{fmt(s['mean_cost_usd'], 3):>8} {fmt(s['mean_turns'], 1):>6} {fmt(s['mean_duration_s'], 0):>7}")
     for arm, d in result.get("deltas_vs_vanilla", {}).items():
         line = f"delta {arm:<10} - vanilla: pass {d['pass_rate']:+.2f} CI {d['ci95']}"
         if "score" in d:
             line += f"  score {d['score']:+.2f} CI {d['score_ci95']}"
+        if "bugs_opened" in d:
+            line += f"  bugs {d['bugs_opened']:+.1f}"
+        print(line)
+    from ab_lean import lean_lines  # type: ignore[import]
+    for line in lean_lines(result.get("lean")):
         print(line)

package/scripts/lib/ab_vsm.py

@@ -0,0 +1,122 @@
+#!/usr/bin/env python3
+# story: e58s08
+"""Value stream map of one A/B run: credit each model turn's time, tokens and cost to the lean stage
+of the action it takes (plan, design, build, test, deploy, maintain; wrap-up when it takes none).
+
+Tools are only 1-3% of a run's time, so time is credited per turn, not per tool call. pi stamps an
+assistant message when it starts, so a turn lasts until the next assistant message starts: the model
+time plus the tools it ran.
+Rules: specs/benchmarks/ab/vsm.yaml.
+
+CLI: python3 scripts/lib/ab_vsm.py <trace.jsonl.gz>...   (one VSM line per saved transcript)
+"""
+from __future__ import annotations
+
+import gzip
+import json
+import re
+import sys
+from pathlib import Path
+
+import yaml
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+RULES_PATH = REPO_ROOT / "specs" / "benchmarks" / "ab" / "vsm.yaml"
+WRAP_UP = "wrap-up"
+
+
+def load_rules(path: Path = RULES_PATH) -> dict:
+    rules = yaml.safe_load(path.read_text())
+    rules["compiled"] = [(r["stage"], re.compile(r["tools"]), re.compile(r["target"])) for r in rules["rules"]]
+    return rules
+
+
+def classify_call(tool: str, args: dict | None, rules: dict) -> str:
+    args = args or {}
+    target = str(args.get("command") or args.get("path") or args.get("file_path") or "")
+    for stage, tools, pattern in rules["compiled"]:
+        if tools.search(tool or "") and pattern.search(target):
+            return stage
+    return "design"
+
+
+def turn_stage(message: dict, rules: dict) -> str:
+    calls = [c for c in message.get("content") or [] if isinstance(c, dict) and c.get("type") == "toolCall"]
+    stages = {classify_call(c.get("name", ""), c.get("arguments"), rules) for c in calls}
+    return next((s for s in rules["turn_priority"] if s in stages), WRAP_UP)
+
+
+def pi_messages(stdout: str) -> list[dict]:
+    messages = []
+    for line in stdout.splitlines():
+        try:
+            event = json.loads(line)
+        except json.JSONDecodeError:
+            continue
+        if isinstance(event, dict) and event.get("type") == "message_end":
+            message = event.get("message") or {}
+            if isinstance(message.get("timestamp"), (int, float)):
+                messages.append(message)
+    return messages
+
+
+def vsm_from_pi(stdout: str, rules: dict) -> dict | None:
+    messages = pi_messages(stdout)
+    if not any(m.get("role") == "assistant" for m in messages):
+        return None
+    start = messages[0]["timestamp"]
+    stages: dict[str, dict] = {}
+    sequence, first_build = [], None
+    turns = [m for m in messages if m.get("role") == "assistant"]
+    for i, message in enumerate(turns):
+        stage = turn_stage(message, rules)
+        end = turns[i + 1]["timestamp"] if i + 1 < len(turns) else messages[-1]["timestamp"]
+        usage = message.get("usage") or {}
+        slot = stages.setdefault(stage, {"s": 0.0, "cost_usd": 0.0, "output_tokens": 0, "turns": 0})
+        slot["s"] = round(slot["s"] + (end - message["timestamp"]) / 1000, 3)
+        slot["cost_usd"] = round(slot["cost_usd"] + ((usage.get("cost") or {}).get("total") or 0), 6)
+        slot["output_tokens"] += usage.get("output", 0)
+        slot["turns"] += 1
+        sequence.append(stage)
+        if stage == "build" and first_build is None:
+            first_build = round((message["timestamp"] - start) / 1000, 3)
+    return {
+        "lead_time_s": round((messages[-1]["timestamp"] - start) / 1000, 3),
+        "time_to_first_build_s": first_build,
+        "switches": sum(1 for a, b in zip(sequence, sequence[1:]) if a != b),
+        "sequence": sequence,
+        "stages": stages,
+    }
+
+
+def vsm_from(parser: str, stdout: str, rules: dict | None = None) -> dict | None:
+    """vsm block for a run record; None when the adapter's transcript format is not supported."""
+    if parser != "pi-json":
+        return None
+    return vsm_from_pi(stdout, rules or load_rules())
+
+
+def describe(name: str, block: dict | None) -> str:
+    if not block:
+        return f"{name:<32} no VSM (unsupported or empty transcript)"
+    lead = block["lead_time_s"] or 1
+    parts = sorted(block["stages"].items(), key=lambda kv: -kv[1]["s"])
+    shares = "  ".join(f"{stage} {v['s']:.0f}s {100 * v['s'] / lead:.0f}%" for stage, v in parts if v["turns"])
+    return f"{name:<32} lead {block['lead_time_s']:6.1f}s  first build {block['time_to_first_build_s']}s  " \
+           f"switches {block['switches']:>2}  | {shares}"
+
+
+def main(argv: list[str]) -> int:
+    if not argv:
+        print("usage: python3 scripts/lib/ab_vsm.py <trace.jsonl.gz>...")
+        return 2
+    rules = load_rules()
+    for name in argv:
+        path = Path(name)
+        text = gzip.decompress(path.read_bytes()).decode() if path.suffix == ".gz" else path.read_text()
+        print(describe(path.name, vsm_from_pi(text, rules)))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))

package/scripts/lib/ab_waste.py

@@ -0,0 +1,173 @@
+#!/usr/bin/env python3
+# story: e58s09 e58s10
+"""Lean waste metrics for one A/B run, from its final patch and its pi transcript.
+
+Patch (any adapter): changed lines per file class, the reference solution's size, extra files,
+leftover scratch files and the review load a person would face. Transcript (pi): relearning
+(rereads, repeated commands), defects and rework (failing test runs after the first build edit,
+re-edits), first-time-through and process cycle efficiency. See LEAN-VSM_LATEST.md and
+specs/benchmarks/ab/vsm.yaml (file classes reuse the stage rules; `leftovers` regex).
+"""
+from __future__ import annotations
+
+import functools
+import gzip
+import json
+import re
+import shutil
+import subprocess
+import tempfile
+from collections import Counter
+from pathlib import Path
+
+from ab_vsm import classify_call, load_rules, vsm_from_pi  # type: ignore[import]
+
+CLASS_OF_STAGE = {"build": "product", "test": "test", "maintain": "doc", "plan": "plan"}
+FAILED = re.compile(r'"exit_code":\s*[1-9]|(?:^|\\n|\n)\s*(?:ℹ|#)\s*fail\s+[1-9]|(?:^|\\n|\n)not ok ')
+
+
+def git(cwd: Path, *args: str) -> str:
+    res = subprocess.run(["git", "-c", "user.name=ab", "-c", "user.email=ab@local", *args],
+                         cwd=cwd, capture_output=True, text=True)
+    return res.stdout
+
+
+def file_class(path: str, rules: dict) -> str:
+    cls = CLASS_OF_STAGE.get(classify_call("edit", {"path": path}, rules), "product")
+    return "doc" if cls == "product" and path.endswith(".md") else cls
+
+
+def numstat(cwd: Path, base: str) -> dict[str, int]:
+    changed = {}
+    for line in git(cwd, "diff", "--cached", "--numstat", base).splitlines():
+        added, deleted, path = line.split("\t", 2)
+        changed[path] = sum(int(n) for n in (added, deleted) if n.isdigit())
+    return changed
+
+
+@functools.lru_cache(maxsize=None)
+def solution_files(task_dir: str) -> dict[str, int] | None:
+    """Changed lines per file of the reference solution, applied over the fixture."""
+    task = Path(task_dir)
+    if not (task / "solution").is_dir():
+        return None
+    work = Path(tempfile.mkdtemp(prefix="ab-solution-"))
+    try:
+        shutil.copytree(task / "fixture", work, dirs_exist_ok=True)
+        git(work, "init", "-q")
+        git(work, "add", "-A")
+        git(work, "commit", "-q", "--no-verify", "-m", "fixture")
+        shutil.copytree(task / "solution", work, dirs_exist_ok=True)
+        git(work, "add", "-A")
+        return numstat(work, "HEAD")
+    finally:
+        shutil.rmtree(work, ignore_errors=True)
+
+
+def patch_waste(workdir: Path, base: str, task_dir: Path, patch_path: Path | None, rules: dict) -> dict:
+    git(workdir, "add", "-A")
+    patch = git(workdir, "diff", "--cached", base)
+    if patch_path is not None:
+        patch_path.parent.mkdir(parents=True, exist_ok=True)
+        patch_path.write_bytes(gzip.compress(patch.encode()))
+    changed = numstat(workdir, base)
+    lines = Counter()
+    for path, n in changed.items():
+        lines[file_class(path, rules)] += n
+    added = [line[2:] for line in git(workdir, "diff", "--cached", "--name-status", base).splitlines()
+             if line.startswith("A\t")]
+    solution = solution_files(str(task_dir))
+    solution_lines = sum(solution.values()) if solution else None
+    product = [p for p in changed if file_class(p, rules) == "product"]
+    doc_words = sum(len(line[1:].split()) for line in md_added_lines(patch))
+    return {
+        "product_lines": lines["product"], "test_lines": lines["test"],
+        "doc_lines": lines["doc"], "plan_lines": lines["plan"],
+        "solution_lines": solution_lines,
+        "diff_ratio": round(lines["product"] / solution_lines, 3) if solution_lines else None,
+        "extra_files": sorted(p for p in product if solution is not None and p not in solution),
+        "leftovers": sorted(p for p in added if re.search(rules["leftovers"], p)),
+        "review_load": sum(changed.values()) + len(changed) + doc_words,
+    }
+
+
+def md_added_lines(patch: str) -> list[str]:
+    out, in_md = [], False
+    for line in patch.splitlines():
+        if line.startswith("+++ "):
+            in_md = line.endswith(".md")
+        elif in_md and line.startswith("+"):
+            out.append(line)
+    return out
+
+
+def tool_calls(stdout: str) -> list[dict]:
+    """Ordered tool calls with their stage and whether a test run failed."""
+    calls, by_id = [], {}
+    for line in stdout.splitlines():
+        try:
+            event = json.loads(line)
+        except json.JSONDecodeError:
+            continue
+        if not isinstance(event, dict):
+            continue
+        if event.get("type") == "tool_execution_start":
+            call = {"tool": event.get("toolName", ""), "args": event.get("args") or {}, "failed": False}
+            by_id[event.get("toolCallId")] = call
+            calls.append(call)
+        elif event.get("type") == "tool_execution_end" and event.get("toolCallId") in by_id:
+            by_id[event["toolCallId"]]["failed"] = bool(FAILED.search(json.dumps(event.get("result"), ensure_ascii=False)))
+    return calls
+
+
+def transcript_waste(stdout: str, rules: dict | None = None) -> dict | None:
+    rules = rules or load_rules()
+    vsm = vsm_from_pi(stdout, rules)
+    calls = tool_calls(stdout)
+    if vsm is None and not calls:
+        return None
+    for call in calls:
+        call["stage"] = classify_call(call["tool"], call["args"], rules)
+    reads = Counter(c["args"].get("path") for c in calls if c["tool"] == "read")
+    commands = Counter(c["args"].get("command") for c in calls if c["tool"] == "bash" and c["stage"] != "test")
+    edits = Counter(c["args"].get("path") for c in calls if c["tool"] in ("edit", "write"))
+    first_build = next((i for i, c in enumerate(calls) if c["stage"] == "build"), None)
+    runs = [c for c in calls if c["tool"] == "bash" and c["stage"] == "test"]
+    after = [c for c in calls[first_build:] if c in runs] if first_build is not None else []
+    rework = sum(c["failed"] for c in after)
+    redundant = redundant_runs(calls)
+    stages = (vsm or {}).get("stages", {})
+    lead = (vsm or {}).get("lead_time_s") or 0
+    value_add = sum(stages.get(s, {}).get("s", 0) for s in ("build", "test"))
+    return {
+        "rereads": sum(n - 1 for n in reads.values()),
+        "repeated_commands": sum(n - 1 for n in commands.values()),
+        "test_runs": len(runs), "redundant_test_runs": redundant, "rework_loops": rework,
+        "reedits": sum(n - 1 for n in edits.values()),
+        "ftt": bool(after) and rework == 0,
+        "pce": round(value_add / lead, 3) if lead else None,
+    }
+
+
+def redundant_runs(calls: list[dict]) -> int:
+    """Passing test runs with no file edited since the previous passing run (overprocessing, e58s10)."""
+    redundant, green, dirty = 0, False, False
+    for call in calls:
+        if call["tool"] in ("edit", "write"):
+            dirty = True
+        elif call["tool"] == "bash" and call["stage"] == "test" and not call["failed"]:
+            redundant += green and not dirty
+            green, dirty = True, False
+    return redundant
+
+
+EMPTY_TRANSCRIPT = dict.fromkeys(["rereads", "repeated_commands", "test_runs", "redundant_test_runs",
+                                  "rework_loops", "reedits", "ftt", "pce"])
+
+
+def waste_for(workdir: Path, base: str, stdout: str, parser: str, task_dir: Path,
+              patch_path: Path | None = None) -> dict:
+    """The run record's waste block; transcript metrics are None for non-pi adapters."""
+    rules = load_rules()
+    transcript = transcript_waste(stdout, rules) if parser == "pi-json" else None
+    return {**patch_waste(workdir, base, task_dir, patch_path, rules), **(transcript or EMPTY_TRANSCRIPT)}

package/scripts/lib/git-argv.py

@@ -0,0 +1,168 @@
+#!/usr/bin/env python3
+"""Normalize git argv before applying the guard-git deny rules (#184).
+
+The old guard matched literal substrings, so `git -C repo reset --hard`,
+`git reset -q --hard` and `git checkout -- .` all slipped through: the
+destructive tokens are still there, only their position or a global flag
+changed. Matching on the raw command string cannot see that.
+
+This module tokenizes the command (quote-aware), finds every `git` invocation,
+strips the global options that precede the subcommand (`-C <path>`,
+`--no-pager`, `-c k=v`, `--git-dir=…`, …), and then decides on the
+(subcommand, arguments) pair. `extensions/omp-hooks.ts` implements the same
+rules in TypeScript; `scripts/test-guard-git-normalize.sh` fails if the two
+disagree on any fixture row.
+
+Rule table (keep in sync with the TS port):
+
+  reset     block when any argument is --hard
+  clean     block when any short flag carries `f` (-f, -fd, -fdx) or --force
+  branch    block on -D, or on --delete together with --force/-f
+  checkout  block on -f/--force, or on a `.` pathspec (with or without --)
+  restore   block on --worktree, or on a `.` pathspec
+  push      block on -f/--force/+refspec, unless --force-with-lease is given
+
+CLI:  git-argv.py --command "<shell command>"   → prints `block: <reason>` or `allow`
+Exit: 0 allow, 10 block, 2 usage error.
+"""
+from __future__ import annotations
+
+import shlex
+import sys
+
+# Global options that consume the next token as their value.
+VALUE_FLAGS = {
+    "-C",
+    "-c",
+    "--exec-path",
+    "--git-dir",
+    "--work-tree",
+    "--namespace",
+    "--super-prefix",
+    "--config-env",
+}
+
+# Global options that stand alone.
+BARE_FLAGS = {
+    "-p",
+    "--paginate",
+    "--no-pager",
+    "--bare",
+    "--no-replace-objects",
+    "--literal-pathspecs",
+    "--glob-pathspecs",
+    "--icase-pathspecs",
+    "--no-optional-locks",
+    "--version",
+    "--help",
+}
+
+# Command separators: the guard judges each git invocation on its own.
+SEPARATORS = {";", "&&", "||", "|", "&", "(", ")", "\n"}
+
+
+def tokenize(command: str) -> list[str]:
+    """Quote-aware split. Bad quoting degrades to whitespace splitting, never to eval."""
+    try:
+        return shlex.split(command, comments=False, posix=True)
+    except ValueError:
+        return command.split()
+
+
+def git_invocations(tokens: list[str]) -> list[tuple[str, list[str], int]]:
+    """Yield (subcommand, args, index) for every git call in the token stream."""
+    found: list[tuple[str, list[str], int]] = []
+    i = 0
+    while i < len(tokens):
+        token = tokens[i]
+        if token.rsplit("/", 1)[-1] != "git":
+            i += 1
+            continue
+        start = i
+        j = i + 1
+        # Skip global options between `git` and the subcommand.
+        while j < len(tokens):
+            opt = tokens[j]
+            if opt in SEPARATORS:
+                break
+            name = opt.split("=", 1)[0]
+            if name in VALUE_FLAGS:
+                j += 2 if "=" not in opt else 1
+                continue
+            if name in BARE_FLAGS or (opt.startswith("-") and not opt.startswith("--") and len(opt) > 1):
+                j += 1
+                continue
+            break
+        if j >= len(tokens) or tokens[j] in SEPARATORS:
+            i = j + 1
+            continue
+        sub = tokens[j]
+        args: list[str] = []
+        k = j + 1
+        while k < len(tokens) and tokens[k] not in SEPARATORS:
+            args.append(tokens[k])
+            k += 1
+        found.append((sub, args, start))
+        i = k
+    return found
+
+
+def _short_flag_has(args: list[str], letter: str) -> bool:
+    for arg in args:
+        if arg.startswith("-") and not arg.startswith("--") and letter in arg[1:]:
+            return True
+    return False
+
+
+def classify_git(sub: str, args: list[str]) -> str | None:
+    """Return a block reason, or None to allow."""
+    if sub == "reset" and "--hard" in args:
+        return "git reset --hard discards uncommitted work"
+    if sub == "clean" and (_short_flag_has(args, "f") or "--force" in args):
+        return "git clean -f deletes untracked files"
+    if sub == "branch":
+        forced = "-D" in args or ("--delete" in args and ("--force" in args or _short_flag_has(args, "f")))
+        if forced:
+            return "git branch -D deletes an unmerged branch"
+    if sub == "checkout":
+        if "-f" in args or "--force" in args:
+            return "git checkout -f discards local changes"
+        if "." in args:
+            return "git checkout . discards local changes"
+    if sub == "restore":
+        if "--worktree" in args or "." in args:
+            return "git restore discards local changes"
+    if sub == "push":
+        leased = any(a == "--force-with-lease" or a.startswith("--force-with-lease=") for a in args)
+        forced = (
+            "-f" in args
+            or "--force" in args
+            or any(a.startswith("+") and not a.startswith("++") for a in args)
+        )
+        if forced and not leased:
+            return "force push rewrites remote history"
+    return None
+
+
+def classify(command: str) -> str | None:
+    for sub, args, _idx in git_invocations(tokenize(command)):
+        reason = classify_git(sub, args)
+        if reason:
+            return reason
+    return None
+
+
+def main(argv: list[str]) -> int:
+    if len(argv) == 3 and argv[1] == "--command":
+        reason = classify(argv[2])
+        if reason:
+            print(f"block: {reason}")
+            return 10
+        print("allow")
+        return 0
+    print(__doc__, file=sys.stderr)
+    return 2
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv))