bigpowers 2.89.1 → 2.90.0

package/scripts/lib/ab_progress.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python3
+# story: e58s05
+"""Live view of an A/B eval while it runs: reads every ab-live.jsonl under the given folders
+(one per shard) and prints runs, pass rate, score and cost per task and arm, each arm's delta
+against vanilla with its bootstrap CI, and how those deltas evolved as runs accumulated.
+
+Usage: python3 scripts/lib/ab_progress.py <out-dir>... [--watch SECONDS]
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+import time
+from datetime import datetime
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from ab_report import aggregate, mean  # type: ignore[import]  # noqa: E402
+
+CHECKPOINTS = (5, 10, 20, 30, 50, 75, 100)
+
+
+def load(dirs: list[Path]) -> list[dict]:
+    records = []
+    for folder in dirs:
+        for live in sorted(folder.rglob("ab-live.jsonl")):
+            records += [json.loads(line) for line in live.read_text().splitlines() if line.strip()]
+    return records
+
+
+def ordered(values: list[str], preferred: str) -> list[str]:
+    unique = sorted(set(values))
+    return [preferred] + [v for v in unique if v != preferred] if preferred in unique else unique
+
+
+def fmt(value, places=2) -> str:
+    return "-" if value is None else f"{value:.{places}f}"
+
+
+def table(records: list[dict], tasks: list[str], arms: list[str]) -> list[str]:
+    lines = [f"{'task':<8} {'arm':<10} {'runs':>5} {'pass':>6} {'score':>6} {'cost$':>7} {'errors':>6}"]
+    for task in tasks:
+        for arm in arms:
+            runs = [r for r in records if r["task"] == task and r["arm"] == arm]
+            if runs:
+                lines.append(f"{task:<8} {arm:<10} {len(runs):>5} {fmt(mean([float(r['pass']) for r in runs])):>6} "
+                             f"{fmt(mean([r.get('score') for r in runs])):>6} {fmt(mean([r['cost_usd'] for r in runs]), 3):>7} "
+                             f"{sum(r['agent_error'] for r in runs):>6}")
+    return lines
+
+
+def delta_line(name: str, d: dict) -> str:
+    line = f"  {name} - vanilla: pass {d['pass_rate']:+.2f} {d['ci95']}"
+    if "score" in d:
+        line += f"  score {d['score']:+.3f} {d['score_ci95']}"
+    return line
+
+
+def evolution(records: list[dict], tasks: list[str], arms: list[str]) -> list[str]:
+    """Deltas using only runs with index <= each checkpoint: shows the CI narrowing over time."""
+    top = max(r["run"] for r in records)
+    lines = []
+    for cp in [c for c in CHECKPOINTS if c < top] + [top]:
+        subset = [r for r in records if r["run"] <= cp]
+        for name, d in aggregate(subset, arms, tasks)["deltas_vs_vanilla"].items():
+            lines.append(f"  runs<={cp:<4}{delta_line(name, d)[1:]}")
+    return lines
+
+
+def render(records: list[dict]) -> str:
+    tasks = sorted({r["task"] for r in records})
+    arms = ordered([r["arm"] for r in records], "vanilla")
+    models = sorted({str(r.get("model")) for r in records})
+    out = [f"A/B progress — {len(records)} runs, model {', '.join(models)}, at {datetime.now():%H:%M:%S}", ""]
+    out += table(records, tasks, arms)
+    result = aggregate(records, arms, tasks)
+    if result["deltas_vs_vanilla"]:
+        out += ["", "deltas (all runs so far, 95% CI):"]
+        out += [delta_line(n, d) for n, d in result["deltas_vs_vanilla"].items()]
+        out += ["", "evolution (pass and score delta by run index):"] + evolution(records, tasks, arms)
+    return "\n".join(out)
+
+
+def main(argv: list[str] | None = None) -> int:
+    p = argparse.ArgumentParser(description="Live A/B eval progress across shards")
+    p.add_argument("dirs", nargs="+", type=Path)
+    p.add_argument("--watch", type=int, help="repeat every N seconds")
+    args = p.parse_args(argv)
+    while True:
+        records = load(args.dirs)
+        if not records:
+            print("no ab-live.jsonl found under: " + ", ".join(map(str, args.dirs)))
+            return 2
+        print(render(records), flush=True)
+        if not args.watch:
+            return 0
+        time.sleep(args.watch)
+        print()
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/scripts/lib/ab_report.py

@@ -0,0 +1,116 @@
+#!/usr/bin/env python3
+# story: e58s01 e58s03 e58s04
+"""A/B eval aggregation and reports — pure functions over run records (no agent, no subprocess
+except reading the bigpowers git SHA). Split from ab-eval.py to keep both under the 300-line cap."""
+from __future__ import annotations
+
+import json
+import random
+import subprocess
+from datetime import date
+from pathlib import Path
+
+import yaml
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+BOOTSTRAP_RESAMPLES = 1000
+BOOTSTRAP_SEED = 58
+
+
+# ── aggregation ──────────────────────────────────────────────────────────────
+
+def mean(values: list) -> float | None:
+    vals = [v for v in values if v is not None]
+    return round(sum(vals) / len(vals), 4) if vals else None
+
+
+def bootstrap_delta_ci(treatment: list[float], control: list[float]) -> list[float]:
+    """Seeded 95% CI of mean(treatment) - mean(control); bools or scores in [0, 1]."""
+    rng = random.Random(BOOTSTRAP_SEED)
+    deltas = []
+    for _ in range(BOOTSTRAP_RESAMPLES):
+        t = [rng.choice(treatment) for _ in treatment]
+        c = [rng.choice(control) for _ in control]
+        deltas.append(sum(t) / len(t) - sum(c) / len(c))
+    deltas.sort()
+    lo = deltas[int(0.025 * BOOTSTRAP_RESAMPLES)]
+    hi = deltas[int(0.975 * BOOTSTRAP_RESAMPLES) - 1]
+    return [round(lo, 4), round(hi, 4)]
+
+
+def arm_summary(runs: list[dict]) -> dict:
+    return {
+        "runs": len(runs),
+        "pass_rate": mean([float(r["pass"]) for r in runs]),
+        "mean_score": mean([r.get("score") for r in runs]),
+        "tamper_rate": mean([float(r["tampered"]) for r in runs]),
+        "agent_error_rate": mean([float(r["agent_error"]) for r in runs]),
+        "mean_cost_usd": mean([r["cost_usd"] for r in runs]),
+        "mean_input_tokens": mean([r["input_tokens"] for r in runs]),
+        "mean_output_tokens": mean([r["output_tokens"] for r in runs]),
+        "mean_turns": mean([r["turns"] for r in runs]),
+        "mean_duration_s": mean([r["duration_s"] for r in runs]),
+        "mean_diff_lines": mean([r["diff_lines"] for r in runs]),
+    }
+
+
+def aggregate(records: list[dict], arms: list[str], tasks: list[str]) -> dict:
+    by_arm = {a: [r for r in records if r["arm"] == a] for a in arms}
+    summary = {a: arm_summary(runs) for a, runs in by_arm.items()}
+    per_task = {t: {a: mean([float(r["pass"]) for r in by_arm[a] if r["task"] == t]) for a in arms}
+                for t in tasks}
+    per_task_score = {t: {a: mean([r.get("score") for r in by_arm[a] if r["task"] == t]) for a in arms}
+                      for t in tasks}
+    deltas = {a: arm_delta(by_arm, summary, a) for a in arms if a != "vanilla"} if "vanilla" in arms else {}
+    return {"arms_summary": summary, "per_task": per_task, "per_task_score": per_task_score,
+            "delta": deltas.get("bigpowers"), "deltas_vs_vanilla": deltas}
+
+
+def arm_delta(by_arm: dict, summary: dict, arm: str) -> dict:
+    """arm - vanilla: pass rate and score with seeded bootstrap CIs, tamper rate and cost (e58s04)."""
+    tr, va = summary[arm], summary["vanilla"]
+    cost = None
+    if tr["mean_cost_usd"] is not None and va["mean_cost_usd"] is not None:
+        cost = round(tr["mean_cost_usd"] - va["mean_cost_usd"], 4)
+    delta = {
+        "pass_rate": round(tr["pass_rate"] - va["pass_rate"], 4),
+        "ci95": bootstrap_delta_ci([r["pass"] for r in by_arm[arm]], [r["pass"] for r in by_arm["vanilla"]]),
+        "tamper_rate": round(tr["tamper_rate"] - va["tamper_rate"], 4),
+        "cost_usd": cost,
+    }
+    scores = {a: [r["score"] for r in by_arm[a] if r.get("score") is not None] for a in (arm, "vanilla")}
+    if scores[arm] and scores["vanilla"]:
+        delta["score"] = round(tr["mean_score"] - va["mean_score"], 4)
+        delta["score_ci95"] = bootstrap_delta_ci(scores[arm], scores["vanilla"])
+    return delta
+
+
+# ── reporting ────────────────────────────────────────────────────────────────
+
+def bigpowers_sha() -> str:
+    res = subprocess.run(["git", "rev-parse", "--short", "HEAD"], cwd=REPO_ROOT, capture_output=True, text=True)
+    return res.stdout.strip() or "unknown"
+
+
+def write_reports(out: Path, meta: dict, result: dict, records: list[dict]) -> Path:
+    out.mkdir(parents=True, exist_ok=True)
+    today = date.today().isoformat()
+    report = out / f"AB-{today}.yaml"
+    report.write_text(yaml.safe_dump({**meta, **result}, sort_keys=False))
+    with (out / f"ab-runs-{today}.jsonl").open("w") as f:
+        for rec in records:
+            f.write(json.dumps({**rec, "bigpowers_sha": meta["bigpowers_sha"]}) + "\n")
+    return report
+
+
+def print_summary(result: dict) -> None:
+    print(f"{'arm':<10} {'pass':>6} {'score':>6} {'tamper':>7} {'cost$':>8} {'turns':>6} {'secs':>7}")
+    for arm, s in result["arms_summary"].items():
+        fmt = lambda v, p=2: "-" if v is None else f"{v:.{p}f}"
+        print(f"{arm:<10} {fmt(s['pass_rate']):>6} {fmt(s['mean_score']):>6} {fmt(s['tamper_rate']):>7} "
+              f"{fmt(s['mean_cost_usd'], 3):>8} {fmt(s['mean_turns'], 1):>6} {fmt(s['mean_duration_s'], 0):>7}")
+    for arm, d in result.get("deltas_vs_vanilla", {}).items():
+        line = f"delta {arm:<10} - vanilla: pass {d['pass_rate']:+.2f} CI {d['ci95']}"
+        if "score" in d:
+            line += f"  score {d['score']:+.2f} CI {d['score_ci95']}"
+        print(line)

package/scripts/lib/plan-consistency-check.sh

@@ -69,7 +69,7 @@ for tasks in "${TASKS[@]+"${TASKS[@]}"}"; do
   sid="$(echo "$base" | grep -oE 'e[0-9]+s[0-9]+' | head -1)"
   [[ -n "$sid" ]] || continue
 
-  spec_name="$(grep -E '^spec:' "$tasks" | head -1 | sed -E 's/^spec:[[:space:]]*//; s/^"(.*)"$/\1/')"
+  spec_name="$(grep -E '^spec:' "$tasks" | head -1 | sed -E 's/^spec:[[:space:]]*//; s/^"(.*)"$/\1/' || true)"
   spec="$CAPSULE/$spec_name"
   if [[ -z "$spec_name" || ! -f "$spec" ]]; then
     report CRITICAL "Story $sid tasks.yaml declares no valid spec: file"
@@ -92,7 +92,9 @@ for tasks in "${TASKS[@]+"${TASKS[@]}"}"; do
     report CRITICAL "Story $sid tasks.yaml missing runnable verify: commands"
   else
     verify_cmd="$("$PYTHON_BIN" "$SCRIPT_LIB_DIR/extract-story-verify.py" "$REPO_ROOT" 2>/dev/null \
-      | awk -F'\t' -v s="$sid" '$1 == s { print $4; exit }')"
+      | awk -F'\t' -v s="$sid" '$1 == s && !found { print $4; found = 1 }')"
+    # awk reads to the end: exiting early closes the pipe, Python fails to flush and exits 120,
+    # and pipefail turns that into a random failure of this check.
     if [[ -n "$verify_cmd" ]]; then
       is_executable_verify "$verify_cmd" \
         || report CRITICAL "Story $sid verify: is prose, not a runnable command: $verify_cmd"

package/scripts/lib/run-benchmark.py

@@ -60,26 +60,44 @@ def scenario_pass_rate(scenario: dict, runs: int, with_skill: bool) -> tuple[flo
     if not command:
         return (-1.0, "missing_command")
     if not with_skill:
-        command = f"test -f /dev/null"  # bare agent: code graders cannot pass without skill context
+        # No agent runs here, so there is no honest without-skill number (e58s01).
+        # Real with/without deltas come from scripts/run-ab-eval.sh.
+        return (-1.0, "unmeasured")
     passes = sum(1 for _ in range(runs) if run_code_grader(command, REPO_ROOT))
     return (passes / runs, "code")
 
 
-def pass_at_k(scenarios: list[dict], runs: int, with_skill: bool) -> float:
+def pass_at_k(scenarios: list[dict], runs: int, with_skill: bool) -> float | None:
     total_weight = 0.0
     weighted = 0.0
     for sc in scenarios:
         rate, status = scenario_pass_rate(sc, runs, with_skill)
-        if status == "pending_agent_eval":
+        if status in ("pending_agent_eval", "unmeasured"):
             continue
         weight = float(sc.get("weight", 1.0))
         total_weight += weight
         weighted += weight * max(rate, 0.0)
     if total_weight == 0:
-        return 0.0
+        return None
     return round(weighted / total_weight, 2)
 
 
+def delta_of(with_score: float | None, without_score: float | None) -> float | None:
+    if with_score is None or without_score is None:
+        return None
+    return round(with_score - without_score, 2)
+
+
+def delta_summary(val_delta: float | None) -> str:
+    if val_delta is None:
+        return "UNMEASURED: no without-skill agent run — use scripts/run-ab-eval.sh for a real delta"
+    if val_delta < 0:
+        return f"REGRESSION: validation Δ {val_delta} — do NOT ship"
+    if val_delta < 0.05:
+        return f"WARN: validation Δ {val_delta} — marginal contribution"
+    return f"OK: validation Δ {val_delta}"
+
+
 def partition_scenarios(scenarios: list[dict]) -> tuple[list[dict], list[dict]]:
     train, validation = [], []
     for sc in scenarios:
@@ -93,13 +111,13 @@ def partition_scenarios(scenarios: list[dict]) -> tuple[list[dict], list[dict]]:
 
 def build_split_report(scenarios: list[dict], runs: int) -> dict:
     if not scenarios:
-        return {"with_skill": 0.0, "without_skill": 0.0, "delta": 0.0, "scenarios": []}
+        return {"with_skill": None, "without_skill": None, "delta": None, "scenarios": []}
     with_score = pass_at_k(scenarios, runs, with_skill=True)
     without_score = pass_at_k(scenarios, runs, with_skill=False)
     return {
         "with_skill": with_score,
         "without_skill": without_score,
-        "delta": round(with_score - without_score, 2),
+        "delta": delta_of(with_score, without_score),
         "scenarios": [sc["id"] for sc in scenarios],
     }
 
@@ -119,13 +137,15 @@ def scenario_details(scenarios: list[dict], runs: int) -> list[dict]:
                 }
             )
             continue
+        with_value = round(max(with_rate, 0.0), 2)
+        without_value = None if without_rate < 0 else round(without_rate, 2)
         rows.append(
             {
                 "id": sc["id"],
                 "split": sc.get("split", "validation"),
-                "with_pass_rate": round(max(with_rate, 0.0), 2),
-                "without_pass_rate": round(max(without_rate, 0.0), 2),
-                "delta": round(max(with_rate, 0.0) - max(without_rate, 0.0), 2),
+                "with_pass_rate": with_value,
+                "without_pass_rate": without_value,
+                "delta": delta_of(with_value, without_value),
                 "weight": sc.get("weight", 1.0),
             }
         )
@@ -180,13 +200,7 @@ def run_benchmark(skill: str, baseline: bool = False) -> dict:
         shutil.copy(yaml_path, REPORTS_DIR / f"BASELINE-{skill}.yaml")
         shutil.copy(json_path, REPORTS_DIR / f"baseline-{skill}.json")
         print(f"Pinned baseline for {skill}")
-    val_delta = report["validation"]["delta"]
-    if val_delta < 0:
-        print(f"REGRESSION: validation Δ {val_delta} — do NOT ship")
-    elif val_delta < 0.05:
-        print(f"WARN: validation Δ {val_delta} — marginal contribution")
-    else:
-        print(f"OK: validation Δ {val_delta}")
+    print(delta_summary(report["validation"]["delta"]))
     return report
 
 

package/scripts/run-ab-eval.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+# story: e58s01
+# run-ab-eval.sh — run the same tasks with and without bigpowers; grade only the outcome.
+# Advisory: exits 0 after a completed run, 2 on configuration errors. See specs/benchmarks/SCHEMA.md.
+set -euo pipefail
+source "$(dirname "${BASH_SOURCE[0]}")/lib/python-env.sh"
+exec "$PYTHON" "$(dirname "${BASH_SOURCE[0]}")/lib/ab-eval.py" "$@"

package/scripts/run-story-verify.sh

@@ -82,6 +82,13 @@ run_story_verify() {
   fi
   echo "FAIL: $sid [$src]$note — $cmd"
   echo "      output: $(echo "$RUN_OUT" | tail -3 | tr '\n' ' ')"
+  # Name the failing test. unittest/pytest footers ("FAILED (failures=1)") say a test
+  # broke but not which one, so a red gate is undiagnosable from CI logs alone
+  # (BUG-2026-11-03-story-verify-output-hides-failing-test).
+  local detail
+  detail="$(echo "$RUN_OUT" | grep -E '^(FAIL|ERROR): |\.\.\. (FAIL|ERROR)$|^AssertionError|^[A-Za-z]*Error: |^Traceback' \
+    | head -3 | tr '\n' ' ')"
+  [ -n "$detail" ] && echo "      detail: $detail"
   FAIL=$((FAIL + 1))
   return 1
 }

package/tests/test_ab_ablation.py

@@ -0,0 +1,91 @@
+#!/usr/bin/env python3
+# story: e58s04
+"""tests/test_ab_ablation.py — ablation arms: skill subsets, seed choice, per-arm deltas.
+
+Run: python3 tests/test_ab_ablation.py
+"""
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from test_ab_eval import REPO_ROOT, rep  # noqa: E402
+
+ARM_SCRIPT = REPO_ROOT / "scripts" / "lib" / "ab-arm-bigpowers.sh"
+
+
+class TestArmSetup(unittest.TestCase):
+    def setup_arm(self, **env):
+        work = Path(tempfile.mkdtemp(prefix="ab-arm-"))
+        self.addCleanup(shutil.rmtree, work, ignore_errors=True)
+        full_env = {**os.environ, "AB_BP_ROOT": str(REPO_ROOT), **env}
+        subprocess.run(["bash", str(ARM_SCRIPT)], cwd=work, env=full_env, check=True, capture_output=True)
+        return work
+
+    def skills(self, work, folder):
+        path = work / folder / "skills"
+        return sorted(p.name for p in path.iterdir()) if path.is_dir() else []
+
+    def test_default_installs_every_skill_and_the_seed(self):
+        work = self.setup_arm()
+        self.assertEqual(len(self.skills(work, ".agents")), len(list((REPO_ROOT / "skills").glob("*/"))))
+        self.assertTrue((work / "AGENTS.md").is_file())
+        self.assertTrue((work / "scripts").is_dir())
+
+    def test_skill_subset(self):
+        work = self.setup_arm(AB_BP_SKILLS="develop-tdd,verify-work")
+        self.assertEqual(self.skills(work, ".agents"), ["develop-tdd", "verify-work"])
+        self.assertEqual(self.skills(work, ".claude"), ["develop-tdd", "verify-work"])
+        self.assertTrue((work / "scripts").is_dir())
+
+    def test_unknown_skill_fails_setup(self):
+        with self.assertRaises(subprocess.CalledProcessError):
+            self.setup_arm(AB_BP_SKILLS="no-such-skill")
+
+    def test_seed_only(self):
+        work = self.setup_arm(AB_BP_SKILLS="none")
+        self.assertTrue((work / "AGENTS.md").is_file())
+        self.assertEqual(self.skills(work, ".agents"), [])
+        self.assertFalse((work / "scripts").exists())
+
+    def test_skills_without_seed(self):
+        work = self.setup_arm(AB_BP_SEED="none")
+        self.assertFalse((work / "AGENTS.md").exists())
+        self.assertFalse((work / "CLAUDE.md").exists())
+        self.assertGreater(len(self.skills(work, ".agents")), 0)
+
+    def test_other_seed_file(self):
+        work = self.setup_arm(AB_BP_SKILLS="none", AB_BP_SEED="specs/benchmarks/ab/arms/tdd-line/AGENTS.md")
+        self.assertNotIn("bigpowers", (work / "AGENTS.md").read_text().lower())
+
+
+class TestDeltasVsVanilla(unittest.TestCase):
+    def rec(self, arm, passed, score=None):
+        return {"arm": arm, "task": "t", "pass": passed, "score": score, "tampered": False, "agent_error": False,
+                "cost_usd": 0.01, "input_tokens": None, "output_tokens": None, "turns": None,
+                "duration_s": 1.0, "diff_lines": 1}
+
+    def test_every_arm_gets_a_delta(self):
+        records = [self.rec("vanilla", False, 0.5), self.rec("vanilla", True, 1.0),
+                   self.rec("bigpowers", True, 1.0), self.rec("bigpowers", True, 1.0),
+                   self.rec("bp-seed", False, 0.5), self.rec("bp-seed", False, 0.5)]
+        result = rep.aggregate(records, ["vanilla", "bigpowers", "bp-seed"], ["t"])
+        deltas = result["deltas_vs_vanilla"]
+        self.assertEqual(sorted(deltas), ["bigpowers", "bp-seed"])
+        self.assertEqual(deltas["bigpowers"]["pass_rate"], 0.5)
+        self.assertEqual(deltas["bp-seed"]["pass_rate"], -0.5)
+        self.assertEqual(deltas["bp-seed"]["score"], -0.25)
+        self.assertEqual(len(deltas["bp-seed"]["ci95"]), 2)
+        self.assertEqual(result["delta"], deltas["bigpowers"])
+
+    def test_no_vanilla_means_no_deltas(self):
+        result = rep.aggregate([self.rec("bigpowers", True)], ["bigpowers"], ["t"])
+        self.assertEqual(result["deltas_vs_vanilla"], {})
+
+
+if __name__ == "__main__":
+    unittest.main(verbosity=1)