bigpowers 2.91.1 → 2.93.0

package/scripts/verify-adversarial.sh

@@ -109,9 +109,16 @@ fi
 DEFECTS_FOUND=0
 mkdir -p specs/verifications
 
+# A JS/TS source file that exists and is not a test: the files the boundary scan inspects.
+is_scannable_source() {
+  local file="$1"
+  [[ "$file" =~ \.(js|ts|mjs)$ && -f "$file" ]] || return 1
+  [[ ! "$file" =~ (^|/)test/ && ! "$file" =~ \.test\. ]]
+}
+
 # 2. Check for common unsafe property access patterns on JS/TS files
 for file in $CHANGED_FILES; do
-  if [[ "$file" =~ \.(js|ts|mjs)$ ]] && [[ -f "$file" ]] && [[ ! "$file" =~ (^|/)test/ ]] && [[ ! "$file" =~ \.test\. ]]; then
+  if is_scannable_source "$file"; then
     # Look for unsafe chained property access like item.brand.toLowerCase() or item.prop.subprop
     if grep -nE '\.[a-zA-Z0-9_]+\.(toLowerCase|toUpperCase|trim|length|slice)\(' "$file" | grep -v '\?\.' >/dev/null 2>&1; then
       echo "WARNING [boundary-mutation]: Potential unhandled undefined access without optional chaining in $file:"

package/tests/test_ab_eval.py

@@ -9,7 +9,6 @@ import importlib.util
 import io
 import json
 import shutil
-import subprocess
 import sys
 import tempfile
 import unittest
@@ -249,49 +248,6 @@ class TestCli(Sandbox):
         self.assertIsNone(report["delta"])
 
 
-@unittest.skipUnless(shutil.which("node"), "node is needed for the starter tasks")
-class TestStarterTasks(unittest.TestCase):
-    """Each shipped task must fail as given and pass with its reference solution."""
-
-    TASKS = REPO_ROOT / "specs" / "benchmarks" / "ab" / "tasks"
-
-    def grade_copy(self, task_dir: Path, with_solution: bool) -> bool:
-        task = yaml.safe_load((task_dir / "task.yaml").read_text())
-        work = Path(tempfile.mkdtemp(prefix="ab-starter-"))
-        try:
-            shutil.copytree(task_dir / "fixture", work, dirs_exist_ok=True)
-            if with_solution:
-                shutil.copytree(task_dir / "solution", work, dirs_exist_ok=True)
-            shutil.copytree(task_dir / "hidden", work, dirs_exist_ok=True)
-            res = subprocess.run(["bash", "-c", task["grade"]], cwd=work, capture_output=True)
-            return res.returncode == 0
-        finally:
-            shutil.rmtree(work, ignore_errors=True)
-
-    def test_eight_tasks_ship(self):
-        self.assertGreaterEqual(len(list(self.TASKS.glob("*/task.yaml"))), 8)
-
-    def test_fixture_fails_and_solution_passes(self):
-        for task_yaml in sorted(self.TASKS.glob("*/task.yaml")):
-            with self.subTest(task=task_yaml.parent.name):
-                self.assertFalse(self.grade_copy(task_yaml.parent, with_solution=False))
-                self.assertTrue(self.grade_copy(task_yaml.parent, with_solution=True))
-
-    def test_prompts_never_name_bigpowers(self):
-        for task_yaml in self.TASKS.glob("*/task.yaml"):
-            self.assertNotIn("bigpowers", yaml.safe_load(task_yaml.read_text())["prompt"].lower())
-
-    def test_grade_commands_never_pass_a_bare_directory(self):
-        """`node --test test/` exits 1 on node 24 (it resolves the directory as a module path), so
-        such a task can never pass however good the solution is, while a hidden-only `score`
-        command still looks healthy (BUG found in e58s07: ab-11 and ab-13 read pass=0 forever)."""
-        for task_yaml in self.TASKS.glob("*/task.yaml"):
-            with self.subTest(task=task_yaml.parent.name):
-                for token in yaml.safe_load(task_yaml.read_text())["grade"].split():
-                    self.assertNotIn(token.rstrip("/"), ("test", "test-hidden"),
-                                     f"{task_yaml.parent.name} grades a directory, not a glob")
-
-
 class TestRunBenchmarkHonesty(unittest.TestCase):
     def test_without_skill_arm_is_unmeasured_not_zero(self):
         sc = {"id": "s1", "grader": {"type": "code", "command": "true"}}

package/tests/test_ab_lean.py

@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+# story: e58s10
+"""tests/test_ab_lean.py — lean views: per-arm VSM and waste summary, deltas, bars, redundant tests.
+
+Run: python3 tests/test_ab_lean.py
+"""
+import sys
+import unittest
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from test_ab_eval import load_module, rep  # noqa: E402
+from test_ab_waste import Stream, waste  # noqa: E402
+
+lean = load_module("ab_lean", "ab_lean.py")
+
+
+def run(arm, lead, build, test, passed=True, cost=0.01, test_runs=3, rework=0):
+    stages = {"plan": {"s": lead * 0.1}, "design": {"s": lead * 0.2}, "build": {"s": build}, "test": {"s": test}}
+    vsm = {"lead_time_s": lead, "time_to_first_build_s": lead * 0.3, "switches": 4, "stages": stages}
+    w = {"test_runs": test_runs, "redundant_test_runs": 1, "rework_loops": rework, "reedits": 0,
+         "rereads": 0, "repeated_commands": 0, "review_load": 50, "diff_ratio": 1.2, "ftt": rework == 0,
+         "pce": round((build + test) / lead, 3), "leftovers": [], "extra_files": []}
+    return {"task": "t", "arm": arm, "pass": passed, "score": None, "tampered": False, "agent_error": False,
+            "cost_usd": cost, "input_tokens": None, "output_tokens": None, "turns": 5, "duration_s": lead,
+            "diff_lines": 5, "vsm": vsm, "waste": w}
+
+
+def records():
+    return ([run("vanilla", 70, 28, 21) for _ in range(5)]
+            + [run("bigpowers", 140, 20, 70, cost=0.02, test_runs=8, rework=1) for _ in range(5)])
+
+
+class TestLeanSummary(unittest.TestCase):
+    def test_per_arm_summary_and_deltas(self):
+        summary = lean.lean_summary(records(), ["vanilla", "bigpowers"])
+        va, bp = summary["arms"]["vanilla"], summary["arms"]["bigpowers"]
+        self.assertEqual(va["lead_time_s"], 70)
+        self.assertEqual(round(va["stage_share"]["build"], 2), 0.4)
+        self.assertEqual(bp["waste"]["test_runs"], 8)
+        self.assertEqual(bp["ftt_rate"], 0.0)
+        self.assertEqual(va["cost_per_pass"], 0.01)
+        d = summary["deltas"]["bigpowers"]
+        self.assertEqual(d["lead_time_s"]["delta"], 70)
+        self.assertEqual(d["test_runs"]["delta"], 5)
+        self.assertEqual(len(d["build_share"]["ci95"]), 2)
+
+    def test_old_records_give_none(self):
+        old = [{k: v for k, v in r.items() if k not in ("vsm", "waste")} for r in records()]
+        self.assertIsNone(lean.lean_summary(old, ["vanilla", "bigpowers"]))
+        self.assertEqual(lean.lean_lines(None), [])
+
+    def test_report_carries_lean(self):
+        result = rep.aggregate(records(), ["vanilla", "bigpowers"], ["t"])
+        self.assertIn("bigpowers", result["lean"]["deltas"])
+
+
+class TestViews(unittest.TestCase):
+    def test_bars_scale_with_lead_time(self):
+        lines = lean.lean_lines(lean.lean_summary(records(), ["vanilla", "bigpowers"]))
+        bars = {line.split()[0]: line for line in lines if line.split() and line.split()[0] in ("vanilla", "bigpowers")}
+        va_len = bars["vanilla"].count("|") and len(bars["vanilla"].split("|")[1].rstrip())
+        bp_len = len(bars["bigpowers"].split("|")[1].rstrip())
+        self.assertAlmostEqual(bp_len / va_len, 2, delta=0.2)
+        self.assertIn("T", bars["bigpowers"])
+        self.assertTrue(any("rework" in line for line in lines))
+
+
+class TestRedundantTestRuns(unittest.TestCase):
+    def test_repeated_green_run_without_edit_is_redundant(self):
+        stream = (Stream()
+                  .call("edit", {"path": "/work/src/a.js"})
+                  .call("bash", {"command": "npm test"}, "# pass 3\n# fail 0\n")
+                  .call("bash", {"command": "npm test 2>&1 | tail -3"}, "# pass 3\n# fail 0\n")
+                  .call("edit", {"path": "/work/src/a.js"})
+                  .call("bash", {"command": "npm test"}, "# pass 3\n# fail 0\n"))
+        self.assertEqual(waste.transcript_waste(stream.text())["redundant_test_runs"], 1)
+
+
+if __name__ == "__main__":
+    unittest.main(verbosity=1)

package/tests/test_ab_score.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# story: e58s03
+# story: e58s03 e58s11
 """tests/test_ab_score.py — partial-credit scoring and the hard tasks (ab-11, ab-13).
 
 Run: python3 tests/test_ab_score.py
@@ -18,7 +18,17 @@ sys.path.insert(0, str(Path(__file__).resolve().parent))
 from test_ab_eval import REPO_ROOT, Sandbox, met, rep  # noqa: E402
 
 TASKS = REPO_ROOT / "specs" / "benchmarks" / "ab" / "tasks"
-HARD_TASKS = ["ab-11-ledger-rounding", "ab-13-orders-split"]
+HARD_TASKS = [
+    "ab-09-inventory-reservations",
+    "ab-11-ledger-rounding",
+    "ab-12-todo-due-dates",
+    "ab-13-orders-split",
+    "ab-15-config-precedence",
+    "ab-16-site-tags-rss",
+    "ab-17-payment-alias",
+    "ab-19-catalog-segregation",
+    "ab-20-brand-discount",
+]
 
 
 class TestTapScore(unittest.TestCase):

package/tests/test_ab_tasks.py

@@ -0,0 +1,75 @@
+#!/usr/bin/env python3
+# story: e58s01 e58s02 e58s07 e58s09
+"""tests/test_ab_tasks.py — every shipped A/B task is calibrated: its fixture fails, its reference
+solution passes, its prompt never names bigpowers, and its grade command uses file globs.
+
+Run: python3 tests/test_ab_tasks.py
+Split from test_ab_eval.py (300-line cap).
+"""
+import json
+import shutil
+import subprocess
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+import yaml
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from test_ab_eval import REPO_ROOT  # noqa: E402
+
+
+@unittest.skipUnless(shutil.which("node"), "node is needed for the starter tasks")
+class TestStarterTasks(unittest.TestCase):
+    """Each shipped task must fail as given and pass with its reference solution."""
+
+    TASKS = REPO_ROOT / "specs" / "benchmarks" / "ab" / "tasks"
+
+    def grade_copy(self, task_dir: Path, with_solution: bool) -> bool:
+        task = yaml.safe_load((task_dir / "task.yaml").read_text())
+        work = Path(tempfile.mkdtemp(prefix="ab-starter-"))
+        try:
+            shutil.copytree(task_dir / "fixture", work, dirs_exist_ok=True)
+            if with_solution:
+                shutil.copytree(task_dir / "solution", work, dirs_exist_ok=True)
+            shutil.copytree(task_dir / "hidden", work, dirs_exist_ok=True)
+            res = subprocess.run(["bash", "-c", task["grade"]], cwd=work, capture_output=True)
+            return res.returncode == 0
+        finally:
+            shutil.rmtree(work, ignore_errors=True)
+
+    def test_eight_tasks_ship(self):
+        self.assertGreaterEqual(len(list(self.TASKS.glob("*/task.yaml"))), 8)
+
+    def test_fixture_fails_and_solution_passes(self):
+        for task_yaml in sorted(self.TASKS.glob("*/task.yaml")):
+            with self.subTest(task=task_yaml.parent.name):
+                self.assertFalse(self.grade_copy(task_yaml.parent, with_solution=False))
+                self.assertTrue(self.grade_copy(task_yaml.parent, with_solution=True))
+
+    def test_prompts_never_name_bigpowers(self):
+        for task_yaml in self.TASKS.glob("*/task.yaml"):
+            self.assertNotIn("bigpowers", yaml.safe_load(task_yaml.read_text())["prompt"].lower())
+
+    def test_fixture_test_scripts_never_pass_a_bare_directory(self):
+        """`npm test` must work in every fixture: a `node --test test/` script fails on node 24, and
+        agents then spend the run repairing the harness (found by e58s09 waste metrics)."""
+        for package in self.TASKS.glob("*/fixture/package.json"):
+            with self.subTest(task=package.parent.parent.name):
+                script = json.loads(package.read_text()).get("scripts", {}).get("test", "")
+                self.assertNotRegex(script, r"--test\s+\S*/(\s|$)", script)
+
+    def test_grade_commands_never_pass_a_bare_directory(self):
+        """`node --test test/` exits 1 on node 24 (it resolves the directory as a module path), so
+        such a task can never pass however good the solution is, while a hidden-only `score`
+        command still looks healthy (BUG found in e58s07: ab-11 and ab-13 read pass=0 forever)."""
+        for task_yaml in self.TASKS.glob("*/task.yaml"):
+            with self.subTest(task=task_yaml.parent.name):
+                for token in yaml.safe_load(task_yaml.read_text())["grade"].split():
+                    self.assertNotIn(token.rstrip("/"), ("test", "test-hidden"),
+                                     f"{task_yaml.parent.name} grades a directory, not a glob")
+
+
+if __name__ == "__main__":
+    unittest.main(verbosity=1)

package/tests/test_ab_vsm.py

@@ -0,0 +1,109 @@
+#!/usr/bin/env python3
+# story: e58s08
+"""tests/test_ab_vsm.py — value stream map per run: stage rules, per-turn time, run records.
+
+Run: python3 tests/test_ab_vsm.py
+"""
+import json
+import sys
+import unittest
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from test_ab_eval import Sandbox, load_module  # noqa: E402
+
+vsm = load_module("ab_vsm", "ab_vsm.py")
+STAGES = {"plan", "design", "build", "test", "deploy", "maintain", "wrap-up"}
+
+
+def assistant(ts_s, cost=0.001, out=10, calls=()):
+    content = [{"type": "toolCall", "name": tool, "arguments": args} for tool, args in calls]
+    usage = {"input": 100, "output": out, "cacheRead": 0, "cacheWrite": 0, "cost": {"total": cost}}
+    return {"type": "message_end", "message": {"role": "assistant", "timestamp": int(ts_s * 1000),
+                                               "content": content, "usage": usage}}
+
+
+def tool_result(ts_s):
+    return {"type": "message_end", "message": {"role": "toolResult", "timestamp": int(ts_s * 1000)}}
+
+
+def transcript():
+    events = [
+        {"type": "session", "id": "s"},
+        {"type": "message_end", "message": {"role": "user", "timestamp": 0}},
+        assistant(0, calls=[("read", {"path": "/work/ISSUE.md"})]), tool_result(9),
+        assistant(10, calls=[("bash", {"command": "cat src/reports.js"})]), tool_result(24),
+        assistant(25, calls=[("edit", {"path": "/work/src/reports.js"})]), tool_result(39),
+        assistant(40, calls=[("bash", {"command": "npm test 2>&1 | tail"})]), tool_result(49),
+        assistant(50, calls=[("bash", {"command": "git commit -am fix"})]), tool_result(54),
+        assistant(55, out=40),
+        {"type": "agent_settled", "aborted": False},
+    ]
+    return "\n".join(json.dumps(e) for e in events)
+
+
+class TestRules(unittest.TestCase):
+    def setUp(self):
+        self.rules = vsm.load_rules()
+
+    def test_rules_file_uses_known_stages(self):
+        self.assertTrue({r["stage"] for r in self.rules["rules"]} <= STAGES)
+        self.assertTrue(set(self.rules["turn_priority"]) <= STAGES)
+
+    def test_calls_are_classified(self):
+        cases = [
+            ("read", {"path": "/work/ISSUE.md"}, "plan"),
+            ("bash", {"command": "sed -n 1,80p .agents/skills/fix-bug/SKILL.md"}, "plan"),
+            ("read", {"path": "/work/src/reports.js"}, "design"),
+            ("edit", {"path": "/work/src/reports.js"}, "build"),
+            ("edit", {"path": "/work/test/ledger.test.js"}, "test"),
+            ("bash", {"command": "npm test 2>&1 | tail -20"}, "test"),
+            ("bash", {"command": "node --test test/"}, "test"),
+            ("bash", {"command": "git checkout -b fix/rounding"}, "deploy"),
+            ("bash", {"command": "git commit -am 'fix'"}, "deploy"),
+            ("write", {"path": "/work/specs/bugs/BUG-001.md"}, "maintain"),
+            ("edit", {"path": "/work/docs/LEDGER.md"}, "maintain"),
+            ("write", {"path": "/work/specs/plan.md"}, "plan"),
+        ]
+        for tool, args, expected in cases:
+            with self.subTest(tool=tool, args=args):
+                self.assertEqual(vsm.classify_call(tool, args, self.rules), expected)
+
+
+class TestStageTime(unittest.TestCase):
+    def test_turn_time_goes_to_the_stage_of_its_action(self):
+        block = vsm.vsm_from("pi-json", transcript())
+        seconds = {stage: round(v["s"], 1) for stage, v in block["stages"].items()}
+        self.assertEqual(seconds, {"plan": 10.0, "design": 15.0, "build": 15.0, "test": 10.0,
+                                   "deploy": 5.0, "wrap-up": 0.0})
+        self.assertEqual(block["lead_time_s"], 55.0)
+        self.assertEqual(block["time_to_first_build_s"], 25.0)
+        self.assertEqual(block["switches"], 5)
+        self.assertEqual(block["sequence"], ["plan", "design", "build", "test", "deploy", "wrap-up"])
+        self.assertEqual(block["stages"]["wrap-up"]["output_tokens"], 40)
+        self.assertAlmostEqual(sum(v["cost_usd"] for v in block["stages"].values()), 0.006)
+
+    def test_build_wins_a_turn_with_several_calls(self):
+        events = [assistant(0, calls=[("read", {"path": "/work/ISSUE.md"}), ("edit", {"path": "/work/src/a.js"})]),
+                  tool_result(5)]
+        block = vsm.vsm_from("pi-json", "\n".join(json.dumps(e) for e in events))
+        self.assertEqual(block["sequence"], ["build"])
+
+    def test_unsupported_or_empty_is_none(self):
+        self.assertIsNone(vsm.vsm_from("none", transcript()))
+        self.assertIsNone(vsm.vsm_from("pi-json", "not json"))
+
+
+class TestRunRecord(Sandbox):
+    def test_record_carries_vsm(self):
+        script = self.tmp / "transcript.jsonl"
+        script.write_text(transcript() + "\n")
+        self.run_main("--adapter-cmd", f"cat {script}; echo 42 > answer.txt", "--parser", "pi-json",
+                      "--arms", "vanilla")
+        rec = self.records()[0]
+        self.assertEqual(rec["vsm"]["stages"]["build"]["turns"], 1)
+        self.assertEqual(rec["vsm"]["lead_time_s"], 55.0)
+
+
+if __name__ == "__main__":
+    unittest.main(verbosity=1)

package/tests/test_ab_waste.py

@@ -0,0 +1,106 @@
+#!/usr/bin/env python3
+# story: e58s09
+"""tests/test_ab_waste.py — lean waste metrics: relearning, rework, first-time-through, patch metrics.
+
+Run: python3 tests/test_ab_waste.py
+"""
+import gzip
+import json
+import sys
+import unittest
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from test_ab_eval import Sandbox, load_module  # noqa: E402
+
+waste = load_module("ab_waste", "ab_waste.py")
+
+
+class Stream:
+    """Builds a pi JSON transcript: assistant turns with one tool call each, plus tool results."""
+
+    def __init__(self):
+        self.events, self.t, self.n = [], 0, 0
+
+    def call(self, tool, args, result_text="ok", exit_code=0):
+        self.n += 1
+        cid = f"c{self.n}"
+        self.events.append({"type": "message_end", "message": {
+            "role": "assistant", "timestamp": self.t * 1000, "usage": {"output": 5, "cost": {"total": 0.001}},
+            "content": [{"type": "toolCall", "name": tool, "arguments": args}]}})
+        self.events.append({"type": "tool_execution_start", "toolCallId": cid, "toolName": tool, "args": args})
+        result = {"content": [{"type": "text", "text": result_text}], "details": {"exit_code": exit_code}}
+        self.events.append({"type": "tool_execution_end", "toolCallId": cid, "toolName": tool,
+                            "result": result, "isError": False})
+        self.events.append({"type": "message_end", "message": {"role": "toolResult",
+                                                               "timestamp": (self.t + 1) * 1000}})
+        self.t += 10
+        return self
+
+    def text(self):
+        return "\n".join(json.dumps(e) for e in self.events)
+
+
+class TestTranscriptWaste(unittest.TestCase):
+    def test_relearning_and_rework_are_counted(self):
+        stream = (Stream()
+                  .call("read", {"path": "/work/src/a.js"})
+                  .call("read", {"path": "/work/src/a.js"})
+                  .call("bash", {"command": "ls"})
+                  .call("bash", {"command": "ls"})
+                  .call("edit", {"path": "/work/src/a.js"})
+                  .call("bash", {"command": "npm test 2>&1 | tail"}, "ℹ pass 3\nℹ fail 1\n")
+                  .call("edit", {"path": "/work/src/a.js"})
+                  .call("bash", {"command": "npm test 2>&1 | tail"}, "ℹ pass 4\nℹ fail 0\n"))
+        got = waste.transcript_waste(stream.text())
+        self.assertEqual((got["rereads"], got["repeated_commands"], got["test_runs"], got["rework_loops"],
+                          got["reedits"], got["ftt"]), (1, 1, 2, 1, 1, False))
+
+    def test_non_zero_exit_is_a_failing_run(self):
+        stream = Stream().call("edit", {"path": "/work/src/a.js"}).call(
+            "bash", {"command": "node --test test/"}, "boom", exit_code=1)
+        self.assertEqual(waste.transcript_waste(stream.text())["rework_loops"], 1)
+
+    def test_clean_run_is_first_time_through(self):
+        stream = Stream().call("edit", {"path": "/work/src/a.js"}).call(
+            "bash", {"command": "npm test"}, "# pass 5\n# fail 0\n")
+        got = waste.transcript_waste(stream.text())
+        self.assertEqual((got["rework_loops"], got["ftt"]), (0, True))
+        self.assertGreater(got["pce"], 0)
+
+    def test_no_transcript_gives_none(self):
+        self.assertIsNone(waste.transcript_waste("not json"))
+
+
+class TestPatchWaste(Sandbox):
+    AGENT = ("echo 42 > answer.txt; printf 'a\\nb\\n' > extra.js; "
+             "printf 'one two three\\n' > notes.md; echo x > debug.tmp.js")
+
+    def setUp(self):
+        super().setUp()
+        solution = self.tasks_dir / "t-01" / "solution"
+        solution.mkdir()
+        (solution / "answer.txt").write_text("42\n")
+
+    def test_patch_metrics_and_saved_patch(self):
+        self.run_main("--adapter-cmd", self.AGENT, "--arms", "vanilla")
+        got = self.records()[0]["waste"]
+        self.assertEqual(got["solution_lines"], 1)
+        self.assertIn("extra.js", got["extra_files"])
+        self.assertNotIn("answer.txt", got["extra_files"])
+        self.assertEqual(got["leftovers"], ["debug.tmp.js"])
+        self.assertEqual(got["doc_lines"], 1)
+        self.assertGreater(got["diff_ratio"], 1)
+        self.assertGreaterEqual(got["review_load"], got["product_lines"] + 3)
+        patch = self.out / "traces" / "t-01-vanilla-1.patch.gz"
+        self.assertIn("extra.js", gzip.decompress(patch.read_bytes()).decode())
+
+    def test_non_pi_adapter_has_patch_metrics_only(self):
+        self.run_main("--adapter-cmd", self.AGENT, "--arms", "vanilla")
+        got = self.records()[0]["waste"]
+        self.assertIsNone(got["rework_loops"])
+        self.assertIsNotNone(got["review_load"])
+
+
+if __name__ == "__main__":
+    unittest.main(verbosity=1)