bigpowers 2.89.1 → 2.90.0

package/tests/test_ab_eval.py

@@ -0,0 +1,309 @@
+#!/usr/bin/env python3
+# story: e58s01
+"""tests/test_ab_eval.py — A/B eval runner (vanilla vs bigpowers) with a fake adapter.
+
+Run: python3 tests/test_ab_eval.py
+No agent is called: every adapter here is a small shell command.
+"""
+import importlib.util
+import io
+import json
+import shutil
+import subprocess
+import sys
+import tempfile
+import unittest
+from contextlib import redirect_stdout
+from pathlib import Path
+
+import yaml
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+LIB = REPO_ROOT / "scripts" / "lib"
+
+
+def load_module(name: str, filename: str):
+    spec = importlib.util.spec_from_file_location(name, LIB / filename)
+    assert spec is not None and spec.loader is not None
+    mod = importlib.util.module_from_spec(spec)
+    sys.modules[name] = mod
+    spec.loader.exec_module(mod)
+    return mod
+
+
+ab = load_module("ab_eval", "ab-eval.py")
+rep = load_module("ab_report", "ab_report.py")
+met = load_module("ab_metrics", "ab_metrics.py")
+bench = load_module("run_benchmark", "run-benchmark.py")
+
+SOLVE_IF_SKILLS = 'if [ -d .claude/skills ]; then echo 42 > answer.txt; fi'
+CLAUDE_JSON = json.dumps({
+    "type": "result", "subtype": "success", "is_error": False, "num_turns": 7,
+    "total_cost_usd": 0.1234, "usage": {"input_tokens": 1500, "output_tokens": 300},
+    "modelUsage": {"claude-opus-5-5": {"costUSD": 0.1}, "claude-haiku-5-5": {"costUSD": 0.0234}},
+})
+
+
+class Sandbox(unittest.TestCase):
+    """Builds a throwaway config + one task, so tests never touch specs/benchmarks/ab/."""
+
+    def setUp(self):
+        self.tmp = Path(tempfile.mkdtemp(prefix="ab-test-"))
+        self.tasks_dir = self.tmp / "tasks"
+        task = self.tasks_dir / "t-01"
+        (task / "fixture" / "test").mkdir(parents=True)
+        (task / "fixture" / "test" / "a.test").write_text("keep me\n")
+        (task / "hidden").mkdir()
+        (task / "hidden" / "secret.check").write_text('test "$(cat answer.txt 2>/dev/null)" = 42\n')
+        (task / "task.yaml").write_text(yaml.safe_dump({
+            "id": "t-01", "title": "answer", "kind": "feature",
+            "prompt": "Write 42 to answer.txt",
+            "protected": ["test/a.test"],
+            "grade": "sh secret.check",
+            "timeout_s": 30,
+        }))
+        self.config = self.tmp / "config.yaml"
+        self.config.write_text(yaml.safe_dump({
+            "k": 1,
+            "arms": {"vanilla": {"setup": "true"}, "bigpowers": {"setup": "mkdir -p .claude/skills"}},
+            "adapters": {"claude": {"command": "false", "parser": "claude-json"}},
+        }))
+        self.out = self.tmp / "out"
+
+    def tearDown(self):
+        shutil.rmtree(self.tmp, ignore_errors=True)
+
+    def run_main(self, *extra):
+        argv = ["--config", str(self.config), "--tasks-dir", str(self.tasks_dir),
+                "--out", str(self.out), *extra]
+        buf = io.StringIO()
+        with redirect_stdout(buf):
+            code = ab.main(argv)
+        return code, buf.getvalue()
+
+    def records(self):
+        lines = next(self.out.glob("ab-runs-*.jsonl")).read_text().splitlines()
+        return [json.loads(line) for line in lines]
+
+
+class TestFairComparison(Sandbox):
+    def test_skill_dependent_agent_shows_full_delta(self):
+        code, _ = self.run_main("--adapter-cmd", SOLVE_IF_SKILLS, "--k", "2")
+        self.assertEqual(code, 0)
+        report = yaml.safe_load(next(self.out.glob("AB-*.yaml")).read_text())
+        self.assertEqual(report["arms_summary"]["bigpowers"]["pass_rate"], 1.0)
+        self.assertEqual(report["arms_summary"]["vanilla"]["pass_rate"], 0.0)
+        self.assertEqual(report["delta"]["pass_rate"], 1.0)
+        self.assertEqual(report["per_task"]["t-01"]["bigpowers"], 1.0)
+
+    def test_hidden_tests_are_not_visible_to_agent(self):
+        peek = "find . -name 'secret*' | grep -q . && exit 1; echo 42 > answer.txt"
+        self.run_main("--adapter-cmd", peek, "--arms", "vanilla")
+        self.assertTrue(all(r["pass"] for r in self.records()))
+
+    def test_workdir_lives_outside_repo(self):
+        where = f"pwd > {self.tmp}/where.txt"
+        self.run_main("--adapter-cmd", where, "--arms", "vanilla")
+        workdir = Path((self.tmp / "where.txt").read_text().strip()).resolve()
+        self.assertNotIn(str(REPO_ROOT.resolve()), str(workdir))
+
+
+class TestRunRecord(Sandbox):
+    def test_deleting_protected_test_is_tampering(self):
+        self.run_main("--adapter-cmd", "rm test/a.test; echo 42 > answer.txt", "--arms", "vanilla")
+        rec = self.records()[0]
+        self.assertTrue(rec["tampered"])
+        self.assertTrue(rec["pass"])
+
+    def test_untouched_protected_test_is_not_tampering(self):
+        self.run_main("--adapter-cmd", "echo 42 > answer.txt", "--arms", "vanilla")
+        self.assertFalse(self.records()[0]["tampered"])
+
+    def test_appending_tests_is_not_tampering(self):  # TDD arms add tests to visible files
+        self.run_main("--adapter-cmd", "echo more >> test/a.test; echo 42 > answer.txt", "--arms", "vanilla")
+        rec = self.records()[0]
+        self.assertFalse(rec["tampered"])
+        self.assertEqual(rec["protected_lines_added"], 1)
+
+    def test_editing_a_protected_line_is_tampering(self):
+        self.run_main("--adapter-cmd", "echo weak > test/a.test; echo 42 > answer.txt", "--arms", "vanilla")
+        self.assertTrue(self.records()[0]["tampered"])
+
+    def test_diff_lines_count_agent_changes_only(self):
+        self.run_main("--adapter-cmd", "printf '42\\nx\\n' > answer.txt", "--arms", "bigpowers")
+        self.assertEqual(self.records()[0]["diff_lines"], 2)
+
+    def test_agent_failure_is_recorded_not_fatal(self):
+        code, _ = self.run_main("--adapter-cmd", "exit 1", "--arms", "vanilla")
+        self.assertEqual(code, 0)
+        rec = self.records()[0]
+        self.assertTrue(rec["agent_error"])
+        self.assertFalse(rec["pass"])
+
+    def test_prompt_reaches_adapter(self):
+        self.run_main("--adapter-cmd", 'printf "%s" "$AB_PROMPT" > got.txt; cp got.txt ' + str(self.tmp),
+                      "--arms", "vanilla")
+        self.assertEqual((self.tmp / "got.txt").read_text(), "Write 42 to answer.txt")
+
+
+class TestMetrics(unittest.TestCase):
+    def test_claude_json_fields_are_parsed(self):
+        m = met.parse_metrics("claude-json", "noise line\n" + CLAUDE_JSON + "\n")
+        self.assertEqual(m["cost_usd"], 0.1234)
+        self.assertEqual(m["input_tokens"], 1500)
+        self.assertEqual(m["output_tokens"], 300)
+        self.assertEqual(m["turns"], 7)
+        self.assertEqual(m["models"], ["claude-haiku-5-5", "claude-opus-5-5"])
+        self.assertFalse(m["agent_reported_error"])
+
+    def test_unparseable_output_gives_nulls(self):
+        m = met.parse_metrics("claude-json", "not json")
+        self.assertIsNone(m["cost_usd"])
+
+    def test_bootstrap_is_deterministic_and_brackets_estimate(self):
+        a = [True, True, False, True, True, False]
+        b = [False, True, False, False, True, False]
+        ci1 = rep.bootstrap_delta_ci(a, b)
+        ci2 = rep.bootstrap_delta_ci(a, b)
+        self.assertEqual(ci1, ci2)
+        self.assertLessEqual(ci1[0], 2 / 6)
+        self.assertGreaterEqual(ci1[1], 2 / 6)
+
+
+OPENCODE_JSONL = "\n".join(json.dumps(e) for e in [
+    {"type": "step_start", "part": {"type": "step-start"}},
+    {"type": "step_finish", "part": {"type": "step-finish", "cost": 0.01,
+                                     "tokens": {"input": 100, "output": 20, "reasoning": 5,
+                                                "cache": {"read": 300, "write": 50}}}},
+    {"type": "text", "part": {"type": "text", "text": "done"}},
+    {"type": "step_finish", "part": {"type": "step-finish", "cost": 0.02,
+                                     "tokens": {"input": 10, "output": 2, "reasoning": 0,
+                                                "cache": {"read": 400, "write": 0}}}},
+])
+
+
+class TestOpencodeParser(unittest.TestCase):
+    def test_opencode_jsonl_sums_steps(self):
+        m = met.parse_metrics("opencode-jsonl", "log noise\n" + OPENCODE_JSONL + "\n")
+        self.assertAlmostEqual(m["cost_usd"], 0.03)
+        self.assertEqual(m["input_tokens"], 100 + 300 + 50 + 10 + 400)
+        self.assertEqual(m["output_tokens"], 20 + 5 + 2)
+        self.assertEqual(m["turns"], 2)
+        self.assertFalse(m["agent_reported_error"])
+
+    def test_opencode_error_event_is_agent_error(self):
+        out = OPENCODE_JSONL + "\n" + json.dumps({"type": "error", "error": {"name": "APIError"}})
+        self.assertTrue(met.parse_metrics("opencode-jsonl", out)["agent_reported_error"])
+
+    def test_opencode_without_steps_gives_nulls(self):
+        self.assertIsNone(met.parse_metrics("opencode-jsonl", "")["cost_usd"])
+
+
+class TestRunEnvironment(Sandbox):
+    def test_each_run_gets_fresh_home_removed_afterwards(self):
+        cmd = (f'ls -A "$AB_HOME" | wc -l | tr -d " " > {self.tmp}/n.txt; '
+               f'echo "$AB_HOME" > {self.tmp}/home.txt; echo 42 > answer.txt')
+        self.run_main("--adapter-cmd", cmd, "--arms", "vanilla")
+        self.assertEqual((self.tmp / "n.txt").read_text().strip(), "0")
+        home = Path((self.tmp / "home.txt").read_text().strip())
+        self.assertTrue(str(home))
+        self.assertFalse(home.exists())
+
+    def test_adapter_stdin_is_closed(self):
+        self.run_main("--adapter-cmd", "cat > /dev/null; echo 42 > answer.txt", "--arms", "vanilla")
+        self.assertTrue(self.records()[0]["pass"])
+
+    def test_adapter_default_model_is_used_and_recorded(self):
+        cfg = yaml.safe_load(self.config.read_text())
+        cfg["adapters"]["fake"] = {
+            "command": f'printf "%s" "$AB_MODEL" > {self.tmp}/model.txt; echo 42 > answer.txt',
+            "parser": "none", "default_model": "prov/model-x"}
+        self.config.write_text(yaml.safe_dump(cfg))
+        self.run_main("--adapter", "fake", "--arms", "vanilla")
+        self.assertEqual((self.tmp / "model.txt").read_text(), "prov/model-x")
+        report = yaml.safe_load(next(self.out.glob("AB-*.yaml")).read_text())
+        self.assertEqual(report["model"], "prov/model-x")
+
+
+class TestCli(Sandbox):
+    def test_unknown_arm_exits_2(self):
+        code, out = self.run_main("--arms", "nope")
+        self.assertEqual(code, 2)
+        self.assertIn("vanilla", out)
+
+    def test_unknown_task_exits_2(self):
+        code, _ = self.run_main("--tasks", "missing")
+        self.assertEqual(code, 2)
+
+    def test_dry_run_spends_nothing(self):
+        marker = self.tmp / "ran.txt"
+        code, out = self.run_main("--adapter-cmd", f"touch {marker}", "--dry-run")
+        self.assertEqual(code, 0)
+        self.assertFalse(marker.exists())
+        self.assertIn("t-01", out)
+        self.assertFalse(self.out.exists())
+
+    def test_single_arm_has_no_delta(self):
+        self.run_main("--adapter-cmd", "true", "--arms", "vanilla")
+        report = yaml.safe_load(next(self.out.glob("AB-*.yaml")).read_text())
+        self.assertIsNone(report["delta"])
+
+
+@unittest.skipUnless(shutil.which("node"), "node is needed for the starter tasks")
+class TestStarterTasks(unittest.TestCase):
+    """Each shipped task must fail as given and pass with its reference solution."""
+
+    TASKS = REPO_ROOT / "specs" / "benchmarks" / "ab" / "tasks"
+
+    def grade_copy(self, task_dir: Path, with_solution: bool) -> bool:
+        task = yaml.safe_load((task_dir / "task.yaml").read_text())
+        work = Path(tempfile.mkdtemp(prefix="ab-starter-"))
+        try:
+            shutil.copytree(task_dir / "fixture", work, dirs_exist_ok=True)
+            if with_solution:
+                shutil.copytree(task_dir / "solution", work, dirs_exist_ok=True)
+            shutil.copytree(task_dir / "hidden", work, dirs_exist_ok=True)
+            res = subprocess.run(["bash", "-c", task["grade"]], cwd=work, capture_output=True)
+            return res.returncode == 0
+        finally:
+            shutil.rmtree(work, ignore_errors=True)
+
+    def test_eight_tasks_ship(self):
+        self.assertGreaterEqual(len(list(self.TASKS.glob("*/task.yaml"))), 8)
+
+    def test_fixture_fails_and_solution_passes(self):
+        for task_yaml in sorted(self.TASKS.glob("*/task.yaml")):
+            with self.subTest(task=task_yaml.parent.name):
+                self.assertFalse(self.grade_copy(task_yaml.parent, with_solution=False))
+                self.assertTrue(self.grade_copy(task_yaml.parent, with_solution=True))
+
+    def test_prompts_never_name_bigpowers(self):
+        for task_yaml in self.TASKS.glob("*/task.yaml"):
+            self.assertNotIn("bigpowers", yaml.safe_load(task_yaml.read_text())["prompt"].lower())
+
+    def test_grade_commands_never_pass_a_bare_directory(self):
+        """`node --test test/` exits 1 on node 24 (it resolves the directory as a module path), so
+        such a task can never pass however good the solution is, while a hidden-only `score`
+        command still looks healthy (BUG found in e58s07: ab-11 and ab-13 read pass=0 forever)."""
+        for task_yaml in self.TASKS.glob("*/task.yaml"):
+            with self.subTest(task=task_yaml.parent.name):
+                for token in yaml.safe_load(task_yaml.read_text())["grade"].split():
+                    self.assertNotIn(token.rstrip("/"), ("test", "test-hidden"),
+                                     f"{task_yaml.parent.name} grades a directory, not a glob")
+
+
+class TestRunBenchmarkHonesty(unittest.TestCase):
+    def test_without_skill_arm_is_unmeasured_not_zero(self):
+        sc = {"id": "s1", "grader": {"type": "code", "command": "true"}}
+        _, status = bench.scenario_pass_rate(sc, runs=1, with_skill=False)
+        self.assertEqual(status, "unmeasured")
+        report = bench.build_split_report([sc], runs=1)
+        self.assertIsNone(report["without_skill"])
+        self.assertIsNone(report["delta"])
+
+    def test_summary_says_unmeasured(self):
+        self.assertIn("UNMEASURED", bench.delta_summary(None))
+
+
+if __name__ == "__main__":
+    unittest.main(verbosity=2)

package/tests/test_ab_pi.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+# story: e58s06
+"""tests/test_ab_pi.py — pi adapter: pi-json metrics parser and the config default adapter.
+
+Run: python3 tests/test_ab_pi.py
+"""
+import json
+import sys
+import unittest
+from pathlib import Path
+
+import yaml
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from test_ab_eval import Sandbox, met  # noqa: E402
+
+
+def assistant_end(inp, out, cache_read, cost, stop="toolUse"):
+    usage = {"input": inp, "output": out, "cacheRead": cache_read, "cacheWrite": 0, "cost": {"total": cost}}
+    return {"type": "message_end", "message": {"role": "assistant", "stopReason": stop, "usage": usage}}
+
+
+def stream(*events):
+    return "\n".join(json.dumps(e) for e in events)
+
+
+class TestPiParser(unittest.TestCase):
+    def test_sums_assistant_usage(self):
+        out = stream({"type": "session", "id": "x"},
+                     {"type": "message_end", "message": {"role": "user", "content": "hi"}},
+                     assistant_end(1390, 56, 256, 0.000243),
+                     assistant_end(55, 3, 1664, 0.000015, stop="stop"),
+                     {"type": "agent_settled", "aborted": False})
+        m = met.parse_metrics("pi-json", out)
+        self.assertEqual((m["input_tokens"], m["output_tokens"], m["turns"]), (1390 + 256 + 55 + 1664, 59, 2))
+        self.assertAlmostEqual(m["cost_usd"], 0.000258)
+        self.assertFalse(m["agent_reported_error"])
+
+    def test_provider_error_is_agent_error(self):
+        out = stream(assistant_end(10, 0, 0, 0.0, stop="error"), {"type": "agent_settled", "aborted": False})
+        self.assertTrue(met.parse_metrics("pi-json", out)["agent_reported_error"])
+
+    def test_aborted_run_is_agent_error(self):
+        out = stream(assistant_end(10, 1, 0, 0.0, stop="stop"), {"type": "agent_settled", "aborted": True})
+        self.assertTrue(met.parse_metrics("pi-json", out)["agent_reported_error"])
+
+    def test_no_events_is_unknown(self):
+        self.assertIsNone(met.parse_metrics("pi-json", "boom")["agent_reported_error"])
+
+
+class TestDefaultAdapter(Sandbox):
+    def test_config_default_adapter_is_used(self):
+        config = yaml.safe_load(self.config.read_text())
+        config["adapters"]["fakepi"] = {"command": "echo 42 > answer.txt", "parser": "none"}
+        config["default_adapter"] = "fakepi"
+        self.config.write_text(yaml.safe_dump(config))
+        _, out = self.run_main("--arms", "vanilla")
+        self.assertIn("adapter 'fakepi'", out)
+        self.assertTrue(self.records()[0]["pass"])
+
+
+if __name__ == "__main__":
+    unittest.main(verbosity=1)

package/tests/test_ab_progress.py

@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+# story: e58s05
+"""tests/test_ab_progress.py — live records, interleaved runs, saved transcripts, ab_progress.
+
+Run: python3 tests/test_ab_progress.py
+"""
+import gzip
+import io
+import json
+import sys
+import unittest
+from contextlib import redirect_stdout
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from test_ab_eval import Sandbox, load_module  # noqa: E402
+
+prog = load_module("ab_progress", "ab_progress.py")
+
+
+class TestLiveRun(Sandbox):
+    def live(self):
+        return [json.loads(line) for line in (self.out / "ab-live.jsonl").read_text().splitlines()]
+
+    def test_records_stream_with_sha_and_model(self):
+        self.run_main("--adapter-cmd", "echo 42 > answer.txt", "--k", "2", "--model", "m-1")
+        live = self.live()
+        self.assertEqual(len(live), 4)
+        self.assertTrue(all(r["bigpowers_sha"] and r["model"] == "m-1" for r in live))
+
+    def test_runs_are_interleaved(self):
+        self.run_main("--adapter-cmd", "echo 42 > answer.txt", "--k", "2")
+        order = [(r["run"], r["arm"]) for r in self.live()]
+        self.assertEqual(order, [(1, "vanilla"), (1, "bigpowers"), (2, "vanilla"), (2, "bigpowers")])
+
+    def test_transcripts_are_saved(self):
+        self.run_main("--adapter-cmd", "echo transcript-line; echo 42 > answer.txt", "--arms", "vanilla")
+        traces = sorted((self.out / "traces").glob("*.jsonl.gz"))
+        self.assertEqual([t.name for t in traces], ["t-01-vanilla-1.jsonl.gz"])
+        self.assertIn("transcript-line", gzip.decompress(traces[0].read_bytes()).decode())
+
+
+class TestProgress(unittest.TestCase):
+    def shard(self, tmp: Path, name: str, records: list[dict]) -> Path:
+        folder = tmp / name
+        folder.mkdir()
+        (folder / "ab-live.jsonl").write_text("".join(json.dumps(r) + "\n" for r in records))
+        return folder
+
+    def rec(self, arm, run, passed, score):
+        return {"task": "ab-11", "arm": arm, "run": run, "pass": passed, "score": score, "tampered": False,
+                "agent_error": False, "cost_usd": 0.01, "input_tokens": None, "output_tokens": None,
+                "turns": 5, "duration_s": 1.0, "diff_lines": 3, "bigpowers_sha": "abc", "model": "m"}
+
+    def test_reads_shards_and_prints_evolution(self):
+        import tempfile
+        tmp = Path(tempfile.mkdtemp(prefix="ab-prog-"))
+        a = self.shard(tmp, "s1", [self.rec("vanilla", i, i % 2 == 0, 0.5) for i in range(1, 11)]
+                       + [self.rec("bigpowers", i, True, 1.0) for i in range(1, 11)])
+        b = self.shard(tmp, "s2", [self.rec("vanilla", i, False, 0.4) for i in range(1, 6)]
+                       + [self.rec("bigpowers", i, True, 1.0) for i in range(1, 6)])
+        buf = io.StringIO()
+        with redirect_stdout(buf):
+            code = prog.main([str(a), str(b)])
+        text = buf.getvalue()
+        self.assertEqual(code, 0)
+        self.assertIn("30 runs", text)
+        self.assertRegex(text, r"ab-11\s+vanilla\s+15")
+        self.assertRegex(text, r"ab-11\s+bigpowers\s+15")
+        self.assertIn("bigpowers - vanilla", text)
+        self.assertIn("evolution", text)
+
+    def test_no_live_file_exits_2(self):
+        import tempfile
+        buf = io.StringIO()
+        with redirect_stdout(buf):
+            self.assertEqual(prog.main([tempfile.mkdtemp()]), 2)
+
+
+if __name__ == "__main__":
+    unittest.main(verbosity=1)

package/tests/test_ab_sandbox.py

@@ -0,0 +1,94 @@
+#!/usr/bin/env python3
+# story: e58s07
+"""tests/test_ab_sandbox.py — Docker sandbox: grade_wrapper, AB_TIMEOUT, container isolation.
+
+Run: python3 tests/test_ab_sandbox.py
+The isolation tests need Docker and `bash scripts/ab-sandbox.sh up`; they skip otherwise.
+"""
+import shutil
+import subprocess
+import sys
+import unittest
+from pathlib import Path
+
+import yaml
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from test_ab_eval import REPO_ROOT, Sandbox  # noqa: E402
+
+AGENT_IMAGE = "bp-ab-agent:pi-1.5.0"
+
+
+def sandbox_up() -> bool:
+    if not shutil.which("docker"):
+        return False
+    res = subprocess.run(["bash", str(REPO_ROOT / "scripts" / "ab-sandbox.sh"), "status"], capture_output=True)
+    return res.returncode == 0
+
+
+class TestGradeWrapper(Sandbox):
+    def add_adapter(self, **extra):
+        config = yaml.safe_load(self.config.read_text())
+        config["adapters"]["boxed"] = {"command": "echo $AB_TIMEOUT > timeout.txt; cp timeout.txt "
+                                       + str(self.tmp) + "/; echo 42 > answer.txt", "parser": "none", **extra}
+        self.config.write_text(yaml.safe_dump(config))
+
+    def test_grade_and_score_go_through_the_wrapper(self):
+        log = self.tmp / "wrapped.log"
+        self.add_adapter(grade_wrapper=f'echo "$AB_CMD" >> {log}; bash -c "$AB_CMD"')
+        task_yaml = self.tasks_dir / "t-01" / "task.yaml"
+        task = yaml.safe_load(task_yaml.read_text())
+        task["score"] = "printf '# pass 1\\n# fail 0\\n'"
+        task_yaml.write_text(yaml.safe_dump(task))
+        self.run_main("--adapter", "boxed", "--arms", "vanilla")
+        rec = self.records()[0]
+        self.assertTrue(rec["pass"])
+        self.assertEqual(rec["score"], 1.0)
+        self.assertEqual(len(log.read_text().splitlines()), 2)
+
+    def test_timeout_is_exported_with_margin(self):
+        self.add_adapter()
+        self.run_main("--adapter", "boxed", "--arms", "vanilla")
+        self.assertEqual((self.tmp / "timeout.txt").read_text().strip(), "10")
+
+
+class TestSandboxScript(unittest.TestCase):
+    def test_sandbox_script_syntax_and_usage(self):
+        script = REPO_ROOT / "scripts" / "ab-sandbox.sh"
+        self.assertTrue(script.exists())
+        self.assertTrue(bool(script.stat().st_mode & 0o111), "ab-sandbox.sh must be executable")
+        res = subprocess.run(["bash", "-n", str(script)], capture_output=True, text=True)
+        self.assertEqual(res.returncode, 0, res.stderr)
+        res = subprocess.run(["bash", str(script), "unknown-cmd"], capture_output=True, text=True)
+        self.assertEqual(res.returncode, 1)
+        self.assertIn("usage: ab-sandbox.sh up|down|status", res.stderr)
+
+
+@unittest.skipUnless(sandbox_up(), "Docker sandbox not up (bash scripts/ab-sandbox.sh up)")
+class TestIsolation(unittest.TestCase):
+    def agent(self, script: str, network: str = "bp-ab-internal") -> subprocess.CompletedProcess:
+        return subprocess.run(
+            ["docker", "run", "--rm", "--network", network,
+             "-e", "HTTPS_PROXY=http://bp-ab-proxy:8888", "-e", "HTTP_PROXY=http://bp-ab-proxy:8888",
+             AGENT_IMAGE, "bash", "-c", script], capture_output=True, text=True, timeout=120)
+
+    def test_host_paths_are_invisible(self):
+        res = self.agent(f'test ! -e /Users && test ! -e "{REPO_ROOT}" && echo hidden')
+        self.assertEqual(res.stdout.strip(), "hidden", res.stderr)
+
+    def test_runs_as_non_root(self):
+        self.assertNotEqual(self.agent("id -u").stdout.strip(), "0")
+
+    def test_only_the_model_provider_is_reachable(self):
+        probe = "node -e \"fetch(process.argv[1]).then(r=>console.log('ok',r.status)).catch(e=>console.log('blocked'))\""
+        self.assertIn("blocked", self.agent(f"{probe} https://example.com").stdout)
+        self.assertIn("ok", self.agent(f"{probe} https://opencode.ai").stdout)
+
+    def test_grading_container_has_no_network(self):
+        res = self.agent("node -e \"fetch('https://opencode.ai').then(()=>console.log('open'))"
+                         ".catch(()=>console.log('none'))\"", network="none")
+        self.assertIn("none", res.stdout)
+
+
+if __name__ == "__main__":
+    unittest.main(verbosity=1)

package/tests/test_ab_score.py

@@ -0,0 +1,113 @@
+#!/usr/bin/env python3
+# story: e58s03
+"""tests/test_ab_score.py — partial-credit scoring and the hard tasks (ab-11, ab-13).
+
+Run: python3 tests/test_ab_score.py
+Split from test_ab_eval.py (300-line cap); reuses its Sandbox and module loaders.
+"""
+import shutil
+import subprocess
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+import yaml
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from test_ab_eval import REPO_ROOT, Sandbox, met, rep  # noqa: E402
+
+TASKS = REPO_ROOT / "specs" / "benchmarks" / "ab" / "tasks"
+HARD_TASKS = ["ab-11-ledger-rounding", "ab-13-orders-split"]
+
+
+class TestTapScore(unittest.TestCase):
+    def test_share_of_passing_tests(self):
+        self.assertEqual(met.tap_score("# tests 4\n# pass 3\n# fail 1\n"), 0.75)
+
+    def test_several_summaries_are_summed(self):
+        self.assertEqual(met.tap_score("# pass 2\n# fail 0\nnoise\n# pass 1\n# fail 1\n"), 0.75)
+
+    def test_no_summary_is_none(self):
+        self.assertIsNone(met.tap_score("SyntaxError: boom\n"))
+
+
+class TestScoreInRuns(Sandbox):
+    def set_score(self, command):
+        task_yaml = self.tasks_dir / "t-01" / "task.yaml"
+        task = yaml.safe_load(task_yaml.read_text())
+        task["score"] = command
+        task_yaml.write_text(yaml.safe_dump(task))
+
+    def test_score_command_sets_score(self):
+        self.set_score("printf '# pass 3\\n# fail 1\\n'; exit 1")
+        self.run_main("--adapter-cmd", "echo 42 > answer.txt", "--arms", "vanilla")
+        self.assertEqual(self.records()[0]["score"], 0.75)
+
+    def test_no_score_key_means_null(self):
+        self.run_main("--adapter-cmd", "echo 42 > answer.txt")
+        self.assertIsNone(self.records()[0]["score"])
+        report = yaml.safe_load(next(self.out.glob("AB-*.yaml")).read_text())
+        self.assertIsNone(report["delta"].get("score"))
+
+
+class TestScoreInReport(unittest.TestCase):
+    def rec(self, arm, task, score):
+        return {"arm": arm, "task": task, "pass": score == 1.0, "tampered": False, "agent_error": False,
+                "cost_usd": None, "input_tokens": None, "output_tokens": None, "turns": None,
+                "duration_s": 1.0, "diff_lines": 1, "score": score}
+
+    def test_mean_score_and_delta(self):
+        records = [self.rec("vanilla", "t", 0.5), self.rec("vanilla", "t", 0.7),
+                   self.rec("bigpowers", "t", 0.9), self.rec("bigpowers", "t", 1.0)]
+        result = rep.aggregate(records, ["vanilla", "bigpowers"], ["t"])
+        self.assertEqual(result["arms_summary"]["vanilla"]["mean_score"], 0.6)
+        self.assertEqual(result["per_task_score"]["t"]["bigpowers"], 0.95)
+        self.assertEqual(result["delta"]["score"], 0.35)
+        lo, hi = result["delta"]["score_ci95"]
+        self.assertLessEqual(lo, 0.35)
+        self.assertGreaterEqual(hi, 0.35)
+
+
+@unittest.skipUnless(shutil.which("node"), "node is needed for the hard tasks")
+class TestHardTasks(unittest.TestCase):
+    """Each hard task must fail as given, pass with its solution, and give partial credit."""
+
+    def grade(self, name: str, with_solution: bool) -> tuple[bool, float | None]:
+        task_dir = TASKS / name
+        task = yaml.safe_load((task_dir / "task.yaml").read_text())
+        work = Path(tempfile.mkdtemp(prefix="ab-hard-"))
+        try:
+            shutil.copytree(task_dir / "fixture", work, dirs_exist_ok=True)
+            if with_solution:
+                shutil.copytree(task_dir / "solution", work, dirs_exist_ok=True)
+            shutil.copytree(task_dir / "hidden", work, dirs_exist_ok=True)
+            run = lambda cmd: subprocess.run(["bash", "-c", cmd], cwd=work, capture_output=True, text=True)
+            passed = run(task["grade"]).returncode == 0
+            return passed, met.tap_score(run(task["score"]).stdout)
+        finally:
+            shutil.rmtree(work, ignore_errors=True)
+
+    def test_fixture_fails_with_partial_score(self):
+        for name in HARD_TASKS:
+            with self.subTest(task=name):
+                passed, score = self.grade(name, with_solution=False)
+                self.assertFalse(passed)
+                assert score is not None
+                self.assertLess(score, 1.0)
+
+    def test_solution_passes_with_full_score(self):
+        for name in HARD_TASKS:
+            with self.subTest(task=name):
+                self.assertEqual(self.grade(name, with_solution=True), (True, 1.0))
+
+    def test_score_runs_hidden_tests_only(self):
+        for name in HARD_TASKS:
+            with self.subTest(task=name):
+                score_cmd = yaml.safe_load((TASKS / name / "task.yaml").read_text())["score"]
+                self.assertIn("test-hidden", score_cmd)
+                self.assertNotRegex(score_cmd, r"(^|\s)test/")
+
+
+if __name__ == "__main__":
+    unittest.main(verbosity=1)