dsh-eval-guard 0.7.1 → 0.7.3

package/eval/legacy/bomberman.py

@@ -107,13 +107,16 @@ def main() -> int:
     cmd = [sys.executable, "-X", "utf8", str(task / "tournament" / "run.py"), "--output", str(run_dir),
            "--bots", *bots, "--sizes", *sizes, "--seeds", str(seeds), "--workers", str(max(1, args.workers))]
     print("tier=%s sizes=%s seeds=%d bots=%d" % (args.tier, sizes, seeds, len(bots)), flush=True)
-    completed = subprocess.run(cmd, capture_output=True, text=True, encoding="utf-8", errors="replace")
+    completed = subprocess.run(cmd, capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=3600)
     if completed.returncode != 0:
         print(completed.stdout[-4000:], completed.stderr[-4000:], file=sys.stderr)
         return 2
     ratings_doc = load_json(run_dir / "bt.json") or {}
+    if ratings_doc.get("unavailable"):
+        print("bomberman rating unavailable: " + str(ratings_doc.get("unavailable")), file=sys.stderr)
+        return 2
     ratings = ratings_doc.get("ratings", {})
-    prices = load_json(ROOT / "prices.json") or {}
+    prices = load_json(ROOT / "bomberman-prices.json") or load_json(ROOT / "prices.json") or {}
 
     rows = []
     for name, program in programs.items():

package/eval/legacy/ctf.py

@@ -123,14 +123,17 @@ def main() -> int:
     print("tier=%s sizes=%s maps=%d players=%d" % (args.tier, sizes, maps_per_size, len(players)), flush=True)
     for step in steps:
         completed = subprocess.run([sys.executable, "-X", "utf8", tournament, *step], capture_output=True, text=True,
-                                   encoding="utf-8", errors="replace")
+                                   encoding="utf-8", errors="replace", timeout=1800)
         if completed.returncode != 0:
             print(completed.stdout[-4000:], completed.stderr[-4000:], file=sys.stderr)
             return 2
     scores = load_json(games_dir / "成绩.json") or {}
+    if scores.get("unavailable"):
+        print("ctf rating unavailable: " + str(scores.get("unavailable")), file=sys.stderr)
+        return 2
     raw_ratings = scores.get("ratings", {})
     ratings = {name: (value[0] if isinstance(value, list) and value else value) for name, value in raw_ratings.items()}
-    prices = load_json(ROOT / "prices.json") or {}
+    prices = load_json(ROOT / "ctf-prices.json") or load_json(ROOT / "prices.json") or {}
 
     rows = []
     for name, program in players.items():

package/eval/legacy/featurebench.py

@@ -4,8 +4,9 @@
 Submission layout:
   submissions/<name>/workspace/   (copy of the agent workspace, including .venv)
 
-The hidden feature test file is copied from the plugin asset; scoring runs the
-instance's fail-to-pass and pass-to-pass files. Missing run/ statistics are skipped.
+The hidden feature test file is copied from the plugin asset; scoring checks the
+fail-to-pass file and every pass-to-pass file explicitly (a skipped or removed
+test does not count as resolved).
 """
 from __future__ import annotations
 
@@ -23,6 +24,7 @@ ASSET = ROOT.parents[1] / "assets" / "featurebench"
 PASSED = re.compile(r"(\d+) passed")
 FAILED = re.compile(r"(\d+) failed")
 ERRORS = re.compile(r"(\d+) error")
+RESULT = re.compile(r"^(PASSED|FAILED|ERROR|SKIPPED|XFAIL|XPASS) (\S+)", re.MULTILINE)
 
 
 def load_json(path: Path):
@@ -39,18 +41,29 @@ def find_workspace(directory: Path) -> Path | None:
     return None
 
 
-def install_hidden_test(workspace: Path) -> None:
+def install_hidden_test(workspace: Path) -> str:
     source = ASSET / "hidden" / "test_regression.py"
-    if source.is_file():
-        shutil.copy2(source, workspace / "tests" / "test_regression.py")
+    if not source.is_file():
+        return "hidden regression test is missing"
+    shutil.copy2(source, workspace / "tests" / "test_regression.py")
+    return ""
 
 
-def run_tests(workspace: Path, instance: dict) -> tuple[int, int, int, str]:
+def matches(node: str, selectors: list[str]) -> bool:
+    for selector in selectors:
+        if node == selector or node.startswith(selector + "::") or node.startswith(selector + "/"):
+            return True
+    return False
+
+
+def run_tests(workspace: Path, instance: dict) -> tuple[int, int, int, str, dict]:
     venv = workspace / ".venv" / "bin" / "python"
     python = str(venv) if venv.is_file() else "python3"
-    tests = list(instance.get("FAIL_TO_PASS", [])) + list(instance.get("PASS_TO_PASS", []))
+    fail_to_pass = list(instance.get("FAIL_TO_PASS", []))
+    pass_to_pass = list(instance.get("PASS_TO_PASS", []))
+    tests = fail_to_pass + pass_to_pass
     env = {k: v for k, v in __import__("os").environ.items() if k not in {"PYTHONPATH", "PYTHONHOME"}}
-    completed = subprocess.run([python, "-m", "pytest", *tests, "-q", "--tb=no"], cwd=workspace,
+    completed = subprocess.run([python, "-m", "pytest", *tests, "-q", "--tb=no", "-rA"], cwd=workspace,
                                capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=1800, env=env)
     output = (completed.stdout or "") + "\n" + (completed.stderr or "")
     passed = int(PASSED.search(output).group(1)) if PASSED.search(output) else 0
@@ -58,7 +71,31 @@ def run_tests(workspace: Path, instance: dict) -> tuple[int, int, int, str]:
     errors = int(ERRORS.search(output).group(1)) if ERRORS.search(output) else 0
     if completed.returncode != 0 and passed == 0 and failed == 0 and errors == 0:
         errors = 1
-    return passed, failed, errors, output[-3000:]
+    outcomes = {node: status for status, node in RESULT.findall(output)}
+    f2p_total = f2p_passed = 0
+    p2p_total = p2p_passed = 0
+    for node, status in outcomes.items():
+        if matches(node, fail_to_pass):
+            f2p_total += 1
+            f2p_passed += 1 if status == "PASSED" else 0
+        elif matches(node, pass_to_pass):
+            p2p_total += 1
+            p2p_passed += 1 if status == "PASSED" else 0
+    resolved = (
+        len(fail_to_pass) > 0
+        and f2p_total > 0
+        and f2p_passed == f2p_total
+        and p2p_total > 0
+        and p2p_passed == p2p_total
+        and completed.returncode == 0
+    )
+    return passed, failed, errors, output[-3000:], {
+        "resolved": resolved,
+        "f2p_total": f2p_total,
+        "f2p_passed": f2p_passed,
+        "p2p_total": p2p_total,
+        "p2p_passed": p2p_passed,
+    }
 
 
 def main() -> int:
@@ -78,20 +115,23 @@ def main() -> int:
     for directory in directories:
         workspace = find_workspace(directory)
         if workspace is None:
-            rows.append({"name": directory.name, "status": "missing-workspace", "score": 0.0, "passed": 0, "failed": 0, "errors": 0})
+            rows.append({"name": directory.name, "status": "missing-workspace", "score": 0.0, "passed": 0, "failed": 0, "errors": 0, "resolved": False})
             continue
         try:
-            install_hidden_test(workspace)
-            passed, failed, errors, detail = run_tests(workspace, instance)
+            hidden_error = install_hidden_test(workspace)
+            if hidden_error:
+                rows.append({"name": directory.name, "status": "hidden-test-failed", "score": 0.0, "passed": 0, "failed": 0, "errors": 1, "resolved": False, "detail": hidden_error})
+                continue
+            passed, failed, errors, detail, extra = run_tests(workspace, instance)
             total = passed + failed + errors
             score = 100.0 * passed / total if total else 0.0
-            rows.append({"name": directory.name, "status": "ok", "score": round(score, 2), "passed": passed, "failed": failed, "errors": errors, "detail": detail})
+            rows.append({"name": directory.name, "status": "ok", "score": round(score, 2), "passed": passed, "failed": failed, "errors": errors, "detail": detail, **extra})
         except Exception as error:
-            rows.append({"name": directory.name, "status": "error", "score": 0.0, "passed": 0, "failed": 0, "errors": 1, "detail": str(error)})
+            rows.append({"name": directory.name, "status": "error", "score": 0.0, "passed": 0, "failed": 0, "errors": 1, "resolved": False, "detail": str(error)})
     rows.sort(key=lambda row: (-row["score"], row["name"]))
     payload = {"generatedAt": time.strftime("%Y-%m-%dT%H:%M:%S%z"), "benchmark": "featurebench", "instance": instance.get("instance_id"), "results": rows}
     (out / "summary.json").write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
-    lines = ["# FeatureBench · 单 feature 排行", "", "- 实例：%s" % instance.get("instance_id", ""), "- 指标：隐藏 fail-to-pass + pass-to-pass 测试通过率", "", "| 名次 | 名称 | 得分 | 通过 | 失败 | 错误 | 状态 |", "|---:|---|---:|---:|---:|---:|---|"]
+    lines = ["# FeatureBench · 单 feature 排行", "", "- 实例：%s" % instance.get("instance_id", ""), "- 指标：隐藏 feature 测试与全部回归文件通过才 resolved", "", "| 名次 | 名称 | 得分 | 通过 | 失败 | 错误 | 状态 |", "|---:|---|---:|---:|---:|---:|---|"]
     for rank, row in enumerate(rows, 1):
         lines.append("| %d | %s | %.1f | %d | %d | %d | %s |" % (rank, row["name"], row["score"], row["passed"], row["failed"], row["errors"], row["status"]))
     (out / "summary.md").write_text("\n".join(lines) + "\n", encoding="utf-8")

package/eval/legacy/land.py

@@ -112,20 +112,23 @@ def main() -> int:
     if melee == 0:
         cmd += ["--only", "duel"]
     print("tier=%s maps=%d melee=%d entries=%d" % (args.tier, maps, melee, len(list(entries.iterdir()))), flush=True)
-    completed = subprocess.run(cmd, capture_output=True, text=True, encoding="utf-8", errors="replace")
+    completed = subprocess.run(cmd, capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=1800)
     if completed.returncode != 0:
         print(completed.stdout[-4000:], completed.stderr[-4000:], file=sys.stderr)
         return 2
     ratings_prefix = out / "ratings"
     rate = subprocess.run([sys.executable, "-X", "utf8", str(task / "tournament" / "rate.py"), str(results_jsonl),
                            "--entries", str(entries), "--out", str(ratings_prefix)], capture_output=True, text=True,
-                          encoding="utf-8", errors="replace")
+                          encoding="utf-8", errors="replace", timeout=1800)
     if rate.returncode != 0:
         print(rate.stdout[-4000:], rate.stderr[-4000:], file=sys.stderr)
         return 2
     ratings = load_json(Path(str(ratings_prefix) + ".json")) or {}
+    if ratings.get("unavailable"):
+        print("land rating unavailable: " + str(ratings.get("unavailable")), file=sys.stderr)
+        return 2
     models = ratings.get("models", {})
-    prices = load_json(ROOT / "prices.json") or {}
+    prices = load_json(ROOT / "land-prices.json") or load_json(ROOT / "prices.json") or {}
 
     rows = []
     all_names = list(names) + [name for name in models if name not in names]

package/eval/legacy/swe_bench_pro.py

@@ -4,8 +4,9 @@
 Submission layout:
   submissions/<name>/workspace/   (copy of the agent workspace, including .venv)
 
-The hidden test patch is applied from the plugin asset; scoring runs the
-instance's fail-to-pass and pass-to-pass tests. Missing run/ statistics are skipped.
+The hidden test patch is applied from the plugin asset; scoring checks the
+instance's fail-to-pass and pass-to-pass node ids explicitly (a skipped or
+renamed fail-to-pass test does not count as resolved).
 """
 from __future__ import annotations
 
@@ -22,6 +23,7 @@ ASSET = ROOT.parents[1] / "assets" / "swe-bench-pro"
 PASSED = re.compile(r"(\d+) passed")
 FAILED = re.compile(r"(\d+) failed")
 ERRORS = re.compile(r"(\d+) error")
+RESULT = re.compile(r"^(PASSED|FAILED|ERROR|SKIPPED|XFAIL|XPASS) (\S+)", re.MULTILINE)
 
 
 def load_json(path: Path):
@@ -48,22 +50,34 @@ def find_workspace(directory: Path) -> Path | None:
     return None
 
 
-def apply_hidden_tests(workspace: Path) -> None:
+def apply_hidden_tests(workspace: Path) -> str:
     patch = ASSET / "hidden" / "test.patch"
     if not patch.is_file():
-        return
-    subprocess.run(["patch", "-p1", "--forward", "-i", str(patch)], cwd=workspace,
-                   capture_output=True, text=True, encoding="utf-8", errors="replace")
+        return "hidden test patch is missing"
+    completed = subprocess.run(["patch", "-p1", "--forward", "-i", str(patch)], cwd=workspace,
+                               capture_output=True, text=True, encoding="utf-8", errors="replace")
+    if completed.returncode != 0:
+        return "hidden test patch failed to apply: " + (completed.stdout + completed.stderr)[-2000:]
+    return ""
 
 
-def run_tests(workspace: Path, instance: dict) -> tuple[int, int, int, str]:
+def matches(node: str, selectors: list[str]) -> bool:
+    for selector in selectors:
+        if node == selector or node.startswith(selector + "::") or node.startswith(selector + "/"):
+            return True
+    return False
+
+
+def run_tests(workspace: Path, instance: dict) -> tuple[int, int, int, str, dict]:
     venv = workspace / ".venv" / "bin" / "python"
     python = str(venv) if venv.is_file() else "python3"
-    tests = as_list(instance.get("fail_to_pass")) + as_list(instance.get("pass_to_pass"))
+    fail_to_pass = as_list(instance.get("fail_to_pass"))
+    pass_to_pass = as_list(instance.get("pass_to_pass"))
+    tests = fail_to_pass + pass_to_pass
     if not tests:
         tests = as_list(instance.get("selected_test_files_to_run"))
     env = {k: v for k, v in __import__("os").environ.items() if k not in {"PYTHONPATH", "PYTHONHOME"}}
-    completed = subprocess.run([python, "-m", "pytest", *tests, "-q", "--tb=no"], cwd=workspace,
+    completed = subprocess.run([python, "-m", "pytest", *tests, "-q", "--tb=no", "-rA"], cwd=workspace,
                                capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=1800, env=env)
     output = (completed.stdout or "") + "\n" + (completed.stderr or "")
     passed = int(PASSED.search(output).group(1)) if PASSED.search(output) else 0
@@ -71,7 +85,31 @@ def run_tests(workspace: Path, instance: dict) -> tuple[int, int, int, str]:
     errors = int(ERRORS.search(output).group(1)) if ERRORS.search(output) else 0
     if completed.returncode != 0 and passed == 0 and failed == 0 and errors == 0:
         errors = 1
-    return passed, failed, errors, output[-3000:]
+    outcomes = {node: status for status, node in RESULT.findall(output)}
+    f2p_total = f2p_passed = 0
+    p2p_total = p2p_passed = 0
+    for node, status in outcomes.items():
+        if matches(node, fail_to_pass):
+            f2p_total += 1
+            f2p_passed += 1 if status == "PASSED" else 0
+        elif matches(node, pass_to_pass):
+            p2p_total += 1
+            p2p_passed += 1 if status == "PASSED" else 0
+    resolved = (
+        len(fail_to_pass) > 0
+        and f2p_total > 0
+        and f2p_passed == f2p_total
+        and p2p_total > 0
+        and p2p_passed == p2p_total
+        and completed.returncode == 0
+    )
+    return passed, failed, errors, output[-3000:], {
+        "resolved": resolved,
+        "f2p_total": f2p_total,
+        "f2p_passed": f2p_passed,
+        "p2p_total": p2p_total,
+        "p2p_passed": p2p_passed,
+    }
 
 
 def main() -> int:
@@ -91,20 +129,23 @@ def main() -> int:
     for directory in directories:
         workspace = find_workspace(directory)
         if workspace is None:
-            rows.append({"name": directory.name, "status": "missing-workspace", "score": 0.0, "passed": 0, "failed": 0, "errors": 0})
+            rows.append({"name": directory.name, "status": "missing-workspace", "score": 0.0, "passed": 0, "failed": 0, "errors": 0, "resolved": False})
             continue
         try:
-            apply_hidden_tests(workspace)
-            passed, failed, errors, detail = run_tests(workspace, instance)
+            patch_error = apply_hidden_tests(workspace)
+            if patch_error:
+                rows.append({"name": directory.name, "status": "hidden-test-failed", "score": 0.0, "passed": 0, "failed": 0, "errors": 1, "resolved": False, "detail": patch_error})
+                continue
+            passed, failed, errors, detail, extra = run_tests(workspace, instance)
             total = passed + failed + errors
             score = 100.0 * passed / total if total else 0.0
-            rows.append({"name": directory.name, "status": "ok", "score": round(score, 2), "passed": passed, "failed": failed, "errors": errors, "detail": detail})
+            rows.append({"name": directory.name, "status": "ok", "score": round(score, 2), "passed": passed, "failed": failed, "errors": errors, "detail": detail, **extra})
         except Exception as error:
-            rows.append({"name": directory.name, "status": "error", "score": 0.0, "passed": 0, "failed": 0, "errors": 1, "detail": str(error)})
+            rows.append({"name": directory.name, "status": "error", "score": 0.0, "passed": 0, "failed": 0, "errors": 1, "resolved": False, "detail": str(error)})
     rows.sort(key=lambda row: (-row["score"], row["name"]))
     payload = {"generatedAt": time.strftime("%Y-%m-%dT%H:%M:%S%z"), "benchmark": "swe-bench-pro", "instance": instance.get("instance_id"), "results": rows}
     (out / "summary.json").write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
-    lines = ["# SWE-Bench Pro · 单任务排行", "", "- 实例：%s" % instance.get("instance_id", ""), "- 指标：隐藏 fail-to-pass + pass-to-pass 测试通过率", "", "| 名次 | 名称 | 得分 | 通过 | 失败 | 错误 | 状态 |", "|---:|---|---:|---:|---:|---:|---|"]
+    lines = ["# SWE-Bench Pro · 单任务排行", "", "- 实例：%s" % instance.get("instance_id", ""), "- 指标：隐藏 fail-to-pass + pass-to-pass 节点全部通过才 resolved", "", "| 名次 | 名称 | 得分 | 通过 | 失败 | 错误 | 状态 |", "|---:|---|---:|---:|---:|---:|---|"]
     for rank, row in enumerate(rows, 1):
         lines.append("| %d | %s | %.1f | %d | %d | %d | %s |" % (rank, row["name"], row["score"], row["passed"], row["failed"], row["errors"], row["status"]))
     (out / "summary.md").write_text("\n".join(lines) + "\n", encoding="utf-8")

package/eval/legacy/tank.py

@@ -258,8 +258,8 @@ def main() -> int:
     if not directories:
         print("no submissions found under " + str(submissions_root), file=sys.stderr)
         return 2
-    prices_path = ROOT / "prices.json"
-    prices = load_json(prices_path) or {}
+    prices_path = ROOT / "tank-prices.json"
+    prices = load_json(prices_path) or load_json(ROOT / "prices.json") or {}
 
     participants: list[Participant] = [Participant("baseline", workspace / "baseline.cpp", "baseline")]
     participants += [load_submission(directory) for directory in directories]

package/eval/legacy/warehouse.py

@@ -2,9 +2,9 @@
 """Independent evaluation for dsh-eval-guard warehouse submissions.
 
 Reads submissions/<name>/{src/*.cpp,run/,cost.json}, compiles with the fixed
-g++ flags, compares every public workload byte-for-byte with the expected
-output, measures CPU time against the baseline, and writes a leaderboard plus
-cost/violation/static-scan reports.
+g++ flags, compares every public workload byte-for-byte (CRLF normalized) with
+the expected output, measures CPU time against the baseline, and writes a
+leaderboard plus cost/violation/static-scan reports.
 
 The evaluator is independent of the DSH plugin: copy the run/ directory from
 $DSH_HOME/eval-guard/runs/<run-id>/ next to src/.
@@ -220,7 +220,7 @@ def main() -> int:
     out_dir = Path(args.out).resolve() if args.out else ROOT / "results" / time.strftime("%Y%m%d-%H%M%S")
     out_dir.mkdir(parents=True, exist_ok=True)
     build_dir = out_dir / "build"
-    prices = load_json(ROOT / "prices.json") or {}
+    prices = load_json(ROOT / "warehouse-prices.json") or load_json(ROOT / "prices.json") or {}
 
     print("tier=%s workloads=%s repeats=%d" % (args.tier, ",".join(names), repeats))
     baseline = Participant("baseline", baseline_src, "baseline")

package/eval/neural-cpp.py

@@ -15,15 +15,12 @@ import sys
 import tempfile
 from pathlib import Path
 
-from common import EVAL_ROOT, clean_copy, emit, fail, parse_workspace
+from common import EVAL_ROOT, child_env, clean_copy, emit, fail, parse_workspace
 
 
 def run(cmd: list[str], cwd: Path, timeout: int = 600) -> subprocess.CompletedProcess[str]:
-    env = dict(os.environ)
-    env.pop('PYTHONPATH', None)
-    env.pop('PYTHONHOME', None)
     return subprocess.run(cmd, cwd=str(cwd), capture_output=True, text=True,
-                          encoding='utf-8', errors='replace', env=env, timeout=timeout)
+                          encoding='utf-8', errors='replace', env=child_env(), timeout=timeout)
 
 
 def main() -> int:
@@ -31,6 +28,9 @@ def main() -> int:
     hidden = EVAL_ROOT.parent / 'assets' / 'neural-cpp' / 'hidden' / 'test_neural.cpp'
     if not hidden.is_file():
         return fail({'task': 'neural-cpp', 'ok': False, 'error': 'hidden test missing'})
+    # Snapshot the hidden test BEFORE the candidate build runs: an agent-authored
+    # CMakeLists can otherwise overwrite the asset and substitute a passing test.
+    hidden_source = hidden.read_bytes()
 
     with tempfile.TemporaryDirectory(prefix='neural-cpp-eval-') as td:
         root = Path(td)
@@ -67,9 +67,12 @@ def main() -> int:
         if not lib.is_file():
             return fail({'task': 'neural-cpp', 'ok': False, 'error': 'build/libnn.a missing'})
 
+        # Compile the in-memory snapshot, not the on-disk asset the build could edit.
+        hidden_copy = root / 'test_neural.cpp'
+        hidden_copy.write_bytes(hidden_source)
         exe = root / 'test_neural'
         cc = run([compiler, '-std=c++17', '-O3', '-pthread',
-                  '-I', str(source / 'include'), str(hidden), str(lib),
+                  '-I', str(source / 'include'), str(hidden_copy), str(lib),
                   '-lm', '-o', str(exe)], root, 300)
         if cc.returncode != 0:
             return fail({'task': 'neural-cpp', 'ok': False, 'error': 'hidden test compile failed',

package/eval/neural-fp32.py

@@ -20,7 +20,7 @@ import numpy as np
 import torch
 import torch.nn.functional as F
 
-from common import EVAL_ROOT, clean_copy, emit, fail, parse_workspace
+from common import EVAL_ROOT, child_env, clean_copy, emit, fail, parse_workspace
 
 torch.set_num_threads(1)
 torch.set_float32_matmul_precision('highest')
@@ -29,6 +29,9 @@ torch.backends.mkldnn.deterministic = True
 torch.manual_seed(1234)
 
 SEEDS = (11, 22, 33)
+# A single CLI invocation budget and a total budget for the whole oracle run.
+INVOCATION_TIMEOUT_SECONDS = 60
+OPS_DEADLINE_SECONDS = 1800
 
 
 def write_tensor(path: Path, t: torch.Tensor) -> None:
@@ -65,11 +68,12 @@ def build_candidate(workspace: Path, root: Path) -> Path:
         shutil.rmtree(build, ignore_errors=True)
     build.mkdir(parents=True)
     cfg = subprocess.run(['cmake', '-S', '.', '-B', 'build', '-DCMAKE_BUILD_TYPE=Release'],
-                         cwd=str(source), capture_output=True, text=True, encoding='utf-8', errors='replace', timeout=180)
+                         cwd=str(source), capture_output=True, text=True, encoding='utf-8', errors='replace',
+                         timeout=180, env=child_env())
     if cfg.returncode != 0:
         raise RuntimeError('cmake configure failed\n' + (cfg.stdout + cfg.stderr)[-4000:])
     made = subprocess.run(['cmake', '--build', 'build', '-j4'], cwd=str(source), capture_output=True,
-                          text=True, encoding='utf-8', errors='replace', timeout=1800)
+                          text=True, encoding='utf-8', errors='replace', timeout=1800, env=child_env())
     if made.returncode != 0:
         raise RuntimeError('build failed\n' + (made.stdout + made.stderr)[-4000:])
     binary = build / 'nn_task'
@@ -87,7 +91,8 @@ class Case:
         self.flops = flops
 
 
-def run_operation(name: str, threshold: float, cases: list[Case], binary: Path, root: Path) -> dict:
+def run_operation(name: str, threshold: float, cases: list[Case], binary: Path, root: Path,
+                  invocation_timeout: float = INVOCATION_TIMEOUT_SECONDS) -> dict:
     op_dir = root / 'ops' / name
     op_dir.mkdir(parents=True, exist_ok=True)
     executable = True
@@ -111,7 +116,8 @@ def run_operation(name: str, threshold: float, cases: list[Case], binary: Path,
         started = time.perf_counter()
         try:
             proc = subprocess.run(cmd, cwd=str(case_dir), capture_output=True, text=True,
-                                  encoding='utf-8', errors='replace', timeout=300)
+                                  encoding='utf-8', errors='replace', timeout=invocation_timeout,
+                                  env=child_env())
         except subprocess.TimeoutExpired:
             executable = False
             first_error = first_error or 'timeout'
@@ -459,7 +465,13 @@ def main() -> int:
         except Exception as exc:
             return fail({'task': 'neural-fp32', 'ok': False, 'error': str(exc)})
         rows = []
+        deadline = time.monotonic() + OPS_DEADLINE_SECONDS
         for name, threshold, builder in OPERATIONS:
+            if time.monotonic() > deadline:
+                rows.append({'name': name, 'executable': False, 'passed': False,
+                             'weighted_mean_relative_error': None, 'threshold': threshold,
+                             'speed_gflops': None, 'error': 'evaluation deadline exceeded'})
+                continue
             try:
                 cases = builder()
                 row = run_operation(name, threshold, cases, binary, root)

package/eval/swe-bench-pro.py

@@ -3,9 +3,12 @@
 from __future__ import annotations
 import os, tempfile
 from pathlib import Path
-from common import EVAL_ROOT, candidate_row, clean_copy, emit, fail, parse_workspace, run_legacy
+from common import EVAL_ROOT, candidate_row, clean_copy, emit, fail, parse_workspace, restore_baseline, run_legacy, verify_venv
 
 workspace = parse_workspace()
+problem = verify_venv(workspace)
+if problem is not None:
+    raise SystemExit(fail({'task': 'swe-bench-pro', 'ok': False, 'error': problem}))
 with tempfile.TemporaryDirectory(prefix='swe-pro-eval-') as td:
     temp = Path(td)
     candidate = temp / 'candidate' / 'workspace'
@@ -13,13 +16,14 @@ with tempfile.TemporaryDirectory(prefix='swe-pro-eval-') as td:
     clean_copy(workspace, candidate, ignore_venv=True)
     venv = workspace / '.venv'
     if venv.is_dir():
-        os.symlink(venv, candidate / '.venv')
+        try:
+            os.symlink(venv, candidate / '.venv')
+        except FileExistsError:
+            pass
+    restore_baseline(candidate)
     result = run_legacy(EVAL_ROOT / 'legacy' / 'swe_bench_pro.py', ['--submissions', str(temp)], temp / 'out')
     if not result.get('ok'):
         raise SystemExit(fail({'task': 'swe-bench-pro', 'ok': False, 'error': result.get('error', 'evaluation failed')}))
     row = candidate_row(result['summary'])
-    passed = row.get('passed', 0)
-    failed = row.get('failed', 0)
-    errors = row.get('errors', 0)
-    resolved = failed == 0 and errors == 0 and passed > 0
-    emit({'task': 'swe-bench-pro', 'ok': True, 'score': 100.0 if resolved else 0.0, 'resolved': resolved, 'passRate': row.get('score', 0), 'passed': passed, 'failed': failed, 'errors': errors, 'status': row.get('status')})
+    resolved = row.get('resolved') is True
+    emit({'task': 'swe-bench-pro', 'ok': True, 'score': 100.0 if resolved else 0.0, 'resolved': resolved, 'passRate': row.get('score', 0), 'passed': row.get('passed', 0), 'failed': row.get('failed', 0), 'errors': row.get('errors', 0), 'f2pPassed': row.get('f2p_passed', 0), 'f2pTotal': row.get('f2p_total', 0), 'status': row.get('status')})

package/eval/tank.py

@@ -16,5 +16,14 @@ with tempfile.TemporaryDirectory(prefix='tank-eval-') as td:
     result = run_legacy(EVAL_ROOT / 'legacy' / 'tank.py', ['--submissions', str(temp), '--arena', str(EVAL_ROOT / 'arena' / 'tank'), '--tier', 'quick'], temp / 'out')
     if not result.get('ok'):
         raise SystemExit(fail({'task': 'tank', 'ok': False, 'error': result.get('error', 'evaluation failed')}))
-    row = candidate_row(result['summary'])
-    emit({'task': 'tank', 'ok': True, 'score': row.get('score', row.get('rating', row.get('theta', 0))), 'rating': row.get('rating', row.get('theta', 0)), 'wins': row.get('wins', 0), 'losses': row.get('losses', 0), 'draws': row.get('draws', 0)})
+    summary = result['summary']
+    row = candidate_row(summary)
+    if not row:
+        raise SystemExit(fail({'task': 'tank', 'ok': False, 'error': 'candidate row missing from evaluator summary'}))
+    wins = draws = losses = 0
+    for pair in summary.get('pairwise') or []:
+        if pair.get('a') == 'candidate':
+            wins += int(pair.get('wins', 0)); draws += int(pair.get('draws', 0)); losses += int(pair.get('losses', 0))
+        elif pair.get('b') == 'candidate':
+            wins += int(pair.get('losses', 0)); draws += int(pair.get('draws', 0)); losses += int(pair.get('wins', 0))
+    emit({'task': 'tank', 'ok': True, 'score': row.get('rating', 0), 'rating': row.get('rating', 0), 'wins': wins, 'losses': losses, 'draws': draws, 'games': row.get('games', 0)})

package/eval/warehouse.py

@@ -17,4 +17,20 @@ with tempfile.TemporaryDirectory(prefix='warehouse-eval-') as td:
     if not result.get('ok'):
         raise SystemExit(fail({'task': 'warehouse', 'ok': False, 'error': result.get('error', 'evaluation failed')}))
     row = candidate_row(result['summary'])
-    emit({'task': 'warehouse', 'ok': True, 'score': row.get('speedup', 0), 'speedup': row.get('speedup', 0), 'correct': row.get('correct'), 'status': row.get('status')})
+    if not row:
+        raise SystemExit(fail({'task': 'warehouse', 'ok': False, 'error': 'candidate row missing from evaluator summary'}))
+    compile_error = row.get('compileError') or ''
+    correct = row.get('correct', 0)
+    total = row.get('total', 0)
+    status = 'compile-failed' if compile_error else ('ok' if correct == total and total else 'incorrect')
+    emit({
+        'task': 'warehouse',
+        'ok': True,
+        'score': row.get('score', 0),
+        'speedup': row.get('geomeanSpeedup', 0),
+        'geomeanSpeedup': row.get('geomeanSpeedup', 0),
+        'correct': correct,
+        'total': total,
+        'status': status,
+        'error': compile_error,
+    })