dsh-eval-guard 0.7.2 → 0.7.4

package/eval/common.py

@@ -2,6 +2,7 @@
 from __future__ import annotations
 
 import argparse
+import hashlib
 import json
 import os
 import shutil
@@ -12,6 +13,22 @@ from pathlib import Path
 
 EVAL_ROOT = Path(__file__).resolve().parent
 
+# Variables the host injects for the evaluator itself. They must not leak into
+# the legacy evaluator or into any candidate process it starts (a candidate
+# that learns EVAL_RESULT_FILE could overwrite the result).
+HOST_EVAL_KEYS = frozenset({
+    'EVAL_RESULT_FILE', 'EVAL_JOB_DIR', 'EVAL_INSTALL_TOKEN',
+    'EVAL_BASELINE_DIR', 'EVAL_VENV_MANIFEST',
+})
+
+
+def child_env() -> dict[str, str]:
+    """Environment for trusted legacy evaluators and their candidate children."""
+    return {
+        key: value for key, value in os.environ.items()
+        if key not in HOST_EVAL_KEYS and key not in {'PYTHONPATH', 'PYTHONHOME'}
+    }
+
 
 def parse_workspace() -> Path:
     parser = argparse.ArgumentParser(description='single-workspace evaluator')
@@ -28,18 +45,23 @@ def emit(payload: dict) -> None:
     Path(target).write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding='utf-8')
 
 
-def clean_copy(source: Path, target: Path, ignore_venv: bool = False) -> None:
+def clean_copy(source: Path, target: Path, ignore_venv: bool = False,
+               ignore_names: tuple[str, ...] = ()) -> None:
+    ignored = set(ignore_names)
+
     def ignore(directory: str, names: list[str]) -> set[str]:
-        if ignore_venv and Path(directory) == source:
-            return {name for name in names if name == '.venv'}
+        if Path(directory) == source:
+            skip = set(ignored)
+            if ignore_venv:
+                skip.add('.venv')
+            return {name for name in names if name in skip}
         return set()
     shutil.copytree(source, target, symlinks=True, ignore=ignore)
 
 
 def run_legacy(script: Path, args: list[str], out: Path) -> dict:
     cmd = [sys.executable, '-X', 'utf8', str(script), *args, '--out', str(out)]
-    env = {k: v for k, v in os.environ.items() if k not in {'PYTHONPATH', 'PYTHONHOME'}}
-    completed = subprocess.run(cmd, cwd=str(EVAL_ROOT), capture_output=True, text=True, encoding='utf-8', errors='replace', env=env, timeout=1800)
+    completed = subprocess.run(cmd, cwd=str(EVAL_ROOT), capture_output=True, text=True, encoding='utf-8', errors='replace', env=child_env(), timeout=1800)
     if completed.returncode != 0:
         return {'ok': False, 'error': (completed.stderr or completed.stdout or 'evaluator failed')[-4000:]}
     summary = out / 'summary.json'
@@ -59,3 +81,145 @@ def candidate_row(summary: dict, name: str = 'candidate') -> dict:
 def fail(payload: dict) -> int:
     emit(payload)
     return 1
+
+
+def parse_pytest_results(output: str) -> dict[str, str]:
+    """Map pytest node ids to PASSED/FAILED/ERROR/SKIPPED from -rA output."""
+    import re
+    pattern = re.compile(r'^(PASSED|FAILED|ERROR|SKIPPED|XFAIL|XPASS) (\S+)', re.MULTILINE)
+    results: dict[str, str] = {}
+    for match in pattern.finditer(output):
+        results[match.group(2)] = match.group(1)
+    return results
+
+
+def matches_node(node: str, selector: str) -> bool:
+    """Whether a pytest node id belongs to a file/node selector."""
+    if node == selector:
+        return True
+    return node.startswith(selector + '::') or node.startswith(selector + '/')
+
+
+
+# ---------------------------------------------------------------------------
+# Pristine test material and venv integrity
+# ---------------------------------------------------------------------------
+
+def sha256_file(path: Path) -> str:
+    digest = hashlib.sha256()
+    digest.update(path.read_bytes())
+    return digest.hexdigest()
+
+
+def sha256_tree(root: Path) -> str:
+    digest = hashlib.sha256()
+
+    def walk(current: Path) -> None:
+        for entry in sorted(current.iterdir(), key=lambda item: item.name):
+            digest.update(str(entry.relative_to(root)).encode('utf-8'))
+            if entry.is_dir():
+                walk(entry)
+            elif entry.is_file():
+                digest.update(entry.read_bytes())
+            elif entry.is_symlink():
+                digest.update(b'link:' + os.readlink(entry).encode('utf-8'))
+
+    walk(root)
+    return digest.hexdigest()
+
+
+def baseline_dir() -> Path | None:
+    raw = os.environ.get('EVAL_BASELINE_DIR', '')
+    if raw == '':
+        return None
+    path = Path(raw)
+    return path if path.is_dir() else None
+
+
+def restore_baseline(candidate: Path) -> None:
+    """Overwrite candidate test material with the pristine install-time copy.
+
+    Also removes conftest.py files the agent added (the pristine snapshot keeps
+    the project's own ones), so a workspace hook cannot rewrite pytest results.
+    """
+    baseline = baseline_dir()
+    if baseline is None:
+        return
+    candidate = candidate.resolve()
+    baseline = baseline.resolve()
+    for hook in candidate.rglob('conftest.py'):
+        if hook.is_symlink():
+            hook.unlink()
+            continue
+        try:
+            relative = hook.relative_to(candidate)
+        except ValueError:
+            continue
+        if not (baseline / relative).is_file():
+            hook.unlink()
+    for entry in baseline.iterdir():
+        if entry.name in {'manifest.json', 'venv.json'}:
+            continue
+        destination = candidate / entry.name
+        if destination.exists() or destination.is_symlink():
+            if destination.is_dir() and not destination.is_symlink():
+                shutil.rmtree(destination, ignore_errors=True)
+            else:
+                destination.unlink()
+        if entry.is_dir():
+            shutil.copytree(entry, destination, symlinks=True)
+        else:
+            shutil.copy2(entry, destination)
+
+
+def verify_venv(workspace: Path) -> str | None:
+    """Verify the venv control plane against the install-time fingerprint.
+
+    Returns a human-readable reason when the venv was tampered with, else None.
+    """
+    baseline = baseline_dir()
+    if baseline is None:
+        return None
+    manifest_path = baseline / 'venv.json'
+    if not manifest_path.is_file():
+        return None
+    try:
+        manifest = json.loads(manifest_path.read_text(encoding='utf-8'))
+    except Exception:
+        return 'venv manifest is corrupt'
+    venv = workspace / '.venv'
+    if not venv.is_dir():
+        return 'the evaluator requires the installed .venv; it is missing'
+    expected_python = manifest.get('python')
+    if isinstance(expected_python, str) and expected_python != '':
+        python = venv / 'bin' / 'python'
+        try:
+            actual = str(python.resolve())
+        except OSError:
+            actual = ''
+        if actual != expected_python:
+            return 'the .venv python interpreter was replaced'
+    for relative, expected in (manifest.get('files') or {}).items():
+        target = venv / relative
+        if not target.is_file():
+            return f'the .venv file {relative} is missing'
+        try:
+            actual = sha256_file(target)
+        except OSError as error:
+            return f'cannot read the .venv file {relative}: {error}'
+        if actual != expected:
+            return f'the .venv file {relative} was modified'
+    expected_pytest = manifest.get('pytest')
+    if isinstance(expected_pytest, str) and expected_pytest != '':
+        found = False
+        for site in venv.glob('lib*/python*/site-packages/pytest'):
+            found = True
+            try:
+                actual = sha256_tree(site)
+            except OSError as error:
+                return f'cannot read the venv pytest package: {error}'
+            if actual != expected_pytest:
+                return 'the venv pytest package was modified'
+        if not found:
+            return 'the venv pytest package disappeared'
+    return None

package/eval/ctf.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-"""Single-workspace tank evaluator wrapper."""
+"""Single-workspace ctf evaluator wrapper."""
 from __future__ import annotations
 import shutil, tempfile
 from pathlib import Path
@@ -17,4 +17,6 @@ with tempfile.TemporaryDirectory(prefix='ctf-eval-') as td:
     if not result.get('ok'):
         raise SystemExit(fail({'task': 'ctf', 'ok': False, 'error': result.get('error', 'evaluation failed')}))
     row = candidate_row(result['summary'])
-    emit({'task': 'ctf', 'ok': True, 'score': row.get('score', row.get('rating', row.get('theta', 0))), 'rating': row.get('rating', row.get('theta', 0)), 'wins': row.get('wins', 0), 'losses': row.get('losses', 0), 'draws': row.get('draws', 0)})
+    if not row:
+        raise SystemExit(fail({'task': 'ctf', 'ok': False, 'error': 'candidate row missing from evaluator summary'}))
+    emit({'task': 'ctf', 'ok': True, 'score': row.get('score', row.get('rating', 0)), 'rating': row.get('rating', row.get('theta', 0)), 'theta': row.get('theta', 0)})

package/eval/featurebench.py

@@ -1,11 +1,14 @@
 #!/usr/bin/env python3
-"""Single-workspace SWE-Bench Pro evaluator wrapper."""
+"""Single-workspace FeatureBench evaluator wrapper."""
 from __future__ import annotations
 import os, tempfile
 from pathlib import Path
-from common import EVAL_ROOT, candidate_row, clean_copy, emit, fail, parse_workspace, run_legacy
+from common import EVAL_ROOT, candidate_row, clean_copy, emit, fail, parse_workspace, restore_baseline, run_legacy, verify_venv
 
 workspace = parse_workspace()
+problem = verify_venv(workspace)
+if problem is not None:
+    raise SystemExit(fail({'task': 'featurebench', 'ok': False, 'error': problem}))
 with tempfile.TemporaryDirectory(prefix='feature-eval-') as td:
     temp = Path(td)
     candidate = temp / 'candidate' / 'workspace'
@@ -13,13 +16,14 @@ with tempfile.TemporaryDirectory(prefix='feature-eval-') as td:
     clean_copy(workspace, candidate, ignore_venv=True)
     venv = workspace / '.venv'
     if venv.is_dir():
-        os.symlink(venv, candidate / '.venv')
+        try:
+            os.symlink(venv, candidate / '.venv')
+        except FileExistsError:
+            pass
+    restore_baseline(candidate)
     result = run_legacy(EVAL_ROOT / 'legacy' / 'featurebench.py', ['--submissions', str(temp)], temp / 'out')
     if not result.get('ok'):
         raise SystemExit(fail({'task': 'featurebench', 'ok': False, 'error': result.get('error', 'evaluation failed')}))
     row = candidate_row(result['summary'])
-    passed = row.get('passed', 0)
-    failed = row.get('failed', 0)
-    errors = row.get('errors', 0)
-    resolved = failed == 0 and errors == 0 and passed > 0
-    emit({'task': 'featurebench', 'ok': True, 'score': 100.0 if resolved else 0.0, 'resolved': resolved, 'passRate': row.get('score', 0), 'passed': passed, 'failed': failed, 'errors': errors, 'status': row.get('status')})
+    resolved = row.get('resolved') is True
+    emit({'task': 'featurebench', 'ok': True, 'score': 100.0 if resolved else 0.0, 'resolved': resolved, 'passRate': row.get('score', 0), 'passed': row.get('passed', 0), 'failed': row.get('failed', 0), 'errors': row.get('errors', 0), 'f2pPassed': row.get('f2p_passed', 0), 'f2pTotal': row.get('f2p_total', 0), 'status': row.get('status')})

package/eval/land.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-"""Single-workspace tank evaluator wrapper."""
+"""Single-workspace land evaluator wrapper."""
 from __future__ import annotations
 import shutil, tempfile
 from pathlib import Path
@@ -17,4 +17,6 @@ with tempfile.TemporaryDirectory(prefix='land-eval-') as td:
     if not result.get('ok'):
         raise SystemExit(fail({'task': 'land', 'ok': False, 'error': result.get('error', 'evaluation failed')}))
     row = candidate_row(result['summary'])
-    emit({'task': 'land', 'ok': True, 'score': row.get('score', row.get('rating', row.get('theta', 0))), 'rating': row.get('rating', row.get('theta', 0)), 'wins': row.get('wins', 0), 'losses': row.get('losses', 0), 'draws': row.get('draws', 0)})
+    if not row:
+        raise SystemExit(fail({'task': 'land', 'ok': False, 'error': 'candidate row missing from evaluator summary'}))
+    emit({'task': 'land', 'ok': True, 'score': row.get('score', row.get('rating', 0)), 'rating': row.get('rating', row.get('theta', 0)), 'theta': row.get('theta', 0)})

package/eval/legacy/bomberman.py

@@ -107,13 +107,16 @@ def main() -> int:
     cmd = [sys.executable, "-X", "utf8", str(task / "tournament" / "run.py"), "--output", str(run_dir),
            "--bots", *bots, "--sizes", *sizes, "--seeds", str(seeds), "--workers", str(max(1, args.workers))]
     print("tier=%s sizes=%s seeds=%d bots=%d" % (args.tier, sizes, seeds, len(bots)), flush=True)
-    completed = subprocess.run(cmd, capture_output=True, text=True, encoding="utf-8", errors="replace")
+    completed = subprocess.run(cmd, capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=3600)
     if completed.returncode != 0:
         print(completed.stdout[-4000:], completed.stderr[-4000:], file=sys.stderr)
         return 2
     ratings_doc = load_json(run_dir / "bt.json") or {}
+    if ratings_doc.get("unavailable"):
+        print("bomberman rating unavailable: " + str(ratings_doc.get("unavailable")), file=sys.stderr)
+        return 2
     ratings = ratings_doc.get("ratings", {})
-    prices = load_json(ROOT / "prices.json") or {}
+    prices = load_json(ROOT / "bomberman-prices.json") or load_json(ROOT / "prices.json") or {}
 
     rows = []
     for name, program in programs.items():

package/eval/legacy/ctf.py

@@ -123,14 +123,17 @@ def main() -> int:
     print("tier=%s sizes=%s maps=%d players=%d" % (args.tier, sizes, maps_per_size, len(players)), flush=True)
     for step in steps:
         completed = subprocess.run([sys.executable, "-X", "utf8", tournament, *step], capture_output=True, text=True,
-                                   encoding="utf-8", errors="replace")
+                                   encoding="utf-8", errors="replace", timeout=1800)
         if completed.returncode != 0:
             print(completed.stdout[-4000:], completed.stderr[-4000:], file=sys.stderr)
             return 2
     scores = load_json(games_dir / "成绩.json") or {}
+    if scores.get("unavailable"):
+        print("ctf rating unavailable: " + str(scores.get("unavailable")), file=sys.stderr)
+        return 2
     raw_ratings = scores.get("ratings", {})
     ratings = {name: (value[0] if isinstance(value, list) and value else value) for name, value in raw_ratings.items()}
-    prices = load_json(ROOT / "prices.json") or {}
+    prices = load_json(ROOT / "ctf-prices.json") or load_json(ROOT / "prices.json") or {}
 
     rows = []
     for name, program in players.items():

package/eval/legacy/featurebench.py

@@ -4,8 +4,9 @@
 Submission layout:
   submissions/<name>/workspace/   (copy of the agent workspace, including .venv)
 
-The hidden feature test file is copied from the plugin asset; scoring runs the
-instance's fail-to-pass and pass-to-pass files. Missing run/ statistics are skipped.
+The hidden feature test file is copied from the plugin asset; scoring checks the
+fail-to-pass file and every pass-to-pass file explicitly (a skipped or removed
+test does not count as resolved).
 """
 from __future__ import annotations
 
@@ -23,6 +24,7 @@ ASSET = ROOT.parents[1] / "assets" / "featurebench"
 PASSED = re.compile(r"(\d+) passed")
 FAILED = re.compile(r"(\d+) failed")
 ERRORS = re.compile(r"(\d+) error")
+RESULT = re.compile(r"^(PASSED|FAILED|ERROR|SKIPPED|XFAIL|XPASS) (\S+)", re.MULTILINE)
 
 
 def load_json(path: Path):
@@ -39,18 +41,29 @@ def find_workspace(directory: Path) -> Path | None:
     return None
 
 
-def install_hidden_test(workspace: Path) -> None:
+def install_hidden_test(workspace: Path) -> str:
     source = ASSET / "hidden" / "test_regression.py"
-    if source.is_file():
-        shutil.copy2(source, workspace / "tests" / "test_regression.py")
+    if not source.is_file():
+        return "hidden regression test is missing"
+    shutil.copy2(source, workspace / "tests" / "test_regression.py")
+    return ""
 
 
-def run_tests(workspace: Path, instance: dict) -> tuple[int, int, int, str]:
+def matches(node: str, selectors: list[str]) -> bool:
+    for selector in selectors:
+        if node == selector or node.startswith(selector + "::") or node.startswith(selector + "/"):
+            return True
+    return False
+
+
+def run_tests(workspace: Path, instance: dict) -> tuple[int, int, int, str, dict]:
     venv = workspace / ".venv" / "bin" / "python"
     python = str(venv) if venv.is_file() else "python3"
-    tests = list(instance.get("FAIL_TO_PASS", [])) + list(instance.get("PASS_TO_PASS", []))
+    fail_to_pass = list(instance.get("FAIL_TO_PASS", []))
+    pass_to_pass = list(instance.get("PASS_TO_PASS", []))
+    tests = fail_to_pass + pass_to_pass
     env = {k: v for k, v in __import__("os").environ.items() if k not in {"PYTHONPATH", "PYTHONHOME"}}
-    completed = subprocess.run([python, "-m", "pytest", *tests, "-q", "--tb=no"], cwd=workspace,
+    completed = subprocess.run([python, "-m", "pytest", *tests, "-q", "--tb=no", "-rA"], cwd=workspace,
                                capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=1800, env=env)
     output = (completed.stdout or "") + "\n" + (completed.stderr or "")
     passed = int(PASSED.search(output).group(1)) if PASSED.search(output) else 0
@@ -58,7 +71,31 @@ def run_tests(workspace: Path, instance: dict) -> tuple[int, int, int, str]:
     errors = int(ERRORS.search(output).group(1)) if ERRORS.search(output) else 0
     if completed.returncode != 0 and passed == 0 and failed == 0 and errors == 0:
         errors = 1
-    return passed, failed, errors, output[-3000:]
+    outcomes = {node: status for status, node in RESULT.findall(output)}
+    f2p_total = f2p_passed = 0
+    p2p_total = p2p_passed = 0
+    for node, status in outcomes.items():
+        if matches(node, fail_to_pass):
+            f2p_total += 1
+            f2p_passed += 1 if status == "PASSED" else 0
+        elif matches(node, pass_to_pass):
+            p2p_total += 1
+            p2p_passed += 1 if status == "PASSED" else 0
+    resolved = (
+        len(fail_to_pass) > 0
+        and f2p_total > 0
+        and f2p_passed == f2p_total
+        and p2p_total > 0
+        and p2p_passed == p2p_total
+        and completed.returncode == 0
+    )
+    return passed, failed, errors, output[-3000:], {
+        "resolved": resolved,
+        "f2p_total": f2p_total,
+        "f2p_passed": f2p_passed,
+        "p2p_total": p2p_total,
+        "p2p_passed": p2p_passed,
+    }
 
 
 def main() -> int:
@@ -78,20 +115,23 @@ def main() -> int:
     for directory in directories:
         workspace = find_workspace(directory)
         if workspace is None:
-            rows.append({"name": directory.name, "status": "missing-workspace", "score": 0.0, "passed": 0, "failed": 0, "errors": 0})
+            rows.append({"name": directory.name, "status": "missing-workspace", "score": 0.0, "passed": 0, "failed": 0, "errors": 0, "resolved": False})
             continue
         try:
-            install_hidden_test(workspace)
-            passed, failed, errors, detail = run_tests(workspace, instance)
+            hidden_error = install_hidden_test(workspace)
+            if hidden_error:
+                rows.append({"name": directory.name, "status": "hidden-test-failed", "score": 0.0, "passed": 0, "failed": 0, "errors": 1, "resolved": False, "detail": hidden_error})
+                continue
+            passed, failed, errors, detail, extra = run_tests(workspace, instance)
             total = passed + failed + errors
             score = 100.0 * passed / total if total else 0.0
-            rows.append({"name": directory.name, "status": "ok", "score": round(score, 2), "passed": passed, "failed": failed, "errors": errors, "detail": detail})
+            rows.append({"name": directory.name, "status": "ok", "score": round(score, 2), "passed": passed, "failed": failed, "errors": errors, "detail": detail, **extra})
         except Exception as error:
-            rows.append({"name": directory.name, "status": "error", "score": 0.0, "passed": 0, "failed": 0, "errors": 1, "detail": str(error)})
+            rows.append({"name": directory.name, "status": "error", "score": 0.0, "passed": 0, "failed": 0, "errors": 1, "resolved": False, "detail": str(error)})
     rows.sort(key=lambda row: (-row["score"], row["name"]))
     payload = {"generatedAt": time.strftime("%Y-%m-%dT%H:%M:%S%z"), "benchmark": "featurebench", "instance": instance.get("instance_id"), "results": rows}
     (out / "summary.json").write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
-    lines = ["# FeatureBench · 单 feature 排行", "", "- 实例：%s" % instance.get("instance_id", ""), "- 指标：隐藏 fail-to-pass + pass-to-pass 测试通过率", "", "| 名次 | 名称 | 得分 | 通过 | 失败 | 错误 | 状态 |", "|---:|---|---:|---:|---:|---:|---|"]
+    lines = ["# FeatureBench · 单 feature 排行", "", "- 实例：%s" % instance.get("instance_id", ""), "- 指标：隐藏 feature 测试与全部回归文件通过才 resolved", "", "| 名次 | 名称 | 得分 | 通过 | 失败 | 错误 | 状态 |", "|---:|---|---:|---:|---:|---:|---|"]
     for rank, row in enumerate(rows, 1):
         lines.append("| %d | %s | %.1f | %d | %d | %d | %s |" % (rank, row["name"], row["score"], row["passed"], row["failed"], row["errors"], row["status"]))
     (out / "summary.md").write_text("\n".join(lines) + "\n", encoding="utf-8")

package/eval/legacy/land.py

@@ -112,20 +112,23 @@ def main() -> int:
     if melee == 0:
         cmd += ["--only", "duel"]
     print("tier=%s maps=%d melee=%d entries=%d" % (args.tier, maps, melee, len(list(entries.iterdir()))), flush=True)
-    completed = subprocess.run(cmd, capture_output=True, text=True, encoding="utf-8", errors="replace")
+    completed = subprocess.run(cmd, capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=1800)
     if completed.returncode != 0:
         print(completed.stdout[-4000:], completed.stderr[-4000:], file=sys.stderr)
         return 2
     ratings_prefix = out / "ratings"
     rate = subprocess.run([sys.executable, "-X", "utf8", str(task / "tournament" / "rate.py"), str(results_jsonl),
                            "--entries", str(entries), "--out", str(ratings_prefix)], capture_output=True, text=True,
-                          encoding="utf-8", errors="replace")
+                          encoding="utf-8", errors="replace", timeout=1800)
     if rate.returncode != 0:
         print(rate.stdout[-4000:], rate.stderr[-4000:], file=sys.stderr)
         return 2
     ratings = load_json(Path(str(ratings_prefix) + ".json")) or {}
+    if ratings.get("unavailable"):
+        print("land rating unavailable: " + str(ratings.get("unavailable")), file=sys.stderr)
+        return 2
     models = ratings.get("models", {})
-    prices = load_json(ROOT / "prices.json") or {}
+    prices = load_json(ROOT / "land-prices.json") or load_json(ROOT / "prices.json") or {}
 
     rows = []
     all_names = list(names) + [name for name in models if name not in names]

package/eval/legacy/swe_bench_pro.py

@@ -4,8 +4,9 @@
 Submission layout:
   submissions/<name>/workspace/   (copy of the agent workspace, including .venv)
 
-The hidden test patch is applied from the plugin asset; scoring runs the
-instance's fail-to-pass and pass-to-pass tests. Missing run/ statistics are skipped.
+The hidden test patch is applied from the plugin asset; scoring checks the
+instance's fail-to-pass and pass-to-pass node ids explicitly (a skipped or
+renamed fail-to-pass test does not count as resolved).
 """
 from __future__ import annotations
 
@@ -22,6 +23,7 @@ ASSET = ROOT.parents[1] / "assets" / "swe-bench-pro"
 PASSED = re.compile(r"(\d+) passed")
 FAILED = re.compile(r"(\d+) failed")
 ERRORS = re.compile(r"(\d+) error")
+RESULT = re.compile(r"^(PASSED|FAILED|ERROR|SKIPPED|XFAIL|XPASS) (\S+)", re.MULTILINE)
 
 
 def load_json(path: Path):
@@ -48,22 +50,34 @@ def find_workspace(directory: Path) -> Path | None:
     return None
 
 
-def apply_hidden_tests(workspace: Path) -> None:
+def apply_hidden_tests(workspace: Path) -> str:
     patch = ASSET / "hidden" / "test.patch"
     if not patch.is_file():
-        return
-    subprocess.run(["patch", "-p1", "--forward", "-i", str(patch)], cwd=workspace,
-                   capture_output=True, text=True, encoding="utf-8", errors="replace")
+        return "hidden test patch is missing"
+    completed = subprocess.run(["patch", "-p1", "--forward", "-i", str(patch)], cwd=workspace,
+                               capture_output=True, text=True, encoding="utf-8", errors="replace")
+    if completed.returncode != 0:
+        return "hidden test patch failed to apply: " + (completed.stdout + completed.stderr)[-2000:]
+    return ""
 
 
-def run_tests(workspace: Path, instance: dict) -> tuple[int, int, int, str]:
+def matches(node: str, selectors: list[str]) -> bool:
+    for selector in selectors:
+        if node == selector or node.startswith(selector + "::") or node.startswith(selector + "/"):
+            return True
+    return False
+
+
+def run_tests(workspace: Path, instance: dict) -> tuple[int, int, int, str, dict]:
     venv = workspace / ".venv" / "bin" / "python"
     python = str(venv) if venv.is_file() else "python3"
-    tests = as_list(instance.get("fail_to_pass")) + as_list(instance.get("pass_to_pass"))
+    fail_to_pass = as_list(instance.get("fail_to_pass"))
+    pass_to_pass = as_list(instance.get("pass_to_pass"))
+    tests = fail_to_pass + pass_to_pass
     if not tests:
         tests = as_list(instance.get("selected_test_files_to_run"))
     env = {k: v for k, v in __import__("os").environ.items() if k not in {"PYTHONPATH", "PYTHONHOME"}}
-    completed = subprocess.run([python, "-m", "pytest", *tests, "-q", "--tb=no"], cwd=workspace,
+    completed = subprocess.run([python, "-m", "pytest", *tests, "-q", "--tb=no", "-rA"], cwd=workspace,
                                capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=1800, env=env)
     output = (completed.stdout or "") + "\n" + (completed.stderr or "")
     passed = int(PASSED.search(output).group(1)) if PASSED.search(output) else 0
@@ -71,7 +85,31 @@ def run_tests(workspace: Path, instance: dict) -> tuple[int, int, int, str]:
     errors = int(ERRORS.search(output).group(1)) if ERRORS.search(output) else 0
     if completed.returncode != 0 and passed == 0 and failed == 0 and errors == 0:
         errors = 1
-    return passed, failed, errors, output[-3000:]
+    outcomes = {node: status for status, node in RESULT.findall(output)}
+    f2p_total = f2p_passed = 0
+    p2p_total = p2p_passed = 0
+    for node, status in outcomes.items():
+        if matches(node, fail_to_pass):
+            f2p_total += 1
+            f2p_passed += 1 if status == "PASSED" else 0
+        elif matches(node, pass_to_pass):
+            p2p_total += 1
+            p2p_passed += 1 if status == "PASSED" else 0
+    resolved = (
+        len(fail_to_pass) > 0
+        and f2p_total > 0
+        and f2p_passed == f2p_total
+        and p2p_total > 0
+        and p2p_passed == p2p_total
+        and completed.returncode == 0
+    )
+    return passed, failed, errors, output[-3000:], {
+        "resolved": resolved,
+        "f2p_total": f2p_total,
+        "f2p_passed": f2p_passed,
+        "p2p_total": p2p_total,
+        "p2p_passed": p2p_passed,
+    }
 
 
 def main() -> int:
@@ -91,20 +129,23 @@ def main() -> int:
     for directory in directories:
         workspace = find_workspace(directory)
         if workspace is None:
-            rows.append({"name": directory.name, "status": "missing-workspace", "score": 0.0, "passed": 0, "failed": 0, "errors": 0})
+            rows.append({"name": directory.name, "status": "missing-workspace", "score": 0.0, "passed": 0, "failed": 0, "errors": 0, "resolved": False})
             continue
         try:
-            apply_hidden_tests(workspace)
-            passed, failed, errors, detail = run_tests(workspace, instance)
+            patch_error = apply_hidden_tests(workspace)
+            if patch_error:
+                rows.append({"name": directory.name, "status": "hidden-test-failed", "score": 0.0, "passed": 0, "failed": 0, "errors": 1, "resolved": False, "detail": patch_error})
+                continue
+            passed, failed, errors, detail, extra = run_tests(workspace, instance)
             total = passed + failed + errors
             score = 100.0 * passed / total if total else 0.0
-            rows.append({"name": directory.name, "status": "ok", "score": round(score, 2), "passed": passed, "failed": failed, "errors": errors, "detail": detail})
+            rows.append({"name": directory.name, "status": "ok", "score": round(score, 2), "passed": passed, "failed": failed, "errors": errors, "detail": detail, **extra})
         except Exception as error:
-            rows.append({"name": directory.name, "status": "error", "score": 0.0, "passed": 0, "failed": 0, "errors": 1, "detail": str(error)})
+            rows.append({"name": directory.name, "status": "error", "score": 0.0, "passed": 0, "failed": 0, "errors": 1, "resolved": False, "detail": str(error)})
     rows.sort(key=lambda row: (-row["score"], row["name"]))
     payload = {"generatedAt": time.strftime("%Y-%m-%dT%H:%M:%S%z"), "benchmark": "swe-bench-pro", "instance": instance.get("instance_id"), "results": rows}
     (out / "summary.json").write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
-    lines = ["# SWE-Bench Pro · 单任务排行", "", "- 实例：%s" % instance.get("instance_id", ""), "- 指标：隐藏 fail-to-pass + pass-to-pass 测试通过率", "", "| 名次 | 名称 | 得分 | 通过 | 失败 | 错误 | 状态 |", "|---:|---|---:|---:|---:|---:|---|"]
+    lines = ["# SWE-Bench Pro · 单任务排行", "", "- 实例：%s" % instance.get("instance_id", ""), "- 指标：隐藏 fail-to-pass + pass-to-pass 节点全部通过才 resolved", "", "| 名次 | 名称 | 得分 | 通过 | 失败 | 错误 | 状态 |", "|---:|---|---:|---:|---:|---:|---|"]
     for rank, row in enumerate(rows, 1):
         lines.append("| %d | %s | %.1f | %d | %d | %d | %s |" % (rank, row["name"], row["score"], row["passed"], row["failed"], row["errors"], row["status"]))
     (out / "summary.md").write_text("\n".join(lines) + "\n", encoding="utf-8")

package/eval/legacy/tank.py

@@ -258,8 +258,8 @@ def main() -> int:
     if not directories:
         print("no submissions found under " + str(submissions_root), file=sys.stderr)
         return 2
-    prices_path = ROOT / "prices.json"
-    prices = load_json(prices_path) or {}
+    prices_path = ROOT / "tank-prices.json"
+    prices = load_json(prices_path) or load_json(ROOT / "prices.json") or {}
 
     participants: list[Participant] = [Participant("baseline", workspace / "baseline.cpp", "baseline")]
     participants += [load_submission(directory) for directory in directories]

package/eval/legacy/warehouse.py

@@ -2,9 +2,9 @@
 """Independent evaluation for dsh-eval-guard warehouse submissions.
 
 Reads submissions/<name>/{src/*.cpp,run/,cost.json}, compiles with the fixed
-g++ flags, compares every public workload byte-for-byte with the expected
-output, measures CPU time against the baseline, and writes a leaderboard plus
-cost/violation/static-scan reports.
+g++ flags, compares every public workload byte-for-byte (CRLF normalized) with
+the expected output, measures CPU time against the baseline, and writes a
+leaderboard plus cost/violation/static-scan reports.
 
 The evaluator is independent of the DSH plugin: copy the run/ directory from
 $DSH_HOME/eval-guard/runs/<run-id>/ next to src/.
@@ -220,7 +220,7 @@ def main() -> int:
     out_dir = Path(args.out).resolve() if args.out else ROOT / "results" / time.strftime("%Y%m%d-%H%M%S")
     out_dir.mkdir(parents=True, exist_ok=True)
     build_dir = out_dir / "build"
-    prices = load_json(ROOT / "prices.json") or {}
+    prices = load_json(ROOT / "warehouse-prices.json") or load_json(ROOT / "prices.json") or {}
 
     print("tier=%s workloads=%s repeats=%d" % (args.tier, ",".join(names), repeats))
     baseline = Participant("baseline", baseline_src, "baseline")