dsh-eval-guard 0.2.3 → 0.3.1

package/eval/bomberman.py

@@ -0,0 +1,20 @@
+#!/usr/bin/env python3
+"""Single-workspace tank evaluator wrapper."""
+from __future__ import annotations
+import shutil, tempfile
+from pathlib import Path
+from common import EVAL_ROOT, candidate_row, emit, fail, parse_workspace, run_legacy
+
+workspace = parse_workspace()
+with tempfile.TemporaryDirectory(prefix='bomber-eval-') as td:
+    temp = Path(td)
+    bot = workspace / 'submission' / 'bot.cpp'
+    if not bot.is_file():
+        raise SystemExit(fail({'task': 'bomberman', 'ok': False, 'error': 'missing submission/bot.cpp'}))
+    (temp / 'candidate').mkdir()
+    shutil.copy2(bot, temp / 'candidate' / 'bot.cpp')
+    result = run_legacy(EVAL_ROOT / 'legacy' / 'bomberman.py', ['--submissions', str(temp), '--task', str(EVAL_ROOT / 'arena' / 'bomberman'), '--tier', 'quick'], temp / 'out')
+    if not result.get('ok'):
+        raise SystemExit(fail({'task': 'bomberman', 'ok': False, 'error': result.get('error', 'evaluation failed')}))
+    row = candidate_row(result['summary'])
+    emit({'task': 'bomberman', 'ok': True, 'score': row.get('score', row.get('rating', row.get('theta', 0))), 'rating': row.get('rating', row.get('theta', 0)), 'wins': row.get('wins', 0), 'losses': row.get('losses', 0), 'draws': row.get('draws', 0)})

package/eval/commit0.py

@@ -0,0 +1,21 @@
+#!/usr/bin/env python3
+"""Single-workspace Commit0 evaluator wrapper."""
+from __future__ import annotations
+import os, shutil, tempfile
+from pathlib import Path
+from common import EVAL_ROOT, candidate_row, clean_copy, emit, fail, parse_workspace, run_legacy
+
+workspace = parse_workspace()
+with tempfile.TemporaryDirectory(prefix='commit0-eval-') as td:
+    temp = Path(td)
+    candidate = temp / 'candidate' / 'workspace'
+    candidate.parent.mkdir(parents=True)
+    clean_copy(workspace, candidate, ignore_venv=True)
+    venv = workspace / '.venv'
+    if venv.is_dir():
+        os.symlink(venv, candidate / '.venv')
+    result = run_legacy(EVAL_ROOT / 'legacy' / 'commit0.py', ['--submissions', str(temp)], temp / 'out')
+    if not result.get('ok'):
+        raise SystemExit(fail({'task': 'commit0', 'ok': False, 'error': result.get('error', 'evaluation failed')}))
+    row = candidate_row(result['summary'])
+    emit({'task': 'commit0', 'ok': True, 'score': row.get('score', 0), 'passed': row.get('passed', 0), 'failed': row.get('failed', 0), 'errors': row.get('errors', 0), 'status': row.get('status')})

package/eval/common.py

@@ -0,0 +1,61 @@
+"""Shared helpers for single-workspace evaluator wrappers."""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+EVAL_ROOT = Path(__file__).resolve().parent
+
+
+def parse_workspace() -> Path:
+    parser = argparse.ArgumentParser(description='single-workspace evaluator')
+    parser.add_argument('--workspace', required=True)
+    args = parser.parse_args()
+    return Path(args.workspace).resolve()
+
+
+def emit(payload: dict) -> None:
+    target = os.environ.get('EVAL_RESULT_FILE')
+    if target is None or target == '':
+        print(json.dumps(payload, ensure_ascii=False, indent=2))
+        return
+    Path(target).write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding='utf-8')
+
+
+def clean_copy(source: Path, target: Path, ignore_venv: bool = False) -> None:
+    def ignore(directory: str, names: list[str]) -> set[str]:
+        if ignore_venv and Path(directory) == source:
+            return {name for name in names if name == '.venv'}
+        return set()
+    shutil.copytree(source, target, symlinks=True, ignore=ignore)
+
+
+def run_legacy(script: Path, args: list[str], out: Path) -> dict:
+    cmd = [sys.executable, '-X', 'utf8', str(script), *args, '--out', str(out)]
+    env = {k: v for k, v in os.environ.items() if k not in {'PYTHONPATH', 'PYTHONHOME'}}
+    completed = subprocess.run(cmd, cwd=str(EVAL_ROOT), capture_output=True, text=True, encoding='utf-8', errors='replace', env=env, timeout=1800)
+    if completed.returncode != 0:
+        return {'ok': False, 'error': (completed.stderr or completed.stdout or 'evaluator failed')[-4000:]}
+    summary = out / 'summary.json'
+    if not summary.is_file():
+        return {'ok': False, 'error': 'summary.json missing'}
+    return {'ok': True, 'summary': json.loads(summary.read_text(encoding='utf-8'))}
+
+
+def candidate_row(summary: dict, name: str = 'candidate') -> dict:
+    rows = summary.get('results') or summary.get('participants') or summary.get('rows') or []
+    for row in rows:
+        if row.get('name') == name:
+            return row
+    return {}
+
+
+def fail(payload: dict) -> int:
+    emit(payload)
+    return 1

package/eval/ctf.py

@@ -0,0 +1,20 @@
+#!/usr/bin/env python3
+"""Single-workspace tank evaluator wrapper."""
+from __future__ import annotations
+import shutil, tempfile
+from pathlib import Path
+from common import EVAL_ROOT, candidate_row, emit, fail, parse_workspace, run_legacy
+
+workspace = parse_workspace()
+with tempfile.TemporaryDirectory(prefix='ctf-eval-') as td:
+    temp = Path(td)
+    bot = workspace / 'submission' / 'bot.cpp'
+    if not bot.is_file():
+        raise SystemExit(fail({'task': 'ctf', 'ok': False, 'error': 'missing submission/bot.cpp'}))
+    (temp / 'candidate').mkdir()
+    shutil.copy2(bot, temp / 'candidate' / 'bot.cpp')
+    result = run_legacy(EVAL_ROOT / 'legacy' / 'ctf.py', ['--submissions', str(temp), '--task', str(EVAL_ROOT / 'arena' / 'ctf'), '--tier', 'quick'], temp / 'out')
+    if not result.get('ok'):
+        raise SystemExit(fail({'task': 'ctf', 'ok': False, 'error': result.get('error', 'evaluation failed')}))
+    row = candidate_row(result['summary'])
+    emit({'task': 'ctf', 'ok': True, 'score': row.get('score', row.get('rating', row.get('theta', 0))), 'rating': row.get('rating', row.get('theta', 0)), 'wins': row.get('wins', 0), 'losses': row.get('losses', 0), 'draws': row.get('draws', 0)})

package/eval/featurebench.py

@@ -0,0 +1,25 @@
+#!/usr/bin/env python3
+"""Single-workspace SWE-Bench Pro evaluator wrapper."""
+from __future__ import annotations
+import os, tempfile
+from pathlib import Path
+from common import EVAL_ROOT, candidate_row, clean_copy, emit, fail, parse_workspace, run_legacy
+
+workspace = parse_workspace()
+with tempfile.TemporaryDirectory(prefix='feature-eval-') as td:
+    temp = Path(td)
+    candidate = temp / 'candidate' / 'workspace'
+    candidate.parent.mkdir(parents=True)
+    clean_copy(workspace, candidate, ignore_venv=True)
+    venv = workspace / '.venv'
+    if venv.is_dir():
+        os.symlink(venv, candidate / '.venv')
+    result = run_legacy(EVAL_ROOT / 'legacy' / 'featurebench.py', ['--submissions', str(temp)], temp / 'out')
+    if not result.get('ok'):
+        raise SystemExit(fail({'task': 'featurebench', 'ok': False, 'error': result.get('error', 'evaluation failed')}))
+    row = candidate_row(result['summary'])
+    passed = row.get('passed', 0)
+    failed = row.get('failed', 0)
+    errors = row.get('errors', 0)
+    resolved = failed == 0 and errors == 0 and passed > 0
+    emit({'task': 'featurebench', 'ok': True, 'score': 100.0 if resolved else 0.0, 'resolved': resolved, 'passRate': row.get('score', 0), 'passed': passed, 'failed': failed, 'errors': errors, 'status': row.get('status')})

package/eval/land.py

@@ -0,0 +1,20 @@
+#!/usr/bin/env python3
+"""Single-workspace tank evaluator wrapper."""
+from __future__ import annotations
+import shutil, tempfile
+from pathlib import Path
+from common import EVAL_ROOT, candidate_row, emit, fail, parse_workspace, run_legacy
+
+workspace = parse_workspace()
+with tempfile.TemporaryDirectory(prefix='land-eval-') as td:
+    temp = Path(td)
+    bot = workspace / 'submission' / 'bot.cpp'
+    if not bot.is_file():
+        raise SystemExit(fail({'task': 'land', 'ok': False, 'error': 'missing submission/bot.cpp'}))
+    (temp / 'candidate').mkdir()
+    shutil.copy2(bot, temp / 'candidate' / 'bot.cpp')
+    result = run_legacy(EVAL_ROOT / 'legacy' / 'land.py', ['--submissions', str(temp), '--task', str(EVAL_ROOT / 'arena' / 'land'), '--tier', 'quick'], temp / 'out')
+    if not result.get('ok'):
+        raise SystemExit(fail({'task': 'land', 'ok': False, 'error': result.get('error', 'evaluation failed')}))
+    row = candidate_row(result['summary'])
+    emit({'task': 'land', 'ok': True, 'score': row.get('score', row.get('rating', row.get('theta', 0))), 'rating': row.get('rating', row.get('theta', 0)), 'wins': row.get('wins', 0), 'losses': row.get('losses', 0), 'draws': row.get('draws', 0)})

package/eval/legacy/bomberman-prices.json

@@ -0,0 +1,4 @@
+{
+  "USD": {},
+  "CNY": {}
+}

package/eval/legacy/bomberman.py

@@ -0,0 +1,161 @@
+#!/usr/bin/env python3
+"""Independent evaluation for bomberman bot submissions.
+
+Reads submissions/<name>/{bot.cpp,run/,cost.json}, then runs the official
+tournament/run.py with the baseline anchor and all submissions. A missing
+run/ directory is fine: only the bot is evaluated.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent
+DEFAULT_TASK = ROOT.parent / "meow-ai-arena" / "bomberman"
+TIERS = {"quick": (["2"], 1), "full": (["2", "5", "10"], 1), "formal": (["2", "5", "10"], 3)}
+STATIC_PATTERNS = [
+    (re.compile(r"#\s*include\s*<(fstream|filesystem|unistd\.h|windows\.h|winsock2?\.h|sys/|netinet/|arpa/|dlfcn\.h)>"), "可疑头文件"),
+    (re.compile(r"\b(fopen|freopen|ifstream|ofstream|filesystem::)\b"), "文件 IO"),
+    (re.compile(r"\b(system|popen|execv?p?e?|fork|posix_spawn|CreateProcess)\b"), "启动进程"),
+    (re.compile(r"\b(socket|connect|bind|listen|accept|sendto|recvfrom|getaddrinfo)\b"), "网络调用"),
+    (re.compile(r"\b(std::thread|pthread_create|_beginthread)\b"), "线程"),
+]
+
+
+def load_json(path: Path):
+    try:
+        return json.loads(path.read_text(encoding="utf-8"))
+    except Exception:
+        return None
+
+
+def static_scan(source: Path) -> list:
+    findings = []
+    for number, line in enumerate(source.read_text(encoding="utf-8", errors="replace").splitlines(), 1):
+        for pattern, label in STATIC_PATTERNS:
+            if pattern.search(line):
+                findings.append({"line": number, "kind": label, "text": line.strip()[:160]})
+    return findings
+
+
+def money(directory: Path, run: dict, usage: dict, cost: dict, prices: dict):
+    if isinstance(cost, dict):
+        try:
+            return float(cost.get("total", 0) or 0), "manual", str(cost.get("currency", "USD"))
+        except (TypeError, ValueError):
+            pass
+    if not usage:
+        return 0.0, "missing", "USD"
+    provider = str(run.get("provider") or "")
+    model = str(run.get("model") or "")
+    table = prices.get(provider + "/" + model) or prices.get(model) or {}
+    if not table:
+        return 0.0, "missing", "USD"
+    def rate(name: str) -> float:
+        return float(table.get(name, 0) or 0) / 1_000_000.0
+    total = float(usage.get("inputTokens", 0) or 0) * rate("input")
+    total += float(usage.get("outputTokens", 0) or 0) * rate("output")
+    total += float(usage.get("cacheReadTokens", 0) or 0) * rate("cacheRead")
+    total += float(usage.get("cacheWriteTokens", 0) or 0) * rate("cacheWrite")
+    return total, "tokens", "USD"
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="bomberman-eval")
+    parser.add_argument("--tier", choices=sorted(TIERS), default="quick")
+    parser.add_argument("--submissions", default=str(ROOT / "submissions"))
+    parser.add_argument("--task", default=str(DEFAULT_TASK))
+    parser.add_argument("--with-history", action="store_true")
+    parser.add_argument("--out", default="")
+    parser.add_argument("--workers", type=int, default=2)
+    args = parser.parse_args()
+
+    task = Path(args.task).resolve()
+    workspace = task / "workspace"
+    if not (task / "tournament" / "run.py").is_file():
+        print("bomberman task not found:", task, file=sys.stderr)
+        return 2
+    submissions = Path(args.submissions).resolve()
+    dirs = [p for p in sorted(submissions.iterdir()) if p.is_dir() and (p / "bot.cpp").is_file()] if submissions.is_dir() else []
+    if not dirs:
+        print("no submissions found under", submissions, file=sys.stderr)
+        return 2
+
+    out = Path(args.out).resolve() if args.out else ROOT / "results" / time.strftime("%Y%m%d-%H%M%S")
+    out.mkdir(parents=True, exist_ok=True)
+    run_dir = out / "tournament"
+    names = {}
+    programs = {"Base": str(workspace / "baseline.cpp")}
+    bots = ["Base=" + programs["Base"]]
+    for directory in dirs:
+        name = re.sub(r"[^A-Za-z0-9_.-]+", "-", directory.name).strip("-") or "submission"
+        names[name] = directory
+        programs[name] = str(directory / "bot.cpp")
+        bots.append(name + "=" + programs[name])
+    if args.with_history:
+        for bot in sorted((task / "bots").glob("*.cpp")):
+            history = "history-" + bot.stem
+            programs[history] = str(bot)
+            bots.append(history + "=" + programs[history])
+
+    sizes, seeds = TIERS[args.tier]
+    cmd = [sys.executable, "-X", "utf8", str(task / "tournament" / "run.py"), "--output", str(run_dir),
+           "--bots", *bots, "--sizes", *sizes, "--seeds", str(seeds), "--workers", str(max(1, args.workers))]
+    print("tier=%s sizes=%s seeds=%d bots=%d" % (args.tier, sizes, seeds, len(bots)), flush=True)
+    completed = subprocess.run(cmd, capture_output=True, text=True, encoding="utf-8", errors="replace")
+    if completed.returncode != 0:
+        print(completed.stdout[-4000:], completed.stderr[-4000:], file=sys.stderr)
+        return 2
+    ratings_doc = load_json(run_dir / "bt.json") or {}
+    ratings = ratings_doc.get("ratings", {})
+    prices = load_json(ROOT / "prices.json") or {}
+
+    rows = []
+    for name, program in programs.items():
+        directory = names.get(name)
+        if directory is not None:
+            sub_run = directory / "run"
+            run = load_json(sub_run / "run.json") or {}
+            usage = load_json(sub_run / "usage.json") or {}
+            cost = load_json(directory / "cost.json") or load_json(sub_run / "cost.json")
+            amount, source, currency = money(directory, run, usage, cost, prices)
+            violations = 0
+            if (sub_run / "violations.jsonl").is_file():
+                violations = len([line for line in (sub_run / "violations.jsonl").read_text(encoding="utf-8").splitlines() if line.strip()])
+            scan = static_scan(directory / "bot.cpp")
+            kind = "submission"
+        else:
+            run = {}
+            amount, source, currency = 0.0, "missing", "USD"
+            violations = 0
+            scan = static_scan(Path(program)) if Path(program).suffix == ".cpp" else []
+            kind = "baseline" if name == "Base" else "history"
+        rows.append({
+            "name": name,
+            "kind": kind,
+            "rating": round(float(ratings.get(name, 0) or 0), 2),
+            "cost": round(amount, 4), "costSource": source, "currency": currency,
+            "violations": violations,
+            "static": scan,
+            "run": run,
+        })
+    rows.sort(key=lambda row: (-row["rating"], row["name"]))
+    payload = {"generatedAt": time.strftime("%Y-%m-%dT%H:%M:%S%z"), "tier": args.tier, "sizes": sizes, "seeds": seeds, "results": rows}
+    (out / "summary.json").write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+    lines = ["# bomberman-eval 结果", "", "- 时间：%s" % payload["generatedAt"], "- 档位：%s（人数 %s，种子数 %d）" % (args.tier, ",".join(sizes), seeds), "", "## 排行榜", "", "| 名次 | 名称 | BT 实力值 | 成本 | 价格来源 | 违规 | 静态扫描 |", "|---:|---|---:|---:|---|---:|---|"]
+    for rank, row in enumerate(rows, 1):
+        scan = "未发现" if not row["static"] else "%d 项" % len(row["static"])
+        lines.append("| %d | %s | %+.2f | %.4f %s | %s | %d | %s |" % (rank, row["name"], row["rating"], row["cost"], row["currency"], row["costSource"], row["violations"], scan))
+    lines += ["", "## 说明", "", "- 使用官方 tournament/run.py 与 BT 评分；Base（baseline）为 0 分锚点。", "- 没有 run/ 时仍评测 bot，只是成本/用量显示 missing。", "- 静态扫描只报告，不取消资格。"]
+    (out / "summary.md").write_text("\n".join(lines) + "\n", encoding="utf-8")
+    print("wrote", out / "summary.md")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/eval/legacy/commit0.py

@@ -0,0 +1,91 @@
+#!/usr/bin/env python3
+"""Evaluate Commit0 single-library submissions.
+
+Submission layout:
+  submissions/<name>/workspace/   (copy of the agent workspace, including .venv)
+
+Score is the unit-test pass rate. Missing run/ statistics are skipped.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent
+SUMMARY = re.compile(r"(\d+) passed")
+FAILED = re.compile(r"(\d+) failed")
+ERRORS = re.compile(r"(\d+) error")
+
+
+def load_json(path: Path):
+    try:
+        return json.loads(path.read_text(encoding="utf-8"))
+    except Exception:
+        return None
+
+
+def find_workspace(directory: Path) -> Path | None:
+    for candidate in (directory / "workspace", directory):
+        if (candidate / "tests").is_dir() and (candidate / "TASK.md").is_file():
+            return candidate
+    return None
+
+
+def run_tests(workspace: Path) -> tuple[int, int, int, str]:
+    venv = workspace / ".venv" / "bin" / "python"
+    python = str(venv) if venv.is_file() else "python3"
+    env = {k: v for k, v in __import__("os").environ.items() if k not in {"PYTHONPATH", "PYTHONHOME"}}
+    completed = subprocess.run([python, "-m", "pytest", "tests", "-q", "--tb=no"], cwd=workspace,
+                               capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=1800, env=env)
+    output = (completed.stdout or "") + "\n" + (completed.stderr or "")
+    passed = int(SUMMARY.search(output).group(1)) if SUMMARY.search(output) else 0
+    failed = int(FAILED.search(output).group(1)) if FAILED.search(output) else 0
+    errors = int(ERRORS.search(output).group(1)) if ERRORS.search(output) else 0
+    if completed.returncode != 0 and passed == 0 and failed == 0 and errors == 0:
+        errors = 1
+    return passed, failed, errors, output[-3000:]
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="commit0-eval")
+    parser.add_argument("--submissions", default=str(ROOT / "submissions"))
+    parser.add_argument("--out", default="")
+    args = parser.parse_args()
+    root = Path(args.submissions).resolve()
+    directories = [p for p in sorted(root.iterdir()) if p.is_dir()] if root.is_dir() else []
+    if not directories:
+        print("no submissions under", root, file=sys.stderr)
+        return 2
+    out = Path(args.out).resolve() if args.out else ROOT / "results" / time.strftime("%Y%m%d-%H%M%S")
+    out.mkdir(parents=True, exist_ok=True)
+    rows = []
+    for directory in directories:
+        workspace = find_workspace(directory)
+        if workspace is None:
+            rows.append({"name": directory.name, "status": "missing-workspace", "score": 0.0, "passed": 0, "failed": 0, "errors": 0})
+            continue
+        try:
+            passed, failed, errors, detail = run_tests(workspace)
+            total = passed + failed + errors
+            score = 100.0 * passed / total if total else 0.0
+            rows.append({"name": directory.name, "status": "ok", "score": round(score, 2), "passed": passed, "failed": failed, "errors": errors, "detail": detail})
+        except Exception as error:
+            rows.append({"name": directory.name, "status": "error", "score": 0.0, "passed": 0, "failed": 0, "errors": 1, "detail": str(error)})
+    rows.sort(key=lambda row: (-row["score"], row["name"]))
+    payload = {"generatedAt": time.strftime("%Y-%m-%dT%H:%M:%S%z"), "benchmark": "commit0", "instance": "commit-0/tinydb", "results": rows}
+    (out / "summary.json").write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+    lines = ["# Commit0 · tinydb 排行", "", "- 指标：单元测试通过率（passed / passed+failed+errors）", "", "| 名次 | 名称 | 得分 | 通过 | 失败 | 错误 | 状态 |", "|---:|---|---:|---:|---:|---:|---|"]
+    for rank, row in enumerate(rows, 1):
+        lines.append("| %d | %s | %.1f | %d | %d | %d | %s |" % (rank, row["name"], row["score"], row["passed"], row["failed"], row["errors"], row["status"]))
+    (out / "summary.md").write_text("\n".join(lines) + "\n", encoding="utf-8")
+    print("wrote", out / "summary.md")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/eval/legacy/ctf-prices.json

@@ -0,0 +1,4 @@
+{
+  "USD": {},
+  "CNY": {}
+}

package/eval/legacy/ctf.py

@@ -0,0 +1,178 @@
+#!/usr/bin/env python3
+"""Independent evaluation for ctf (capture the flag) bot submissions.
+
+Reads submissions/<name>/{bot.cpp,run/,cost.json}, then runs the official
+arena plan/run/rate over submissions plus the baseline anchor. A missing
+run/ directory is fine: only the bot is evaluated.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent
+DEFAULT_TASK = ROOT.parent / "meow-ai-arena" / "ctf"
+TIERS = {
+    "quick": ([2, 3], 2, 2),
+    "full": ([2, 3, 5, 8], 3, 4),
+    "formal": ([2, 3, 4, 5, 6, 8, 10, 13], 8, 7),
+}
+STATIC_PATTERNS = [
+    (re.compile(r"#\s*include\s*<(fstream|filesystem|unistd\.h|windows\.h|winsock2?\.h|sys/|netinet/|arpa/|dlfcn\.h)>"), "可疑头文件"),
+    (re.compile(r"\b(fopen|freopen|ifstream|ofstream|filesystem::)\b"), "文件 IO"),
+    (re.compile(r"\b(system|popen|execv?p?e?|fork|posix_spawn|CreateProcess)\b"), "启动进程"),
+    (re.compile(r"\b(socket|connect|bind|listen|accept|sendto|recvfrom|getaddrinfo)\b"), "网络调用"),
+    (re.compile(r"\b(std::thread|pthread_create|_beginthread)\b"), "线程"),
+]
+
+
+def load_json(path: Path):
+    try:
+        return json.loads(path.read_text(encoding="utf-8"))
+    except Exception:
+        return None
+
+
+def static_scan(source: Path) -> list:
+    findings = []
+    for number, line in enumerate(source.read_text(encoding="utf-8", errors="replace").splitlines(), 1):
+        for pattern, label in STATIC_PATTERNS:
+            if pattern.search(line):
+                findings.append({"line": number, "kind": label, "text": line.strip()[:160]})
+    return findings
+
+
+def money(directory: Path, run: dict, usage: dict, cost: dict, prices: dict):
+    if isinstance(cost, dict):
+        try:
+            return float(cost.get("total", 0) or 0), "manual", str(cost.get("currency", "USD"))
+        except (TypeError, ValueError):
+            pass
+    if not usage:
+        return 0.0, "missing", "USD"
+    provider = str(run.get("provider") or "")
+    model = str(run.get("model") or "")
+    table = prices.get(provider + "/" + model) or prices.get(model) or {}
+    if not table:
+        return 0.0, "missing", "USD"
+    def rate(name: str) -> float:
+        return float(table.get(name, 0) or 0) / 1_000_000.0
+    total = float(usage.get("inputTokens", 0) or 0) * rate("input")
+    total += float(usage.get("outputTokens", 0) or 0) * rate("output")
+    total += float(usage.get("cacheReadTokens", 0) or 0) * rate("cacheRead")
+    total += float(usage.get("cacheWriteTokens", 0) or 0) * rate("cacheWrite")
+    return total, "tokens", "USD"
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="ctf-eval")
+    parser.add_argument("--tier", choices=sorted(TIERS), default="quick")
+    parser.add_argument("--submissions", default=str(ROOT / "submissions"))
+    parser.add_argument("--task", default=str(DEFAULT_TASK))
+    parser.add_argument("--with-history", action="store_true")
+    parser.add_argument("--out", default="")
+    parser.add_argument("--workers", type=int, default=2)
+    args = parser.parse_args()
+
+    task = Path(args.task).resolve()
+    workspace = task / "workspace"
+    arena = task.parent / "arena"
+    if not (arena / "tournament.py").is_file() or not (workspace / "engine.py").is_file():
+        print("ctf task not found:", task, file=sys.stderr)
+        return 2
+    submissions = Path(args.submissions).resolve()
+    dirs = [p for p in sorted(submissions.iterdir()) if p.is_dir() and (p / "bot.cpp").is_file()] if submissions.is_dir() else []
+    if not dirs:
+        print("no submissions found under", submissions, file=sys.stderr)
+        return 2
+
+    out = Path(args.out).resolve() if args.out else ROOT / "results" / time.strftime("%Y%m%d-%H%M%S")
+    games_dir = out / "games"
+    out.mkdir(parents=True, exist_ok=True)
+    players = {"Base": str(workspace / "baseline.cpp")}
+    names = {}
+    for directory in dirs:
+        name = re.sub(r"[^A-Za-z0-9_.-]+", "-", directory.name).strip("-") or "submission"
+        names[name] = directory
+        players[name] = str(directory / "bot.cpp")
+    if args.with_history:
+        for bot in sorted((task / "bots").glob("*.cpp")):
+            players["history-" + bot.stem] = str(bot)
+
+    sizes, maps_per_size, copies = TIERS[args.tier]
+    config = {
+        "game": str(workspace / "engine.py"),
+        "anchor": "Base",
+        "players": players,
+        "sizes": sizes,
+        "maps_per_size": maps_per_size,
+        "copies": copies,
+        "replays": False,
+        "build_dir": str(out / "build"),
+        "seed": 20261017,
+    }
+    config_path = out / "config.json"
+    config_path.write_text(json.dumps(config, ensure_ascii=False, indent=1), encoding="utf-8")
+    tournament = str(arena / "tournament.py")
+    steps = [["plan", str(config_path), str(games_dir)], ["run", str(games_dir), "--workers", str(max(1, args.workers))], ["rate", str(games_dir)]]
+    print("tier=%s sizes=%s maps=%d players=%d" % (args.tier, sizes, maps_per_size, len(players)), flush=True)
+    for step in steps:
+        completed = subprocess.run([sys.executable, "-X", "utf8", tournament, *step], capture_output=True, text=True,
+                                   encoding="utf-8", errors="replace")
+        if completed.returncode != 0:
+            print(completed.stdout[-4000:], completed.stderr[-4000:], file=sys.stderr)
+            return 2
+    scores = load_json(games_dir / "成绩.json") or {}
+    raw_ratings = scores.get("ratings", {})
+    ratings = {name: (value[0] if isinstance(value, list) and value else value) for name, value in raw_ratings.items()}
+    prices = load_json(ROOT / "prices.json") or {}
+
+    rows = []
+    for name, program in players.items():
+        directory = names.get(name)
+        if directory is not None:
+            sub_run = directory / "run"
+            run = load_json(sub_run / "run.json") or {}
+            usage = load_json(sub_run / "usage.json") or {}
+            cost = load_json(directory / "cost.json") or load_json(sub_run / "cost.json")
+            amount, source, currency = money(directory, run, usage, cost, prices)
+            violations = 0
+            if (sub_run / "violations.jsonl").is_file():
+                violations = len([line for line in (sub_run / "violations.jsonl").read_text(encoding="utf-8").splitlines() if line.strip()])
+            scan = static_scan(directory / "bot.cpp")
+            kind = "submission"
+        else:
+            run = {}
+            amount, source, currency = 0.0, "missing", "USD"
+            violations = 0
+            scan = static_scan(Path(program)) if Path(program).suffix == ".cpp" else []
+            kind = "baseline" if name == "Base" else "history"
+        rows.append({
+            "name": name,
+            "kind": kind,
+            "rating": round(float(ratings.get(name, 0) or 0), 2),
+            "cost": round(amount, 4), "costSource": source, "currency": currency,
+            "violations": violations,
+            "static": scan,
+            "run": run,
+        })
+    rows.sort(key=lambda row: (-row["rating"], row["name"]))
+    payload = {"generatedAt": time.strftime("%Y-%m-%dT%H:%M:%S%z"), "tier": args.tier, "sizes": sizes, "mapsPerSize": maps_per_size, "results": rows}
+    (out / "summary.json").write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+    lines = ["# ctf-eval 结果", "", "- 时间：%s" % payload["generatedAt"], "- 档位：%s（人数 %s，每档 %d 张图）" % (args.tier, ",".join(map(str, sizes)), maps_per_size), "", "## 排行榜", "", "| 名次 | 名称 | BT 实力值 | 成本 | 价格来源 | 违规 | 静态扫描 |", "|---:|---|---:|---:|---|---:|---|"]
+    for rank, row in enumerate(rows, 1):
+        scan = "未发现" if not row["static"] else "%d 项" % len(row["static"])
+        lines.append("| %d | %s | %+.2f | %.4f %s | %s | %d | %s |" % (rank, row["name"], row["rating"], row["cost"], row["currency"], row["costSource"], row["violations"], scan))
+    lines += ["", "## 说明", "", "- 使用官方 arena/tournament.py 的 plan/run/rate；Base（baseline）为 0 分锚点。", "- 没有 run/ 时仍评测 bot，只是成本/用量显示 missing。", "- 静态扫描只报告，不取消资格。"]
+    (out / "summary.md").write_text("\n".join(lines) + "\n", encoding="utf-8")
+    print("wrote", out / "summary.md")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())