dsh-eval-guard 0.2.3 → 0.3.0

package/eval/arena/arena/schedule.py

@@ -0,0 +1,79 @@
+"""赛程生成：任意参赛人数、任意单局人数。
+
+每张地图上，把参赛程序分成若干组，每组在同一张地图上把座位完整轮换一圈
+（k 人组打 k 局），所以组内的座位优势完全抵消。分组时优先选出场少、
+彼此交手少的程序，使出场次数最多相差 1、两两交手次数尽量接近。
+"""
+from __future__ import annotations
+
+import itertools
+import random
+from collections import Counter
+
+
+def groups_for_map(players, k, appear, meet, rng):
+    """为一张地图选出若干 k 人组，覆盖所有程序至少一次。"""
+    if k == len(players):
+        order = players[:]
+        rng.shuffle(order)
+        return [order]
+    remaining = set(players)
+    groups = []
+    while remaining:
+        group = []
+        pool = sorted(remaining, key=lambda p: (appear[p], rng.random()))
+        while len(group) < k:
+            if not pool:  # 最后一组不满时，从已分组的程序里补
+                pool = sorted((p for p in players if p not in group),
+                              key=lambda p: (appear[p], rng.random()))
+            best = min(pool, key=lambda p: (appear[p], sum(meet[frozenset((p, q))] for q in group), rng.random()))
+            group.append(best)
+            pool.remove(best)
+            remaining.discard(best)
+        for a, b in itertools.combinations(group, 2):
+            meet[frozenset((a, b))] += k
+        for p in group:
+            appear[p] += k
+        rng.shuffle(group)
+        groups.append(group)
+    return groups
+
+
+def make_schedule(players, sizes, maps_per_size, seed):
+    """返回局列表：{id, size, map_seed, map_index, group, seats}。seats[i] 是第 i 号位的程序名。"""
+    rng = random.Random(seed)
+    used = set()
+    games = []
+    for k in sizes:
+        if not 2 <= k <= len(players):
+            raise ValueError(f"单局人数 {k} 超出范围")
+        appear, meet = Counter(), Counter()
+        for m in range(maps_per_size):
+            map_seed = rng.randrange(1, 2**31)
+            while map_seed in used:
+                map_seed = rng.randrange(1, 2**31)
+            used.add(map_seed)
+            for g, group in enumerate(groups_for_map(players, k, appear, meet, rng)):
+                for shift in range(k):
+                    games.append({"size": k, "map_seed": map_seed, "map_index": m, "group": g,
+                                  "seats": group[shift:] + group[:shift]})
+    for i, game in enumerate(games, 1):
+        game["id"] = f"{game['size']:02d}-{game['map_index']:03d}-{game['group']:02d}-{i:05d}"
+    return games
+
+
+def audit(games, players):
+    """按人数统计出场、座位、交手是否均衡。"""
+    report = {}
+    for k in sorted({g["size"] for g in games}):
+        rows = [g for g in games if g["size"] == k]
+        appear = Counter(p for g in rows for p in g["seats"])
+        seat = Counter((p, i) for g in rows for i, p in enumerate(g["seats"]))
+        meet = Counter(frozenset(pair) for g in rows for pair in itertools.combinations(g["seats"], 2))
+        pairs = [meet[frozenset(pair)] for pair in itertools.combinations(players, 2)]
+        report[k] = {"games": len(rows), "maps": len({g["map_seed"] for g in rows}),
+                     "appear_min": min(appear[p] for p in players), "appear_max": max(appear[p] for p in players),
+                     "seat_min": min(seat[p, i] for p in players for i in range(k) if appear[p]),
+                     "seat_max": max(seat.values()),
+                     "meet_min": min(pairs), "meet_max": max(pairs)}
+    return report

package/eval/arena/arena/tournament.py

@@ -0,0 +1,183 @@
+"""正式赛：定赛程 → 跑比赛（可中断续跑）→ 出成绩。
+
+  python tournament.py plan  赛事配置.json  赛事目录
+  python tournament.py run   赛事目录 --workers 4
+  python tournament.py rate  赛事目录
+
+赛事配置（JSON）：
+  game           游戏引擎文件（提供 Game 类）
+  anchor         基准的名字，实力值以它为 0
+  players        {名字: 程序路径}
+  sizes          要打的单局人数列表
+  maps_per_size  每种人数用几张地图
+  seed           赛程随机种子（可选；不填则随机，写入赛程后冻结）
+  replays        是否保存全部回放（默认 true，gzip 压缩）
+  build_dir      编译结果放在哪里（可选；默认是赛事目录下的 build/）
+  copies         每位选手（含基准）复制几份上场（可选，默认 1）。单局人数超过选手数时用，
+                 复制品名为 名字#2、名字#3……，计分时合并，复制品之间不比较
+
+rate 可以一次读多个赛事目录合并计分，比如 2–6 方和补跑的 7–15 方：
+  python tournament.py rate 赛事目录A 赛事目录B
+成绩写在第一个目录里。
+"""
+from __future__ import annotations
+
+import argparse
+import gzip
+import importlib.util
+import json
+import os
+import secrets
+import sys
+import time
+from concurrent.futures import ProcessPoolExecutor, as_completed
+from pathlib import Path
+
+HERE = Path(__file__).resolve().parent
+sys.path.insert(0, str(HERE))
+import bt  # noqa: E402
+import core  # noqa: E402
+import schedule  # noqa: E402
+
+_GAMES = {}
+
+
+def load_game(path):
+    if path not in _GAMES:
+        spec = importlib.util.spec_from_file_location(f"game_{len(_GAMES)}", path)
+        module = importlib.util.module_from_spec(spec)
+        sys.path.insert(0, str(Path(path).parent))
+        spec.loader.exec_module(module)
+        _GAMES[path] = module.Game
+    return _GAMES[path]
+
+
+def plan(config_path, out):
+    out = Path(out)
+    if (out / "赛程.json").exists():
+        raise SystemExit("赛程已存在，不覆盖。要重排请换一个赛事目录。")
+    cfg = json.loads(Path(config_path).read_text(encoding="utf-8-sig"))  # 兼容记事本存的带 BOM 文件
+    base = Path(config_path).resolve().parent
+    players = {n: str((base / p).resolve()) for n, p in cfg["players"].items()}
+    copies = cfg.get("copies", 1)   # 人数超过选手数时，每位选手复制几份上场
+    players.update({f"{n}#{k}": p for n, p in list(players.items()) for k in range(2, copies + 1)})
+    for n, p in players.items():
+        if not Path(p).is_file():
+            raise SystemExit(f"{n} 的程序不存在：{p}")
+    seed = cfg.get("seed", secrets.randbits(48))
+    games = schedule.make_schedule(sorted(players), cfg["sizes"], cfg["maps_per_size"], seed)
+    doc = {"game": str((base / cfg["game"]).resolve()), "anchor": cfg["anchor"], "players": players,
+           "sizes": cfg["sizes"], "maps_per_size": cfg["maps_per_size"], "seed": seed,
+           "replays": cfg.get("replays", True), "build_dir": cfg.get("build_dir"), "audit": schedule.audit(games, sorted(players)), "games": games}
+    out.mkdir(parents=True, exist_ok=True)
+    (out / "赛程.json").write_text(json.dumps(doc, ensure_ascii=False, indent=1), encoding="utf-8")
+    total = len(games)
+    print(f"已冻结赛程：{total} 局")
+    for k, a in doc["audit"].items():
+        print(f"  {k} 人局 {a['games']} 局，每人出场 {a['appear_min']}–{a['appear_max']}，两两交手 {a['meet_min']}–{a['meet_max']}")
+
+
+def play(game_path, commands, g, replay_dir):
+    Game = load_game(game_path)
+    t = time.time()
+    result, replay = core.run_match(Game, [commands[n] for n in g["seats"]], g["seats"], g["map_seed"],
+                                    record=replay_dir is not None)
+    if replay is not None:
+        path = Path(replay_dir) / f"{g['id']}.json.gz"
+        path.parent.mkdir(parents=True, exist_ok=True)
+        with gzip.open(path, "wt", encoding="utf-8") as f:
+            json.dump(replay, f, ensure_ascii=False, separators=(",", ":"))
+    return {"id": g["id"], "size": g["size"], "map_seed": g["map_seed"], "seats": g["seats"],
+            "score": result["score"], "turns": result.get("turns"), "seconds": round(time.time() - t, 2),
+            "faults": {b["name"]: b["dead"] or len(b["faults"]) for b in result["bots"] if b["faults"] or b["dead"]},
+            "max_ms": {b["name"]: b["max_ms"] for b in result["bots"]},
+            "stats": result.get("stats")}
+
+
+def run(out, workers):
+    out = Path(out)
+    doc = json.loads((out / "赛程.json").read_text(encoding="utf-8"))
+    done_file = out / "结果.jsonl"
+    done = set()
+    if done_file.exists():
+        for line in done_file.read_text(encoding="utf-8").splitlines():
+            if line.strip():
+                done.add(json.loads(line)["id"])
+    todo = [g for g in doc["games"] if g["id"] not in done]
+    print(f"共 {len(doc['games'])} 局，已完成 {len(done)}，本次运行 {len(todo)}，并发 {workers}", flush=True)
+    replay_dir = str(out / "回放") if doc["replays"] else None
+    # 开赛前在主进程里统一编译一次
+    build = Path(doc.get("build_dir") or out / "build")
+    commands = {}
+    for name, path in doc["players"].items():
+        try:
+            commands[name] = core.command_for(path, build)
+        except RuntimeError as exc:
+            raise SystemExit(f"{name}：{exc}")
+    print(f"已编译 {len(commands)} 个程序，放在 {build}", flush=True)
+    start = time.time()
+    with ProcessPoolExecutor(workers) as pool, done_file.open("a", encoding="utf-8") as f:
+        futures = [pool.submit(play, doc["game"], commands, g, replay_dir) for g in todo]
+        for i, fut in enumerate(as_completed(futures), 1):
+            row = fut.result()
+            f.write(json.dumps(row, ensure_ascii=False) + "\n")
+            f.flush()
+            if i % 25 == 0 or i == len(todo):
+                rate_ = i / (time.time() - start)
+                print(f"  {len(done) + i}/{len(doc['games'])}  约剩 {(len(todo) - i) / rate_ / 60:.1f} 分钟", flush=True)
+
+
+def rate(outs):
+    outs = [Path(o) for o in outs]
+    out = outs[0]
+    rows, names, anchor = [], set(), None
+    for o in outs:
+        doc = json.loads((o / "赛程.json").read_text(encoding="utf-8"))
+        part = [json.loads(x) for x in (o / "结果.jsonl").read_text(encoding="utf-8").splitlines() if x.strip()]
+        if len(part) < len(doc["games"]):
+            print(f"注意：{o.name} 只完成了 {len(part)}/{len(doc['games'])} 局，以下为中途成绩。")
+        if anchor not in (None, doc["anchor"]):
+            raise SystemExit("合并计分的几个赛事目录，基准必须相同")
+        anchor = doc["anchor"]
+        rows += part
+        names |= {bt.base(n) for n in doc["players"]}
+    names = sorted(names)
+    doc = {"anchor": anchor}
+    ci, per_size = bt.rate(rows, names, doc["anchor"])
+    order = sorted(names, key=lambda n: -ci[n][0])
+    sizes = sorted(per_size)
+    lines = ["| 名次 | 选手 | 实力值 | 95% 区间 | " + " | ".join(f"{k}人" for k in sizes) + " | 故障局 |",
+             "|---:|---|---:|---|" + "---:|" * len(sizes) + "---:|"]
+    faults = {n: sum(1 for r in rows if any(bt.base(f) == n for f in r["faults"])) for n in names}
+    rank = 0
+    for n in order:
+        if n != doc["anchor"]:
+            rank += 1
+        v, lo, hi = ci[n]
+        lines.append(f"| {'—' if n == doc['anchor'] else rank} | {n} | {v:.0f} | {lo:.0f} ~ {hi:.0f} | " +
+                     " | ".join(f"{per_size[k][n]:.0f}" for k in sizes) + f" | {faults[n]} |")
+    text = (f"# 成绩\n\n{len(rows)} 局。实力值以{doc['anchor']}为 0，强 10 倍记 +400；各人数等权；"
+            f"区间按地图重抽样 200 次。\n\n" + "\n".join(lines) + "\n")
+    (out / "成绩.md").write_text(text, encoding="utf-8")
+    (out / "成绩.json").write_text(json.dumps({"ratings": ci, "per_size": per_size}, ensure_ascii=False, indent=1),
+                                   encoding="utf-8")
+    print(text)
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    sub = ap.add_subparsers(dest="cmd", required=True)
+    p = sub.add_parser("plan"); p.add_argument("config"); p.add_argument("out")
+    p = sub.add_parser("run"); p.add_argument("out"); p.add_argument("--workers", type=int, default=4)
+    p = sub.add_parser("rate"); p.add_argument("out", nargs="+")
+    a = ap.parse_args()
+    if a.cmd == "plan":
+        plan(a.config, a.out)
+    elif a.cmd == "run":
+        run(a.out, a.workers)
+    else:
+        rate(a.out)
+
+
+if __name__ == "__main__":
+    main()

package/eval/arena/bomberman/bots/README.md

@@ -0,0 +1,16 @@
+# 各模型的交卷代码
+
+都是模型在比赛中独立写出、交卷后封存的原样代码，主办方没有改动。基准是 `../workspace/baseline.cpp`（选手包里公开的，可修改的练习对手）。
+
+| 文件 | 模型 | 工具 | 正式赛名次 | BT 实力值 | SHA-256（前 16 位） |
+|---|---|---|---:|---:|---|
+| `gpt-6-astra.cpp` | GPT-6 Astra | Codex | 1 | 488.6 | `ffa1355ef2a13874` |
+| `opus-5.5.cpp` | Opus 5.5 | Claude Code | 2 | 321.8 | `5bb8cbb8c99f8d56` |
+| `gpt-6-sol.cpp` | GPT-6 Sol | Codex | 3 | 301.9 | `6cf1e62beeb04a78` |
+| `deepseek-v4.1-flash.cpp` | DeepSeek V4.1 Flash | ZCode | 4 | 270.7 | `893ca913a06e3a36` |
+| `kimi-k3.cpp` | Kimi K3 | ZCode | 5 | 230.4 | `204e89bcc828cd0a` |
+| `qwen-3.8-max.cpp` | Qwen 3.8 Max | ZCode | 6 | 226.5 | `37fa03b4f491362d` |
+| `glm-5.3.cpp` | GLM 5.3 | ZCode | 7 | 175.0 | `62cfd8c25179af31` |
+| `doubao-seed-2.1.cpp` | 豆包 Seed 2.1 | ZCode | 8 | 95.4 | `230cea2edfdaaf24` |
+| `mimo-v2.6-pro.cpp` | MiMo V2.6 Pro | ZCode | 9 | 39.4 | `00cc917e49b93b59` |
+| `../workspace/baseline.cpp` | 基准 | — | — | 0 | `a4ca3e21d2f8a75c` |